From 007f4f7314eabd9cc3a2b0d11889de49ad3c682a Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Sat, 12 Nov 2022 15:12:15 +0300
Subject: [PATCH 01/11] Tests cleaned up

---
 launch.py                |   5 ++++-
 test/server_poll.py      |   7 ++++---
 test/test_files/empty.pt | Bin 0 -> 431 bytes
 test/txt2img_test.py     |   4 +++-
 test/utils_test.py       |  18 +++++++++---------
 5 files changed, 20 insertions(+), 14 deletions(-)
 create mode 100644 test/test_files/empty.pt

diff --git a/launch.py b/launch.py
index 8e65676d3..6822a01de 100644
--- a/launch.py
+++ b/launch.py
@@ -229,6 +229,9 @@ def prepare_enviroment():
 def tests(argv):
     if "--api" not in argv:
         argv.append("--api")
+    if "--ckpt" not in argv:
+        argv.append("--ckpt")
+        argv.append("./test/test_files/empty.pt")
 
     print(f"Launching Web UI in another process for testing with arguments: {' '.join(argv[1:])}")
 
@@ -236,7 +239,7 @@ def tests(argv):
         proc = subprocess.Popen([sys.executable, *argv], stdout=stdout, stderr=stderr)
 
     import test.server_poll
-    test.server_poll.run_tests()
+    test.server_poll.run_tests(proc)
 
     print(f"Stopping Web UI process with id {proc.pid}")
     proc.kill()
diff --git a/test/server_poll.py b/test/server_poll.py
index eeefb7ebc..8e63b4502 100644
--- a/test/server_poll.py
+++ b/test/server_poll.py
@@ -3,7 +3,7 @@ import requests
 import time
 
 
-def run_tests():
+def run_tests(proc):
     timeout_threshold = 240
     start_time = time.time()
     while time.time()-start_time < timeout_threshold:
@@ -11,8 +11,9 @@ def run_tests():
             requests.head("http://localhost:7860/")
             break
         except requests.exceptions.ConnectionError:
-            pass
-    if time.time()-start_time < timeout_threshold:
+            if proc.poll() is not None:
+                break
+    if proc.poll() is None:
         suite = unittest.TestLoader().discover('', pattern='*_test.py')
         result = unittest.TextTestRunner(verbosity=2).run(suite)
     else:
diff --git a/test/test_files/empty.pt b/test/test_files/empty.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c6ac59eb01fcb778290a85f12bdb7867de3dfdd1
GIT binary patch
literal 431
zcmWIWW@cev;NW1u00Im`42ea_8JT6N`YDMeiFyUuIc`pT3{fbcfvL8TK`+3Yo#WF)
zrlV{?Q$RQXr>Xo5ws2F+Qj3Z+^Yh%CEYS=_u>n8Fm<we0#d+zWOh9`<IKZ0`M8W-p
z>@ES21PVa+A-Zm4llf6}h5>mn-B6zdc(bwTKo!X`>%x_T-2>#o=xV6UB`6Kl#|~op
WGC~AERDd@tC?tV;m>59nA!-3{+(-BT

literal 0
HcmV?d00001

diff --git a/test/txt2img_test.py b/test/txt2img_test.py
index 1936e07e2..ce7520858 100644
--- a/test/txt2img_test.py
+++ b/test/txt2img_test.py
@@ -53,13 +53,15 @@ class TestTxt2ImgWorking(unittest.TestCase):
         self.simple_txt2img["restore_faces"] = True
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
 
-    def test_txt2img_with_tiling_faces_performed(self):
+    def test_txt2img_with_tiling_performed(self):
         self.simple_txt2img["tiling"] = True
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
 
     def test_txt2img_with_vanilla_sampler_performed(self):
         self.simple_txt2img["sampler_index"] = "PLMS"
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+        self.simple_txt2img["sampler_index"] = "DDIM"
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
 
     def test_txt2img_multiple_batches_performed(self):
         self.simple_txt2img["n_iter"] = 2
diff --git a/test/utils_test.py b/test/utils_test.py
index 65d3d177d..be9e6bf8c 100644
--- a/test/utils_test.py
+++ b/test/utils_test.py
@@ -18,19 +18,19 @@ class UtilsTests(unittest.TestCase):
   def test_options_get(self):
     self.assertEqual(requests.get(self.url_options).status_code, 200)
 
-  def test_options_write(self):
-    response = requests.get(self.url_options)
-    self.assertEqual(response.status_code, 200)
+  # def test_options_write(self):
+  #   response = requests.get(self.url_options)
+  #   self.assertEqual(response.status_code, 200)
     
-    pre_value = response.json()["send_seed"]
+  #   pre_value = response.json()["send_seed"]
 
-    self.assertEqual(requests.post(self.url_options, json={"send_seed":not pre_value}).status_code, 200)
+  #   self.assertEqual(requests.post(self.url_options, json={"send_seed":not pre_value}).status_code, 200)
 
-    response = requests.get(self.url_options)
-    self.assertEqual(response.status_code, 200)
-    self.assertEqual(response.json()["send_seed"], not pre_value)
+  #   response = requests.get(self.url_options)
+  #   self.assertEqual(response.status_code, 200)
+  #   self.assertEqual(response.json()["send_seed"], not pre_value)
 
-    requests.post(self.url_options, json={"send_seed": pre_value})
+  #   requests.post(self.url_options, json={"send_seed": pre_value})
 
   def test_cmd_flags(self):
     self.assertEqual(requests.get(self.url_cmd_flags).status_code, 200)

From 93d6c0209ae55632b72751cf82740e32a0cd81bc Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 14 Nov 2022 13:39:22 +0300
Subject: [PATCH 02/11] Tests separated for github-actions CI

---
 .github/workflows/run_tests.yaml            | 26 ++++++++++++
 launch.py                                   | 36 ++++++++++------
 test/advanced_features/__init__.py          |  0
 test/{ => advanced_features}/extras_test.py |  4 +-
 test/advanced_features/txt2img_test.py      | 47 +++++++++++++++++++++
 test/basic_features/__init__.py             |  0
 test/{ => basic_features}/img2img_test.py   |  4 --
 test/{ => basic_features}/txt2img_test.py   |  4 --
 test/{ => basic_features}/utils_test.py     |  6 ++-
 test/server_poll.py                         |  6 ++-
 10 files changed, 108 insertions(+), 25 deletions(-)
 create mode 100644 .github/workflows/run_tests.yaml
 create mode 100644 test/advanced_features/__init__.py
 rename test/{ => advanced_features}/extras_test.py (90%)
 create mode 100644 test/advanced_features/txt2img_test.py
 create mode 100644 test/basic_features/__init__.py
 rename test/{ => basic_features}/img2img_test.py (96%)
 rename test/{ => basic_features}/txt2img_test.py (97%)
 rename test/{ => basic_features}/utils_test.py (97%)

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
new file mode 100644
index 000000000..a56a81103
--- /dev/null
+++ b/.github/workflows/run_tests.yaml
@@ -0,0 +1,26 @@
+name: Run tests on CPU with empty model
+
+on:
+  - push
+  - pull_request
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.10.6
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Run tests
+        run: |
+          export COMMANDLINE_ARGS="--tests basic_features --no-half --disable-opt-split-attention --use-cpu all"
+          python launch.py
diff --git a/launch.py b/launch.py
index 6822a01de..d0f502c2b 100644
--- a/launch.py
+++ b/launch.py
@@ -17,6 +17,19 @@ def extract_arg(args, name):
     return [x for x in args if x != name], name in args
 
 
+def extract_opt(args, name):
+    opt = None
+    is_present = False
+    if name in args:
+        is_present = True
+        idx = args.index(name)
+        del args[idx]
+        if idx < len(args) and args[idx][0] != "-":
+            opt = args[idx]
+            del args[idx]
+    return args, is_present, opt
+
+
 def run(command, desc=None, errdesc=None, custom_env=None):
     if desc is not None:
         print(desc)
@@ -151,12 +164,11 @@ def prepare_enviroment():
     blip_commit_hash = os.environ.get('BLIP_COMMIT_HASH', "48211a1594f1321b00f14c9f7a5b4813144b2fb9")
 
     sys.argv += shlex.split(commandline_args)
-    test_argv = [x for x in sys.argv if x != '--tests']
 
     sys.argv, skip_torch_cuda_test = extract_arg(sys.argv, '--skip-torch-cuda-test')
     sys.argv, reinstall_xformers = extract_arg(sys.argv, '--reinstall-xformers')
     sys.argv, update_check = extract_arg(sys.argv, '--update-check')
-    sys.argv, run_tests = extract_arg(sys.argv, '--tests')
+    sys.argv, run_tests, test_dir = extract_opt(sys.argv, '--tests')
     xformers = '--xformers' in sys.argv
     deepdanbooru = '--deepdanbooru' in sys.argv
     ngrok = '--ngrok' in sys.argv
@@ -222,24 +234,24 @@ def prepare_enviroment():
         exit(0)
 
     if run_tests:
-        tests(test_argv)
+        tests(test_dir)
         exit(0)
 
 
-def tests(argv):
-    if "--api" not in argv:
-        argv.append("--api")
-    if "--ckpt" not in argv:
-        argv.append("--ckpt")
-        argv.append("./test/test_files/empty.pt")
+def tests(test_dir):
+    if "--api" not in sys.argv:
+        sys.argv.append("--api")
+    if "--ckpt" not in sys.argv:
+        sys.argv.append("--ckpt")
+        sys.argv.append("./test/test_files/empty.pt")
 
-    print(f"Launching Web UI in another process for testing with arguments: {' '.join(argv[1:])}")
+    print(f"Launching Web UI in another process for testing with arguments: {' '.join(sys.argv[1:])}")
 
     with open('test/stdout.txt', "w", encoding="utf8") as stdout, open('test/stderr.txt', "w", encoding="utf8") as stderr:
-        proc = subprocess.Popen([sys.executable, *argv], stdout=stdout, stderr=stderr)
+        proc = subprocess.Popen([sys.executable, *sys.argv], stdout=stdout, stderr=stderr)
 
     import test.server_poll
-    test.server_poll.run_tests(proc)
+    test.server_poll.run_tests(proc, test_dir)
 
     print(f"Stopping Web UI process with id {proc.pid}")
     proc.kill()
diff --git a/test/advanced_features/__init__.py b/test/advanced_features/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/extras_test.py b/test/advanced_features/extras_test.py
similarity index 90%
rename from test/extras_test.py
rename to test/advanced_features/extras_test.py
index 9b8ce0f03..8763f8ed1 100644
--- a/test/extras_test.py
+++ b/test/advanced_features/extras_test.py
@@ -11,8 +11,8 @@ class TestExtrasWorking(unittest.TestCase):
             "codeformer_visibility": 0,
             "codeformer_weight": 0,
             "upscaling_resize": 2,
-            "upscaling_resize_w": 512,
-            "upscaling_resize_h": 512,
+            "upscaling_resize_w": 128,
+            "upscaling_resize_h": 128,
             "upscaling_crop": True,
             "upscaler_1": "None",
             "upscaler_2": "None",
diff --git a/test/advanced_features/txt2img_test.py b/test/advanced_features/txt2img_test.py
new file mode 100644
index 000000000..36ed7b9a9
--- /dev/null
+++ b/test/advanced_features/txt2img_test.py
@@ -0,0 +1,47 @@
+import unittest
+import requests
+
+
+class TestTxt2ImgWorking(unittest.TestCase):
+    def setUp(self):
+        self.url_txt2img = "http://localhost:7860/sdapi/v1/txt2img"
+        self.simple_txt2img = {
+            "enable_hr": False,
+            "denoising_strength": 0,
+            "firstphase_width": 0,
+            "firstphase_height": 0,
+            "prompt": "example prompt",
+            "styles": [],
+            "seed": -1,
+            "subseed": -1,
+            "subseed_strength": 0,
+            "seed_resize_from_h": -1,
+            "seed_resize_from_w": -1,
+            "batch_size": 1,
+            "n_iter": 1,
+            "steps": 3,
+            "cfg_scale": 7,
+            "width": 64,
+            "height": 64,
+            "restore_faces": False,
+            "tiling": False,
+            "negative_prompt": "",
+            "eta": 0,
+            "s_churn": 0,
+            "s_tmax": 0,
+            "s_tmin": 0,
+            "s_noise": 1,
+            "sampler_index": "Euler a"
+        }
+
+    def test_txt2img_with_restore_faces_performed(self):
+        self.simple_txt2img["restore_faces"] = True
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
+
+class TestTxt2ImgCorrectness(unittest.TestCase):
+    pass
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/basic_features/__init__.py b/test/basic_features/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/test/img2img_test.py b/test/basic_features/img2img_test.py
similarity index 96%
rename from test/img2img_test.py
rename to test/basic_features/img2img_test.py
index 012a95809..0a9c1e8ad 100644
--- a/test/img2img_test.py
+++ b/test/basic_features/img2img_test.py
@@ -51,9 +51,5 @@ class TestImg2ImgWorking(unittest.TestCase):
         self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200)
 
 
-class TestImg2ImgCorrectness(unittest.TestCase):
-    pass
-
-
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/txt2img_test.py b/test/basic_features/txt2img_test.py
similarity index 97%
rename from test/txt2img_test.py
rename to test/basic_features/txt2img_test.py
index ce7520858..fe4af9991 100644
--- a/test/txt2img_test.py
+++ b/test/basic_features/txt2img_test.py
@@ -68,9 +68,5 @@ class TestTxt2ImgWorking(unittest.TestCase):
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
 
 
-class TestTxt2ImgCorrectness(unittest.TestCase):
-    pass
-
-
 if __name__ == "__main__":
     unittest.main()
diff --git a/test/utils_test.py b/test/basic_features/utils_test.py
similarity index 97%
rename from test/utils_test.py
rename to test/basic_features/utils_test.py
index be9e6bf8c..9706db8b4 100644
--- a/test/utils_test.py
+++ b/test/basic_features/utils_test.py
@@ -60,4 +60,8 @@ class UtilsTests(unittest.TestCase):
     self.assertEqual(requests.get(self.url_artist_categories).status_code, 200)
 
   def test_artists(self):
-    self.assertEqual(requests.get(self.url_artists).status_code, 200)
\ No newline at end of file
+    self.assertEqual(requests.get(self.url_artists).status_code, 200)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/test/server_poll.py b/test/server_poll.py
index 8e63b4502..c71e906a6 100644
--- a/test/server_poll.py
+++ b/test/server_poll.py
@@ -3,7 +3,7 @@ import requests
 import time
 
 
-def run_tests(proc):
+def run_tests(proc, test_dir):
     timeout_threshold = 240
     start_time = time.time()
     while time.time()-start_time < timeout_threshold:
@@ -14,7 +14,9 @@ def run_tests(proc):
             if proc.poll() is not None:
                 break
     if proc.poll() is None:
-        suite = unittest.TestLoader().discover('', pattern='*_test.py')
+        if test_dir is None:
+            test_dir = ""
+        suite = unittest.TestLoader().discover(test_dir, pattern="*_test.py", top_level_dir="test")
         result = unittest.TextTestRunner(verbosity=2).run(suite)
     else:
         print("Launch unsuccessful")

From 3ffc1c6ceee169fac767a956fd0d4f153b005dbf Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 14 Nov 2022 13:45:21 +0300
Subject: [PATCH 03/11] skip cuda test

---
 .github/workflows/run_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
index a56a81103..f30486817 100644
--- a/.github/workflows/run_tests.yaml
+++ b/.github/workflows/run_tests.yaml
@@ -22,5 +22,5 @@ jobs:
             ${{ runner.os }}-pip-
       - name: Run tests
         run: |
-          export COMMANDLINE_ARGS="--tests basic_features --no-half --disable-opt-split-attention --use-cpu all"
+          export COMMANDLINE_ARGS="--tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test"
           python launch.py

From 0646040667b59526ac8346d53efd14dc0e75b01e Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 14 Nov 2022 14:36:07 +0300
Subject: [PATCH 04/11] Propagate test error and try it without localhost

---
 launch.py                              |  7 ++++---
 test/advanced_features/extras_test.py  |  2 +-
 test/advanced_features/txt2img_test.py |  2 +-
 test/basic_features/img2img_test.py    |  2 +-
 test/basic_features/txt2img_test.py    |  2 +-
 test/basic_features/utils_test.py      | 22 +++++++++++-----------
 test/server_poll.py                    |  4 +++-
 7 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/launch.py b/launch.py
index d0f502c2b..8ed1dffca 100644
--- a/launch.py
+++ b/launch.py
@@ -234,8 +234,8 @@ def prepare_enviroment():
         exit(0)
 
     if run_tests:
-        tests(test_dir)
-        exit(0)
+        exitcode = tests(test_dir)
+        exit(exitcode)
 
 
 def tests(test_dir):
@@ -251,10 +251,11 @@ def tests(test_dir):
         proc = subprocess.Popen([sys.executable, *sys.argv], stdout=stdout, stderr=stderr)
 
     import test.server_poll
-    test.server_poll.run_tests(proc, test_dir)
+    exitcode = test.server_poll.run_tests(proc, test_dir)
 
     print(f"Stopping Web UI process with id {proc.pid}")
     proc.kill()
+    return exitcode
 
 
 def start():
diff --git a/test/advanced_features/extras_test.py b/test/advanced_features/extras_test.py
index 8763f8ed1..abdd5aa21 100644
--- a/test/advanced_features/extras_test.py
+++ b/test/advanced_features/extras_test.py
@@ -3,7 +3,7 @@ import unittest
 
 class TestExtrasWorking(unittest.TestCase):
     def setUp(self):
-        self.url_img2img = "http://localhost:7860/sdapi/v1/extra-single-image"
+        self.url_img2img = "http://127.0.0.1:7860/sdapi/v1/extra-single-image"
         self.simple_extras = {
             "resize_mode": 0,
             "show_extras_results": True,
diff --git a/test/advanced_features/txt2img_test.py b/test/advanced_features/txt2img_test.py
index 36ed7b9a9..6ab5a2422 100644
--- a/test/advanced_features/txt2img_test.py
+++ b/test/advanced_features/txt2img_test.py
@@ -4,7 +4,7 @@ import requests
 
 class TestTxt2ImgWorking(unittest.TestCase):
     def setUp(self):
-        self.url_txt2img = "http://localhost:7860/sdapi/v1/txt2img"
+        self.url_txt2img = "http://127.0.0.1:7860/sdapi/v1/txt2img"
         self.simple_txt2img = {
             "enable_hr": False,
             "denoising_strength": 0,
diff --git a/test/basic_features/img2img_test.py b/test/basic_features/img2img_test.py
index 0a9c1e8ad..b9a341135 100644
--- a/test/basic_features/img2img_test.py
+++ b/test/basic_features/img2img_test.py
@@ -6,7 +6,7 @@ from PIL import Image
 
 class TestImg2ImgWorking(unittest.TestCase):
     def setUp(self):
-        self.url_img2img = "http://localhost:7860/sdapi/v1/img2img"
+        self.url_img2img = "http://127.0.0.1:7860/sdapi/v1/img2img"
         self.simple_img2img = {
             "init_images": [encode_pil_to_base64(Image.open(r"test/test_files/img2img_basic.png"))],
             "resize_mode": 0,
diff --git a/test/basic_features/txt2img_test.py b/test/basic_features/txt2img_test.py
index fe4af9991..0be675d92 100644
--- a/test/basic_features/txt2img_test.py
+++ b/test/basic_features/txt2img_test.py
@@ -4,7 +4,7 @@ import requests
 
 class TestTxt2ImgWorking(unittest.TestCase):
     def setUp(self):
-        self.url_txt2img = "http://localhost:7860/sdapi/v1/txt2img"
+        self.url_txt2img = "http://127.0.0.1:7860/sdapi/v1/txt2img"
         self.simple_txt2img = {
             "enable_hr": False,
             "denoising_strength": 0,
diff --git a/test/basic_features/utils_test.py b/test/basic_features/utils_test.py
index 9706db8b4..592f76196 100644
--- a/test/basic_features/utils_test.py
+++ b/test/basic_features/utils_test.py
@@ -3,17 +3,17 @@ import requests
 
 class UtilsTests(unittest.TestCase):
   def setUp(self):
-    self.url_options = "http://localhost:7860/sdapi/v1/options"
-    self.url_cmd_flags = "http://localhost:7860/sdapi/v1/cmd-flags"
-    self.url_samplers = "http://localhost:7860/sdapi/v1/samplers"
-    self.url_upscalers = "http://localhost:7860/sdapi/v1/upscalers"
-    self.url_sd_models = "http://localhost:7860/sdapi/v1/sd-models"
-    self.url_hypernetworks = "http://localhost:7860/sdapi/v1/hypernetworks"
-    self.url_face_restorers = "http://localhost:7860/sdapi/v1/face-restorers"
-    self.url_realesrgan_models = "http://localhost:7860/sdapi/v1/realesrgan-models"
-    self.url_prompt_styles = "http://localhost:7860/sdapi/v1/prompt-styles"
-    self.url_artist_categories = "http://localhost:7860/sdapi/v1/artist-categories"
-    self.url_artists = "http://localhost:7860/sdapi/v1/artists"
+    self.url_options = "http://127.0.0.1:7860/sdapi/v1/options"
+    self.url_cmd_flags = "http://127.0.0.1:7860/sdapi/v1/cmd-flags"
+    self.url_samplers = "http://127.0.0.1:7860/sdapi/v1/samplers"
+    self.url_upscalers = "http://127.0.0.1:7860/sdapi/v1/upscalers"
+    self.url_sd_models = "http://127.0.0.1:7860/sdapi/v1/sd-models"
+    self.url_hypernetworks = "http://127.0.0.1:7860/sdapi/v1/hypernetworks"
+    self.url_face_restorers = "http://127.0.0.1:7860/sdapi/v1/face-restorers"
+    self.url_realesrgan_models = "http://127.0.0.1:7860/sdapi/v1/realesrgan-models"
+    self.url_prompt_styles = "http://127.0.0.1:7860/sdapi/v1/prompt-styles"
+    self.url_artist_categories = "http://127.0.0.1:7860/sdapi/v1/artist-categories"
+    self.url_artists = "http://127.0.0.1:7860/sdapi/v1/artists"
 
   def test_options_get(self):
     self.assertEqual(requests.get(self.url_options).status_code, 200)
diff --git a/test/server_poll.py b/test/server_poll.py
index c71e906a6..028fd476b 100644
--- a/test/server_poll.py
+++ b/test/server_poll.py
@@ -8,7 +8,7 @@ def run_tests(proc, test_dir):
     start_time = time.time()
     while time.time()-start_time < timeout_threshold:
         try:
-            requests.head("http://localhost:7860/")
+            requests.head("http://127.0.0.1:7860/")
             break
         except requests.exceptions.ConnectionError:
             if proc.poll() is not None:
@@ -18,5 +18,7 @@ def run_tests(proc, test_dir):
             test_dir = ""
         suite = unittest.TestLoader().discover(test_dir, pattern="*_test.py", top_level_dir="test")
         result = unittest.TextTestRunner(verbosity=2).run(suite)
+        return len(result.failures)
     else:
         print("Launch unsuccessful")
+        return 1

From 7416ac8d3cadcc6a53bbcc41e1cd184fa1587afd Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 14 Nov 2022 14:55:39 +0300
Subject: [PATCH 05/11] Use localhost with 80 port, count errors as well

---
 test/advanced_features/extras_test.py  |  2 +-
 test/advanced_features/txt2img_test.py |  2 +-
 test/basic_features/img2img_test.py    |  2 +-
 test/basic_features/txt2img_test.py    |  2 +-
 test/basic_features/utils_test.py      | 22 +++++++++++-----------
 test/server_poll.py                    |  4 ++--
 6 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/test/advanced_features/extras_test.py b/test/advanced_features/extras_test.py
index abdd5aa21..4b8ae25ae 100644
--- a/test/advanced_features/extras_test.py
+++ b/test/advanced_features/extras_test.py
@@ -3,7 +3,7 @@ import unittest
 
 class TestExtrasWorking(unittest.TestCase):
     def setUp(self):
-        self.url_img2img = "http://127.0.0.1:7860/sdapi/v1/extra-single-image"
+        self.url_img2img = "http://localhost:80/sdapi/v1/extra-single-image"
         self.simple_extras = {
             "resize_mode": 0,
             "show_extras_results": True,
diff --git a/test/advanced_features/txt2img_test.py b/test/advanced_features/txt2img_test.py
index 6ab5a2422..e6c3531a7 100644
--- a/test/advanced_features/txt2img_test.py
+++ b/test/advanced_features/txt2img_test.py
@@ -4,7 +4,7 @@ import requests
 
 class TestTxt2ImgWorking(unittest.TestCase):
     def setUp(self):
-        self.url_txt2img = "http://127.0.0.1:7860/sdapi/v1/txt2img"
+        self.url_txt2img = "http://localhost:80/sdapi/v1/txt2img"
         self.simple_txt2img = {
             "enable_hr": False,
             "denoising_strength": 0,
diff --git a/test/basic_features/img2img_test.py b/test/basic_features/img2img_test.py
index b9a341135..c4c9a90fe 100644
--- a/test/basic_features/img2img_test.py
+++ b/test/basic_features/img2img_test.py
@@ -6,7 +6,7 @@ from PIL import Image
 
 class TestImg2ImgWorking(unittest.TestCase):
     def setUp(self):
-        self.url_img2img = "http://127.0.0.1:7860/sdapi/v1/img2img"
+        self.url_img2img = "http://localhost:80/sdapi/v1/img2img"
         self.simple_img2img = {
             "init_images": [encode_pil_to_base64(Image.open(r"test/test_files/img2img_basic.png"))],
             "resize_mode": 0,
diff --git a/test/basic_features/txt2img_test.py b/test/basic_features/txt2img_test.py
index 0be675d92..4f1fc77d3 100644
--- a/test/basic_features/txt2img_test.py
+++ b/test/basic_features/txt2img_test.py
@@ -4,7 +4,7 @@ import requests
 
 class TestTxt2ImgWorking(unittest.TestCase):
     def setUp(self):
-        self.url_txt2img = "http://127.0.0.1:7860/sdapi/v1/txt2img"
+        self.url_txt2img = "http://localhost:80/sdapi/v1/txt2img"
         self.simple_txt2img = {
             "enable_hr": False,
             "denoising_strength": 0,
diff --git a/test/basic_features/utils_test.py b/test/basic_features/utils_test.py
index 592f76196..fdb72b9aa 100644
--- a/test/basic_features/utils_test.py
+++ b/test/basic_features/utils_test.py
@@ -3,17 +3,17 @@ import requests
 
 class UtilsTests(unittest.TestCase):
   def setUp(self):
-    self.url_options = "http://127.0.0.1:7860/sdapi/v1/options"
-    self.url_cmd_flags = "http://127.0.0.1:7860/sdapi/v1/cmd-flags"
-    self.url_samplers = "http://127.0.0.1:7860/sdapi/v1/samplers"
-    self.url_upscalers = "http://127.0.0.1:7860/sdapi/v1/upscalers"
-    self.url_sd_models = "http://127.0.0.1:7860/sdapi/v1/sd-models"
-    self.url_hypernetworks = "http://127.0.0.1:7860/sdapi/v1/hypernetworks"
-    self.url_face_restorers = "http://127.0.0.1:7860/sdapi/v1/face-restorers"
-    self.url_realesrgan_models = "http://127.0.0.1:7860/sdapi/v1/realesrgan-models"
-    self.url_prompt_styles = "http://127.0.0.1:7860/sdapi/v1/prompt-styles"
-    self.url_artist_categories = "http://127.0.0.1:7860/sdapi/v1/artist-categories"
-    self.url_artists = "http://127.0.0.1:7860/sdapi/v1/artists"
+    self.url_options = "http://localhost:80/sdapi/v1/options"
+    self.url_cmd_flags = "http://localhost:80/sdapi/v1/cmd-flags"
+    self.url_samplers = "http://localhost:80/sdapi/v1/samplers"
+    self.url_upscalers = "http://localhost:80/sdapi/v1/upscalers"
+    self.url_sd_models = "http://localhost:80/sdapi/v1/sd-models"
+    self.url_hypernetworks = "http://localhost:80/sdapi/v1/hypernetworks"
+    self.url_face_restorers = "http://localhost:80/sdapi/v1/face-restorers"
+    self.url_realesrgan_models = "http://localhost:80/sdapi/v1/realesrgan-models"
+    self.url_prompt_styles = "http://localhost:80/sdapi/v1/prompt-styles"
+    self.url_artist_categories = "http://localhost:80/sdapi/v1/artist-categories"
+    self.url_artists = "http://localhost:80/sdapi/v1/artists"
 
   def test_options_get(self):
     self.assertEqual(requests.get(self.url_options).status_code, 200)
diff --git a/test/server_poll.py b/test/server_poll.py
index 028fd476b..e4462b6c9 100644
--- a/test/server_poll.py
+++ b/test/server_poll.py
@@ -8,7 +8,7 @@ def run_tests(proc, test_dir):
     start_time = time.time()
     while time.time()-start_time < timeout_threshold:
         try:
-            requests.head("http://127.0.0.1:7860/")
+            requests.head("http://localhost:80/")
             break
         except requests.exceptions.ConnectionError:
             if proc.poll() is not None:
@@ -18,7 +18,7 @@ def run_tests(proc, test_dir):
             test_dir = ""
         suite = unittest.TestLoader().discover(test_dir, pattern="*_test.py", top_level_dir="test")
         result = unittest.TextTestRunner(verbosity=2).run(suite)
-        return len(result.failures)
+        return len(result.failures) + len(result.errors)
     else:
         print("Launch unsuccessful")
         return 1

From 5808241dd76983212ab8e27b07f72866671f0b2d Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 14 Nov 2022 15:14:52 +0300
Subject: [PATCH 06/11] Use 80 port on launch

---
 .github/workflows/run_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
index f30486817..223a31b93 100644
--- a/.github/workflows/run_tests.yaml
+++ b/.github/workflows/run_tests.yaml
@@ -22,5 +22,5 @@ jobs:
             ${{ runner.os }}-pip-
       - name: Run tests
         run: |
-          export COMMANDLINE_ARGS="--tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test"
+          export COMMANDLINE_ARGS="--tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test --port 80"
           python launch.py

From 9e4f68acad4697fdf10004eb85d3f6f769c2c70b Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 14 Nov 2022 18:40:15 +0300
Subject: [PATCH 07/11] Stop exporting cl args and upload stdout and stderr as
 artifacts

---
 .github/workflows/run_tests.yaml       | 11 +++++++++--
 launch.py                              |  2 ++
 test/advanced_features/extras_test.py  |  2 +-
 test/advanced_features/txt2img_test.py |  2 +-
 test/basic_features/img2img_test.py    |  2 +-
 test/basic_features/txt2img_test.py    |  2 +-
 test/basic_features/utils_test.py      | 22 +++++++++++-----------
 test/server_poll.py                    |  2 +-
 8 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
index 223a31b93..558b0c615 100644
--- a/.github/workflows/run_tests.yaml
+++ b/.github/workflows/run_tests.yaml
@@ -22,5 +22,12 @@ jobs:
             ${{ runner.os }}-pip-
       - name: Run tests
         run: |
-          export COMMANDLINE_ARGS="--tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test --port 80"
-          python launch.py
+          python launch.py --tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test
+      - name: Upload main app stdout-stderr
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: stdout-stderr
+          path: |
+            ./test/stdout.txt
+            ./test/stderr.txt
diff --git a/launch.py b/launch.py
index 8ed1dffca..ed21d0e68 100644
--- a/launch.py
+++ b/launch.py
@@ -244,6 +244,8 @@ def tests(test_dir):
     if "--ckpt" not in sys.argv:
         sys.argv.append("--ckpt")
         sys.argv.append("./test/test_files/empty.pt")
+    if "--skip-torch-cuda-test" not in sys.argv:
+        sys.argv.append("--skip-torch-cuda-test")
 
     print(f"Launching Web UI in another process for testing with arguments: {' '.join(sys.argv[1:])}")
 
diff --git a/test/advanced_features/extras_test.py b/test/advanced_features/extras_test.py
index 4b8ae25ae..8763f8ed1 100644
--- a/test/advanced_features/extras_test.py
+++ b/test/advanced_features/extras_test.py
@@ -3,7 +3,7 @@ import unittest
 
 class TestExtrasWorking(unittest.TestCase):
     def setUp(self):
-        self.url_img2img = "http://localhost:80/sdapi/v1/extra-single-image"
+        self.url_img2img = "http://localhost:7860/sdapi/v1/extra-single-image"
         self.simple_extras = {
             "resize_mode": 0,
             "show_extras_results": True,
diff --git a/test/advanced_features/txt2img_test.py b/test/advanced_features/txt2img_test.py
index e6c3531a7..36ed7b9a9 100644
--- a/test/advanced_features/txt2img_test.py
+++ b/test/advanced_features/txt2img_test.py
@@ -4,7 +4,7 @@ import requests
 
 class TestTxt2ImgWorking(unittest.TestCase):
     def setUp(self):
-        self.url_txt2img = "http://localhost:80/sdapi/v1/txt2img"
+        self.url_txt2img = "http://localhost:7860/sdapi/v1/txt2img"
         self.simple_txt2img = {
             "enable_hr": False,
             "denoising_strength": 0,
diff --git a/test/basic_features/img2img_test.py b/test/basic_features/img2img_test.py
index c4c9a90fe..0a9c1e8ad 100644
--- a/test/basic_features/img2img_test.py
+++ b/test/basic_features/img2img_test.py
@@ -6,7 +6,7 @@ from PIL import Image
 
 class TestImg2ImgWorking(unittest.TestCase):
     def setUp(self):
-        self.url_img2img = "http://localhost:80/sdapi/v1/img2img"
+        self.url_img2img = "http://localhost:7860/sdapi/v1/img2img"
         self.simple_img2img = {
             "init_images": [encode_pil_to_base64(Image.open(r"test/test_files/img2img_basic.png"))],
             "resize_mode": 0,
diff --git a/test/basic_features/txt2img_test.py b/test/basic_features/txt2img_test.py
index 4f1fc77d3..fe4af9991 100644
--- a/test/basic_features/txt2img_test.py
+++ b/test/basic_features/txt2img_test.py
@@ -4,7 +4,7 @@ import requests
 
 class TestTxt2ImgWorking(unittest.TestCase):
     def setUp(self):
-        self.url_txt2img = "http://localhost:80/sdapi/v1/txt2img"
+        self.url_txt2img = "http://localhost:7860/sdapi/v1/txt2img"
         self.simple_txt2img = {
             "enable_hr": False,
             "denoising_strength": 0,
diff --git a/test/basic_features/utils_test.py b/test/basic_features/utils_test.py
index fdb72b9aa..9706db8b4 100644
--- a/test/basic_features/utils_test.py
+++ b/test/basic_features/utils_test.py
@@ -3,17 +3,17 @@ import requests
 
 class UtilsTests(unittest.TestCase):
   def setUp(self):
-    self.url_options = "http://localhost:80/sdapi/v1/options"
-    self.url_cmd_flags = "http://localhost:80/sdapi/v1/cmd-flags"
-    self.url_samplers = "http://localhost:80/sdapi/v1/samplers"
-    self.url_upscalers = "http://localhost:80/sdapi/v1/upscalers"
-    self.url_sd_models = "http://localhost:80/sdapi/v1/sd-models"
-    self.url_hypernetworks = "http://localhost:80/sdapi/v1/hypernetworks"
-    self.url_face_restorers = "http://localhost:80/sdapi/v1/face-restorers"
-    self.url_realesrgan_models = "http://localhost:80/sdapi/v1/realesrgan-models"
-    self.url_prompt_styles = "http://localhost:80/sdapi/v1/prompt-styles"
-    self.url_artist_categories = "http://localhost:80/sdapi/v1/artist-categories"
-    self.url_artists = "http://localhost:80/sdapi/v1/artists"
+    self.url_options = "http://localhost:7860/sdapi/v1/options"
+    self.url_cmd_flags = "http://localhost:7860/sdapi/v1/cmd-flags"
+    self.url_samplers = "http://localhost:7860/sdapi/v1/samplers"
+    self.url_upscalers = "http://localhost:7860/sdapi/v1/upscalers"
+    self.url_sd_models = "http://localhost:7860/sdapi/v1/sd-models"
+    self.url_hypernetworks = "http://localhost:7860/sdapi/v1/hypernetworks"
+    self.url_face_restorers = "http://localhost:7860/sdapi/v1/face-restorers"
+    self.url_realesrgan_models = "http://localhost:7860/sdapi/v1/realesrgan-models"
+    self.url_prompt_styles = "http://localhost:7860/sdapi/v1/prompt-styles"
+    self.url_artist_categories = "http://localhost:7860/sdapi/v1/artist-categories"
+    self.url_artists = "http://localhost:7860/sdapi/v1/artists"
 
   def test_options_get(self):
     self.assertEqual(requests.get(self.url_options).status_code, 200)
diff --git a/test/server_poll.py b/test/server_poll.py
index e4462b6c9..d4df697b2 100644
--- a/test/server_poll.py
+++ b/test/server_poll.py
@@ -8,7 +8,7 @@ def run_tests(proc, test_dir):
     start_time = time.time()
     while time.time()-start_time < timeout_threshold:
         try:
-            requests.head("http://localhost:80/")
+            requests.head("http://localhost:7860/")
             break
         except requests.exceptions.ConnectionError:
             if proc.poll() is not None:

From 4a35c3744c68be470e6caf72689322ed58f90aac Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 14 Nov 2022 18:57:14 +0300
Subject: [PATCH 08/11] remove test requiring codeformers

---
 test/basic_features/txt2img_test.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/test/basic_features/txt2img_test.py b/test/basic_features/txt2img_test.py
index fe4af9991..1c2674b2a 100644
--- a/test/basic_features/txt2img_test.py
+++ b/test/basic_features/txt2img_test.py
@@ -49,10 +49,6 @@ class TestTxt2ImgWorking(unittest.TestCase):
         self.simple_txt2img["enable_hr"] = True
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
 
-    def test_txt2img_with_restore_faces_performed(self):
-        self.simple_txt2img["restore_faces"] = True
-        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
-
     def test_txt2img_with_tiling_performed(self):
         self.simple_txt2img["tiling"] = True
         self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)

From a07107900019c8f18ed0f8d07e39435ee3189e79 Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Mon, 14 Nov 2022 19:22:06 +0300
Subject: [PATCH 09/11] Use empty model as CLIP weights

---
 .github/workflows/run_tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
index 558b0c615..4aeeab9c7 100644
--- a/.github/workflows/run_tests.yaml
+++ b/.github/workflows/run_tests.yaml
@@ -22,7 +22,7 @@ jobs:
             ${{ runner.os }}-pip-
       - name: Run tests
         run: |
-          python launch.py --tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test
+          python launch.py --tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test --clip-models-path ./test/test_files/empty.pt
       - name: Upload main app stdout-stderr
         uses: actions/upload-artifact@v3
         if: always()

From 694611cbd8151533cf18e01a8ead32771ea12ff3 Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Sat, 19 Nov 2022 00:56:08 +0300
Subject: [PATCH 10/11] Apply suggestions from code review

Use last version of setup-python action
Remove unnecesarry multicommand from run
Remove current directory from artifact paths

Co-authored-by: Margen67 <Margen67@users.noreply.github.com>
---
 .github/workflows/run_tests.yaml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
index 4aeeab9c7..6c6f15a89 100644
--- a/.github/workflows/run_tests.yaml
+++ b/.github/workflows/run_tests.yaml
@@ -11,7 +11,7 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v3
       - name: Set up Python 3.10
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v4
         with:
           python-version: 3.10.6
       - uses: actions/cache@v2
@@ -21,13 +21,12 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Run tests
-        run: |
-          python launch.py --tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test --clip-models-path ./test/test_files/empty.pt
+        run: python launch.py --tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test --clip-models-path ./test/test_files/empty.pt
       - name: Upload main app stdout-stderr
         uses: actions/upload-artifact@v3
         if: always()
         with:
           name: stdout-stderr
           path: |
-            ./test/stdout.txt
-            ./test/stderr.txt
+            test/stdout.txt
+            test/stderr.txt

From 14dfede8ddbf6b82bb290d5c4292d52b6b68e3ca Mon Sep 17 00:00:00 2001
From: Vladimir Repin <32306715+mezotaken@users.noreply.github.com>
Date: Sat, 19 Nov 2022 14:15:10 +0300
Subject: [PATCH 11/11] Minor fixes

Remove unused test completely
Change job name
Don't use empty.pt as CLIP weights - it wont work.
Use latest version of actions/cache
---
 .github/workflows/run_tests.yaml  |  9 ++++-----
 test/basic_features/utils_test.py | 14 --------------
 2 files changed, 4 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml
index 6c6f15a89..49dc92bd9 100644
--- a/.github/workflows/run_tests.yaml
+++ b/.github/workflows/run_tests.yaml
@@ -1,4 +1,4 @@
-name: Run tests on CPU with empty model
+name: Run basic features tests on CPU with empty SD model
 
 on:
   - push
@@ -14,14 +14,13 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.10.6
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+          restore-keys: ${{ runner.os }}-pip-
       - name: Run tests
-        run: python launch.py --tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test --clip-models-path ./test/test_files/empty.pt
+        run: python launch.py --tests basic_features --no-half --disable-opt-split-attention --use-cpu all --skip-torch-cuda-test
       - name: Upload main app stdout-stderr
         uses: actions/upload-artifact@v3
         if: always()
diff --git a/test/basic_features/utils_test.py b/test/basic_features/utils_test.py
index 9706db8b4..765470c90 100644
--- a/test/basic_features/utils_test.py
+++ b/test/basic_features/utils_test.py
@@ -18,20 +18,6 @@ class UtilsTests(unittest.TestCase):
   def test_options_get(self):
     self.assertEqual(requests.get(self.url_options).status_code, 200)
 
-  # def test_options_write(self):
-  #   response = requests.get(self.url_options)
-  #   self.assertEqual(response.status_code, 200)
-    
-  #   pre_value = response.json()["send_seed"]
-
-  #   self.assertEqual(requests.post(self.url_options, json={"send_seed":not pre_value}).status_code, 200)
-
-  #   response = requests.get(self.url_options)
-  #   self.assertEqual(response.status_code, 200)
-  #   self.assertEqual(response.json()["send_seed"], not pre_value)
-
-  #   requests.post(self.url_options, json={"send_seed": pre_value})
-
   def test_cmd_flags(self):
     self.assertEqual(requests.get(self.url_cmd_flags).status_code, 200)