Merge pull request #3941 from mezotaken/master

Automated testing through API [WIP, feedback required]

run_tests.bat

@@ -0,0 +1,15 @@
+@echo off
+set ERROR_REPORTING=FALSE
+set COMMANDLINE_ARGS= --api
+echo Launching SDWebUI...
+start "SDWebUITest" webui.bat
+
+if not defined PYTHON (set PYTHON=python)
+if not defined VENV_DIR (set VENV_DIR=venv)
+set PYTHON="%~dp0%VENV_DIR%\Scripts\Python.exe"
+%PYTHON% test/server_poll.py
+for /f "tokens=2 delims=," %%a in ('tasklist /v /fo csv ^| findstr /i "SDWebUITest"') do set "$PID=%%a"
+
+taskkill /PID %$PID% >nul 2>&1
+
+pause

test/extras_test.py

@@ -0,0 +1,29 @@
+import unittest
+import requests
+from gradio.processing_utils import encode_pil_to_base64
+from PIL import Image
+
+class TestExtrasWorking(unittest.TestCase):
+    def setUp(self):
+        self.url_img2img = "http://localhost:7860/sdapi/v1/extra-single-image"
+        self.simple_extras = {
+            "resize_mode": 0,
+            "show_extras_results": True,
+            "gfpgan_visibility": 0,
+            "codeformer_visibility": 0,
+            "codeformer_weight": 0,
+            "upscaling_resize": 2,
+            "upscaling_resize_w": 512,
+            "upscaling_resize_h": 512,
+            "upscaling_crop": True,
+            "upscaler_1": "None",
+            "upscaler_2": "None",
+            "extras_upscaler_2_visibility": 0,
+            "image": ""
+            }
+
+class TestExtrasCorrectness(unittest.TestCase):
+    pass
+
+if __name__ == "__main__":
+    unittest.main()

test/img2img_test.py

@@ -0,0 +1,59 @@
+import unittest
+import requests
+from gradio.processing_utils import encode_pil_to_base64
+from PIL import Image
+
+class TestImg2ImgWorking(unittest.TestCase):
+    def setUp(self):
+        self.url_img2img = "http://localhost:7860/sdapi/v1/img2img"
+        self.simple_img2img = {
+            "init_images": [
+                encode_pil_to_base64(Image.open(r"test/test_files/img2img_basic.png"))
+                ],
+            "resize_mode": 0,
+            "denoising_strength": 0.75,
+            "mask": None,
+            "mask_blur": 4,
+            "inpainting_fill": 0,
+            "inpaint_full_res": False,
+            "inpaint_full_res_padding": 0,
+            "inpainting_mask_invert": 0,
+            "prompt": "example prompt",
+            "styles": [
+                ""
+                ],
+            "seed": -1,
+            "subseed": -1,
+            "subseed_strength": 0,
+            "seed_resize_from_h": -1,
+            "seed_resize_from_w": -1,
+            "batch_size": 1,
+            "n_iter": 1,
+            "steps": 3,
+            "cfg_scale": 7,
+            "width": 64,
+            "height": 64,
+            "restore_faces": False,
+            "tiling": False,
+            "negative_prompt": "",
+            "eta": 0,
+            "s_churn": 0,
+            "s_tmax": 0,
+            "s_tmin": 0,
+            "s_noise": 1,
+            "override_settings": {},
+            "sampler_index": "Euler a",
+            "include_init_images": False
+            }
+    def test_img2img_simple_performed(self):
+        self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200)
+
+    def test_inpainting_masked_performed(self):
+        self.simple_img2img["mask"] = encode_pil_to_base64(Image.open(r"test/test_files/mask_basic.png"))
+        self.assertEqual(requests.post(self.url_img2img, json=self.simple_img2img).status_code, 200)
+
+class TestImg2ImgCorrectness(unittest.TestCase):
+    pass
+
+if __name__ == "__main__":
+    unittest.main()

test/server_poll.py

@@ -0,0 +1,17 @@
+import unittest
+import requests
+import time
+
+timeout_threshold = 240
+start_time = time.time()
+while time.time()-start_time < timeout_threshold:
+    try:
+        requests.head("http://localhost:7860/")
+        break
+    except requests.exceptions.ConnectionError:
+        pass
+if time.time()-start_time < timeout_threshold:
+    suite = unittest.TestLoader().discover('', pattern='*_test.py')
+    result = unittest.TextTestRunner(verbosity=2).run(suite)
+else:
+    print("Launch unsuccessful")

test/txt2img_test.py

@@ -0,0 +1,72 @@
+import unittest
+import requests
+
+class TestTxt2ImgWorking(unittest.TestCase):
+    def setUp(self):
+        self.url_txt2img = "http://localhost:7860/sdapi/v1/txt2img"
+        self.simple_txt2img = {
+            "enable_hr": False,
+            "denoising_strength": 0,
+            "firstphase_width": 0,
+            "firstphase_height": 0,
+            "prompt": "example prompt",
+            "styles": [
+            ""
+            ],
+            "seed": -1,
+            "subseed": -1,
+            "subseed_strength": 0,
+            "seed_resize_from_h": -1,
+            "seed_resize_from_w": -1,
+            "batch_size": 1,
+            "n_iter": 1,
+            "steps": 3,
+            "cfg_scale": 7,
+            "width": 64,
+            "height": 64,
+            "restore_faces": False,
+            "tiling": False,
+            "negative_prompt": "",
+            "eta": 0,
+            "s_churn": 0,
+            "s_tmax": 0,
+            "s_tmin": 0,
+            "s_noise": 1,
+            "sampler_index": "Euler a"
+        }
+    def test_txt2img_simple_performed(self):
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
+    def test_txt2img_with_negative_prompt_performed(self):
+        self.simple_txt2img["negative_prompt"] = "example negative prompt"
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
+    def test_txt2img_not_square_image_performed(self):
+        self.simple_txt2img["height"] = 128
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
+    def test_txt2img_with_hrfix_performed(self):
+        self.simple_txt2img["enable_hr"] = True
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
+    def test_txt2img_with_restore_faces_performed(self):
+        self.simple_txt2img["restore_faces"] = True
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
+    def test_txt2img_with_tiling_faces_performed(self):
+        self.simple_txt2img["tiling"] = True
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
+    def test_txt2img_with_vanilla_sampler_performed(self):
+        self.simple_txt2img["sampler_index"] = "PLMS"
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
+    def test_txt2img_multiple_batches_performed(self):
+        self.simple_txt2img["n_iter"] = 2
+        self.assertEqual(requests.post(self.url_txt2img, json=self.simple_txt2img).status_code, 200)
+
+class TestTxt2ImgCorrectness(unittest.TestCase):
+    pass
+
+if __name__ == "__main__":
+    unittest.main()