Fix infinite loop on bridge mode when seed == 0 (#1406)

The bridge will keep looping on the same generation because the
evaluation of "while not seed" will always be False when seed is 0 (or
00000000 etc)

This fixes this. Also allows to request more verbosity on the webui
command

Co-authored-by: hlky <106811348+hlky@users.noreply.github.com>
Co-authored-by: lukas5450 <46075099+lukas5450@users.noreply.github.com>

docs/1.windows-installation.md

@@ -107,7 +107,7 @@ There are three more models that we need to download in order to get the most ou
 
 ### GFPGAN
 1. If you want to use GFPGAN to improve generated faces, you need to install it separately.
-1. Download [GFPGANv1.3.pth](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth) and put it
+1. Download [GFPGANv1.3.pth](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth) and [GFPGANv1.4.pth](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.4/GFPGANv1.4.pth) and put it
 into the `/stable-diffusion-webui/src/gfpgan/experiments/pretrained_models` directory. 
 
 ### RealESRGAN

scripts/relauncher.py

@@ -40,6 +40,7 @@ tiling = False
 additional_arguments = ""
 
 parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-v', '--verbosity', action='count', default=0, help="The default logging level is ERROR or higher. This value increases the amount of logging seen in your screen")
 parser.add_argument('-n', '--horde_name', action="store", required=False, type=str, help="The server name for the Horde. It will be shown to the world and there can be only one.")
 parser.add_argument('--bridge', action="store_true", required=False, default=False, help="When specified, start the stable horde bridge instead of the webui.")
 args = parser.parse_args()
@@ -48,6 +49,10 @@ if args.bridge:
     additional_arguments += f' --bridge'
     if args.horde_name:
         additional_arguments += f' --horde_name "{args.horde_name}"'
+    if args.verbosity:
+        for iter in range(args.verbosity):
+            additional_arguments += ' -v'
+    print(f"Additional args: {additional_arguments}")
 
 
 

scripts/webui.py

@@ -2708,8 +2708,12 @@ def run_bridge(interval, api_key, horde_name, horde_url, priority_usernames, hor
                 current_id = None
                 current_payload = None
                 current_generation = None
+                loop_retry = 0
                 time.sleep(10)
                 continue
+        # In bridge-mode, matrix is prepared on the horde and split in multiple nodes
+        if 'toggles' in current_payload and 0 in current_payload['toggles']:
+            current_payload['toggles'].remove(0)
         images, seed, info, stats = txt2img(**current_payload)
         buffer = BytesIO()
         # We send as WebP to avoid using all the horde bandwidth
@@ -2723,19 +2727,19 @@ def run_bridge(interval, api_key, horde_name, horde_url, priority_usernames, hor
             "max_pixels": horde_max_pixels,
         }
         current_generation = seed
-        while current_id and current_generation:
+        while current_id and current_generation != None:
             try:
                 submit_req = requests.post(horde_url + '/api/v2/generate/submit', json = submit_dict, headers = headers)
                 try:
                     submit = submit_req.json()
                 except json.decoder.JSONDecodeError:
-                    logger.error(f"Something has gone wrong with {horde_url} during submit. Please inform its administrator!")
+                    logger.error(f"Something has gone wrong with {horde_url} during submit. Please inform its administrator!  (Retry {loop_retry}/10)")
                     time.sleep(interval)
                     continue
                 if submit_req.status_code == 404:
                     logger.warning(f"The generation we were working on got stale. Aborting!")
                 elif not submit_req.ok:
-                    logger.warning(f"During gen submit, server {horde_url} responded with status code {submit_req.status_code}: {submit['message']}. Waiting for 10 seconds...")
+                    logger.warning(f"During gen submit, server {horde_url} responded with status code {submit_req.status_code}: {submit['message']}. Waiting for 10 seconds...  (Retry {loop_retry}/10)")
                     if 'errors' in submit:
                         logger.warning(f"Detailed Request Errors: {submit['errors']}")
                     time.sleep(10)
@@ -2745,10 +2749,19 @@ def run_bridge(interval, api_key, horde_name, horde_url, priority_usernames, hor
                 current_id = None
                 current_payload = None
                 current_generation = None
+                loop_retry = 0
             except requests.exceptions.ConnectionError:
-                logger.warning(f"Server {horde_url} unavailable during submit. Waiting 10 seconds...")
+                logger.warning(f"Server {horde_url} unavailable during submit. Waiting 10 seconds...  (Retry {loop_retry}/10)")
                 time.sleep(10)
                 continue
+        if loop_retry > 10 and current_id:
+            logger.error(f"Exceeded retry count {loop_retry} for generation id {current_id}. Aborting generation!")
+            current_id = None
+            current_payload = None
+            current_generation = None
+            loop_retry = 0
+        elif current_id:
+            logger.debug(f"Retrying ({loop_retry}/10) for generation id {current_id}...")
         time.sleep(interval)