diff --git a/docs/1.windows-installation.md b/docs/1.windows-installation.md index db327e5..05a980d 100644 --- a/docs/1.windows-installation.md +++ b/docs/1.windows-installation.md @@ -107,7 +107,7 @@ There are three more models that we need to download in order to get the most ou ### GFPGAN 1. If you want to use GFPGAN to improve generated faces, you need to install it separately. -1. Download [GFPGANv1.3.pth](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth) and put it +1. Download [GFPGANv1.3.pth](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.3.pth) and [GFPGANv1.4.pth](https://github.com/TencentARC/GFPGAN/releases/download/v1.3.4/GFPGANv1.4.pth) and put it into the `/stable-diffusion-webui/src/gfpgan/experiments/pretrained_models` directory. ### RealESRGAN diff --git a/scripts/relauncher.py b/scripts/relauncher.py index 3ce2e91..e56cde9 100644 --- a/scripts/relauncher.py +++ b/scripts/relauncher.py @@ -40,6 +40,7 @@ tiling = False additional_arguments = "" parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('-v', '--verbosity', action='count', default=0, help="The default logging level is ERROR or higher. This value increases the amount of logging seen in your screen") parser.add_argument('-n', '--horde_name', action="store", required=False, type=str, help="The server name for the Horde. It will be shown to the world and there can be only one.") parser.add_argument('--bridge', action="store_true", required=False, default=False, help="When specified, start the stable horde bridge instead of the webui.") args = parser.parse_args() @@ -48,6 +49,10 @@ if args.bridge: additional_arguments += f' --bridge' if args.horde_name: additional_arguments += f' --horde_name "{args.horde_name}"' + if args.verbosity: + for iter in range(args.verbosity): + additional_arguments += ' -v' + print(f"Additional args: {additional_arguments}") diff --git a/scripts/webui.py b/scripts/webui.py index 4178246..c090180 100644 --- a/scripts/webui.py +++ b/scripts/webui.py @@ -2708,8 +2708,12 @@ def run_bridge(interval, api_key, horde_name, horde_url, priority_usernames, hor current_id = None current_payload = None current_generation = None + loop_retry = 0 time.sleep(10) continue + # In bridge-mode, matrix is prepared on the horde and split in multiple nodes + if 'toggles' in current_payload and 0 in current_payload['toggles']: + current_payload['toggles'].remove(0) images, seed, info, stats = txt2img(**current_payload) buffer = BytesIO() # We send as WebP to avoid using all the horde bandwidth @@ -2723,19 +2727,19 @@ def run_bridge(interval, api_key, horde_name, horde_url, priority_usernames, hor "max_pixels": horde_max_pixels, } current_generation = seed - while current_id and current_generation: + while current_id and current_generation != None: try: submit_req = requests.post(horde_url + '/api/v2/generate/submit', json = submit_dict, headers = headers) try: submit = submit_req.json() except json.decoder.JSONDecodeError: - logger.error(f"Something has gone wrong with {horde_url} during submit. Please inform its administrator!") + logger.error(f"Something has gone wrong with {horde_url} during submit. Please inform its administrator! (Retry {loop_retry}/10)") time.sleep(interval) continue if submit_req.status_code == 404: logger.warning(f"The generation we were working on got stale. Aborting!") elif not submit_req.ok: - logger.warning(f"During gen submit, server {horde_url} responded with status code {submit_req.status_code}: {submit['message']}. Waiting for 10 seconds...") + logger.warning(f"During gen submit, server {horde_url} responded with status code {submit_req.status_code}: {submit['message']}. Waiting for 10 seconds... (Retry {loop_retry}/10)") if 'errors' in submit: logger.warning(f"Detailed Request Errors: {submit['errors']}") time.sleep(10) @@ -2745,10 +2749,19 @@ def run_bridge(interval, api_key, horde_name, horde_url, priority_usernames, hor current_id = None current_payload = None current_generation = None + loop_retry = 0 except requests.exceptions.ConnectionError: - logger.warning(f"Server {horde_url} unavailable during submit. Waiting 10 seconds...") + logger.warning(f"Server {horde_url} unavailable during submit. Waiting 10 seconds... (Retry {loop_retry}/10)") time.sleep(10) continue + if loop_retry > 10 and current_id: + logger.error(f"Exceeded retry count {loop_retry} for generation id {current_id}. Aborting generation!") + current_id = None + current_payload = None + current_generation = None + loop_retry = 0 + elif current_id: + logger.debug(f"Retrying ({loop_retry}/10) for generation id {current_id}...") time.sleep(interval)