fix memory leak when generation fails

modules/call_queue.py

@@ -3,7 +3,7 @@ import html
 import threading
 import time
 
-from modules import shared, progress, errors
+from modules import shared, progress, errors, devices
 
 queue_lock = threading.Lock()
 
@@ -75,6 +75,8 @@ def wrap_gradio_call(func, extra_outputs=None, add_stats=False):
             error_message = f'{type(e).__name__}: {e}'
             res = extra_outputs_array + [f"<div class='error'>{html.escape(error_message)}</div>"]
 
+        devices.torch_gc()
+
         shared.state.skipped = False
         shared.state.interrupted = False
         shared.state.job_count = 0

modules/errors.py

@@ -14,7 +14,8 @@ def record_exception():
     if exception_records and exception_records[-1] == e:
         return
 
-    exception_records.append((e, tb))
+    from modules import sysinfo
+    exception_records.append(sysinfo.format_exception(e, tb))
 
     if len(exception_records) > 5:
         exception_records.pop(0)

modules/sysinfo.py

@@ -109,11 +109,15 @@ def format_traceback(tb):
     return [[f"{x.filename}, line {x.lineno}, {x.name}", x.line] for x in traceback.extract_tb(tb)]
 
 
+def format_exception(e, tb):
+    return {"exception": str(e), "traceback": format_traceback(tb)}
+
+
 def get_exceptions():
     try:
         from modules import errors
 
-        return [{"exception": str(e), "traceback": format_traceback(tb)} for e, tb in reversed(errors.exception_records)]
+        return list(reversed(errors.exception_records))
     except Exception as e:
         return str(e)