cache git extension repo information

modules/cache.py

@@ -0,0 +1,96 @@
+import json
+import os.path
+
+import filelock
+
+from modules.paths import data_path, script_path
+
+cache_filename = os.path.join(data_path, "cache.json")
+cache_data = None
+
+
+def dump_cache():
+    """
+    Saves all cache data to a file.
+    """
+
+    with filelock.FileLock(f"{cache_filename}.lock"):
+        with open(cache_filename, "w", encoding="utf8") as file:
+            json.dump(cache_data, file, indent=4)
+
+
+def cache(subsection):
+    """
+    Retrieves or initializes a cache for a specific subsection.
+
+    Parameters:
+        subsection (str): The subsection identifier for the cache.
+
+    Returns:
+        dict: The cache data for the specified subsection.
+    """
+
+    global cache_data
+
+    if cache_data is None:
+        with filelock.FileLock(f"{cache_filename}.lock"):
+            if not os.path.isfile(cache_filename):
+                cache_data = {}
+            else:
+                try:
+                    with open(cache_filename, "r", encoding="utf8") as file:
+                        cache_data = json.load(file)
+                except Exception:
+                    os.replace(cache_filename, os.path.join(script_path, "tmp", "cache.json"))
+                    print('[ERROR] issue occurred while trying to read cache.json, move current cache to tmp/cache.json and create new cache')
+                    cache_data = {}
+
+    s = cache_data.get(subsection, {})
+    cache_data[subsection] = s
+
+    return s
+
+
+def cached_data_for_file(subsection, title, filename, func):
+    """
+    Retrieves or generates data for a specific file, using a caching mechanism.
+
+    Parameters:
+        subsection (str): The subsection of the cache to use.
+        title (str): The title of the data entry in the subsection of the cache.
+        filename (str): The path to the file to be checked for modifications.
+        func (callable): A function that generates the data if it is not available in the cache.
+
+    Returns:
+        dict or None: The cached or generated data, or None if data generation fails.
+
+    The `cached_data_for_file` function implements a caching mechanism for data stored in files.
+    It checks if the data associated with the given `title` is present in the cache and compares the
+    modification time of the file with the cached modification time. If the file has been modified,
+    the cache is considered invalid and the data is regenerated using the provided `func`.
+    Otherwise, the cached data is returned.
+
+    If the data generation fails, None is returned to indicate the failure. Otherwise, the generated
+    or cached data is returned as a dictionary.
+    """
+
+    existing_cache = cache(subsection)
+    ondisk_mtime = os.path.getmtime(filename)
+
+    entry = existing_cache.get(title)
+    if entry:
+        cached_mtime = existing_cache[title].get("mtime", 0)
+        if ondisk_mtime > cached_mtime:
+            entry = None
+
+    if not entry:
+        entry = func()
+        if entry is None:
+            return None
+
+        entry['mtime'] = ondisk_mtime
+        existing_cache[title] = entry
+
+        dump_cache()
+
+    return entry

modules/extensions.py

@@ -1,7 +1,7 @@
 import os
 import threading
 
-from modules import shared, errors
+from modules import shared, errors, cache
 from modules.gitpython_hack import Repo
 from modules.paths_internal import extensions_dir, extensions_builtin_dir, script_path  # noqa: F401
 
@@ -21,6 +21,7 @@ def active():
 
 class Extension:
     lock = threading.Lock()
+    cached_fields = ['remote', 'commit_date', 'branch', 'commit_hash', 'version']
 
     def __init__(self, name, path, enabled=True, is_builtin=False):
         self.name = name
@@ -36,15 +37,29 @@ class Extension:
         self.remote = None
         self.have_info_from_repo = False
 
+    def to_dict(self):
+        return {x: getattr(self, x) for x in self.cached_fields}
+
+    def from_dict(self, d):
+        for field in self.cached_fields:
+            setattr(self, field, d[field])
+
     def read_info_from_repo(self):
         if self.is_builtin or self.have_info_from_repo:
             return
 
-        with self.lock:
+        def read_from_repo():
-            if self.have_info_from_repo:
+            with self.lock:
-                return
+                if self.have_info_from_repo:
+                    return
 
-            self.do_read_info_from_repo()
+                self.do_read_info_from_repo()
+
+                return self.to_dict()
+
+        d = cache.cached_data_for_file('extensions-git', self.name, os.path.join(self.path, ".git"), read_from_repo)
+        self.from_dict(d)
+        self.status = 'unknown'
 
     def do_read_info_from_repo(self):
         repo = None
@@ -58,7 +73,6 @@ class Extension:
             self.remote = None
         else:
             try:
-                self.status = 'unknown'
                 self.remote = next(repo.remote().urls, None)
                 commit = repo.head.commit
                 self.commit_date = commit.committed_date

modules/hashes.py

@@ -1,43 +1,11 @@
 import hashlib
-import json
 import os.path
 
-import filelock
-
 from modules import shared
-from modules.paths import data_path, script_path
+import modules.cache
 
-
+dump_cache = modules.cache.dump_cache
-cache_filename = os.path.join(data_path, "cache.json")
+cache = modules.cache.cache
-cache_data = None
-
-
-def dump_cache():
-    with filelock.FileLock(f"{cache_filename}.lock"):
-        with open(cache_filename, "w", encoding="utf8") as file:
-            json.dump(cache_data, file, indent=4)
-
-
-def cache(subsection):
-    global cache_data
-
-    if cache_data is None:
-        with filelock.FileLock(f"{cache_filename}.lock"):
-            if not os.path.isfile(cache_filename):
-                cache_data = {}
-            else:
-                try:
-                    with open(cache_filename, "r", encoding="utf8") as file:
-                        cache_data = json.load(file)
-                except Exception:
-                    os.replace(cache_filename, os.path.join(script_path, "tmp", "cache.json"))
-                    print('[ERROR] issue occurred while trying to read cache.json, move current cache to tmp/cache.json and create new cache')
-                    cache_data = {}
-
-    s = cache_data.get(subsection, {})
-    cache_data[subsection] = s
-
-    return s
 
 
 def calculate_sha256(filename):

modules/ui_extensions.py

@@ -513,14 +513,8 @@ def refresh_available_extensions_from_data(hide_tags, sort_column, filter_text="
 
 
 def preload_extensions_git_metadata():
-    t0 = time.time()
     for extension in extensions.extensions:
         extension.read_info_from_repo()
-    print(
-        f"preload_extensions_git_metadata for "
-        f"{len(extensions.extensions)} extensions took "
-        f"{time.time() - t0:.2f}s"
-    )
 
 
 def create_ui():