diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index ddc19d7426e5..0ef93ac569ab 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -21,7 +21,7 @@ import xml.etree.ElementTree as ET from datetime import datetime from multiprocessing.dummy import Pool from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union, Any +from typing import Dict, List, Optional, Tuple, Union, Any, Callable from urllib.parse import urljoin, urlparse from tempfile import NamedTemporaryFile @@ -33,6 +33,42 @@ ROOT = Path(__file__).parent DEFAULT_IN = ROOT.joinpath("vim-plugin-names") DEFAULT_OUT = ROOT.joinpath("generated.nix") +import time +from functools import wraps + + +def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2): + """Retry calling the decorated function using an exponential backoff. + + http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ + original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry + (BSD licensed) + + :param ExceptionToCheck: the exception on which to retry + :param tries: number of times to try (not retry) before giving up + :param delay: initial delay between retries in seconds + :param backoff: backoff multiplier e.g. value of 2 will double the delay + each retry + """ + + def deco_retry(f: Callable) -> Callable: + @wraps(f) + def f_retry(*args: Any, **kwargs: Any) -> Any: + mtries, mdelay = tries, delay + while mtries > 1: + try: + return f(*args, **kwargs) + except ExceptionToCheck as e: + print(f"{str(e)}, Retrying in {mdelay} seconds...") + time.sleep(mdelay) + mtries -= 1 + mdelay *= backoff + return f(*args, **kwargs) + + return f_retry # true decorator + + return deco_retry + class Repo: def __init__(self, owner: str, name: str) -> None: @@ -45,9 +81,12 @@ class Repo: def __repr__(self) -> str: return f"Repo({self.owner}, {self.name})" + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def has_submodules(self) -> bool: try: - urllib.request.urlopen(self.url("blob/master/.gitmodules")).close() + urllib.request.urlopen( + self.url("blob/master/.gitmodules"), timeout=10 + ).close() except urllib.error.HTTPError as e: if e.code == 404: return False @@ -55,8 +94,9 @@ class Repo: raise return True + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def latest_commit(self) -> Tuple[str, datetime]: - with urllib.request.urlopen(self.url("commits/master.atom")) as req: + with urllib.request.urlopen(self.url("commits/master.atom"), timeout=10) as req: xml = req.read() root = ET.fromstring(xml) latest_entry = root.find(ATOM_ENTRY)