From 61c60bad222dc8a59ac6f65a4f7643334f9cd86d Mon Sep 17 00:00:00 2001 From: Timo Kaufmann Date: Sat, 16 Nov 2019 16:15:08 +0100 Subject: [PATCH 1/2] vimPlugins: fix formatting and typing in update.py --- pkgs/misc/vim-plugins/update.py | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index 92a47bc5d131..ddc19d7426e5 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -25,9 +25,9 @@ from typing import Dict, List, Optional, Tuple, Union, Any from urllib.parse import urljoin, urlparse from tempfile import NamedTemporaryFile -ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" -ATOM_LINK = "{http://www.w3.org/2005/Atom}link" -ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" +ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" # " vim gets confused here +ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # " +ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # " ROOT = Path(__file__).parent DEFAULT_IN = ROOT.joinpath("vim-plugin-names") @@ -69,7 +69,7 @@ class Repo: updated_tag is not None and updated_tag.text is not None ), f"No updated tag found feed entry {xml}" updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ") - return Path(url.path).name, updated + return Path(str(url.path)).name, updated def prefetch_git(self, ref: str) -> str: data = subprocess.check_output( @@ -210,20 +210,17 @@ def check_results( sys.exit(1) -def parse_plugin_line(line: str) -> Tuple[str, str, str]: +def parse_plugin_line(line: str) -> Tuple[str, str, Optional[str]]: + name, repo = line.split("/") try: - name, repo = line.split("/") - try: - repo, alias = repo.split(" as ") - return (name, repo, alias.strip()) - except ValueError: - # no alias defined - return (name, repo.strip(), None) + repo, alias = repo.split(" as ") + return (name, repo, alias.strip()) except ValueError: - return (None, None, None) + # no alias defined + return (name, repo.strip(), None) -def load_plugin_spec(plugin_file: str) -> List[Tuple[str, str]]: +def load_plugin_spec(plugin_file: str) -> List[Tuple[str, str, Optional[str]]]: plugins = [] with open(plugin_file) as f: for line in f: @@ -385,7 +382,7 @@ def main() -> None: try: # synchronous variant for debugging - # results = map(prefetch_with_cache, plugins) + # results = list(map(prefetch_with_cache, plugin_names)) pool = Pool(processes=30) results = pool.map(prefetch_with_cache, plugin_names) finally: From 8c757f4dc93d927fa12099900f839c3c47dded0c Mon Sep 17 00:00:00 2001 From: Timo Kaufmann Date: Sat, 16 Nov 2019 16:15:42 +0100 Subject: [PATCH 2/2] vimPlugins: backoff on timeout in update.py Updating vim-plugins recently started timing out regularly for me. It may have to do with an ISP switch on my side, but I don't think that should cause timeouts. I guess I'm probably not the only one experiencing this, so in this comment I introduce exponential backoff. Every request will be retried up to 3 times (3 seconds, 6 seconds, 12 seconds delay). --- pkgs/misc/vim-plugins/update.py | 46 ++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/pkgs/misc/vim-plugins/update.py b/pkgs/misc/vim-plugins/update.py index ddc19d7426e5..0ef93ac569ab 100755 --- a/pkgs/misc/vim-plugins/update.py +++ b/pkgs/misc/vim-plugins/update.py @@ -21,7 +21,7 @@ import xml.etree.ElementTree as ET from datetime import datetime from multiprocessing.dummy import Pool from pathlib import Path -from typing import Dict, List, Optional, Tuple, Union, Any +from typing import Dict, List, Optional, Tuple, Union, Any, Callable from urllib.parse import urljoin, urlparse from tempfile import NamedTemporaryFile @@ -33,6 +33,42 @@ ROOT = Path(__file__).parent DEFAULT_IN = ROOT.joinpath("vim-plugin-names") DEFAULT_OUT = ROOT.joinpath("generated.nix") +import time +from functools import wraps + + +def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2): + """Retry calling the decorated function using an exponential backoff. + + http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/ + original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry + (BSD licensed) + + :param ExceptionToCheck: the exception on which to retry + :param tries: number of times to try (not retry) before giving up + :param delay: initial delay between retries in seconds + :param backoff: backoff multiplier e.g. value of 2 will double the delay + each retry + """ + + def deco_retry(f: Callable) -> Callable: + @wraps(f) + def f_retry(*args: Any, **kwargs: Any) -> Any: + mtries, mdelay = tries, delay + while mtries > 1: + try: + return f(*args, **kwargs) + except ExceptionToCheck as e: + print(f"{str(e)}, Retrying in {mdelay} seconds...") + time.sleep(mdelay) + mtries -= 1 + mdelay *= backoff + return f(*args, **kwargs) + + return f_retry # true decorator + + return deco_retry + class Repo: def __init__(self, owner: str, name: str) -> None: @@ -45,9 +81,12 @@ class Repo: def __repr__(self) -> str: return f"Repo({self.owner}, {self.name})" + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def has_submodules(self) -> bool: try: - urllib.request.urlopen(self.url("blob/master/.gitmodules")).close() + urllib.request.urlopen( + self.url("blob/master/.gitmodules"), timeout=10 + ).close() except urllib.error.HTTPError as e: if e.code == 404: return False @@ -55,8 +94,9 @@ class Repo: raise return True + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def latest_commit(self) -> Tuple[str, datetime]: - with urllib.request.urlopen(self.url("commits/master.atom")) as req: + with urllib.request.urlopen(self.url("commits/master.atom"), timeout=10) as req: xml = req.read() root = ET.fromstring(xml) latest_entry = root.find(ATOM_ENTRY)