@@ -12,11 +12,10 @@ to update all non-pinned libraries in that folder.
1212"""
1313
1414import argparse
15−import logging
1615import os
16+import pathlib
1717import re
1818import requests
19−import toolz
2019from concurrent.futures import ThreadPoolExecutor as Pool
2120from packaging.version import Version as _Version
2221from packaging.version import InvalidVersion
@@ -34,6 +33,8 @@ PRERELEASES = False
3433
3534GIT = "git"
3635
36+NIXPGKS_ROOT = str(pathlib.Path(__file__).absolute().parents[5])
37+
3738import logging
3839logging.basicConfig(level=logging.INFO)
3940
@@ -101,6 +102,7 @@ def _replace_value(attribute, value, text):
101102 new_text = text.replace(old_line, new_line)
102103 return new_text
103104
105+
104106def _fetch_page(url):
105107 r = requests.get(url)
106108 if r.status_code == requests.codes.ok:
@@ -109,6 +111,19 @@ def _fetch_page(url):
109111 raise ValueError("request for {} failed".format(url))
110112
111113
114+def _fetch_github(url):
115+ headers = {}
116+ token = os.environ.get('GITHUB_API_TOKEN')
117+ if token:
118+ headers["Authorization"] = f"token {token}"
119+ r = requests.get(url, headers=headers)
120+
121+ if r.status_code == requests.codes.ok:
122+ return r.json()
123+ else:
124+ raise ValueError("request for {} failed".format(url))
125+
126+
112127SEMVER = {
113128 'major' : 0,
114129 'minor' : 1,
@@ -167,11 +182,42 @@ def _get_latest_version_pypi(package, extension, current_version, target):
167182 break
168183 else:
169184 sha256 = None
170− return version, sha256
185+ return version, sha256, None
171186
172187
173188def _get_latest_version_github(package, extension, current_version, target):
174− raise ValueError("updating from GitHub is not yet supported.")
189+ def strip_prefix(tag):
190+ return re.sub("^[^0-9]*", "", tag)
191+
192+ def get_prefix(string):
193+ matches = re.findall(r"^([^0-9]*)", string)
194+ return next(iter(matches), "")
195+
196+ try:
197+ homepage = subprocess.check_output(
198+ ["nix", "eval", "-f", f"{NIXPGKS_ROOT}/default.nix", "--raw", f"python3Packages.{package}.src.meta.homepage"])\
199+ .decode('utf-8')
200+ except Exception as e:
201+ raise ValueError(f"Unable to determine homepage: {e}")
202+ owner_repo = homepage[len("https://github.com/"):] # remove prefix
203+ owner, repo = owner_repo.split("/")
204+
205+ url = f"https://api.github.com/repos/{owner}/{repo}/releases"
206+ all_releases = _fetch_github(url)
207+ releases = list(filter(lambda x: not x['prerelease'], all_releases))
208+
209+ if len(releases) == 0:
210+ raise ValueError(f"{homepage} does not contain any stable releases")
211+
212+ versions = map(lambda x: strip_prefix(x['tag_name']), releases)
213+ version = _determine_latest_version(current_version, target, versions)
214+
215+ release = next(filter(lambda x: strip_prefix(x['tag_name']) == version, releases))
216+ prefix = get_prefix(release['tag_name'])
217+ sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", f"{release['tarball_url']}"], stderr=subprocess.DEVNULL)\
218+ .decode('utf-8').strip()
219+
220+ return version, sha256, prefix
175221
176222
177223FETCHERS = {
@@ -240,7 +286,9 @@ def _determine_extension(text, fetcher):
240286 raise ValueError('url does not point to PyPI.')
241287
242288 elif fetcher == 'fetchFromGitHub':
243− raise ValueError('updating from GitHub is not yet implemented.')
289+ if "fetchSubmodules" in text:
290+ raise ValueError("fetchFromGitHub fetcher doesn't support submodules")
291+ extension = "tar.gz"
244292
245293 return extension
246294
@@ -262,7 +310,7 @@ def _update_package(path, target):
262310
263311 extension = _determine_extension(text, fetcher)
264312
265− new_version, new_sha256 = FETCHERS[fetcher](pname, extension, version, target)
313+ new_version, new_sha256, prefix = FETCHERS[fetcher](pname, extension, version, target)
266314
267315 if new_version == version:
268316 logging.info("Path {}: no update available for {}.".format(path, pname))
@@ -274,6 +322,10 @@ def _update_package(path, target):
274322
275323 text = _replace_value('version', new_version, text)
276324 text = _replace_value('sha256', new_sha256, text)
325+ if fetcher == 'fetchFromGitHub':
326+ text = _replace_value('rev', f"{prefix}${{version}}", text)
327+ # incase there's no prefix, just rewrite without interpolation
328+ text = text.replace('"${version}";', 'version;')
277329
278330 with open(path, 'w') as f:
279331 f.write(text)
@@ -333,7 +385,11 @@ def _commit(path, pname, old_version, new_version, pkgs_prefix="python: ", **kwa
333385
334386def main():
335387
336− parser = argparse.ArgumentParser()
388+ epilog = """
389+environment variables:
390+ GITHUB_API_TOKEN\tGitHub API token used when updating github packages
391+ """
392+ parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog)
337393 parser.add_argument('package', type=str, nargs='+')
338394 parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major')
339395 parser.add_argument('--commit', action='store_true', help='Create a commit for each package update')