4eaf246 · typeable/nixpkgs

git.iko.soy/typeable/nixpkgs

on master help sign in

update-python-libraries: add github fetcher

4eaf246Jonathan Ringer6 years ago

pkgs/development/interpreters/python/update-python-libraries/update-python-libraries.py+63−7

compare with master

unifiedsplit

@@ -12,11 +12,10 @@ to update all non-pinned libraries in that folder.

12"""

14import argparse

15−import logging

16import os

15import os

16+import pathlib

17import re

18import requests

19−import toolz

20from concurrent.futures import ThreadPoolExecutor as Pool

19from concurrent.futures import ThreadPoolExecutor as Pool

21from packaging.version import Version as _Version

20from packaging.version import Version as _Version

22from packaging.version import InvalidVersion

21from packaging.version import InvalidVersion

@@ -34,6 +33,8 @@ PRERELEASES = False

35GIT = "git"

34GIT = "git"

36+NIXPGKS_ROOT = str(pathlib.Path(__file__).absolute().parents[5])

37+

37import logging

38import logging

38logging.basicConfig(level=logging.INFO)

39logging.basicConfig(level=logging.INFO)

@@ -101,6 +102,7 @@ def _replace_value(attribute, value, text):

101 new_text = text.replace(old_line, new_line)

102 new_text = text.replace(old_line, new_line)

102 return new_text

103 return new_text

103

104

105+

104def _fetch_page(url):

106def _fetch_page(url):

105 r = requests.get(url)

107 r = requests.get(url)

106 if r.status_code == requests.codes.ok:

108 if r.status_code == requests.codes.ok:

@@ -109,6 +111,19 @@ def _fetch_page(url):

109 raise ValueError("request for {} failed".format(url))

111 raise ValueError("request for {} failed".format(url))

110

112

111

113

114+def _fetch_github(url):

115+ headers = {}

116+ token = os.environ.get('GITHUB_API_TOKEN')

117+ if token:

118+ headers["Authorization"] = f"token {token}"

119+ r = requests.get(url, headers=headers)

120+

121+ if r.status_code == requests.codes.ok:

122+ return r.json()

123+ else:

124+ raise ValueError("request for {} failed".format(url))

125+

126+

112SEMVER = {

127SEMVER = {

113 'major' : 0,

128 'major' : 0,

114 'minor' : 1,

129 'minor' : 1,

@@ -167,11 +182,42 @@ def _get_latest_version_pypi(package, extension, current_version, target):

167 break

182 break

168 else:

183 else:

169 sha256 = None

184 sha256 = None

170− return version, sha256

185+ return version, sha256, None

171

186

172

187

173def _get_latest_version_github(package, extension, current_version, target):

188def _get_latest_version_github(package, extension, current_version, target):

174− raise ValueError("updating from GitHub is not yet supported.")

189+ def strip_prefix(tag):

190+ return re.sub("^[^0-9]*", "", tag)

191+

192+ def get_prefix(string):

193+ matches = re.findall(r"^([^0-9]*)", string)

194+ return next(iter(matches), "")

195+

196+ try:

197+ homepage = subprocess.check_output(

198+ ["nix", "eval", "-f", f"{NIXPGKS_ROOT}/default.nix", "--raw", f"python3Packages.{package}.src.meta.homepage"])\

199+ .decode('utf-8')

200+ except Exception as e:

201+ raise ValueError(f"Unable to determine homepage: {e}")

202+ owner_repo = homepage[len("https://github.com/"):] # remove prefix

203+ owner, repo = owner_repo.split("/")

204+

205+ url = f"https://api.github.com/repos/{owner}/{repo}/releases"

206+ all_releases = _fetch_github(url)

207+ releases = list(filter(lambda x: not x['prerelease'], all_releases))

208+

209+ if len(releases) == 0:

210+ raise ValueError(f"{homepage} does not contain any stable releases")

211+

212+ versions = map(lambda x: strip_prefix(x['tag_name']), releases)

213+ version = _determine_latest_version(current_version, target, versions)

214+

215+ release = next(filter(lambda x: strip_prefix(x['tag_name']) == version, releases))

216+ prefix = get_prefix(release['tag_name'])

217+ sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", f"{release['tarball_url']}"], stderr=subprocess.DEVNULL)\

218+ .decode('utf-8').strip()

219+

220+ return version, sha256, prefix

175

221

176

222

177FETCHERS = {

223FETCHERS = {

@@ -240,7 +286,9 @@ def _determine_extension(text, fetcher):

240 raise ValueError('url does not point to PyPI.')

286 raise ValueError('url does not point to PyPI.')

241

287

242 elif fetcher == 'fetchFromGitHub':

288 elif fetcher == 'fetchFromGitHub':

243− raise ValueError('updating from GitHub is not yet implemented.')

289+ if "fetchSubmodules" in text:

290+ raise ValueError("fetchFromGitHub fetcher doesn't support submodules")

291+ extension = "tar.gz"

244

292

245 return extension

293 return extension

246

294

@@ -262,7 +310,7 @@ def _update_package(path, target):

262

310

263 extension = _determine_extension(text, fetcher)

311 extension = _determine_extension(text, fetcher)

264

312

265− new_version, new_sha256 = FETCHERS[fetcher](pname, extension, version, target)

313+ new_version, new_sha256, prefix = FETCHERS[fetcher](pname, extension, version, target)

266

314

267 if new_version == version:

315 if new_version == version:

268 logging.info("Path {}: no update available for {}.".format(path, pname))

316 logging.info("Path {}: no update available for {}.".format(path, pname))

@@ -274,6 +322,10 @@ def _update_package(path, target):

274

322

275 text = _replace_value('version', new_version, text)

323 text = _replace_value('version', new_version, text)

276 text = _replace_value('sha256', new_sha256, text)

324 text = _replace_value('sha256', new_sha256, text)

325+ if fetcher == 'fetchFromGitHub':

326+ text = _replace_value('rev', f"{prefix}${{version}}", text)

327+ # incase there's no prefix, just rewrite without interpolation

328+ text = text.replace('"${version}";', 'version;')

277

329

278 with open(path, 'w') as f:

330 with open(path, 'w') as f:

279 f.write(text)

331 f.write(text)

@@ -333,7 +385,11 @@ def _commit(path, pname, old_version, new_version, pkgs_prefix="python: ", **kwa

333

385

334def main():

386def main():

335

387

388+ epilog = """

389+environment variables:

390+ GITHUB_API_TOKEN\tGitHub API token used when updating github packages

391+ """

336− parser = argparse.ArgumentParser()

392+ parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog)

337 parser.add_argument('package', type=str, nargs='+')

393 parser.add_argument('package', type=str, nargs='+')

338 parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major')

394 parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major')

339 parser.add_argument('--commit', action='store_true', help='Create a commit for each package update')

395 parser.add_argument('--commit', action='store_true', help='Create a commit for each package update')