git.iko.soy/typeable/nixpkgs
on master help sign in
Merge pull request #98304 from jtojnar/updateScript-commit3

maintainers/scripts/update.nix: Add support for auto-commiting changes

commit74c547209048372ca10fa92660108b03363c3489 authorJan Tojnar <jtojnar@gmail.com> dateFri 2 Oct 2020, 06:16 (6 years ago) parent1b17aea74a5bb4
4 files changed+275−84
doc/stdenv/stdenv.xml+6−3 maintainers/scripts/update.nix+80−45 maintainers/scripts/update.py+178−35 pkgs/common-updater/scripts/update-source-version+11−1
doc/stdenv/stdenv.xml+6−3
@@ -475,9 +475,12 @@ passthru.updateScript = writeScript "update-zoom-us" ''
475475<programlisting>
476476passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ];
477477</programlisting>
478− </para>
479− <para>
480− The script will be usually run from the root of the Nixpkgs repository but you should not rely on that. Also note that the update scripts will be run in parallel by default; you should avoid running <command>git commit</command> or any other commands that cannot handle that.
478+ The script will be run with <variable>UPDATE_NIX_ATTR_PATH</variable> environment variable set to the attribute path it is supposed to update.
479+ <note>
480+ <para>
481+ The script will be usually run from the root of the Nixpkgs repository but you should not rely on that. Also note that the update scripts will be run in parallel by default; you should avoid running <command>git commit</command> or any other commands that cannot handle that.
482+ </para>
483+ </note>
481484 </para>
482485 <para>
483486 For information about how to run the updates, execute <command>nix-shell maintainers/scripts/update.nix</command>.
maintainers/scripts/update.nix+80−45
@@ -4,6 +4,7 @@
44, max-workers ? null
55, include-overlays ? false
66, keep-going ? null
7+, commit ? null
78}:
89
910# TODO: add assert statements
@@ -31,30 +32,47 @@ let
3132 in
3233 [x] ++ nubOn f xs;
3334
34− packagesWithPath = relativePath: cond: return: pathContent:
35− let
36− result = builtins.tryEval pathContent;
35+ /* Recursively find all packages (derivations) in `pkgs` matching `cond` predicate.
3736
38− dedupResults = lst: nubOn (pkg: pkg.updateScript) (lib.concatLists lst);
39− in
40− if result.success then
37+ Type: packagesWithPath :: AttrPath → (AttrPath → derivation → bool) → (AttrSet | List) → List<AttrSet{attrPath :: str; package :: derivation; }>
38+ AttrPath :: [str]
39+
40+ The packages will be returned as a list of named pairs comprising of:
41+ - attrPath: stringified attribute path (based on `rootPath`)
42+ - package: corresponding derivation
43+ */
44+ packagesWithPath = rootPath: cond: pkgs:
45+ let
46+ packagesWithPathInner = path: pathContent:
4147 let
42− pathContent = result.value;
48+ result = builtins.tryEval pathContent;
49+
50+ dedupResults = lst: nubOn ({ package, attrPath }: package.updateScript) (lib.concatLists lst);
4351 in
44− if lib.isDerivation pathContent then
45− lib.optional (cond relativePath pathContent) (return relativePath pathContent)
46− else if lib.isAttrs pathContent then
47− # If user explicitly points to an attrSet or it is marked for recursion, we recur.
48− if relativePath == [] || pathContent.recurseForDerivations or false || pathContent.recurseForRelease or false then
49− dedupResults (lib.mapAttrsToList (name: elem: packagesWithPath (relativePath ++ [name]) cond return elem) pathContent)
50− else []
51− else if lib.isList pathContent then
52− dedupResults (lib.imap0 (i: elem: packagesWithPath (relativePath ++ [i]) cond return elem) pathContent)
53− else []
54− else [];
52+ if result.success then
53+ let
54+ evaluatedPathContent = result.value;
55+ in
56+ if lib.isDerivation evaluatedPathContent then
57+ lib.optional (cond path evaluatedPathContent) { attrPath = lib.concatStringsSep "." path; package = evaluatedPathContent; }
58+ else if lib.isAttrs evaluatedPathContent then
59+ # If user explicitly points to an attrSet or it is marked for recursion, we recur.
60+ if path == rootPath || evaluatedPathContent.recurseForDerivations or false || evaluatedPathContent.recurseForRelease or false then
61+ dedupResults (lib.mapAttrsToList (name: elem: packagesWithPathInner (path ++ [name]) elem) evaluatedPathContent)
62+ else []
63+ else if lib.isList evaluatedPathContent then
64+ dedupResults (lib.imap0 (i: elem: packagesWithPathInner (path ++ [i]) elem) evaluatedPathContent)
65+ else []
66+ else [];
67+ in
68+ packagesWithPathInner rootPath pkgs;
5569
70+ /* Recursively find all packages (derivations) in `pkgs` matching `cond` predicate.
71+ */
5672 packagesWith = packagesWithPath [];
5773
74+ /* Recursively find all packages in `pkgs` with updateScript by given maintainer.
75+ */
5876 packagesWithUpdateScriptAndMaintainer = maintainer':
5977 let
6078 maintainer =
@@ -63,47 +81,51 @@ let
6381 else
6482 builtins.getAttr maintainer' lib.maintainers;
6583 in
66− packagesWith (relativePath: pkg: builtins.hasAttr "updateScript" pkg &&
67− (if builtins.hasAttr "maintainers" pkg.meta
68− then (if builtins.isList pkg.meta.maintainers
69− then builtins.elem maintainer pkg.meta.maintainers
70− else maintainer == pkg.meta.maintainers
71− )
72− else false
73− )
74− )
75− (relativePath: pkg: pkg)
76− pkgs;
77−
78− packagesWithUpdateScript = path:
84+ packagesWith (path: pkg: builtins.hasAttr "updateScript" pkg &&
85+ (if builtins.hasAttr "maintainers" pkg.meta
86+ then (if builtins.isList pkg.meta.maintainers
87+ then builtins.elem maintainer pkg.meta.maintainers
88+ else maintainer == pkg.meta.maintainers
89+ )
90+ else false
91+ )
92+ );
93+
94+ /* Recursively find all packages under `path` in `pkgs` with updateScript.
95+ */
96+ packagesWithUpdateScript = path: pkgs:
7997 let
80− pathContent = lib.attrByPath (lib.splitString "." path) null pkgs;
98+ prefix = lib.splitString "." path;
99+ pathContent = lib.attrByPath prefix null pkgs;
81100 in
82101 if pathContent == null then
83102 builtins.throw "Attribute path `${path}` does not exists."
84103 else
85− packagesWith (relativePath: pkg: builtins.hasAttr "updateScript" pkg)
86− (relativePath: pkg: pkg)
104+ packagesWithPath prefix (path: pkg: builtins.hasAttr "updateScript" pkg)
87105 pathContent;
88106
89− packageByName = name:
107+ /* Find a package under `path` in `pkgs` and require that it has an updateScript.
108+ */
109+ packageByName = path: pkgs:
90110 let
91− package = lib.attrByPath (lib.splitString "." name) null pkgs;
111+ package = lib.attrByPath (lib.splitString "." path) null pkgs;
92112 in
93113 if package == null then
94− builtins.throw "Package with an attribute name `${name}` does not exists."
114+ builtins.throw "Package with an attribute name `${path}` does not exists."
95115 else if ! builtins.hasAttr "updateScript" package then
96− builtins.throw "Package with an attribute name `${name}` does not have a `passthru.updateScript` attribute defined."
116+ builtins.throw "Package with an attribute name `${path}` does not have a `passthru.updateScript` attribute defined."
97117 else
98− package;
118+ { attrPath = path; inherit package; };
99119
120+ /* List of packages matched based on the CLI arguments.
121+ */
100122 packages =
101123 if package != null then
102− [ (packageByName package) ]
124+ [ (packageByName package pkgs) ]
103125 else if maintainer != null then
104− packagesWithUpdateScriptAndMaintainer maintainer
126+ packagesWithUpdateScriptAndMaintainer maintainer pkgs
105127 else if path != null then
106− packagesWithUpdateScript path
128+ packagesWithUpdateScript path pkgs
107129 else
108130 builtins.throw "No arguments provided.\n\n${helpText}";
109131
@@ -132,19 +154,32 @@ let
132154 --argstr keep-going true
133155
134156 to continue running when a single update fails.
157+
158+ You can also make the updater automatically commit on your behalf from updateScripts
159+ that support it by adding
160+
161+ --argstr commit true
135162 '';
136163
137− packageData = package: {
164+ /* Transform a matched package into an object for update.py.
165+ */
166+ packageData = { package, attrPath }: {
138167 name = package.name;
139168 pname = lib.getName package;
140− updateScript = map builtins.toString (lib.toList package.updateScript);
169+ oldVersion = lib.getVersion package;
170+ updateScript = map builtins.toString (lib.toList (package.updateScript.command or package.updateScript));
171+ supportedFeatures = package.updateScript.supportedFeatures or [];
172+ attrPath = package.updateScript.attrPath or attrPath;
141173 };
142174
175+ /* JSON file with data for update.py.
176+ */
143177 packagesJson = pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages));
144178
145179 optionalArgs =
146180 lib.optional (max-workers != null) "--max-workers=${max-workers}"
147− ++ lib.optional (keep-going == "true") "--keep-going";
181+ ++ lib.optional (keep-going == "true") "--keep-going"
182+ ++ lib.optional (commit == "true") "--commit";
148183
149184 args = [ packagesJson ] ++ optionalArgs;
150185
maintainers/scripts/update.py+178−35
@@ -1,23 +1,189 @@
1+from __future__ import annotations
2+from typing import Dict, Generator, List, Optional, Tuple
13import argparse
2−import concurrent.futures
4+import asyncio
5+import contextlib
36import json
47import os
8+import re
59import subprocess
610import sys
11+import tempfile
712
8−updates = {}
13+class CalledProcessError(Exception):
14+ process: asyncio.subprocess.Process
915
1016def eprint(*args, **kwargs):
1117 print(*args, file=sys.stderr, **kwargs)
1218
13−def run_update_script(package):
19+async def check_subprocess(*args, **kwargs):
20+ """
21+ Emulate check argument of subprocess.run function.
22+ """
23+ process = await asyncio.create_subprocess_exec(*args, **kwargs)
24+ returncode = await process.wait()
25+
26+ if returncode != 0:
27+ error = CalledProcessError()
28+ error.process = process
29+
30+ raise error
31+
32+ return process
33+
34+async def run_update_script(nixpkgs_root: str, merge_lock: asyncio.Lock, temp_dir: Optional[Tuple[str, str]], package: Dict, keep_going: bool):
35+ worktree: Optional[str] = None
36+
37+ update_script_command = package['updateScript']
38+
39+ if temp_dir is not None:
40+ worktree, _branch = temp_dir
41+
42+ # Update scripts can use $(dirname $0) to get their location but we want to run
43+ # their clones in the git worktree, not in the main nixpkgs repo.
44+ update_script_command = map(lambda arg: re.sub(r'^{0}'.format(re.escape(nixpkgs_root)), worktree, arg), update_script_command)
45+
1446 eprint(f" - {package['name']}: UPDATING ...")
1547
16− subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
48+ try:
49+ update_process = await check_subprocess('env', f"UPDATE_NIX_ATTR_PATH={package['attrPath']}", *update_script_command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=worktree)
50+ update_info = await update_process.stdout.read()
51+
52+ await merge_changes(merge_lock, package, update_info, temp_dir)
53+ except KeyboardInterrupt as e:
54+ eprint('Cancelling…')
55+ raise asyncio.exceptions.CancelledError()
56+ except CalledProcessError as e:
57+ eprint(f" - {package['name']}: ERROR")
58+ eprint()
59+ eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
60+ eprint()
61+ stderr = await e.process.stderr.read()
62+ eprint(stderr.decode('utf-8'))
63+ with open(f"{package['pname']}.log", 'wb') as logfile:
64+ logfile.write(stderr)
65+ eprint()
66+ eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
67+
68+ if not keep_going:
69+ raise asyncio.exceptions.CancelledError()
70+
71+@contextlib.contextmanager
72+def make_worktree() -> Generator[Tuple[str, str], None, None]:
73+ with tempfile.TemporaryDirectory() as wt:
74+ branch_name = f'update-{os.path.basename(wt)}'
75+ target_directory = f'{wt}/nixpkgs'
76+
77+ subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory])
78+ yield (target_directory, branch_name)
79+ subprocess.run(['git', 'worktree', 'remove', '--force', target_directory])
80+ subprocess.run(['git', 'branch', '-D', branch_name])
81+
82+async def commit_changes(name: str, merge_lock: asyncio.Lock, worktree: str, branch: str, changes: List[Dict]) -> None:
83+ for change in changes:
84+ # Git can only handle a single index operation at a time
85+ async with merge_lock:
86+ await check_subprocess('git', 'add', *change['files'], cwd=worktree)
87+ commit_message = '{attrPath}: {oldVersion} → {newVersion}'.format(**change)
88+ await check_subprocess('git', 'commit', '--quiet', '-m', commit_message, cwd=worktree)
89+ await check_subprocess('git', 'cherry-pick', branch)
90+
91+async def check_changes(package: Dict, worktree: str, update_info: str):
92+ if 'commit' in package['supportedFeatures']:
93+ changes = json.loads(update_info)
94+ else:
95+ changes = [{}]
96+
97+ # Try to fill in missing attributes when there is just a single change.
98+ if len(changes) == 1:
99+ # Dynamic data from updater take precedence over static data from passthru.updateScript.
100+ if 'attrPath' not in changes[0]:
101+ # update.nix is always passing attrPath
102+ changes[0]['attrPath'] = package['attrPath']
103+
104+ if 'oldVersion' not in changes[0]:
105+ # update.nix is always passing oldVersion
106+ changes[0]['oldVersion'] = package['oldVersion']
107+
108+ if 'newVersion' not in changes[0]:
109+ attr_path = changes[0]['attrPath']
110+ obtain_new_version_process = await check_subprocess('nix-instantiate', '--expr', f'with import ./. {{}}; lib.getVersion {attr_path}', '--eval', '--strict', '--json', stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=worktree)
111+ changes[0]['newVersion'] = json.loads((await obtain_new_version_process.stdout.read()).decode('utf-8'))
112+
113+ if 'files' not in changes[0]:
114+ changed_files_process = await check_subprocess('git', 'diff', '--name-only', stdout=asyncio.subprocess.PIPE, cwd=worktree)
115+ changed_files = (await changed_files_process.stdout.read()).splitlines()
116+ changes[0]['files'] = changed_files
117+
118+ if len(changed_files) == 0:
119+ return []
120+
121+ return changes
122+
123+async def merge_changes(merge_lock: asyncio.Lock, package: Dict, update_info: str, temp_dir: Optional[Tuple[str, str]]) -> None:
124+ if temp_dir is not None:
125+ worktree, branch = temp_dir
126+ changes = await check_changes(package, worktree, update_info)
127+
128+ if len(changes) > 0:
129+ await commit_changes(package['name'], merge_lock, worktree, branch, changes)
130+ else:
131+ eprint(f" - {package['name']}: DONE, no changes.")
132+ else:
133+ eprint(f" - {package['name']}: DONE.")
17134
135+async def updater(nixpkgs_root: str, temp_dir: Optional[Tuple[str, str]], merge_lock: asyncio.Lock, packages_to_update: asyncio.Queue[Optional[Dict]], keep_going: bool, commit: bool):
136+ while True:
137+ package = await packages_to_update.get()
138+ if package is None:
139+ # A sentinel received, we are done.
140+ return
18141
19−def main(max_workers, keep_going, packages):
20− with open(sys.argv[1]) as f:
142+ if not ('commit' in package['supportedFeatures'] or 'attrPath' in package):
143+ temp_dir = None
144+
145+ await run_update_script(nixpkgs_root, merge_lock, temp_dir, package, keep_going)
146+
147+async def start_updates(max_workers: int, keep_going: bool, commit: bool, packages: List[Dict]):
148+ merge_lock = asyncio.Lock()
149+ packages_to_update: asyncio.Queue[Optional[Dict]] = asyncio.Queue()
150+
151+ with contextlib.ExitStack() as stack:
152+ temp_dirs: List[Optional[Tuple[str, str]]] = []
153+
154+ # Do not create more workers than there are packages.
155+ num_workers = min(max_workers, len(packages))
156+
157+ nixpkgs_root_process = await check_subprocess('git', 'rev-parse', '--show-toplevel', stdout=asyncio.subprocess.PIPE)
158+ nixpkgs_root = (await nixpkgs_root_process.stdout.read()).decode('utf-8').strip()
159+
160+ # Set up temporary directories when using auto-commit.
161+ for i in range(num_workers):
162+ temp_dir = stack.enter_context(make_worktree()) if commit else None
163+ temp_dirs.append(temp_dir)
164+
165+ # Fill up an update queue,
166+ for package in packages:
167+ await packages_to_update.put(package)
168+
169+ # Add sentinels, one for each worker.
170+ # A workers will terminate when it gets sentinel from the queue.
171+ for i in range(num_workers):
172+ await packages_to_update.put(None)
173+
174+ # Prepare updater workers for each temp_dir directory.
175+ # At most `num_workers` instances of `run_update_script` will be running at one time.
176+ updaters = asyncio.gather(*[updater(nixpkgs_root, temp_dir, merge_lock, packages_to_update, keep_going, commit) for temp_dir in temp_dirs])
177+
178+ try:
179+ # Start updater workers.
180+ await updaters
181+ except asyncio.exceptions.CancelledError as e:
182+ # When one worker is cancelled, cancel the others too.
183+ updaters.cancel()
184+
185+def main(max_workers: int, keep_going: bool, commit: bool, packages_path: str) -> None:
186+ with open(packages_path) as f:
21187 packages = json.load(f)
22188
23189 eprint()
@@ -31,29 +197,7 @@ def main(max_workers, keep_going, packages):
31197 eprint()
32198 eprint('Running update for:')
33199
34− with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
35− for package in packages:
36− updates[executor.submit(run_update_script, package)] = package
37−
38− for future in concurrent.futures.as_completed(updates):
39− package = updates[future]
40−
41− try:
42− future.result()
43− eprint(f" - {package['name']}: DONE.")
44− except subprocess.CalledProcessError as e:
45− eprint(f" - {package['name']}: ERROR")
46− eprint()
47− eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
48− eprint()
49− eprint(e.stdout.decode('utf-8'))
50− with open(f"{package['pname']}.log", 'wb') as f:
51− f.write(e.stdout)
52− eprint()
53− eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
54−
55− if not keep_going:
56− sys.exit(1)
200+ asyncio.run(start_updates(max_workers, keep_going, commit, packages))
57201
58202 eprint()
59203 eprint('Packages updated!')
@@ -65,15 +209,14 @@ def main(max_workers, keep_going, packages):
65209parser = argparse.ArgumentParser(description='Update packages')
66210parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
67211parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
212+parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes')
68213parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
69214
70215if __name__ == '__main__':
71216 args = parser.parse_args()
72217
73218 try:
74− main(args.max_workers, args.keep_going, args.packages)
75− except (KeyboardInterrupt, SystemExit) as e:
76− for update in updates:
77− update.cancel()
78−
79− sys.exit(e.code if isinstance(e, SystemExit) else 130)
219+ main(args.max_workers, args.keep_going, args.commit, args.packages)
220+ except KeyboardInterrupt as e:
221+ # Let’s cancel outside of the main loop too.
222+ sys.exit(130)
pkgs/common-updater/scripts/update-source-version+11−1
@@ -11,7 +11,7 @@ die() {
1111usage() {
1212 echo "Usage: $scriptName <attr> <version> [<new-source-hash>] [<new-source-url>]"
1313 echo " [--version-key=<version-key>] [--system=<system>] [--file=<file-to-update>]"
14− echo " [--ignore-same-hash]"
14+ echo " [--ignore-same-hash] [--print-changes]"
1515}
1616
1717args=()
@@ -33,6 +33,9 @@ for arg in "$@"; do
3333 --ignore-same-hash)
3434 ignoreSameHash="true"
3535 ;;
36+ --print-changes)
37+ printChanges="true"
38+ ;;
3639 --help)
3740 usage
3841 exit 0
@@ -102,6 +105,9 @@ fi
102105
103106if [[ "$oldVersion" = "$newVersion" ]]; then
104107 echo "$scriptName: New version same as old version, nothing to do." >&2
108+ if [ -n "$printChanges" ]; then
109+ printf '[]\n'
110+ fi
105111 exit 0
106112fi
107113
@@ -197,3 +203,7 @@ fi
197203
198204rm -f "$nixFile.bak"
199205rm -f "$attr.fetchlog"
206+
207+if [ -n "$printChanges" ]; then
208+ printf '[{"attrPath":"%s","oldVersion":"%s","newVersion":"%s","files":["%s"]}]\n' "$attr" "$oldVersion" "$newVersion" "$nixFile"
209+fi