@@ -1,23 +1,189 @@
1+from __future__ import annotations
2+from typing import Dict, Generator, List, Optional, Tuple
13import argparse
2−import concurrent.futures
4+import asyncio
5+import contextlib
36import json
47import os
8+import re
59import subprocess
610import sys
11+import tempfile
712
8−updates = {}
13+class CalledProcessError(Exception):
14+ process: asyncio.subprocess.Process
915
1016def eprint(*args, **kwargs):
1117 print(*args, file=sys.stderr, **kwargs)
1218
13−def run_update_script(package):
19+async def check_subprocess(*args, **kwargs):
20+ """
21+ Emulate check argument of subprocess.run function.
22+ """
23+ process = await asyncio.create_subprocess_exec(*args, **kwargs)
24+ returncode = await process.wait()
25+
26+ if returncode != 0:
27+ error = CalledProcessError()
28+ error.process = process
29+
30+ raise error
31+
32+ return process
33+
34+async def run_update_script(nixpkgs_root: str, merge_lock: asyncio.Lock, temp_dir: Optional[Tuple[str, str]], package: Dict, keep_going: bool):
35+ worktree: Optional[str] = None
36+
37+ update_script_command = package['updateScript']
38+
39+ if temp_dir is not None:
40+ worktree, _branch = temp_dir
41+
42+ # Update scripts can use $(dirname $0) to get their location but we want to run
43+ # their clones in the git worktree, not in the main nixpkgs repo.
44+ update_script_command = map(lambda arg: re.sub(r'^{0}'.format(re.escape(nixpkgs_root)), worktree, arg), update_script_command)
45+
1446 eprint(f" - {package['name']}: UPDATING ...")
1547
16− subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
48+ try:
49+ update_process = await check_subprocess('env', f"UPDATE_NIX_ATTR_PATH={package['attrPath']}", *update_script_command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=worktree)
50+ update_info = await update_process.stdout.read()
51+
52+ await merge_changes(merge_lock, package, update_info, temp_dir)
53+ except KeyboardInterrupt as e:
54+ eprint('Cancelling…')
55+ raise asyncio.exceptions.CancelledError()
56+ except CalledProcessError as e:
57+ eprint(f" - {package['name']}: ERROR")
58+ eprint()
59+ eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
60+ eprint()
61+ stderr = await e.process.stderr.read()
62+ eprint(stderr.decode('utf-8'))
63+ with open(f"{package['pname']}.log", 'wb') as logfile:
64+ logfile.write(stderr)
65+ eprint()
66+ eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
67+
68+ if not keep_going:
69+ raise asyncio.exceptions.CancelledError()
70+
71+@contextlib.contextmanager
72+def make_worktree() -> Generator[Tuple[str, str], None, None]:
73+ with tempfile.TemporaryDirectory() as wt:
74+ branch_name = f'update-{os.path.basename(wt)}'
75+ target_directory = f'{wt}/nixpkgs'
76+
77+ subprocess.run(['git', 'worktree', 'add', '-b', branch_name, target_directory])
78+ yield (target_directory, branch_name)
79+ subprocess.run(['git', 'worktree', 'remove', '--force', target_directory])
80+ subprocess.run(['git', 'branch', '-D', branch_name])
81+
82+async def commit_changes(name: str, merge_lock: asyncio.Lock, worktree: str, branch: str, changes: List[Dict]) -> None:
83+ for change in changes:
84+ # Git can only handle a single index operation at a time
85+ async with merge_lock:
86+ await check_subprocess('git', 'add', *change['files'], cwd=worktree)
87+ commit_message = '{attrPath}: {oldVersion} → {newVersion}'.format(**change)
88+ await check_subprocess('git', 'commit', '--quiet', '-m', commit_message, cwd=worktree)
89+ await check_subprocess('git', 'cherry-pick', branch)
90+
91+async def check_changes(package: Dict, worktree: str, update_info: str):
92+ if 'commit' in package['supportedFeatures']:
93+ changes = json.loads(update_info)
94+ else:
95+ changes = [{}]
96+
97+ # Try to fill in missing attributes when there is just a single change.
98+ if len(changes) == 1:
99+ # Dynamic data from updater take precedence over static data from passthru.updateScript.
100+ if 'attrPath' not in changes[0]:
101+ # update.nix is always passing attrPath
102+ changes[0]['attrPath'] = package['attrPath']
103+
104+ if 'oldVersion' not in changes[0]:
105+ # update.nix is always passing oldVersion
106+ changes[0]['oldVersion'] = package['oldVersion']
107+
108+ if 'newVersion' not in changes[0]:
109+ attr_path = changes[0]['attrPath']
110+ obtain_new_version_process = await check_subprocess('nix-instantiate', '--expr', f'with import ./. {{}}; lib.getVersion {attr_path}', '--eval', '--strict', '--json', stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=worktree)
111+ changes[0]['newVersion'] = json.loads((await obtain_new_version_process.stdout.read()).decode('utf-8'))
112+
113+ if 'files' not in changes[0]:
114+ changed_files_process = await check_subprocess('git', 'diff', '--name-only', stdout=asyncio.subprocess.PIPE, cwd=worktree)
115+ changed_files = (await changed_files_process.stdout.read()).splitlines()
116+ changes[0]['files'] = changed_files
117+
118+ if len(changed_files) == 0:
119+ return []
120+
121+ return changes
122+
123+async def merge_changes(merge_lock: asyncio.Lock, package: Dict, update_info: str, temp_dir: Optional[Tuple[str, str]]) -> None:
124+ if temp_dir is not None:
125+ worktree, branch = temp_dir
126+ changes = await check_changes(package, worktree, update_info)
127+
128+ if len(changes) > 0:
129+ await commit_changes(package['name'], merge_lock, worktree, branch, changes)
130+ else:
131+ eprint(f" - {package['name']}: DONE, no changes.")
132+ else:
133+ eprint(f" - {package['name']}: DONE.")
17134
135+async def updater(nixpkgs_root: str, temp_dir: Optional[Tuple[str, str]], merge_lock: asyncio.Lock, packages_to_update: asyncio.Queue[Optional[Dict]], keep_going: bool, commit: bool):
136+ while True:
137+ package = await packages_to_update.get()
138+ if package is None:
139+ # A sentinel received, we are done.
140+ return
18141
19−def main(max_workers, keep_going, packages):
20− with open(sys.argv[1]) as f:
142+ if not ('commit' in package['supportedFeatures'] or 'attrPath' in package):
143+ temp_dir = None
144+
145+ await run_update_script(nixpkgs_root, merge_lock, temp_dir, package, keep_going)
146+
147+async def start_updates(max_workers: int, keep_going: bool, commit: bool, packages: List[Dict]):
148+ merge_lock = asyncio.Lock()
149+ packages_to_update: asyncio.Queue[Optional[Dict]] = asyncio.Queue()
150+
151+ with contextlib.ExitStack() as stack:
152+ temp_dirs: List[Optional[Tuple[str, str]]] = []
153+
154+ # Do not create more workers than there are packages.
155+ num_workers = min(max_workers, len(packages))
156+
157+ nixpkgs_root_process = await check_subprocess('git', 'rev-parse', '--show-toplevel', stdout=asyncio.subprocess.PIPE)
158+ nixpkgs_root = (await nixpkgs_root_process.stdout.read()).decode('utf-8').strip()
159+
160+ # Set up temporary directories when using auto-commit.
161+ for i in range(num_workers):
162+ temp_dir = stack.enter_context(make_worktree()) if commit else None
163+ temp_dirs.append(temp_dir)
164+
165+ # Fill up an update queue,
166+ for package in packages:
167+ await packages_to_update.put(package)
168+
169+ # Add sentinels, one for each worker.
170+ # A workers will terminate when it gets sentinel from the queue.
171+ for i in range(num_workers):
172+ await packages_to_update.put(None)
173+
174+ # Prepare updater workers for each temp_dir directory.
175+ # At most `num_workers` instances of `run_update_script` will be running at one time.
176+ updaters = asyncio.gather(*[updater(nixpkgs_root, temp_dir, merge_lock, packages_to_update, keep_going, commit) for temp_dir in temp_dirs])
177+
178+ try:
179+ # Start updater workers.
180+ await updaters
181+ except asyncio.exceptions.CancelledError as e:
182+ # When one worker is cancelled, cancel the others too.
183+ updaters.cancel()
184+
185+def main(max_workers: int, keep_going: bool, commit: bool, packages_path: str) -> None:
186+ with open(packages_path) as f:
21187 packages = json.load(f)
22188
23189 eprint()
@@ -31,29 +197,7 @@ def main(max_workers, keep_going, packages):
31197 eprint()
32198 eprint('Running update for:')
33199
34− with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
35− for package in packages:
36− updates[executor.submit(run_update_script, package)] = package
37−
38− for future in concurrent.futures.as_completed(updates):
39− package = updates[future]
40−
41− try:
42− future.result()
43− eprint(f" - {package['name']}: DONE.")
44− except subprocess.CalledProcessError as e:
45− eprint(f" - {package['name']}: ERROR")
46− eprint()
47− eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
48− eprint()
49− eprint(e.stdout.decode('utf-8'))
50− with open(f"{package['pname']}.log", 'wb') as f:
51− f.write(e.stdout)
52− eprint()
53− eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
54−
55− if not keep_going:
56− sys.exit(1)
200+ asyncio.run(start_updates(max_workers, keep_going, commit, packages))
57201
58202 eprint()
59203 eprint('Packages updated!')
@@ -65,15 +209,14 @@ def main(max_workers, keep_going, packages):
65209parser = argparse.ArgumentParser(description='Update packages')
66210parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
67211parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
212+parser.add_argument('--commit', '-c', dest='commit', action='store_true', help='Commit the changes')
68213parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
69214
70215if __name__ == '__main__':
71216 args = parser.parse_args()
72217
73218 try:
74− main(args.max_workers, args.keep_going, args.packages)
75− except (KeyboardInterrupt, SystemExit) as e:
76− for update in updates:
77− update.cancel()
78−
79− sys.exit(e.code if isinstance(e, SystemExit) else 130)
219+ main(args.max_workers, args.keep_going, args.commit, args.packages)
220+ except KeyboardInterrupt as e:
221+ # Let’s cancel outside of the main loop too.
222+ sys.exit(130)