git.iko.soy/typeable/nixpkgs
on master help sign in
Merge pull request #50977 from jtojnar/parallel-update.nix

update.nix: Run update scripts in parallel

commit4920f0c0adebf632345d90fe134e1cde265f21f5 authorJan Tojnar <jtojnar@gmail.com> dateSun 2 Dec 2018, 01:23 (8 years ago) parent8a5c98830f090f
17 files changed+169−50
doc/stdenv.xml+37 maintainers/scripts/update.nix+19−34 maintainers/scripts/update.py+79 pkgs/applications/networking/browsers/firefox-bin/default.nix+1−1 pkgs/applications/networking/browsers/firefox-bin/update.nix+3−1 pkgs/applications/networking/browsers/firefox/update.nix+2 pkgs/applications/office/mendeley/default.nix+1−1 pkgs/applications/office/mendeley/update.nix+2−1 pkgs/desktops/deepin/update.nix+2−1 pkgs/desktops/gnome-3/update.nix+12−7 pkgs/development/libraries/safefile/default.nix+1 pkgs/development/tools/build-managers/gup/default.nix+1 pkgs/development/web/nodejs/update.nix+3−1 pkgs/os-specific/linux/tp_smapi/default.nix+1−1 pkgs/os-specific/linux/tp_smapi/update.nix+2−1 pkgs/tools/package-management/nix-pin/default.nix+1 pkgs/tools/package-management/nix-update-source/default.nix+2−1
doc/stdenv.xml+37
@@ -671,6 +671,43 @@ passthru = {
671671 </para>
672672 </listitem>
673673 </varlistentry>
674+ <varlistentry>
675+ <term>
676+ <varname>passthru.updateScript</varname>
677+ </term>
678+ <listitem>
679+ <para>
680+ A script to be run by <filename>maintainers/scripts/update.nix</filename> when
681+ the package is matched. It needs to be an executable file, either on the file
682+ system:
683+<programlisting>
684+passthru.updateScript = ./update.sh;
685+</programlisting>
686+ or inside the expression itself:
687+<programlisting>
688+passthru.updateScript = writeScript "update-zoom-us" ''
689+ #!/usr/bin/env nix-shell
690+ #!nix-shell -i bash -p curl pcre common-updater-scripts
691+
692+ set -eu -o pipefail
693+
694+ version="$(curl -sI https://zoom.us/client/latest/zoom_x86_64.tar.xz | grep -Fi 'Location:' | pcregrep -o1 '/(([0-9]\.?)+)/')"
695+ update-source-version zoom-us "$version"
696+'';
697+</programlisting>
698+ The attribute can also contain a list, a script followed by arguments to be passed to it:
699+<programlisting>
700+passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ];
701+</programlisting>
702+ Note that the update scripts will be run in parallel by default; you should avoid running <command>git commit</command> or any other commands that cannot handle that.
703+ </para>
704+
705+ <para>
706+ For information about how to run the updates, execute
707+ <cmdsynopsis><command>nix-shell</command> <arg>maintainers/scripts/update.nix</arg></cmdsynopsis>.
708+ </para>
709+ </listitem>
710+ </varlistentry>
674711 </variablelist>
675712 </section>
676713 <section xml:id="sec-stdenv-phases">
maintainers/scripts/update.nix+19−34
@@ -1,6 +1,8 @@
11{ package ? null
22, maintainer ? null
33, path ? null
4+, max-workers ? null
5+, keep-going ? null
46}:
57
68# TODO: add assert statements
@@ -105,27 +107,24 @@ let
105107 % nix-shell maintainers/scripts/update.nix --argstr path gnome3
106108
107109 to run update script for all package under an attribute path.
108− '';
109110
110− runUpdateScript = package: ''
111− echo -ne " - ${package.name}: UPDATING ..."\\r
112− ${package.updateScript} &> ${(builtins.parseDrvName package.name).name}.log
113− CODE=$?
114− if [ "$CODE" != "0" ]; then
115− echo " - ${package.name}: ERROR "
116− echo ""
117− echo "--- SHOWING ERROR LOG FOR ${package.name} ----------------------"
118− echo ""
119− cat ${(builtins.parseDrvName package.name).name}.log
120− echo ""
121− echo "--- SHOWING ERROR LOG FOR ${package.name} ----------------------"
122− exit $CODE
123− else
124− rm ${(builtins.parseDrvName package.name).name}.log
125− fi
126− echo " - ${package.name}: DONE. "
111+ You can also add
112+
113+ --argstr max-workers 8
114+
115+ to increase the number of jobs in parallel, or
116+
117+ --argstr keep-going true
118+
119+ to continue running when a single update fails.
127120 '';
128121
122+ packageData = package: {
123+ name = package.name;
124+ pname = (builtins.parseDrvName package.name).name;
125+ updateScript = pkgs.lib.toList package.updateScript;
126+ };
127+
129128in pkgs.stdenv.mkDerivation {
130129 name = "nixpkgs-update-script";
131130 buildCommand = ''
@@ -139,21 +138,7 @@ in pkgs.stdenv.mkDerivation {
139138 exit 1
140139 '';
141140 shellHook = ''
142− echo ""
143− echo "Going to be running update for following packages:"
144− echo "${builtins.concatStringsSep "\n" (map (x: " - ${x.name}") packages)}"
145− echo ""
146− read -n1 -r -p "Press space to continue..." confirm
147− if [ "$confirm" = "" ]; then
148− echo ""
149− echo "Running update for:"
150− ${builtins.concatStringsSep "\n" (map runUpdateScript packages)}
151− echo ""
152− echo "Packages updated!"
153− exit 0
154− else
155− echo "Aborting!"
156− exit 1
157− fi
141+ unset shellHook # do not contaminate nested shells
142+ exec ${pkgs.python3.interpreter} ${./update.py} ${pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages))}${pkgs.lib.optionalString (max-workers != null) " --max-workers=${max-workers}"}${pkgs.lib.optionalString (keep-going == "true") " --keep-going"}
158143 '';
159144}
maintainers/scripts/update.pynew file+79
@@ -0,0 +1,79 @@
1+import argparse
2+import concurrent.futures
3+import json
4+import os
5+import subprocess
6+import sys
7+
8+updates = {}
9+
10+def eprint(*args, **kwargs):
11+ print(*args, file=sys.stderr, **kwargs)
12+
13+def run_update_script(package):
14+ eprint(f" - {package['name']}: UPDATING ...")
15+
16+ subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
17+
18+
19+def main(max_workers, keep_going, packages):
20+ with open(sys.argv[1]) as f:
21+ packages = json.load(f)
22+
23+ eprint()
24+ eprint('Going to be running update for following packages:')
25+ for package in packages:
26+ eprint(f" - {package['name']}")
27+ eprint()
28+
29+ confirm = input('Press Enter key to continue...')
30+ if confirm == '':
31+ eprint()
32+ eprint('Running update for:')
33+
34+ with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
35+ for package in packages:
36+ updates[executor.submit(run_update_script, package)] = package
37+
38+ for future in concurrent.futures.as_completed(updates):
39+ package = updates[future]
40+
41+ try:
42+ future.result()
43+ eprint(f" - {package['name']}: DONE.")
44+ except subprocess.CalledProcessError as e:
45+ eprint(f" - {package['name']}: ERROR")
46+ eprint()
47+ eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
48+ eprint()
49+ eprint(e.stdout.decode('utf-8'))
50+ with open(f"{package['pname']}.log", 'wb') as f:
51+ f.write(e.stdout)
52+ eprint()
53+ eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
54+
55+ if not keep_going:
56+ sys.exit(1)
57+
58+ eprint()
59+ eprint('Packages updated!')
60+ sys.exit()
61+ else:
62+ eprint('Aborting!')
63+ sys.exit(130)
64+
65+parser = argparse.ArgumentParser(description='Update packages')
66+parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
67+parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
68+parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
69+
70+if __name__ == '__main__':
71+ args = parser.parse_args()
72+
73+ try:
74+ main(args.max_workers, args.keep_going, args.packages)
75+ except (KeyboardInterrupt, SystemExit) as e:
76+ for update in updates:
77+ update.cancel()
78+
79+ sys.exit(e.code if isinstance(e, SystemExit) else 130)
pkgs/applications/networking/browsers/firefox-bin/default.nix+1−1
@@ -191,7 +191,7 @@ stdenv.mkDerivation {
191191 # update with:
192192 # $ nix-shell maintainers/scripts/update.nix --argstr package firefox-bin-unwrapped
193193 passthru.updateScript = import ./update.nix {
194− inherit name channel writeScript xidel coreutils gnused gnugrep gnupg curl;
194+ inherit stdenv name channel writeScript xidel coreutils gnused gnugrep gnupg curl;
195195 baseUrl =
196196 if channel == "devedition"
197197 then "http://archive.mozilla.org/pub/devedition/releases/"
pkgs/applications/networking/browsers/firefox-bin/update.nix+3−1
@@ -1,4 +1,5 @@
1−{ name
1+{ stdenv
2+, name
23, channel
34, writeScript
45, xidel
@@ -17,6 +18,7 @@ let
1718 channel != "release";
1819
1920in writeScript "update-${name}" ''
21+ #!${stdenv.shell}
2022 PATH=${coreutils}/bin:${gnused}/bin:${gnugrep}/bin:${xidel}/bin:${curl}/bin:${gnupg}/bin
2123 set -eux
2224 pushd ${basePath}
pkgs/applications/networking/browsers/firefox/update.nix+2
@@ -1,4 +1,5 @@
11{ writeScript
2+, stdenv
23, lib
34, xidel
45, common-updater-scripts
@@ -13,6 +14,7 @@
1314}:
1415
1516writeScript "update-${attrPath}" ''
17+ #!${stdenv.shell}
1618 PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep gnused xidel ]}
1719
1820 url=${baseUrl}
pkgs/applications/office/mendeley/default.nix+1−1
@@ -131,7 +131,7 @@ stdenv.mkDerivation {
131131 dontStrip = true;
132132 dontPatchElf = true;
133133
134− updateScript = import ./update.nix { inherit writeScript; };
134+ updateScript = import ./update.nix { inherit stdenv writeScript; };
135135
136136 meta = with stdenv.lib; {
137137 homepage = http://www.mendeley.com;
pkgs/applications/office/mendeley/update.nix+2−1
@@ -1,6 +1,7 @@
1−{ writeScript }:
1+{ stdenv, writeScript }:
22
33writeScript "update-mendeley" ''
4+ #!${stdenv.shell}
45 function follow() {
56 local URL=$1
67 while true; do
pkgs/desktops/deepin/update.nix+2−1
@@ -1,4 +1,4 @@
1−{ lib, writeScript, coreutils, curl, gnugrep, gnused, jq, common-updater-scripts, nix }:
1+{ stdenv, lib, writeScript, coreutils, curl, gnugrep, gnused, jq, common-updater-scripts, nix }:
22{ name, ignored-versions ? "^2014\\.|^v[0-9]+" }:
33
44let
@@ -9,6 +9,7 @@ let
99in
1010
1111writeScript "update-${packageName}" ''
12+ #!${stdenv.shell}
1213 set -o errexit
1314 set -x
1415
pkgs/desktops/gnome-3/update.nix+12−7
@@ -1,11 +1,16 @@
1−{ lib, writeScript, python3, common-updater-scripts, coreutils, gnugrep, gnused }:
1+{ stdenv, lib, writeScript, python3, common-updater-scripts, coreutils, gnugrep, gnused }:
22{ packageName, attrPath ? packageName, versionPolicy ? "odd-unstable" }:
33
44let
55 python = python3.withPackages (p: [ p.requests ]);
6−in writeScript "update-${packageName}" ''
7− set -o errexit
8− PATH=${lib.makeBinPath [ common-updater-scripts coreutils gnugrep gnused python ]}
9− latest_tag=$(python "${./find-latest-version.py}" "${packageName}" "${versionPolicy}" "stable")
10− update-source-version "${attrPath}" "$latest_tag"
11−''
6+ updateScript = writeScript "gnome-update-script" ''
7+ #!${stdenv.shell}
8+ set -o errexit
9+ package_name="$1"
10+ attr_path="$2"
11+ version_policy="$3"
12+ PATH=${lib.makeBinPath [ common-updater-scripts coreutils gnugrep gnused python ]}
13+ latest_tag=$(python "${./find-latest-version.py}" "$package_name" "$version_policy" "stable")
14+ update-source-version "$attr_path" "$latest_tag"
15+ '';
16+in [ updateScript packageName attrPath versionPolicy ]
pkgs/development/libraries/safefile/default.nix+1
@@ -13,6 +13,7 @@ stdenv.mkDerivation rec {
1313
1414 passthru = {
1515 updateScript = ''
16+ #!${stdenv.shell}
1617 cd ${toString ./.}
1718 ${toString path}/pkgs/build-support/upstream-updater/update-walker.sh default.nix
1819 '';
pkgs/development/tools/build-managers/gup/default.nix+1
@@ -16,6 +16,7 @@ stdenv.mkDerivation rec {
1616 cp -r python/bin $out/bin
1717 '';
1818 passthru.updateScript = ''
19+ #!${stdenv.shell}
1920 set -e
2021 echo
2122 cd ${toString ./.}
pkgs/development/web/nodejs/update.nix+3−1
@@ -1,4 +1,5 @@
1−{ lib
1+{ stdenv
2+, lib
23, writeScript
34, coreutils
45, curl
@@ -11,6 +12,7 @@
1112}:
1213
1314writeScript "update-nodejs" ''
15+ #!${stdenv.shell}
1416 PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep jq gnupg nix ]}
1517
1618 HOME=`mktemp -d`
pkgs/os-specific/linux/tp_smapi/default.nix+1−1
@@ -34,7 +34,7 @@ stdenv.mkDerivation rec {
3434 enableParallelBuilding = true;
3535
3636 passthru.updateScript = import ./update.nix {
37− inherit lib writeScript coreutils gnugrep jq curl common-updater-scripts;
37+ inherit stdenv lib writeScript coreutils gnugrep jq curl common-updater-scripts;
3838 };
3939
4040 meta = {
pkgs/os-specific/linux/tp_smapi/update.nix+2−1
@@ -1,6 +1,7 @@
1−{ lib, writeScript, coreutils, curl, gnugrep, jq, common-updater-scripts }:
1+{ stdenv, lib, writeScript, coreutils, curl, gnugrep, jq, common-updater-scripts }:
22
33writeScript "update-tp_smapi" ''
4+#!${stdenv.shell}
45PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep jq ]}
56
67tags=`curl -s https://api.github.com/repos/evgeni/tp_smapi/tags`
pkgs/tools/package-management/nix-pin/default.nix+1
@@ -26,6 +26,7 @@ let self = stdenv.mkDerivation rec {
2626 let impl = import "${self}/share/nix/api.nix" { inherit pkgs pinConfig; }; in
2727 { inherit (impl) augmentedPkgs pins callPackage; };
2828 updateScript = ''
29+ #!${stdenv.shell}
2930 set -e
3031 echo
3132 cd ${toString ./.}
pkgs/tools/package-management/nix-update-source/default.nix+2−1
@@ -1,4 +1,4 @@
1−{ lib, pkgs, fetchFromGitHub, python3Packages, nix-prefetch-scripts }:
1+{ stdenv, lib, pkgs, fetchFromGitHub, python3Packages, nix-prefetch-scripts }:
22python3Packages.buildPythonApplication rec {
33 version = "0.6.3";
44 name = "nix-update-source-${version}";
@@ -28,6 +28,7 @@ python3Packages.buildPythonApplication rec {
2828 overrideSrc = drv: lib.overrideDerivation drv (orig: { inherit src; });
2929 };
3030 updateScript = ''
31+ #!${stdenv.shell}
3132 set -e
3233 echo
3334 cd ${toString ./.}