Merge pull request #50977 from jtojnar/parallel-update.nix

update.nix: Run update scripts in parallel
This commit is contained in:
Jan Tojnar 2018-12-02 01:23:39 +01:00 committed by GitHub
commit 4920f0c0ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 169 additions and 50 deletions

View File

@ -671,6 +671,43 @@ passthru = {
</para>
</listitem>
</varlistentry>
<varlistentry>
<term>
<varname>passthru.updateScript</varname>
</term>
<listitem>
<para>
A script to be run by <filename>maintainers/scripts/update.nix</filename> when
the package is matched. It needs to be an executable file, either on the file
system:
<programlisting>
passthru.updateScript = ./update.sh;
</programlisting>
or inside the expression itself:
<programlisting>
passthru.updateScript = writeScript "update-zoom-us" ''
#!/usr/bin/env nix-shell
#!nix-shell -i bash -p curl pcre common-updater-scripts
set -eu -o pipefail
version="$(curl -sI https://zoom.us/client/latest/zoom_x86_64.tar.xz | grep -Fi 'Location:' | pcregrep -o1 '/(([0-9]\.?)+)/')"
update-source-version zoom-us "$version"
'';
</programlisting>
The attribute can also contain a list, a script followed by arguments to be passed to it:
<programlisting>
passthru.updateScript = [ ../../update.sh pname "--requested-release=unstable" ];
</programlisting>
Note that the update scripts will be run in parallel by default; you should avoid running <command>git commit</command> or any other commands that cannot handle that.
</para>
<para>
For information about how to run the updates, execute
<cmdsynopsis><command>nix-shell</command> <arg>maintainers/scripts/update.nix</arg></cmdsynopsis>.
</para>
</listitem>
</varlistentry>
</variablelist>
</section>
<section xml:id="sec-stdenv-phases">

View File

@ -1,6 +1,8 @@
{ package ? null
, maintainer ? null
, path ? null
, max-workers ? null
, keep-going ? null
}:
# TODO: add assert statements
@ -105,26 +107,23 @@ let
% nix-shell maintainers/scripts/update.nix --argstr path gnome3
to run update script for all package under an attribute path.
You can also add
--argstr max-workers 8
to increase the number of jobs in parallel, or
--argstr keep-going true
to continue running when a single update fails.
'';
runUpdateScript = package: ''
echo -ne " - ${package.name}: UPDATING ..."\\r
${package.updateScript} &> ${(builtins.parseDrvName package.name).name}.log
CODE=$?
if [ "$CODE" != "0" ]; then
echo " - ${package.name}: ERROR "
echo ""
echo "--- SHOWING ERROR LOG FOR ${package.name} ----------------------"
echo ""
cat ${(builtins.parseDrvName package.name).name}.log
echo ""
echo "--- SHOWING ERROR LOG FOR ${package.name} ----------------------"
exit $CODE
else
rm ${(builtins.parseDrvName package.name).name}.log
fi
echo " - ${package.name}: DONE. "
'';
packageData = package: {
name = package.name;
pname = (builtins.parseDrvName package.name).name;
updateScript = pkgs.lib.toList package.updateScript;
};
in pkgs.stdenv.mkDerivation {
name = "nixpkgs-update-script";
@ -139,21 +138,7 @@ in pkgs.stdenv.mkDerivation {
exit 1
'';
shellHook = ''
echo ""
echo "Going to be running update for following packages:"
echo "${builtins.concatStringsSep "\n" (map (x: " - ${x.name}") packages)}"
echo ""
read -n1 -r -p "Press space to continue..." confirm
if [ "$confirm" = "" ]; then
echo ""
echo "Running update for:"
${builtins.concatStringsSep "\n" (map runUpdateScript packages)}
echo ""
echo "Packages updated!"
exit 0
else
echo "Aborting!"
exit 1
fi
unset shellHook # do not contaminate nested shells
exec ${pkgs.python3.interpreter} ${./update.py} ${pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages))}${pkgs.lib.optionalString (max-workers != null) " --max-workers=${max-workers}"}${pkgs.lib.optionalString (keep-going == "true") " --keep-going"}
'';
}

View File

@ -0,0 +1,79 @@
import argparse
import concurrent.futures
import json
import os
import subprocess
import sys
updates = {}
def eprint(*args, **kwargs):
print(*args, file=sys.stderr, **kwargs)
def run_update_script(package):
eprint(f" - {package['name']}: UPDATING ...")
subprocess.run(package['updateScript'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=True)
def main(max_workers, keep_going, packages):
with open(sys.argv[1]) as f:
packages = json.load(f)
eprint()
eprint('Going to be running update for following packages:')
for package in packages:
eprint(f" - {package['name']}")
eprint()
confirm = input('Press Enter key to continue...')
if confirm == '':
eprint()
eprint('Running update for:')
with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
for package in packages:
updates[executor.submit(run_update_script, package)] = package
for future in concurrent.futures.as_completed(updates):
package = updates[future]
try:
future.result()
eprint(f" - {package['name']}: DONE.")
except subprocess.CalledProcessError as e:
eprint(f" - {package['name']}: ERROR")
eprint()
eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
eprint()
eprint(e.stdout.decode('utf-8'))
with open(f"{package['pname']}.log", 'wb') as f:
f.write(e.stdout)
eprint()
eprint(f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------")
if not keep_going:
sys.exit(1)
eprint()
eprint('Packages updated!')
sys.exit()
else:
eprint('Aborting!')
sys.exit(130)
parser = argparse.ArgumentParser(description='Update packages')
parser.add_argument('--max-workers', '-j', dest='max_workers', type=int, help='Number of updates to run concurrently', nargs='?', default=4)
parser.add_argument('--keep-going', '-k', dest='keep_going', action='store_true', help='Do not stop after first failure')
parser.add_argument('packages', help='JSON file containing the list of package names and their update scripts')
if __name__ == '__main__':
args = parser.parse_args()
try:
main(args.max_workers, args.keep_going, args.packages)
except (KeyboardInterrupt, SystemExit) as e:
for update in updates:
update.cancel()
sys.exit(e.code if isinstance(e, SystemExit) else 130)

View File

@ -191,7 +191,7 @@ stdenv.mkDerivation {
# update with:
# $ nix-shell maintainers/scripts/update.nix --argstr package firefox-bin-unwrapped
passthru.updateScript = import ./update.nix {
inherit name channel writeScript xidel coreutils gnused gnugrep gnupg curl;
inherit stdenv name channel writeScript xidel coreutils gnused gnugrep gnupg curl;
baseUrl =
if channel == "devedition"
then "http://archive.mozilla.org/pub/devedition/releases/"

View File

@ -1,4 +1,5 @@
{ name
{ stdenv
, name
, channel
, writeScript
, xidel
@ -17,6 +18,7 @@ let
channel != "release";
in writeScript "update-${name}" ''
#!${stdenv.shell}
PATH=${coreutils}/bin:${gnused}/bin:${gnugrep}/bin:${xidel}/bin:${curl}/bin:${gnupg}/bin
set -eux
pushd ${basePath}

View File

@ -1,4 +1,5 @@
{ writeScript
, stdenv
, lib
, xidel
, common-updater-scripts
@ -13,6 +14,7 @@
}:
writeScript "update-${attrPath}" ''
#!${stdenv.shell}
PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep gnused xidel ]}
url=${baseUrl}

View File

@ -131,7 +131,7 @@ stdenv.mkDerivation {
dontStrip = true;
dontPatchElf = true;
updateScript = import ./update.nix { inherit writeScript; };
updateScript = import ./update.nix { inherit stdenv writeScript; };
meta = with stdenv.lib; {
homepage = http://www.mendeley.com;

View File

@ -1,6 +1,7 @@
{ writeScript }:
{ stdenv, writeScript }:
writeScript "update-mendeley" ''
#!${stdenv.shell}
function follow() {
local URL=$1
while true; do

View File

@ -1,4 +1,4 @@
{ lib, writeScript, coreutils, curl, gnugrep, gnused, jq, common-updater-scripts, nix }:
{ stdenv, lib, writeScript, coreutils, curl, gnugrep, gnused, jq, common-updater-scripts, nix }:
{ name, ignored-versions ? "^2014\\.|^v[0-9]+" }:
let
@ -9,6 +9,7 @@ let
in
writeScript "update-${packageName}" ''
#!${stdenv.shell}
set -o errexit
set -x

View File

@ -1,11 +1,16 @@
{ lib, writeScript, python3, common-updater-scripts, coreutils, gnugrep, gnused }:
{ stdenv, lib, writeScript, python3, common-updater-scripts, coreutils, gnugrep, gnused }:
{ packageName, attrPath ? packageName, versionPolicy ? "odd-unstable" }:
let
python = python3.withPackages (p: [ p.requests ]);
in writeScript "update-${packageName}" ''
set -o errexit
PATH=${lib.makeBinPath [ common-updater-scripts coreutils gnugrep gnused python ]}
latest_tag=$(python "${./find-latest-version.py}" "${packageName}" "${versionPolicy}" "stable")
update-source-version "${attrPath}" "$latest_tag"
''
updateScript = writeScript "gnome-update-script" ''
#!${stdenv.shell}
set -o errexit
package_name="$1"
attr_path="$2"
version_policy="$3"
PATH=${lib.makeBinPath [ common-updater-scripts coreutils gnugrep gnused python ]}
latest_tag=$(python "${./find-latest-version.py}" "$package_name" "$version_policy" "stable")
update-source-version "$attr_path" "$latest_tag"
'';
in [ updateScript packageName attrPath versionPolicy ]

View File

@ -13,6 +13,7 @@ stdenv.mkDerivation rec {
passthru = {
updateScript = ''
#!${stdenv.shell}
cd ${toString ./.}
${toString path}/pkgs/build-support/upstream-updater/update-walker.sh default.nix
'';

View File

@ -16,6 +16,7 @@ stdenv.mkDerivation rec {
cp -r python/bin $out/bin
'';
passthru.updateScript = ''
#!${stdenv.shell}
set -e
echo
cd ${toString ./.}

View File

@ -1,4 +1,5 @@
{ lib
{ stdenv
, lib
, writeScript
, coreutils
, curl
@ -11,6 +12,7 @@
}:
writeScript "update-nodejs" ''
#!${stdenv.shell}
PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep jq gnupg nix ]}
HOME=`mktemp -d`

View File

@ -34,7 +34,7 @@ stdenv.mkDerivation rec {
enableParallelBuilding = true;
passthru.updateScript = import ./update.nix {
inherit lib writeScript coreutils gnugrep jq curl common-updater-scripts;
inherit stdenv lib writeScript coreutils gnugrep jq curl common-updater-scripts;
};
meta = {

View File

@ -1,6 +1,7 @@
{ lib, writeScript, coreutils, curl, gnugrep, jq, common-updater-scripts }:
{ stdenv, lib, writeScript, coreutils, curl, gnugrep, jq, common-updater-scripts }:
writeScript "update-tp_smapi" ''
#!${stdenv.shell}
PATH=${lib.makeBinPath [ common-updater-scripts coreutils curl gnugrep jq ]}
tags=`curl -s https://api.github.com/repos/evgeni/tp_smapi/tags`

View File

@ -26,6 +26,7 @@ let self = stdenv.mkDerivation rec {
let impl = import "${self}/share/nix/api.nix" { inherit pkgs pinConfig; }; in
{ inherit (impl) augmentedPkgs pins callPackage; };
updateScript = ''
#!${stdenv.shell}
set -e
echo
cd ${toString ./.}

View File

@ -1,4 +1,4 @@
{ lib, pkgs, fetchFromGitHub, python3Packages, nix-prefetch-scripts }:
{ stdenv, lib, pkgs, fetchFromGitHub, python3Packages, nix-prefetch-scripts }:
python3Packages.buildPythonApplication rec {
version = "0.6.3";
name = "nix-update-source-${version}";
@ -28,6 +28,7 @@ python3Packages.buildPythonApplication rec {
overrideSrc = drv: lib.overrideDerivation drv (orig: { inherit src; });
};
updateScript = ''
#!${stdenv.shell}
set -e
echo
cd ${toString ./.}