ryantm/nixpkgs-update
1
1
Fork 0
mirror of https://github.com/ryantm/nixpkgs-update.git synced 2024-12-16 07:32:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

wip: port to haskell

Browse Source
This commit is contained in:
Jan Tojnar 2018-03-31 05:07:46 +02:00
parent bacff1cc8d
commit faf0c39164
No known key found for this signature in database
GPG Key ID: 7FAB2A15F7A607A4
15 changed files with 378 additions and 514 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -6,16 +6,17 @@ Scripts to try to update nixpkgs packages. Uses `hub` to automatically make PRs.
# Instructions
1. Clone this repo:
1. Clone this repo and build the tool:
```
git clone https://github.com/ryantm/nixpkgs-update && cd nixpkgs-update
nix run nixpkgs.cabal2nix -c cabal2nix --shell --hpack . > shell.nix && nix-build shell.nix
```
2. Get a list of oudated packages and place them in a `packages-to-update.txt` file in the root directory of this repository.
```
git clone https://github.com/ryantm/repology-api.git && cd repology-api
nix run nixpkgs.cabal2nix -c cabal2nix --shell --hpack . > shell.nix && nix-build shell.nix && result/bin/repology-api > ../packages-to-update.txt
```
3. Return back `cd ..` and run `./ups.sh`
3. Return back `cd ..` and run the tool `result/bin/nix-update`
# Prior work

11
old/README.md
View File

@ -1,11 +0,0 @@
# old notes
git ls-remote --refs --tags git://github.com/mattermost/mattermost-server.git | grep -v "rc" | tail -1
nix-instantiate -I nixpkgs=/home/ryantm/p/nixpkgs --eval --expr 'let pkgs = import <nixpkgs> {}; in builtins.elemAt pkgs.mattermost.src.urls 0'
nix-instantiate -I nixpkgs=/home/ryantm/p/nixpkgs --eval --expr 'let pkgs = import <nixpkgs> { config = {permittedInsecurePackages = [ "autotrace-0.31.1" ];}; }; aV = builtins.attrValues pkgs; in map (v: builtins.elemAt v.src.urls 0) (builtins.filter (builtins.hasAttr "src") (builtins.filter builtins.isAttrs aV))'
wget "https://repology.org/api/v1/metapackages/?inrepo=nix_unstable&outdated=on"
wget "https://nixos.org/nixpkgs/packages-unstable.json.gz"

2
old/Setup.hs
View File

@ -1,2 +0,0 @@
import Distribution.Simple
main = defaultMain

24
old/hello.nix
View File

@ -1,24 +0,0 @@
{ stdenv, fetchurl }:
stdenv.mkDerivation rec {
name = "hello-2.10";
src = fetchurl {
url = "mirror://gnu/hello/${name}.tar.gz";
sha256 = "0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i";
};
doCheck = true;
meta = {
description = "A program that produces a familiar, friendly greeting";
longDescription = ''
GNU Hello is a program that prints "Hello, world!" when you run it.
It is fully customizable.
'';
homepage = http://www.gnu.org/software/hello/manual/;
license = stdenv.lib.licenses.gpl3Plus;
maintainers = [ stdenv.lib.maintainers.eelco ];
platforms = stdenv.lib.platforms.all;
};
}

30
old/nix-update.cabal
View File

@ -1,30 +0,0 @@
-- This file has been generated from package.yaml by hpack version 0.18.1.
--
-- see: https://github.com/sol/hpack
name: nix-update
version: 0.1.0.0
category: Web
homepage: https://github.com/ryantm/nix-update#readme
author: Ryan Mulligan
maintainer: ryan@ryantm.com
copyright: 2017 Ryan Mulligan
license: BSD3
license-file: LICENSE
build-type: Simple
cabal-version: >= 1.10
extra-source-files:
README.md
executable nix-update
main-is: Main.hs
hs-source-dirs:
src
build-depends:
base >= 4.7 && < 5
, hnix
, data-fix
, text
, containers
default-language: Haskell2010

24
old/package.yaml
View File

@ -1,24 +0,0 @@
name: nix-update
version: 0.1.0.0
#synopsis:
#description:
homepage: https://github.com/ryantm/nix-update#readme
license: BSD3
author: Ryan Mulligan
maintainer: ryan@ryantm.com
copyright: 2017 Ryan Mulligan
category: Web
extra-source-files:
- README.md
dependencies:
- base >= 4.7 && < 5
- hnix
- data-fix
- text
- containers
executables:
nix-update:
source-dirs: src
main: Main.hs

11
old/src/Main.hs
View File

@ -1,11 +0,0 @@
{-# LANGUAGE OverloadedStrings #-}
module Main where
import Data.Fix
import Data.Map.Strict
import Nix.Eval
import Nix.Atoms
import Nix.Parser
main :: IO ()
main = ourEval

71
old/stack.yaml
View File

@ -1,71 +0,0 @@
# This file was automatically generated by 'stack init'
#
# Some commonly used options have been documented as comments in this file.
# For advanced use and comprehensive documentation of the format, please see:
# https://docs.haskellstack.org/en/stable/yaml_configuration/
# Resolver to choose a 'specific' stackage snapshot or a compiler version.
# A snapshot resolver dictates the compiler version and the set of packages
# to be used for project dependencies. For example:
#
# resolver: lts-3.5
# resolver: nightly-2015-09-21
# resolver: ghc-7.10.2
# resolver: ghcjs-0.1.0_ghc-7.10.2
# resolver:
# name: custom-snapshot
# location: "./custom-snapshot.yaml"
resolver: lts-9.11
# User packages to be built.
# Various formats can be used as shown in the example below.
#
# packages:
# - some-directory
# - https://example.com/foo/bar/baz-0.0.2.tar.gz
# - location:
# git: https://github.com/commercialhaskell/stack.git
# commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a
# - location: https://github.com/commercialhaskell/stack/commit/e7b331f14bcffb8367cd58fbfc8b40ec7642100a
# extra-dep: true
# subdirs:
# - auto-update
# - wai
#
# A package marked 'extra-dep: true' will only be built if demanded by a
# non-dependency (i.e. a user package), and its test suites and benchmarks
# will not be run. This is useful for tweaking upstream packages.
packages:
- .
- location: ../hnix
extra-dep: true
# Dependency packages to be pulled from upstream that are not in the resolver
# (e.g., acme-missiles-0.3)
# extra-deps:
# - hnix-0.3.4
# Override default flag values for local packages and extra-deps
flags: {}
# Extra package databases containing global packages
extra-package-dbs: []
# Control whether we use the GHC we find on the path
# system-ghc: true
#
# Require a specific version of stack, using version ranges
# require-stack-version: -any # Default
# require-stack-version: ">=1.5"
#
# Override the architecture used by stack, especially useful on Windows
# arch: i386
# arch: x86_64
#
# Extra directories used by stack for building
# extra-include-dirs: [/path/to/dir]
# extra-lib-dirs: [/path/to/dir]
#
# Allow a newer minor version of GHC than the snapshot specifies
# compiler-check: newer-minor

22
package.yaml Normal file
View File

@ -0,0 +1,22 @@
name: nix-update
version: 0.2.0
synopsis: Tool for automatic updating of nixpkgs repository
homepage: https://github.com/ryantm/nix-update#readme
license: BSD3
author: Ryan Mulligan
maintainer: ryan@ryantm.com
copyright: 2018 Ryan Mulligan
category: Web
extra-source-files:
- README.md
dependencies:
- base >= 4.7 && < 5
- shelly >= 1.6 && < 1.8
- text
- filepath
executables:
nix-update:
source-dirs: src
main: Main.hs

18
setup-nixpkgs.sh
View File

@ -1,18 +0,0 @@
#!/usr/bin/env bash
set -euxo pipefail
NIXPKGS=$HOME/.cache/nixpkgs
if ! [ -d "$NIXPKGS" ]
then
hub clone nixpkgs "$NIXPKGS" # requires that user has forked nixpkgs
cd "$NIXPKGS"
git remote add upstream https://github.com/NixOS/nixpkgs
git fetch upstream
git fetch origin staging
git fetch upstream staging
fi
export NIXPKGS
cd "$NIXPKGS"

62
src/Main.hs Normal file
View File

@ -0,0 +1,62 @@
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ExtendedDefaultRules #-}
{-# OPTIONS_GHC -fno-warn-type-defaults #-}
import qualified Data.Text as T
import Shelly
import Prelude hiding (log)
import Utils (Options(..), Version, setupNixpkgs, parseUpdates, tRead)
import Data.Text (Text)
import Data.Maybe (isJust)
import Update (updatePackage)
import Data.Semigroup ((<>))
default (T.Text)
workingDir = "~/.nix-update"
logFile = workingDir </> "ups.log"
logSep = appendfile logFile "\n\n"
log msg = do
runDate <- cmd "date" "-Iseconds"
appendfile logFile (runDate <> msg)
makeOptions :: Sh Options
makeOptions = do
dryRun <- isJust <$> get_env "DRY_RUN"
return $ Options dryRun
main :: IO ()
main = shelly $ do
options <- makeOptions
mkdir_p workingDir
touchfile logFile
githubToken <- cmd "cat" "github_token.txt"
updates <- cmd "cat" "packages-to-update.txt"
setupNixpkgs
logSep
log "New run of ups.sh"
loop options (parseUpdates updates) 0
loop :: Options -> [(Text, Version, Version)] -> Int -> Sh ()
loop _ [] _ = log "ups.sh finished"
loop options ((package, oldVersion, newVersion) : moreUpdates) okToPrAt = do
log package
updated <- updatePackage options package oldVersion newVersion okToPrAt
okToPrAt <-
if updated then do
log "SUCCESS"
tRead <$> cmd "date" "+%s" "-d" "+15 minutes"
else do
log "FAIL"
return okToPrAt
loop options moreUpdates okToPrAt

246
src/Update.hs Normal file
View File

@ -0,0 +1,246 @@
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ExtendedDefaultRules #-}
{-# OPTIONS_GHC -fno-warn-type-defaults #-}
module Update (updatePackage) where
import qualified Data.Text as T
import Data.Text (Text)
import Data.Maybe (fromMaybe)
import Shelly
import Utils (Version, Options(..), setupNixpkgs, tRead)
import Data.Semigroup ((<>))
default (T.Text)
cleanup :: Text -> Sh ()
cleanup branchName = do
cmd "git" "reset" "--hard"
cmd "git" "clean" "-fd"
cmd "git" "checkout" "-B" "master" "upstream/master"
cmd "git" "reset" "--hard" "upstream/master"
canFail $ cmd "git" "branch" "-D" branchName
errorExit' :: Text -> Text -> Sh a
errorExit' branchName message = do
cleanup branchName
echo $ "$(date -Iseconds) " <> message
exit 1
isOnBlackList :: (Text -> Sh Text) -> Text -> Sh Text
isOnBlackList errorExit packageName
| "jquery" `T.isInfixOf` packageName = errorExit "this isn't a real package"
| "google-cloud-sdk" `T.isInfixOf` packageName = errorExit "complicated package"
| "github-release" `T.isInfixOf` packageName = errorExit "complicated package"
| "fcitx" `T.isInfixOf` packageName = errorExit "gets stuck in daemons"
| "libxc" `T.isInfixOf` packageName = errorExit "currently people don't want to update this https://github.com/NixOS/nixpkgs/pull/35821"
| "perl" `T.isInfixOf` packageName = errorExit "currently don't know how to update perl"
| "python" `T.isInfixOf` packageName = errorExit "currently don't know how to update python"
| "cdrtools" `T.isInfixOf` packageName = errorExit "We keep downgrading this by accident."
| "gst" `T.isInfixOf` packageName = errorExit "gstreamer plugins are kept in lockstep."
| "electron" `T.isInfixOf` packageName = errorExit "multi-platform srcs in file."
| "linux-headers" `T.isInfixOf` packageName = errorExit "Not updated until many packages depend on it (part of stdenv)."
| "mpich" `T.isInfixOf` packageName = errorExit "Reported on repology.org as mischaracterized newest version"
| "xfce" `T.isInfixOf` packageName = errorExit "@volth asked to not update xfce"
| "cmake-cursesUI-qt4UI" `T.isInfixOf` packageName = errorExit "Derivation file is complicated"
| "varnish" `T.isInfixOf` packageName = errorExit "Temporary blacklist because of multiple versions and slow nixpkgs update"
| "iana-etc" `T.isInfixOf` packageName = errorExit "@mic92 takes care of this package"
isOnBlackList errorExit "isl" = errorExit "multi-version long building package"
isOnBlackList errorExit "tokei" = errorExit "got stuck forever building with no CPU usage"
isOnBlackList _ _ = return ""
rawEval :: Text -> Sh Text
rawEval expr = cmd "nix" "eval" "-f" "." "--raw" expr
canFail :: Sh a -> Sh a
canFail = errExit False
orElse :: Sh a -> Sh a -> Sh a
orElse a b = do
v <- canFail a
status <- lastExitCode
if status == 0 then
return v
else
b
infixl 3 `orElse`
fixSrcUrl :: Text -> Version -> Version -> Text -> Text -> Text -> Sh Text
fixSrcUrl packageName oldVersion newVersion derivationFile attrPath oldSrcUrl = cmd "./fix-src-url.sh" packageName oldVersion newVersion derivationFile attrPath oldSrcUrl
push :: Text -> Options -> Sh ()
push branchName options =
if dryRun options then
return ()
else
cmd "git" "push" "--set-upstream" "origin" branchName "--force"
updatePackage :: Options -> Text -> Version -> Version -> Int -> Sh Bool
updatePackage options packageName oldVersion newVersion okToPrAt = do
nixpkgsPath <- setupNixpkgs
setenv "NIX_PATH" ("nixpkgs=" <> toTextIgnore nixpkgsPath)
let branchName = "auto-update/" <> packageName
let errorExit = errorExit' branchName
versionComparison <- cmd "nix" "eval" "-f" "." ("(builtins.compareVersions \"" <> newVersion <> "\" \"" <> oldVersion <> "\")")
unless (versionComparison == "1") $ do
errorExit $ newVersion <> " is not newer than " <> oldVersion <> " according to Nix"
-- Package blacklist
isOnBlackList errorExit packageName
oneHourAgo <- cmd "date" "+%s" "-d" "-1 hour"
fetchedLast <- cmd "stat" "-c" "%Y" ".git/FETCH_HEAD"
when (fetchedLast < oneHourAgo) $ do
canFail $ cmd "git" "fetch" "--prune" "--multiple" "upstream" "origin"
remotes <- T.lines <$> cmd "git" "branch" "--remote"
when (("origin/auto-update/" <> packageName) `elem` remotes) $ do
errorExit "Update branch already on origin."
cmd "git" "reset" "--hard"
cmd "git" "clean" "-fd"
cmd "git" "checkout" "master"
cmd "git" "reset" "--hard" "upstream/master"
cmd "git" "clean" "-fd"
-- This is extremely slow but will give us better results
attrPath <- head . T.words . head . T.lines <$> cmd "nix-env" "-qa" (packageName <> "-" <> oldVersion) "-f" "." "--attr-path" `orElse`
errorExit "nix-env -q failed to find package name with old version"
-- Temporarily blacklist gnome sources for lockstep update
whenM (("gnome" `T.isInfixOf`) <$> cmd "nix" "eval" "-f" "." ("pkgs." <> attrPath <> ".src.urls")) $ do
errorExit "Packages from gnome are currently blacklisted."
-- Temporarily blacklist lua packages at @teto's request
-- https://github.com/NixOS/nixpkgs/pull/37501#issuecomment-375169646
when (T.isPrefixOf "lua" attrPath) $ do
errorExit "Packages for lua are currently blacklisted."
derivationFile <- cmd "env" "EDITOR=echo" "nix" "edit" attrPath "-f" "." `orElse` errorExit "Couldn't find derivation file."
flip finally_sh (cleanup branchName) $ do
numberOfFetchers <- tRead <$> cmd "grep" "-c" "fetchurl {|fetchgit {|fetchFromGitHub {" derivationFile
unless ((numberOfFetchers :: Int) <= 1) $ do
errorExit $ "More than one fetcher in " <> derivationFile
derivationContents <- cmd "cat" derivationFile
if T.isInfixOf "DO NOT EDIT" derivationContents then
errorExit "Derivation file says not to edit it."
-- Skip packages that have special builders
else if T.isInfixOf "buildGoPackage" derivationContents then
errorExit "Derivation contains buildGoPackage."
else if T.isInfixOf "buildRustCrate" derivationContents then
errorExit "Derivation contains buildRustCrate."
else if T.isInfixOf "buildPythonPackage" derivationContents then
errorExit "Derivation contains buildPythonPackage."
else if T.isInfixOf "buildRubyGem" derivationContents then
errorExit "Derivation contains buildRubyGem."
else if T.isInfixOf "bundlerEnv" derivationContents then
errorExit "Derivation contains bundlerEnv."
else if T.isInfixOf "buildPerlPackage" derivationContents then
errorExit "Derivation contains buildPerlPackage."
else return ()
cmd "./check-attrpath-version.sh" attrPath newVersion `orElse`
errorExit ("Version in attr path " <> attrPath <> " not compatible with " <> newVersion)
-- Make sure it hasn't been updated on master
cmd "grep" oldVersion derivationFile `orElse`
errorExit "Old version not present in master derivation file."
-- Make sure it hasn't been updated on staging
cmd "git" "reset" "--hard"
cmd "git" "clean" "-fd"
cmd "git" "checkout" "-B" "staging" "upstream/staging"
cmd "git" "reset" "--hard" "upstream/staging"
cmd "git" "clean" "-fd"
cmd "grep" oldVersion derivationFile `orElse`
errorExit "Old version not present in staging derivation file."
base <- cmd "git" "merge-base" "upstream/master" "upstream/staging"
cmd "git" "checkout" "-B" branchName base
oldHash <- rawEval ("pkgs." <> attrPath <> ".src.drvAttrs.outputHash") `orElse`
errorExit ("Could not find old output hash at " <> attrPath <> ".src.drvAttrs.outputHash.")
oldSrcUrl <- rawEval ("let pkgs = import ./. {}; in builtins.elemAt pkgs." <> attrPath <> ".src.drvAttrs.urls 0)")
cmd "sed" "-i" ("s/" <> (T.replace "." "\\." oldVersion) <> "/" <> newVersion <> "/g") derivationFile `orElse`
errorExit "Could not replace oldVersion with newVersion."
newSrcUrl <- rawEval ("let pkgs = import ./. {}; in builtins.elemAt pkgs." <> attrPath <> ".src.drvAttrs.urls 0")
when (oldSrcUrl == newSrcUrl) $ do
errorExit "Source url did not change."
newHash <- cmd "nix-prefetch-url" "-A" (attrPath <> ".src") `orElse`
fixSrcUrl packageName oldVersion newVersion derivationFile attrPath oldSrcUrl `orElse`
errorExit "Could not prefetch new version URL."
when (oldHash == newHash) $ do
errorExit "Hashes equal; no update necessary"
cmd "sed" "-i" ("s/" <> oldHash <> "/" <> newHash <> "/g") derivationFile `orElse`
errorExit "Could not replace oldHash with newHash."
cmd "rm" "-f" "result*"
cmd "nix" "build" "-f" "." attrPath `orElse` do
buildLog <- T.unlines . reverse . take 30 . reverse . T.lines <$> cmd "nix" "log" "-f" "." attrPath
errorExit ("nix build failed.\n" <> buildLog)
result <- cmd "readlink" "./result" `orElse` cmd "readlink" "./result-bin" `orElse`
errorExit "Could not find result link."
checkResult <- cmd "./check-result.sh" result newVersion
hasMaintainers <- const True <$> cmd "nix" "eval" ("let pkgs = import ./. {}; in pkgs." <> attrPath <> ".meta.maintainers") `orElse` return False
maintainers <- if hasMaintainers then rawEval ("let pkgs = import ./. {}; gh = m : m.github or \"\"; nonempty = s: s != \"\"; addat = s: \"@\"+s; in builtins.concatStringsSep \" \" (map addat (builtins.filter nonempty (map gh pkgs." <> attrPath <> ".meta.maintainers)))") else return ""
let maintainersCc = if not (T.null maintainers) then "\n\ncc " <> maintainers <> " for review" else ""
cmd "git" "diff"
let
commitMessage :: Text
commitMessage = attrPath <> ": " <> oldVersion <> " → " <> newVersion <> "\n\n"
<> "Semi-automatic update generated by https://github.com/ryantm/nixpkgs-update tools.\n\n"
<> "This update was made based on information from https://repology.org/metapackage/" <> packageName <> "/versions.\n\n"
<> "These checks were done:\n\n"
<> "- built on NixOS\n"
<> checkResult
cmd "git" "commit" "-am" commitMessage
-- Try to push it three times
push branchName options `orElse` push branchName options `orElse` push branchName options
isBroken <- cmd "nix" "eval" "-f" "." ("let pkgs = import ./. {}; in pkgs." <> attrPath <> ".meta.broken or false")
let brokenWarning = if isBroken == "true" then "" else "- WARNING: Package has meta.broken=true; Please manually test this package update and remove the broken attribute."
let prMessage = commitMessage <> brokenWarning <> maintainersCc
unless (dryRun options) $ do
current <- tRead <$> cmd "date" "+%s"
when (current < okToPrAt) $ do
let sleepSeconds = okToPrAt - current
echo $ "Sleeping " <> T.pack (show sleepSeconds) <> " seconds."
sleep sleepSeconds
cmd "hub" "pull-request" "-m" prMessage
cmd "git" "reset" "--hard"
cmd "git" "clean" "-fd"
cmd "git" "checkout" "-B" "master" "upstream/master"
cmd "git" "reset" "--hard"
cmd "git" "clean" "-fd"
exit 0

45
src/Utils.hs Normal file
View File

@ -0,0 +1,45 @@
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ExtendedDefaultRules #-}
{-# OPTIONS_GHC -fno-warn-type-defaults #-}
module Utils (Options(..), Version, setupNixpkgs, tRead, parseUpdates) where
import Prelude hiding (FilePath)
import Data.Text (Text)
import Data.Semigroup ((<>))
import qualified Data.Text as T
import Shelly
default (T.Text)
type Version = Text
data Options = Options {
dryRun :: Bool
}
setupNixpkgs :: Sh FilePath
setupNixpkgs = do
home <- get_env_text "HOME"
let nixpkgsPath = home </> ".cache" </> "nixpkgs"
unlessM (test_e nixpkgsPath) $ do
cmd "hub" "clone" "nixpkgs" nixpkgsPath -- requires that user has forked nixpkgs
cmd "cd" nixpkgsPath
cmd "git" "remote" "add" "upstream" "https://github.com/NixOS/nixpkgs"
cmd "git" "fetch" "upstream"
cmd "git" "fetch" "origin" "staging"
cmd "git" "fetch" "upstream" "staging"
cmd "cd" nixpkgsPath
return nixpkgsPath
parseUpdates :: Text -> [(Text, Version, Version)]
parseUpdates = map (toTriple . T.words) . T.lines where
toTriple :: [Text] -> (Text, Version, Version)
toTriple [package, oldVersion, newVersion] = (package, oldVersion, newVersion)
toTriple line = error $ T.unpack ("Unable to parse update: " <> T.unwords line)
tRead :: Read a => Text -> a
tRead = read . T.unpack

272
up.sh
View File

@ -1,272 +0,0 @@
#!/usr/bin/env bash
set -euxo pipefail
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
PAGER=
export PAGER
# shellcheck source=setup-nixpkgs.sh
source "$SCRIPT_DIR/setup-nixpkgs.sh"
NIX_PATH=nixpkgs="$NIXPKGS"
export NIX_PATH
PACKAGE_NAME=$1
OLD_VERSION=$2
NEW_VERSION=$3
if [ "$#" -eq 4 ]
then
OK_TO_PR_AT=$4
else
OK_TO_PR_AT="0"
fi
BRANCH_NAME="auto-update/$1"
function cleanup {
git reset --hard
git clean -fd
git checkout -B master upstream/master
git reset --hard upstream/master
git branch -D "$BRANCH_NAME" || true
}
function error_exit {
cleanup
if ( true >&3 ) 2> /dev/null
then
echo "$(date -Iseconds) $1" >&3
else
echo "$(date -Iseconds) $1"
fi
exit 1
}
if ! [ "$(nix eval -f . "(builtins.compareVersions \"$NEW_VERSION\" \"$OLD_VERSION\")")" -eq 1 ];
then
error_exit "$NEW_VERSION is not newer than $OLD_VERSION according to Nix"
fi
# Package blacklist
case "$PACKAGE_NAME" in
*jquery*) error_exit "this isn't a real package";;
*google-cloud-sdk*) error_exit "complicated package";;
*github-release*) error_exit "complicated package";;
*fcitx*) error_exit "gets stuck in daemons";;
*libxc*) error_exit "currently people don't want to update this https://github.com/NixOS/nixpkgs/pull/35821";;
*perl*) error_exit "currently don't know how to update perl";;
*python*) error_exit "currently don't know how to update python";;
*cdrtools*) error_exit "We keep downgrading this by accident.";;
*gst*) error_exit "gstreamer plugins are kept in lockstep.";;
*electron*) error_exit "multi-platform srcs in file.";;
*linux-headers*) error_exit "Not updated until many packages depend on it (part of stdenv).";;
*mpich*) error_exit "Reported on repology.org as mischaracterized newest version";;
*xfce*) error_exit "@volth asked to not update xfce";;
*cmake-cursesUI-qt4UI*) error_exit "Derivation file is complicated";;
*varnish*) error_exit "Temporary blacklist because of multiple versions and slow nixpkgs update";;
*iana-etc*) error_exit "@mic92 takes care of this package";;
isl) error_exit "multi-version long building package";;
tokei) error_exit "got stuck forever building with no CPU usage";;
*) true;;
esac || error_exit "Package on blacklist."
ONEHOURAGO=$(date +%s -d "-1 hour")
FETCHEDLAST=$(stat -c %Y .git/FETCH_HEAD)
if (( FETCHEDLAST < ONEHOURAGO ))
then
git fetch --prune --multiple upstream origin || true
fi
if git branch --remote | grep "origin/auto-update/${PACKAGE_NAME}"
then
error_exit "Update branch already on origin."
fi
git reset --hard
git clean -fd
git checkout master
git reset --hard upstream/master
git clean -fd
# This is extremely slow but will give us better results
ATTR_PATH=$(nix-env -qa "$PACKAGE_NAME-$OLD_VERSION" -f . --attr-path | head -n1 | cut -d' ' -f1) || error_exit "nix-env -q failed to find package name with old version"
# Temporarily blacklist gnome sources for lockstep update
if nix eval -f . "pkgs.${ATTR_PATH}.src.urls" | grep "gnome"
then
error_exit "Packages from gnome are currently blacklisted."
fi
# Temporarily blacklist lua packages at @teto's request
# https://github.com/NixOS/nixpkgs/pull/37501#issuecomment-375169646
if echo "$ATTR_PATH" | grep "^lua"
then
error_exit "Packages for lua are currently blacklisted."
fi
DERIVATION_FILE=$(EDITOR="echo" nix edit "$ATTR_PATH" -f .) || error_exit "Couldn't find derivation file."
function error_cleanup {
cleanup
exit 1
}
trap error_cleanup ERR
function interrupt_cleanup {
cleanup
exit 2
}
trap interrupt_cleanup INT
(( $(grep -c "fetchurl {" "$DERIVATION_FILE") + $(grep -c "fetchgit {" "$DERIVATION_FILE") + $(grep -c "fetchFromGitHub {" "$DERIVATION_FILE") <= 1 )) || error_exit "More than one fetcher in $DERIVATION_FILE"
if grep -q "DO NOT EDIT" "$DERIVATION_FILE"
then
error_exit "Derivation file says not to edit it."
fi
# Skip packages that have special builders
if grep -q "buildGoPackage" "$DERIVATION_FILE"
then
error_exit "Derivation contains buildGoPackage."
fi
if grep -q "buildRustCrate" "$DERIVATION_FILE"
then
error_exit "Derivation contains buildRustCrate."
fi
if grep -q "buildPythonPackage" "$DERIVATION_FILE"
then
error_exit "Derivation contains buildPythonPackage."
fi
if grep -q "buildRubyGem" "$DERIVATION_FILE"
then
error_exit "Derivation contains buildRubyGem."
fi
if grep -q "bundlerEnv" "$DERIVATION_FILE"
then
error_exit "Derivation contains bundlerEnv."
fi
if grep -q "buildPerlPackage" "$DERIVATION_FILE"
then
error_exit "Derivation contains buildPerlPackage."
fi
"$SCRIPT_DIR"/check-attrpath-version.sh "$ATTR_PATH" "$NEW_VERSION" || error_exit "Version in attr path $ATTR_PATH not compatible with $NEW_VERSION"
# Make sure it hasn't been updated on master
grep "$OLD_VERSION" "$DERIVATION_FILE" || error_exit "Old version not present in master derivation file."
# Make sure it hasn't been updated on staging
git reset --hard
git clean -fd
git checkout -B staging upstream/staging
git reset --hard upstream/staging
git clean -fd
grep "$OLD_VERSION" "$DERIVATION_FILE" || error_exit "Old version not present in staging derivation file."
git checkout -B "$BRANCH_NAME" "$(git merge-base upstream/master upstream/staging)"
OLD_HASH=$(nix eval -f . --raw "pkgs.$ATTR_PATH.src.drvAttrs.outputHash" || error_exit "Couldn't find old output hash at ATTR_PATH.src.drvAttrs.outputHash.")
OLD_SRC_URL=$(nix eval -f . --raw '(let pkgs = import ./. {}; in builtins.elemAt pkgs.'"$ATTR_PATH"'.src.drvAttrs.urls 0)')
sed -i "s/${OLD_VERSION//\./\\.}/$NEW_VERSION/g" "$DERIVATION_FILE" || error_exit "Could not replace OLD_VERSION with NEW_VERSION."
NEW_SRC_URL=$(nix eval -f . --raw '(let pkgs = import ./. {}; in builtins.elemAt pkgs.'"$ATTR_PATH"'.src.drvAttrs.urls 0)')
if [ "$OLD_SRC_URL" == "$NEW_SRC_URL" ]
then
error_exit "Source url did not change."
fi
NEW_HASH=$(
nix-prefetch-url -A "$ATTR_PATH.src" || \
"$SCRIPT_DIR"/fix-src-url.sh "$PACKAGE_NAME" "$OLD_VERSION" "$NEW_VERSION" "$DERIVATION_FILE" "$ATTR_PATH" "$OLD_SRC_URL" || \
error_exit "Could not prefetch new version URL.")
if [ "$OLD_HASH" = "$NEW_HASH" ]
then
error_exit "Hashes equal; no update necessary"
fi
sed -i "s/$OLD_HASH/$NEW_HASH/g" "$DERIVATION_FILE" || error_exit "Could not replace OLD_HASH with NEW_HASH."
rm -f result*
nix build -f . "$ATTR_PATH" || error_exit "nix build failed.
$(nix log -f . "$ATTR_PATH" | tail -n 30)"
RESULT=$(readlink ./result || readlink ./result-bin || error_exit "Couldn't find result link.")
CHECK_RESULT="$("$SCRIPT_DIR"/check-result.sh "$RESULT" "$NEW_VERSION")"
MAINTAINERS=
if nix eval "(let pkgs = import ./. {}; in pkgs.$ATTR_PATH.meta.maintainers)" > /dev/null 2>&1
then
maintainers=$(nix eval --raw '(let pkgs = import ./. {}; gh = m : m.github or ""; nonempty = s: s != ""; addat = s: "@"+s; in builtins.concatStringsSep " " (map addat (builtins.filter nonempty (map gh pkgs.'"${ATTR_PATH}"'.meta.maintainers))))')
if [ -n "$maintainers" ]
then
MAINTAINERS="
cc $maintainers for review"
fi
fi
git diff
COMMIT_MESSAGE="$ATTR_PATH: $OLD_VERSION -> $NEW_VERSION
Semi-automatic update generated by https://github.com/ryantm/nixpkgs-update tools.
This update was made based on information from https://repology.org/metapackage/$PACKAGE_NAME/versions.
These checks were done:
- built on NixOS
$CHECK_RESULT"
git commit -am "$COMMIT_MESSAGE"
# Try to push it three times
function push() {
if [[ -v DRY_RUN ]]
then
return 0
else
git push --set-upstream origin "$BRANCH_NAME" --force
fi
}
push || push || push
BROKEN_WARNING=
if [ "$(nix eval -f . '(let pkgs = import ./. {}; in pkgs.'"${ATTR_PATH}"'.meta.broken or false)')" == "true" ]
then
BROKEN_WARNING="- WARNING: Package has meta.broken=true; Please manually test this package update and remove the broken attribute."
fi
PR_MESSAGE="$COMMIT_MESSAGE$BROKEN_WARNING$MAINTAINERS"
if [[ -v DRY_RUN ]]
then
true
else
CURRENT=$(date +%s)
if (( CURRENT < OK_TO_PR_AT ))
then
SLEEP_SECONDS=$(( OK_TO_PR_AT - CURRENT ))
echo "Sleeping $SLEEP_SECONDS seconds."
sleep "$SLEEP_SECONDS"
fi
hub pull-request -m "$PR_MESSAGE"
fi
git reset --hard
git clean -fd
git checkout -B master upstream/master
git reset --hard
git clean -fd
exit 0

49
ups.sh
View File

@ -1,49 +0,0 @@
#!/usr/bin/env bash
set -euxo pipefail
LOG_FILE=~/.nix-update/ups.log
mkdir -p "$(dirname $LOG_FILE)"
touch $LOG_FILE
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
GITHUB_TOKEN="$(cat "$SCRIPT_DIR"/github_token.txt)"
export GITHUB_TOKEN
ARGUMENTS=$(cat packages-to-update.txt)
# shellcheck source=setup-nixpkgs.sh
source "$SCRIPT_DIR/setup-nixpkgs.sh"
echo "
$(date -Iseconds) New run of ups.sh" >> $LOG_FILE
OK_TO_PR_AT=0
IFS=$'\n'
for a in $ARGUMENTS
do
unset IFS
echo "$(date -Iseconds) $a" >> $LOG_FILE
if eval "$SCRIPT_DIR/up.sh $a $OK_TO_PR_AT 3>>$LOG_FILE"
then
RESULT=$?
else
RESULT=1
fi
case "$RESULT" in
0)
echo "$(date -Iseconds) SUCCESS" >> $LOG_FILE
OK_TO_PR_AT=$(date +%s -d "+15 minutes")
;;
1)
echo "$(date -Iseconds) FAIL" >> $LOG_FILE
;;
*) exit 1
;;
esac
done
echo "$(date -Iseconds) ups.sh finished" >> $LOG_FILE