From faf0c39164fc1d1111534f1104be55571f0e7fcc Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Sat, 31 Mar 2018 05:07:46 +0200 Subject: [PATCH] wip: port to haskell --- README.md | 5 +- old/README.md | 11 -- old/Setup.hs | 2 - old/hello.nix | 24 ---- old/nix-update.cabal | 30 ----- old/package.yaml | 24 ---- old/src/Main.hs | 11 -- old/stack.yaml | 71 ----------- package.yaml | 22 ++++ setup-nixpkgs.sh | 18 --- src/Main.hs | 62 ++++++++++ src/Update.hs | 246 ++++++++++++++++++++++++++++++++++++++ src/Utils.hs | 45 +++++++ up.sh | 272 ------------------------------------------- ups.sh | 49 -------- 15 files changed, 378 insertions(+), 514 deletions(-) delete mode 100644 old/README.md delete mode 100644 old/Setup.hs delete mode 100644 old/hello.nix delete mode 100644 old/nix-update.cabal delete mode 100644 old/package.yaml delete mode 100644 old/src/Main.hs delete mode 100644 old/stack.yaml create mode 100644 package.yaml delete mode 100755 setup-nixpkgs.sh create mode 100644 src/Main.hs create mode 100644 src/Update.hs create mode 100644 src/Utils.hs delete mode 100755 up.sh delete mode 100755 ups.sh diff --git a/README.md b/README.md index 8406bc7..7923eb9 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,17 @@ Scripts to try to update nixpkgs packages. Uses `hub` to automatically make PRs. # Instructions -1. Clone this repo: +1. Clone this repo and build the tool: ``` git clone https://github.com/ryantm/nixpkgs-update && cd nixpkgs-update + nix run nixpkgs.cabal2nix -c cabal2nix --shell --hpack . > shell.nix && nix-build shell.nix ``` 2. Get a list of oudated packages and place them in a `packages-to-update.txt` file in the root directory of this repository. ``` git clone https://github.com/ryantm/repology-api.git && cd repology-api nix run nixpkgs.cabal2nix -c cabal2nix --shell --hpack . > shell.nix && nix-build shell.nix && result/bin/repology-api > ../packages-to-update.txt ``` -3. Return back `cd ..` and run `./ups.sh` +3. Return back `cd ..` and run the tool `result/bin/nix-update` # Prior work diff --git a/old/README.md b/old/README.md deleted file mode 100644 index 3334f9d..0000000 --- a/old/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# old notes - -git ls-remote --refs --tags git://github.com/mattermost/mattermost-server.git | grep -v "rc" | tail -1 - -nix-instantiate -I nixpkgs=/home/ryantm/p/nixpkgs --eval --expr 'let pkgs = import {}; in builtins.elemAt pkgs.mattermost.src.urls 0' - -nix-instantiate -I nixpkgs=/home/ryantm/p/nixpkgs --eval --expr 'let pkgs = import { config = {permittedInsecurePackages = [ "autotrace-0.31.1" ];}; }; aV = builtins.attrValues pkgs; in map (v: builtins.elemAt v.src.urls 0) (builtins.filter (builtins.hasAttr "src") (builtins.filter builtins.isAttrs aV))' - - -wget "https://repology.org/api/v1/metapackages/?inrepo=nix_unstable&outdated=on" -wget "https://nixos.org/nixpkgs/packages-unstable.json.gz" diff --git a/old/Setup.hs b/old/Setup.hs deleted file mode 100644 index 9a994af..0000000 --- a/old/Setup.hs +++ /dev/null @@ -1,2 +0,0 @@ -import Distribution.Simple -main = defaultMain diff --git a/old/hello.nix b/old/hello.nix deleted file mode 100644 index 8a31c59..0000000 --- a/old/hello.nix +++ /dev/null @@ -1,24 +0,0 @@ -{ stdenv, fetchurl }: - -stdenv.mkDerivation rec { - name = "hello-2.10"; - - src = fetchurl { - url = "mirror://gnu/hello/${name}.tar.gz"; - sha256 = "0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i"; - }; - - doCheck = true; - - meta = { - description = "A program that produces a familiar, friendly greeting"; - longDescription = '' - GNU Hello is a program that prints "Hello, world!" when you run it. - It is fully customizable. - ''; - homepage = http://www.gnu.org/software/hello/manual/; - license = stdenv.lib.licenses.gpl3Plus; - maintainers = [ stdenv.lib.maintainers.eelco ]; - platforms = stdenv.lib.platforms.all; - }; -} diff --git a/old/nix-update.cabal b/old/nix-update.cabal deleted file mode 100644 index 018f1aa..0000000 --- a/old/nix-update.cabal +++ /dev/null @@ -1,30 +0,0 @@ --- This file has been generated from package.yaml by hpack version 0.18.1. --- --- see: https://github.com/sol/hpack - -name: nix-update -version: 0.1.0.0 -category: Web -homepage: https://github.com/ryantm/nix-update#readme -author: Ryan Mulligan -maintainer: ryan@ryantm.com -copyright: 2017 Ryan Mulligan -license: BSD3 -license-file: LICENSE -build-type: Simple -cabal-version: >= 1.10 - -extra-source-files: - README.md - -executable nix-update - main-is: Main.hs - hs-source-dirs: - src - build-depends: - base >= 4.7 && < 5 - , hnix - , data-fix - , text - , containers - default-language: Haskell2010 diff --git a/old/package.yaml b/old/package.yaml deleted file mode 100644 index 105379e..0000000 --- a/old/package.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: nix-update -version: 0.1.0.0 -#synopsis: -#description: -homepage: https://github.com/ryantm/nix-update#readme -license: BSD3 -author: Ryan Mulligan -maintainer: ryan@ryantm.com -copyright: 2017 Ryan Mulligan -category: Web -extra-source-files: -- README.md - -dependencies: - - base >= 4.7 && < 5 - - hnix - - data-fix - - text - - containers - -executables: - nix-update: - source-dirs: src - main: Main.hs diff --git a/old/src/Main.hs b/old/src/Main.hs deleted file mode 100644 index 77669bc..0000000 --- a/old/src/Main.hs +++ /dev/null @@ -1,11 +0,0 @@ -{-# LANGUAGE OverloadedStrings #-} -module Main where - -import Data.Fix -import Data.Map.Strict -import Nix.Eval -import Nix.Atoms -import Nix.Parser - -main :: IO () -main = ourEval diff --git a/old/stack.yaml b/old/stack.yaml deleted file mode 100644 index 81beb78..0000000 --- a/old/stack.yaml +++ /dev/null @@ -1,71 +0,0 @@ -# This file was automatically generated by 'stack init' -# -# Some commonly used options have been documented as comments in this file. -# For advanced use and comprehensive documentation of the format, please see: -# https://docs.haskellstack.org/en/stable/yaml_configuration/ - -# Resolver to choose a 'specific' stackage snapshot or a compiler version. -# A snapshot resolver dictates the compiler version and the set of packages -# to be used for project dependencies. For example: -# -# resolver: lts-3.5 -# resolver: nightly-2015-09-21 -# resolver: ghc-7.10.2 -# resolver: ghcjs-0.1.0_ghc-7.10.2 -# resolver: -# name: custom-snapshot -# location: "./custom-snapshot.yaml" -resolver: lts-9.11 - -# User packages to be built. -# Various formats can be used as shown in the example below. -# -# packages: -# - some-directory -# - https://example.com/foo/bar/baz-0.0.2.tar.gz -# - location: -# git: https://github.com/commercialhaskell/stack.git -# commit: e7b331f14bcffb8367cd58fbfc8b40ec7642100a -# - location: https://github.com/commercialhaskell/stack/commit/e7b331f14bcffb8367cd58fbfc8b40ec7642100a -# extra-dep: true -# subdirs: -# - auto-update -# - wai - -# -# A package marked 'extra-dep: true' will only be built if demanded by a -# non-dependency (i.e. a user package), and its test suites and benchmarks -# will not be run. This is useful for tweaking upstream packages. -packages: -- . -- location: ../hnix - extra-dep: true - -# Dependency packages to be pulled from upstream that are not in the resolver -# (e.g., acme-missiles-0.3) -# extra-deps: -# - hnix-0.3.4 - -# Override default flag values for local packages and extra-deps -flags: {} - -# Extra package databases containing global packages -extra-package-dbs: [] - -# Control whether we use the GHC we find on the path -# system-ghc: true -# -# Require a specific version of stack, using version ranges -# require-stack-version: -any # Default -# require-stack-version: ">=1.5" -# -# Override the architecture used by stack, especially useful on Windows -# arch: i386 -# arch: x86_64 -# -# Extra directories used by stack for building -# extra-include-dirs: [/path/to/dir] -# extra-lib-dirs: [/path/to/dir] -# -# Allow a newer minor version of GHC than the snapshot specifies -# compiler-check: newer-minor diff --git a/package.yaml b/package.yaml new file mode 100644 index 0000000..f21b283 --- /dev/null +++ b/package.yaml @@ -0,0 +1,22 @@ +name: nix-update +version: 0.2.0 +synopsis: Tool for automatic updating of nixpkgs repository +homepage: https://github.com/ryantm/nix-update#readme +license: BSD3 +author: Ryan Mulligan +maintainer: ryan@ryantm.com +copyright: 2018 Ryan Mulligan +category: Web +extra-source-files: +- README.md + +dependencies: + - base >= 4.7 && < 5 + - shelly >= 1.6 && < 1.8 + - text + - filepath + +executables: + nix-update: + source-dirs: src + main: Main.hs diff --git a/setup-nixpkgs.sh b/setup-nixpkgs.sh deleted file mode 100755 index 416636a..0000000 --- a/setup-nixpkgs.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -NIXPKGS=$HOME/.cache/nixpkgs - -if ! [ -d "$NIXPKGS" ] -then - hub clone nixpkgs "$NIXPKGS" # requires that user has forked nixpkgs - cd "$NIXPKGS" - git remote add upstream https://github.com/NixOS/nixpkgs - git fetch upstream - git fetch origin staging - git fetch upstream staging -fi - -export NIXPKGS - -cd "$NIXPKGS" diff --git a/src/Main.hs b/src/Main.hs new file mode 100644 index 0000000..dc824a6 --- /dev/null +++ b/src/Main.hs @@ -0,0 +1,62 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ExtendedDefaultRules #-} +{-# OPTIONS_GHC -fno-warn-type-defaults #-} +import qualified Data.Text as T +import Shelly +import Prelude hiding (log) +import Utils (Options(..), Version, setupNixpkgs, parseUpdates, tRead) +import Data.Text (Text) +import Data.Maybe (isJust) +import Update (updatePackage) +import Data.Semigroup ((<>)) +default (T.Text) +workingDir = "~/.nix-update" +logFile = workingDir "ups.log" + +logSep = appendfile logFile "\n\n" + +log msg = do + runDate <- cmd "date" "-Iseconds" + appendfile logFile (runDate <> msg) + + +makeOptions :: Sh Options +makeOptions = do + dryRun <- isJust <$> get_env "DRY_RUN" + return $ Options dryRun + +main :: IO () +main = shelly $ do + options <- makeOptions + + mkdir_p workingDir + touchfile logFile + + githubToken <- cmd "cat" "github_token.txt" + + updates <- cmd "cat" "packages-to-update.txt" + + setupNixpkgs + + logSep + log "New run of ups.sh" + + loop options (parseUpdates updates) 0 + +loop :: Options -> [(Text, Version, Version)] -> Int -> Sh () +loop _ [] _ = log "ups.sh finished" +loop options ((package, oldVersion, newVersion) : moreUpdates) okToPrAt = do + log package + + updated <- updatePackage options package oldVersion newVersion okToPrAt + + okToPrAt <- + if updated then do + log "SUCCESS" + tRead <$> cmd "date" "+%s" "-d" "+15 minutes" + else do + log "FAIL" + return okToPrAt + + loop options moreUpdates okToPrAt + diff --git a/src/Update.hs b/src/Update.hs new file mode 100644 index 0000000..d9f2ef1 --- /dev/null +++ b/src/Update.hs @@ -0,0 +1,246 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ExtendedDefaultRules #-} +{-# OPTIONS_GHC -fno-warn-type-defaults #-} + +module Update (updatePackage) where + +import qualified Data.Text as T +import Data.Text (Text) +import Data.Maybe (fromMaybe) +import Shelly +import Utils (Version, Options(..), setupNixpkgs, tRead) +import Data.Semigroup ((<>)) +default (T.Text) + +cleanup :: Text -> Sh () +cleanup branchName = do + cmd "git" "reset" "--hard" + cmd "git" "clean" "-fd" + cmd "git" "checkout" "-B" "master" "upstream/master" + cmd "git" "reset" "--hard" "upstream/master" + canFail $ cmd "git" "branch" "-D" branchName + +errorExit' :: Text -> Text -> Sh a +errorExit' branchName message = do + cleanup branchName + echo $ "$(date -Iseconds) " <> message + exit 1 + +isOnBlackList :: (Text -> Sh Text) -> Text -> Sh Text +isOnBlackList errorExit packageName + | "jquery" `T.isInfixOf` packageName = errorExit "this isn't a real package" + | "google-cloud-sdk" `T.isInfixOf` packageName = errorExit "complicated package" + | "github-release" `T.isInfixOf` packageName = errorExit "complicated package" + | "fcitx" `T.isInfixOf` packageName = errorExit "gets stuck in daemons" + | "libxc" `T.isInfixOf` packageName = errorExit "currently people don't want to update this https://github.com/NixOS/nixpkgs/pull/35821" + | "perl" `T.isInfixOf` packageName = errorExit "currently don't know how to update perl" + | "python" `T.isInfixOf` packageName = errorExit "currently don't know how to update python" + | "cdrtools" `T.isInfixOf` packageName = errorExit "We keep downgrading this by accident." + | "gst" `T.isInfixOf` packageName = errorExit "gstreamer plugins are kept in lockstep." + | "electron" `T.isInfixOf` packageName = errorExit "multi-platform srcs in file." + | "linux-headers" `T.isInfixOf` packageName = errorExit "Not updated until many packages depend on it (part of stdenv)." + | "mpich" `T.isInfixOf` packageName = errorExit "Reported on repology.org as mischaracterized newest version" + | "xfce" `T.isInfixOf` packageName = errorExit "@volth asked to not update xfce" + | "cmake-cursesUI-qt4UI" `T.isInfixOf` packageName = errorExit "Derivation file is complicated" + | "varnish" `T.isInfixOf` packageName = errorExit "Temporary blacklist because of multiple versions and slow nixpkgs update" + | "iana-etc" `T.isInfixOf` packageName = errorExit "@mic92 takes care of this package" +isOnBlackList errorExit "isl" = errorExit "multi-version long building package" +isOnBlackList errorExit "tokei" = errorExit "got stuck forever building with no CPU usage" +isOnBlackList _ _ = return "" + +rawEval :: Text -> Sh Text +rawEval expr = cmd "nix" "eval" "-f" "." "--raw" expr + +canFail :: Sh a -> Sh a +canFail = errExit False + +orElse :: Sh a -> Sh a -> Sh a +orElse a b = do + v <- canFail a + status <- lastExitCode + if status == 0 then + return v + else + b + +infixl 3 `orElse` + +fixSrcUrl :: Text -> Version -> Version -> Text -> Text -> Text -> Sh Text +fixSrcUrl packageName oldVersion newVersion derivationFile attrPath oldSrcUrl = cmd "./fix-src-url.sh" packageName oldVersion newVersion derivationFile attrPath oldSrcUrl + +push :: Text -> Options -> Sh () +push branchName options = + if dryRun options then + return () + else + cmd "git" "push" "--set-upstream" "origin" branchName "--force" + +updatePackage :: Options -> Text -> Version -> Version -> Int -> Sh Bool +updatePackage options packageName oldVersion newVersion okToPrAt = do + nixpkgsPath <- setupNixpkgs + + setenv "NIX_PATH" ("nixpkgs=" <> toTextIgnore nixpkgsPath) + + let branchName = "auto-update/" <> packageName + + let errorExit = errorExit' branchName + + versionComparison <- cmd "nix" "eval" "-f" "." ("(builtins.compareVersions \"" <> newVersion <> "\" \"" <> oldVersion <> "\")") + unless (versionComparison == "1") $ do + errorExit $ newVersion <> " is not newer than " <> oldVersion <> " according to Nix" + + -- Package blacklist + isOnBlackList errorExit packageName + + oneHourAgo <- cmd "date" "+%s" "-d" "-1 hour" + fetchedLast <- cmd "stat" "-c" "%Y" ".git/FETCH_HEAD" + when (fetchedLast < oneHourAgo) $ do + canFail $ cmd "git" "fetch" "--prune" "--multiple" "upstream" "origin" + + remotes <- T.lines <$> cmd "git" "branch" "--remote" + when (("origin/auto-update/" <> packageName) `elem` remotes) $ do + errorExit "Update branch already on origin." + + cmd "git" "reset" "--hard" + cmd "git" "clean" "-fd" + cmd "git" "checkout" "master" + cmd "git" "reset" "--hard" "upstream/master" + cmd "git" "clean" "-fd" + + -- This is extremely slow but will give us better results + attrPath <- head . T.words . head . T.lines <$> cmd "nix-env" "-qa" (packageName <> "-" <> oldVersion) "-f" "." "--attr-path" `orElse` + errorExit "nix-env -q failed to find package name with old version" + + -- Temporarily blacklist gnome sources for lockstep update + whenM (("gnome" `T.isInfixOf`) <$> cmd "nix" "eval" "-f" "." ("pkgs." <> attrPath <> ".src.urls")) $ do + errorExit "Packages from gnome are currently blacklisted." + + -- Temporarily blacklist lua packages at @teto's request + -- https://github.com/NixOS/nixpkgs/pull/37501#issuecomment-375169646 + when (T.isPrefixOf "lua" attrPath) $ do + errorExit "Packages for lua are currently blacklisted." + + + derivationFile <- cmd "env" "EDITOR=echo" "nix" "edit" attrPath "-f" "." `orElse` errorExit "Couldn't find derivation file." + + + flip finally_sh (cleanup branchName) $ do + numberOfFetchers <- tRead <$> cmd "grep" "-c" "fetchurl {|fetchgit {|fetchFromGitHub {" derivationFile + unless ((numberOfFetchers :: Int) <= 1) $ do + errorExit $ "More than one fetcher in " <> derivationFile + + derivationContents <- cmd "cat" derivationFile + + if T.isInfixOf "DO NOT EDIT" derivationContents then + errorExit "Derivation file says not to edit it." + -- Skip packages that have special builders + else if T.isInfixOf "buildGoPackage" derivationContents then + errorExit "Derivation contains buildGoPackage." + else if T.isInfixOf "buildRustCrate" derivationContents then + errorExit "Derivation contains buildRustCrate." + else if T.isInfixOf "buildPythonPackage" derivationContents then + errorExit "Derivation contains buildPythonPackage." + else if T.isInfixOf "buildRubyGem" derivationContents then + errorExit "Derivation contains buildRubyGem." + else if T.isInfixOf "bundlerEnv" derivationContents then + errorExit "Derivation contains bundlerEnv." + else if T.isInfixOf "buildPerlPackage" derivationContents then + errorExit "Derivation contains buildPerlPackage." + else return () + + cmd "./check-attrpath-version.sh" attrPath newVersion `orElse` + errorExit ("Version in attr path " <> attrPath <> " not compatible with " <> newVersion) + + -- Make sure it hasn't been updated on master + cmd "grep" oldVersion derivationFile `orElse` + errorExit "Old version not present in master derivation file." + + -- Make sure it hasn't been updated on staging + cmd "git" "reset" "--hard" + cmd "git" "clean" "-fd" + cmd "git" "checkout" "-B" "staging" "upstream/staging" + cmd "git" "reset" "--hard" "upstream/staging" + cmd "git" "clean" "-fd" + + cmd "grep" oldVersion derivationFile `orElse` + errorExit "Old version not present in staging derivation file." + + base <- cmd "git" "merge-base" "upstream/master" "upstream/staging" + cmd "git" "checkout" "-B" branchName base + + oldHash <- rawEval ("pkgs." <> attrPath <> ".src.drvAttrs.outputHash") `orElse` + errorExit ("Could not find old output hash at " <> attrPath <> ".src.drvAttrs.outputHash.") + + oldSrcUrl <- rawEval ("let pkgs = import ./. {}; in builtins.elemAt pkgs." <> attrPath <> ".src.drvAttrs.urls 0)") + + cmd "sed" "-i" ("s/" <> (T.replace "." "\\." oldVersion) <> "/" <> newVersion <> "/g") derivationFile `orElse` + errorExit "Could not replace oldVersion with newVersion." + + newSrcUrl <- rawEval ("let pkgs = import ./. {}; in builtins.elemAt pkgs." <> attrPath <> ".src.drvAttrs.urls 0") + + when (oldSrcUrl == newSrcUrl) $ do + errorExit "Source url did not change." + + newHash <- cmd "nix-prefetch-url" "-A" (attrPath <> ".src") `orElse` + fixSrcUrl packageName oldVersion newVersion derivationFile attrPath oldSrcUrl `orElse` + errorExit "Could not prefetch new version URL." + + when (oldHash == newHash) $ do + errorExit "Hashes equal; no update necessary" + + cmd "sed" "-i" ("s/" <> oldHash <> "/" <> newHash <> "/g") derivationFile `orElse` + errorExit "Could not replace oldHash with newHash." + + cmd "rm" "-f" "result*" + + cmd "nix" "build" "-f" "." attrPath `orElse` do + buildLog <- T.unlines . reverse . take 30 . reverse . T.lines <$> cmd "nix" "log" "-f" "." attrPath + errorExit ("nix build failed.\n" <> buildLog) + + result <- cmd "readlink" "./result" `orElse` cmd "readlink" "./result-bin" `orElse` + errorExit "Could not find result link." + + checkResult <- cmd "./check-result.sh" result newVersion + + hasMaintainers <- const True <$> cmd "nix" "eval" ("let pkgs = import ./. {}; in pkgs." <> attrPath <> ".meta.maintainers") `orElse` return False + maintainers <- if hasMaintainers then rawEval ("let pkgs = import ./. {}; gh = m : m.github or \"\"; nonempty = s: s != \"\"; addat = s: \"@\"+s; in builtins.concatStringsSep \" \" (map addat (builtins.filter nonempty (map gh pkgs." <> attrPath <> ".meta.maintainers)))") else return "" + + let maintainersCc = if not (T.null maintainers) then "\n\ncc " <> maintainers <> " for review" else "" + + cmd "git" "diff" + + let + commitMessage :: Text + commitMessage = attrPath <> ": " <> oldVersion <> " → " <> newVersion <> "\n\n" + <> "Semi-automatic update generated by https://github.com/ryantm/nixpkgs-update tools.\n\n" + <> "This update was made based on information from https://repology.org/metapackage/" <> packageName <> "/versions.\n\n" + <> "These checks were done:\n\n" + <> "- built on NixOS\n" + <> checkResult + + cmd "git" "commit" "-am" commitMessage + + -- Try to push it three times + push branchName options `orElse` push branchName options `orElse` push branchName options + + isBroken <- cmd "nix" "eval" "-f" "." ("let pkgs = import ./. {}; in pkgs." <> attrPath <> ".meta.broken or false") + let brokenWarning = if isBroken == "true" then "" else "- WARNING: Package has meta.broken=true; Please manually test this package update and remove the broken attribute." + + let prMessage = commitMessage <> brokenWarning <> maintainersCc + + unless (dryRun options) $ do + current <- tRead <$> cmd "date" "+%s" + when (current < okToPrAt) $ do + let sleepSeconds = okToPrAt - current + echo $ "Sleeping " <> T.pack (show sleepSeconds) <> " seconds." + sleep sleepSeconds + + cmd "hub" "pull-request" "-m" prMessage + + cmd "git" "reset" "--hard" + cmd "git" "clean" "-fd" + cmd "git" "checkout" "-B" "master" "upstream/master" + cmd "git" "reset" "--hard" + cmd "git" "clean" "-fd" + + exit 0 diff --git a/src/Utils.hs b/src/Utils.hs new file mode 100644 index 0000000..c6319cc --- /dev/null +++ b/src/Utils.hs @@ -0,0 +1,45 @@ +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ExtendedDefaultRules #-} +{-# OPTIONS_GHC -fno-warn-type-defaults #-} +module Utils (Options(..), Version, setupNixpkgs, tRead, parseUpdates) where + +import Prelude hiding (FilePath) +import Data.Text (Text) +import Data.Semigroup ((<>)) +import qualified Data.Text as T +import Shelly + +default (T.Text) + +type Version = Text + +data Options = Options { + dryRun :: Bool +} + +setupNixpkgs :: Sh FilePath +setupNixpkgs = do + home <- get_env_text "HOME" + let nixpkgsPath = home ".cache" "nixpkgs" + + unlessM (test_e nixpkgsPath) $ do + cmd "hub" "clone" "nixpkgs" nixpkgsPath -- requires that user has forked nixpkgs + cmd "cd" nixpkgsPath + cmd "git" "remote" "add" "upstream" "https://github.com/NixOS/nixpkgs" + cmd "git" "fetch" "upstream" + cmd "git" "fetch" "origin" "staging" + cmd "git" "fetch" "upstream" "staging" + + cmd "cd" nixpkgsPath + + return nixpkgsPath + + +parseUpdates :: Text -> [(Text, Version, Version)] +parseUpdates = map (toTriple . T.words) . T.lines where + toTriple :: [Text] -> (Text, Version, Version) + toTriple [package, oldVersion, newVersion] = (package, oldVersion, newVersion) + toTriple line = error $ T.unpack ("Unable to parse update: " <> T.unwords line) + +tRead :: Read a => Text -> a +tRead = read . T.unpack diff --git a/up.sh b/up.sh deleted file mode 100755 index c08926c..0000000 --- a/up.sh +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -PAGER= -export PAGER - -# shellcheck source=setup-nixpkgs.sh -source "$SCRIPT_DIR/setup-nixpkgs.sh" - -NIX_PATH=nixpkgs="$NIXPKGS" -export NIX_PATH - -PACKAGE_NAME=$1 -OLD_VERSION=$2 -NEW_VERSION=$3 - -if [ "$#" -eq 4 ] -then - OK_TO_PR_AT=$4 -else - OK_TO_PR_AT="0" -fi - -BRANCH_NAME="auto-update/$1" - -function cleanup { - git reset --hard - git clean -fd - git checkout -B master upstream/master - git reset --hard upstream/master - git branch -D "$BRANCH_NAME" || true -} - -function error_exit { - cleanup - if ( true >&3 ) 2> /dev/null - then - echo "$(date -Iseconds) $1" >&3 - else - echo "$(date -Iseconds) $1" - fi - exit 1 -} - -if ! [ "$(nix eval -f . "(builtins.compareVersions \"$NEW_VERSION\" \"$OLD_VERSION\")")" -eq 1 ]; -then - error_exit "$NEW_VERSION is not newer than $OLD_VERSION according to Nix" -fi - -# Package blacklist -case "$PACKAGE_NAME" in - *jquery*) error_exit "this isn't a real package";; - *google-cloud-sdk*) error_exit "complicated package";; - *github-release*) error_exit "complicated package";; - *fcitx*) error_exit "gets stuck in daemons";; - *libxc*) error_exit "currently people don't want to update this https://github.com/NixOS/nixpkgs/pull/35821";; - *perl*) error_exit "currently don't know how to update perl";; - *python*) error_exit "currently don't know how to update python";; - *cdrtools*) error_exit "We keep downgrading this by accident.";; - *gst*) error_exit "gstreamer plugins are kept in lockstep.";; - *electron*) error_exit "multi-platform srcs in file.";; - *linux-headers*) error_exit "Not updated until many packages depend on it (part of stdenv).";; - *mpich*) error_exit "Reported on repology.org as mischaracterized newest version";; - *xfce*) error_exit "@volth asked to not update xfce";; - *cmake-cursesUI-qt4UI*) error_exit "Derivation file is complicated";; - *varnish*) error_exit "Temporary blacklist because of multiple versions and slow nixpkgs update";; - *iana-etc*) error_exit "@mic92 takes care of this package";; - isl) error_exit "multi-version long building package";; - tokei) error_exit "got stuck forever building with no CPU usage";; - *) true;; -esac || error_exit "Package on blacklist." - -ONEHOURAGO=$(date +%s -d "-1 hour") -FETCHEDLAST=$(stat -c %Y .git/FETCH_HEAD) -if (( FETCHEDLAST < ONEHOURAGO )) -then - git fetch --prune --multiple upstream origin || true -fi - -if git branch --remote | grep "origin/auto-update/${PACKAGE_NAME}" -then - error_exit "Update branch already on origin." -fi - -git reset --hard -git clean -fd -git checkout master -git reset --hard upstream/master -git clean -fd - -# This is extremely slow but will give us better results -ATTR_PATH=$(nix-env -qa "$PACKAGE_NAME-$OLD_VERSION" -f . --attr-path | head -n1 | cut -d' ' -f1) || error_exit "nix-env -q failed to find package name with old version" - -# Temporarily blacklist gnome sources for lockstep update -if nix eval -f . "pkgs.${ATTR_PATH}.src.urls" | grep "gnome" -then - error_exit "Packages from gnome are currently blacklisted." -fi - -# Temporarily blacklist lua packages at @teto's request -# https://github.com/NixOS/nixpkgs/pull/37501#issuecomment-375169646 -if echo "$ATTR_PATH" | grep "^lua" -then - error_exit "Packages for lua are currently blacklisted." -fi - - -DERIVATION_FILE=$(EDITOR="echo" nix edit "$ATTR_PATH" -f .) || error_exit "Couldn't find derivation file." - -function error_cleanup { - cleanup - exit 1 -} -trap error_cleanup ERR - -function interrupt_cleanup { - cleanup - exit 2 -} -trap interrupt_cleanup INT - -(( $(grep -c "fetchurl {" "$DERIVATION_FILE") + $(grep -c "fetchgit {" "$DERIVATION_FILE") + $(grep -c "fetchFromGitHub {" "$DERIVATION_FILE") <= 1 )) || error_exit "More than one fetcher in $DERIVATION_FILE" - -if grep -q "DO NOT EDIT" "$DERIVATION_FILE" -then - error_exit "Derivation file says not to edit it." -fi - -# Skip packages that have special builders -if grep -q "buildGoPackage" "$DERIVATION_FILE" -then - error_exit "Derivation contains buildGoPackage." -fi -if grep -q "buildRustCrate" "$DERIVATION_FILE" -then - error_exit "Derivation contains buildRustCrate." -fi -if grep -q "buildPythonPackage" "$DERIVATION_FILE" -then - error_exit "Derivation contains buildPythonPackage." -fi -if grep -q "buildRubyGem" "$DERIVATION_FILE" -then - error_exit "Derivation contains buildRubyGem." -fi -if grep -q "bundlerEnv" "$DERIVATION_FILE" -then - error_exit "Derivation contains bundlerEnv." -fi -if grep -q "buildPerlPackage" "$DERIVATION_FILE" -then - error_exit "Derivation contains buildPerlPackage." -fi - -"$SCRIPT_DIR"/check-attrpath-version.sh "$ATTR_PATH" "$NEW_VERSION" || error_exit "Version in attr path $ATTR_PATH not compatible with $NEW_VERSION" - -# Make sure it hasn't been updated on master -grep "$OLD_VERSION" "$DERIVATION_FILE" || error_exit "Old version not present in master derivation file." - -# Make sure it hasn't been updated on staging -git reset --hard -git clean -fd -git checkout -B staging upstream/staging -git reset --hard upstream/staging -git clean -fd -grep "$OLD_VERSION" "$DERIVATION_FILE" || error_exit "Old version not present in staging derivation file." - -git checkout -B "$BRANCH_NAME" "$(git merge-base upstream/master upstream/staging)" - -OLD_HASH=$(nix eval -f . --raw "pkgs.$ATTR_PATH.src.drvAttrs.outputHash" || error_exit "Couldn't find old output hash at ATTR_PATH.src.drvAttrs.outputHash.") - -OLD_SRC_URL=$(nix eval -f . --raw '(let pkgs = import ./. {}; in builtins.elemAt pkgs.'"$ATTR_PATH"'.src.drvAttrs.urls 0)') - -sed -i "s/${OLD_VERSION//\./\\.}/$NEW_VERSION/g" "$DERIVATION_FILE" || error_exit "Could not replace OLD_VERSION with NEW_VERSION." - -NEW_SRC_URL=$(nix eval -f . --raw '(let pkgs = import ./. {}; in builtins.elemAt pkgs.'"$ATTR_PATH"'.src.drvAttrs.urls 0)') - -if [ "$OLD_SRC_URL" == "$NEW_SRC_URL" ] -then - error_exit "Source url did not change." -fi - -NEW_HASH=$( - nix-prefetch-url -A "$ATTR_PATH.src" || \ - "$SCRIPT_DIR"/fix-src-url.sh "$PACKAGE_NAME" "$OLD_VERSION" "$NEW_VERSION" "$DERIVATION_FILE" "$ATTR_PATH" "$OLD_SRC_URL" || \ - error_exit "Could not prefetch new version URL.") - -if [ "$OLD_HASH" = "$NEW_HASH" ] -then - error_exit "Hashes equal; no update necessary" -fi - -sed -i "s/$OLD_HASH/$NEW_HASH/g" "$DERIVATION_FILE" || error_exit "Could not replace OLD_HASH with NEW_HASH." - -rm -f result* - -nix build -f . "$ATTR_PATH" || error_exit "nix build failed. -$(nix log -f . "$ATTR_PATH" | tail -n 30)" - -RESULT=$(readlink ./result || readlink ./result-bin || error_exit "Couldn't find result link.") - -CHECK_RESULT="$("$SCRIPT_DIR"/check-result.sh "$RESULT" "$NEW_VERSION")" - -MAINTAINERS= -if nix eval "(let pkgs = import ./. {}; in pkgs.$ATTR_PATH.meta.maintainers)" > /dev/null 2>&1 -then - maintainers=$(nix eval --raw '(let pkgs = import ./. {}; gh = m : m.github or ""; nonempty = s: s != ""; addat = s: "@"+s; in builtins.concatStringsSep " " (map addat (builtins.filter nonempty (map gh pkgs.'"${ATTR_PATH}"'.meta.maintainers))))') - if [ -n "$maintainers" ] - then - MAINTAINERS=" - -cc $maintainers for review" - fi -fi - -git diff - -COMMIT_MESSAGE="$ATTR_PATH: $OLD_VERSION -> $NEW_VERSION - -Semi-automatic update generated by https://github.com/ryantm/nixpkgs-update tools. - -This update was made based on information from https://repology.org/metapackage/$PACKAGE_NAME/versions. - -These checks were done: - -- built on NixOS -$CHECK_RESULT" - -git commit -am "$COMMIT_MESSAGE" - -# Try to push it three times -function push() { - if [[ -v DRY_RUN ]] - then - return 0 - else - git push --set-upstream origin "$BRANCH_NAME" --force - fi -} -push || push || push - -BROKEN_WARNING= -if [ "$(nix eval -f . '(let pkgs = import ./. {}; in pkgs.'"${ATTR_PATH}"'.meta.broken or false)')" == "true" ] -then - BROKEN_WARNING="- WARNING: Package has meta.broken=true; Please manually test this package update and remove the broken attribute." -fi - -PR_MESSAGE="$COMMIT_MESSAGE$BROKEN_WARNING$MAINTAINERS" - -if [[ -v DRY_RUN ]] -then - true -else - CURRENT=$(date +%s) - if (( CURRENT < OK_TO_PR_AT )) - then - SLEEP_SECONDS=$(( OK_TO_PR_AT - CURRENT )) - echo "Sleeping $SLEEP_SECONDS seconds." - sleep "$SLEEP_SECONDS" - fi - - hub pull-request -m "$PR_MESSAGE" -fi - -git reset --hard -git clean -fd -git checkout -B master upstream/master -git reset --hard -git clean -fd - -exit 0 diff --git a/ups.sh b/ups.sh deleted file mode 100755 index 61fc0b8..0000000 --- a/ups.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail - -LOG_FILE=~/.nix-update/ups.log -mkdir -p "$(dirname $LOG_FILE)" -touch $LOG_FILE - -SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -GITHUB_TOKEN="$(cat "$SCRIPT_DIR"/github_token.txt)" -export GITHUB_TOKEN - -ARGUMENTS=$(cat packages-to-update.txt) - -# shellcheck source=setup-nixpkgs.sh -source "$SCRIPT_DIR/setup-nixpkgs.sh" - -echo " - -$(date -Iseconds) New run of ups.sh" >> $LOG_FILE - -OK_TO_PR_AT=0 -IFS=$'\n' -for a in $ARGUMENTS -do - unset IFS - echo "$(date -Iseconds) $a" >> $LOG_FILE - - if eval "$SCRIPT_DIR/up.sh $a $OK_TO_PR_AT 3>>$LOG_FILE" - then - RESULT=$? - else - RESULT=1 - fi - - case "$RESULT" in - 0) - echo "$(date -Iseconds) SUCCESS" >> $LOG_FILE - OK_TO_PR_AT=$(date +%s -d "+15 minutes") - ;; - 1) - echo "$(date -Iseconds) FAIL" >> $LOG_FILE - ;; - *) exit 1 - ;; - esac -done - -echo "$(date -Iseconds) ups.sh finished" >> $LOG_FILE