diff --git a/README.md b/README.md index 1a92a66..469e61f 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,10 @@ nixpkgs-update supports interactive, single package updates via the * `--cve`—adds CVE vulnerability reporting to the PR message. On first invocation with this option, a CVE database is built. Subsequent invocations will be much faster. +* `--nixpkgs-review`—runs + [nixpkgs-review](https://github.com/Mic92/nixpkgs-review), which + tries to build all the packages that depend on the one being updated + and adds a report. # Batch updates diff --git a/app/Main.hs b/app/Main.hs index 7fbe4e1..878dc26 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -26,6 +26,7 @@ data UpdateOptions { pr :: Bool, cve :: Bool, cachix :: Bool, + nixpkgsReview :: Bool, outpaths :: Bool } @@ -46,6 +47,7 @@ updateOptionsParser = <$> O.flag False True (O.long "pr" <> O.help "Make a pull request using Hub.") <*> O.flag False True (O.long "cve" <> O.help "Make a CVE vulnerability report.") <*> O.flag False True (O.long "cachix" <> O.help "Push changes to Cachix") + <*> O.flag False True (O.long "nixpkgs-review" <> O.help "Runs nixpkgs-review on update commit rev") <*> O.flag False True (O.long "outpaths" <> O.help "Calculate outpaths to determine the branch to target") updateParser :: O.Parser Command @@ -126,19 +128,19 @@ main = do setupNixpkgs token P.setEnv "GITHUB_TOKEN" (T.unpack token) True deleteDone token - UpdateList UpdateOptions {pr, cachix, cve, outpaths} -> do + UpdateList UpdateOptions {pr, cachix, cve, nixpkgsReview, outpaths} -> do token <- getGithubToken updates <- T.readFile "packages-to-update.txt" setupNixpkgs token P.setEnv "PAGER" "" True P.setEnv "GITHUB_TOKEN" (T.unpack token) True - updateAll (Options pr True token cve cachix outpaths) updates - Update UpdateOptions {pr, cve, cachix} update -> do + updateAll (Options pr True token cve cachix nixpkgsReview outpaths) updates + Update UpdateOptions {pr, cve, cachix, nixpkgsReview} update -> do token <- getGithubToken setupNixpkgs token P.setEnv "PAGER" "" True P.setEnv "GITHUB_TOKEN" (T.unpack token) True - result <- updatePackage (Options pr False token cve cachix False) update + result <- updatePackage (Options pr False token cve cachix nixpkgsReview False) update case result of Left e -> T.putStrLn e Right () -> T.putStrLn "Done." @@ -156,12 +158,12 @@ main = do setupNixpkgs undefined report <- cveReport - (UpdateEnv productID oldVersion newVersion Nothing (Options False False undefined False False False)) + (UpdateEnv productID oldVersion newVersion Nothing (Options False False undefined False False False False)) T.putStrLn report SourceGithub -> do token <- getGithubToken updates <- T.readFile "packages-to-update.txt" setupNixpkgs token P.setEnv "GITHUB_TOKEN" (T.unpack token) True - sourceGithubAll (Options False False token False False False) updates + sourceGithubAll (Options False False token False False False False) updates FetchRepology -> Repology.fetch diff --git a/default.nix b/default.nix index ed9f622..1a8f51c 100644 --- a/default.nix +++ b/default.nix @@ -24,5 +24,7 @@ in pkg.overrideAttrs (attrs: { jq tree gist + (import sources.nixpkgs-review { inherit pkgs; }) + cabal-install # just for develpoment ]; }) diff --git a/nix/sources.json b/nix/sources.json index 166b1a5..c138fe4 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -34,5 +34,17 @@ "type": "tarball", "url": "https://github.com/nixos/nixpkgs/archive/78bfdbb291fd20df0f0f65061ee3081610b0a48f.tar.gz", "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs-review": { + "branch": "master", + "description": "Review pull-requests on https://github.com/NixOS/nixpkgs", + "homepage": "", + "owner": "mic92", + "repo": "nixpkgs-review", + "rev": "370e90a8d20640cc8924dacb4f55a86dadcec57f", + "sha256": "026lmwbvqdp7a3nkd08rd0nfyb9yiic36w6s7mh2rpp0ihp7qsd6", + "type": "tarball", + "url": "https://github.com/mic92/nixpkgs-review/archive/370e90a8d20640cc8924dacb4f55a86dadcec57f.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" } } diff --git a/nixpkgs-update.cabal b/nixpkgs-update.cabal index be8d93a..cbc7551 100644 --- a/nixpkgs-update.cabal +++ b/nixpkgs-update.cabal @@ -4,7 +4,7 @@ cabal-version: 2.2 -- -- see: https://github.com/sol/hpack -- --- hash: 09aa8671b0f8077c9c194e15f72bc2aec790275993b5122ab17c192622236836 +-- hash: 7492d0fc2af5df377764d358317e0892c0bda2961943218f8dcdeef179700cd9 name: nixpkgs-update version: 0.2.0 @@ -37,6 +37,7 @@ library GH Git Nix + NixpkgsReview NVD NVDRules OurPrelude diff --git a/src/File.hs b/src/File.hs index e656287..4bb5132 100644 --- a/src/File.hs +++ b/src/File.hs @@ -53,6 +53,6 @@ replaceIO :: MonadIO m => Text -> Text -> FilePath -> m Bool replaceIO find replacement file = liftIO $ runFinal - $ embedToFinal @IO + $ embedToFinal $ runIO $ (replace find replacement file) diff --git a/src/Git.hs b/src/Git.hs index 89776d8..ce97c2f 100644 --- a/src/Git.hs +++ b/src/Git.hs @@ -124,7 +124,7 @@ commit ref = runProcessNoIndexIssue_ (proc "git" ["commit", "-am", T.unpack ref]) headHash :: MonadIO m => ExceptT Text m Text -headHash = readProcessInterleavedNoIndexIssue_ "git rev-parse HEAD" +headHash = T.strip <$> readProcessInterleavedNoIndexIssue_ "git rev-parse HEAD" deleteBranchesEverywhere :: Vector Text -> IO () deleteBranchesEverywhere branches = do diff --git a/src/Nix.hs b/src/Nix.hs index 5153d7f..c0eeb40 100644 --- a/src/Nix.hs +++ b/src/Nix.hs @@ -224,7 +224,7 @@ getHomepageET attrPath = ExceptT . liftIO . runFinal - . embedToFinal @IO + . embedToFinal . Error.runError . Process.runIO $ getHomepage attrPath diff --git a/src/NixpkgsReview.hs b/src/NixpkgsReview.hs new file mode 100644 index 0000000..751664e --- /dev/null +++ b/src/NixpkgsReview.hs @@ -0,0 +1,48 @@ +{-# LANGUAGE OverloadedStrings #-} + +module NixpkgsReview + ( cacheDir, + runReport, + ) +where + +import Data.Text as T +import qualified File as F +import OurPrelude +import qualified Process as P +import System.Environment.XDG.BaseDir (getUserCacheDir) +import Prelude hiding (log) + +cacheDir :: IO FilePath +cacheDir = getUserCacheDir "nixpkgs-review" + +revDir :: FilePath -> Text -> FilePath +revDir cache commit = cache <> "/rev-" <> T.unpack commit + +run :: + Members '[F.File, P.Process] r => + FilePath -> + Text -> + Sem r Text +run cache commit = do + -- TODO: probably just skip running nixpkgs-review if the directory + -- already exists + void $ ourReadProcessInterleavedSem $ + proc "rm" ["-rf", revDir cache commit] + void $ ourReadProcessInterleavedSem $ + proc "nixpkgs-review" ["rev", T.unpack commit, "--no-shell"] + F.read $ (revDir cache commit) <> "/report.md" + +-- Assumes we are already in nixpkgs dir +runReport :: (Text -> IO ()) -> Text -> IO Text +runReport log commit = do + log "[check][nixpkgs-review]" + c <- cacheDir + msg <- + runFinal + . embedToFinal + . F.runIO + . P.runIO + $ NixpkgsReview.run c commit + log msg + return msg diff --git a/src/OurPrelude.hs b/src/OurPrelude.hs index 8aa610c..9a261d9 100644 --- a/src/OurPrelude.hs +++ b/src/OurPrelude.hs @@ -25,6 +25,7 @@ module OurPrelude ourReadProcessInterleavedBS_, ourReadProcessInterleaved, ourReadProcessInterleaved_Sem, + ourReadProcessInterleavedSem, silently, bytestringToText, ) @@ -81,11 +82,11 @@ ourReadProcessInterleaved_ = readProcessInterleaved_ >>> tryIOTextET >>> fmapRT bytestringToText ourReadProcessInterleaved_Sem :: - Members '[P.Process, Error Text] r => + Members '[P.Process] r => ProcessConfig stdin stdoutIgnored stderrIgnored -> Sem r Text ourReadProcessInterleaved_Sem = - P.readInterleaved >>> fmap bytestringToText + P.readInterleaved_ >>> fmap bytestringToText ourReadProcessInterleaved :: MonadIO m => @@ -96,5 +97,13 @@ ourReadProcessInterleaved = >>> tryIOTextET >>> fmapRT (\(a, b) -> (a, bytestringToText b)) +ourReadProcessInterleavedSem :: + Members '[P.Process] r => + ProcessConfig stdin stdoutIgnored stderrIgnored -> + Sem r (ExitCode, Text) +ourReadProcessInterleavedSem = + P.readInterleaved + >>> fmap (\(a, b) -> (a, bytestringToText b)) + silently :: ProcessConfig stdin stdout stderr -> ProcessConfig () () () silently = setStderr closed >>> setStdin closed >>> setStdout closed diff --git a/src/Process.hs b/src/Process.hs index c323354..e0f4385 100644 --- a/src/Process.hs +++ b/src/Process.hs @@ -7,9 +7,11 @@ import qualified Data.ByteString.Lazy as BSL import Polysemy import Polysemy.Input import qualified System.Process.Typed as TP +import System.Exit (ExitCode(..)) data Process m a where - ReadInterleaved :: TP.ProcessConfig stdin stdout stderr -> Process m BSL.ByteString + ReadInterleaved_ :: TP.ProcessConfig stdin stdout stderr -> Process m BSL.ByteString + ReadInterleaved :: TP.ProcessConfig stdin stdout stderr -> Process m (ExitCode, BSL.ByteString) makeSem ''Process @@ -19,7 +21,8 @@ runIO :: Sem r a runIO = interpret $ \case - ReadInterleaved config -> embed $ (TP.readProcessInterleaved_ config :: IO BSL.ByteString) + ReadInterleaved_ config -> embed $ (TP.readProcessInterleaved_ config) + ReadInterleaved config -> embed $ (TP.readProcessInterleaved config) runPure :: [BSL.ByteString] -> @@ -28,4 +31,7 @@ runPure :: runPure outputList = runInputList outputList . reinterpret \case - ReadInterleaved _config -> maybe "" id <$> input + ReadInterleaved_ _config -> maybe "" id <$> input + ReadInterleaved _config -> do + r <- maybe "" id <$> input + return (ExitSuccess, r) diff --git a/src/Rewrite.hs b/src/Rewrite.hs index 6bc1e50..7b6a4e6 100644 --- a/src/Rewrite.hs +++ b/src/Rewrite.hs @@ -101,7 +101,7 @@ quotedUrlsET log rwArgs = ExceptT $ liftIO . runFinal - . embedToFinal @IO + . embedToFinal . Error.runError . Process.runIO . File.runIO diff --git a/src/Update.hs b/src/Update.hs index 8698a6b..730b888 100644 --- a/src/Update.hs +++ b/src/Update.hs @@ -31,6 +31,7 @@ import qualified GH import qualified Git import NVD (getCVEs, withVulnDB) import qualified Nix +import qualified NixpkgsReview import OurPrelude import Outpaths import qualified Rewrite @@ -79,13 +80,19 @@ getLog o = do return log else return T.putStrLn +notifyOptions :: (Text -> IO ()) -> Options -> IO () +notifyOptions log o = do + when (doPR o) $ log "Will do push to origin and do PR on success." + when (pushToCachix o) $ log "Will push to cachix." + when (calculateOutpaths o) $ log "Will calculate outpaths." + when (makeCVEReport o) $ log "Will make a CVE security report." + when (runNixpkgsReview o) $ log "Will run nixpkgs-review." + updateAll :: Options -> Text -> IO () updateAll o updates = do log <- getLog o log "New run of nixpkgs-update" - when (doPR o) $ log "Will do push to origin and do PR on success." - when (pushToCachix o) $ log "Will push to cachix." - when (calculateOutpaths o) $ log "Will calculate outpaths." + notifyOptions log o twoHoursAgo <- runM $ Time.runIO Time.twoHoursAgo mergeBaseOutpathSet <- liftIO $ newIORef (MergeBaseOutpathsInfo twoHoursAgo S.empty) @@ -256,8 +263,7 @@ updatePackageBatch log updateEnv mergeBaseOutpathsContext = Git.cleanAndResetTo "master" publishPackage :: - MonadIO m => - (Text -> m ()) -> + (Text -> IO ()) -> UpdateEnv -> Text -> Text -> @@ -265,7 +271,7 @@ publishPackage :: Text -> Maybe (Set ResultLine) -> [Text] -> - ExceptT Text m () + ExceptT Text IO () publishPackage log updateEnv oldSrcUrl newSrcUrl attrPath result opDiff msgs = do cachixTestInstructions <- doCachix log updateEnv result resultCheckReport <- @@ -304,6 +310,10 @@ publishPackage log updateEnv oldSrcUrl newSrcUrl attrPath result opDiff msgs = d let commitMsg = commitMessage updateEnv attrPath Git.commit commitMsg commitHash <- Git.headHash + nixpkgsReviewMsg <- + if runNixpkgsReview . options $ updateEnv + then liftIO $ NixpkgsReview.runReport log commitHash + else return "" -- Try to push it three times when (doPR . options $ updateEnv) @@ -329,6 +339,7 @@ publishPackage log updateEnv oldSrcUrl newSrcUrl attrPath result opDiff msgs = d (fromMaybe "" (outpathReport <$> opDiff)) cveRep cachixTestInstructions + nixpkgsReviewMsg if (doPR . options $ updateEnv) then do let base = @@ -362,8 +373,9 @@ prMessage :: Text -> Text -> Text -> + Text -> Text -prMessage updateEnv isBroken metaDescription metaHomepage rewriteMessages releaseUrlMessage compareUrlMessage resultCheckReport commitHash attrPath maintainersCc resultPath opReport cveRep cachixTestInstructions = +prMessage updateEnv isBroken metaDescription metaHomepage rewriteMessages releaseUrlMessage compareUrlMessage resultCheckReport commitHash attrPath maintainersCc resultPath opReport cveRep cachixTestInstructions nixpkgsReviewMsg = let brokenMsg = brokenWarning isBroken title = prTitle updateEnv attrPath sourceLinkInfo = maybe "" pattern $ sourceURL updateEnv @@ -417,6 +429,7 @@ prMessage updateEnv isBroken metaDescription metaHomepage rewriteMessages releas
$cveRep + $nixpkgsReviewMsg $maintainersCc |] @@ -540,6 +553,7 @@ updatePackage o updateInfo = do let (p, oldV, newV, url) = head (rights (parseUpdates updateInfo)) let updateEnv = UpdateEnv p oldV newV url o let log = T.putStrLn + liftIO $ notifyOptions log o Nix.assertNewerVersion updateEnv attrPath <- Nix.lookupAttrPath updateEnv Version.assertCompatibleWithPathPin updateEnv attrPath diff --git a/src/Utils.hs b/src/Utils.hs index 5ed34a7..00e539e 100644 --- a/src/Utils.hs +++ b/src/Utils.hs @@ -109,6 +109,7 @@ data Options githubToken :: Text, makeCVEReport :: Bool, pushToCachix :: Bool, + runNixpkgsReview :: Bool, calculateOutpaths :: Bool } deriving (Show) @@ -204,10 +205,10 @@ setupNixpkgs githubt = do & System.Process.Typed.setEnv -- requires that user has forked nixpkgs [("GITHUB_TOKEN" :: String, githubt & T.unpack)] & runProcess_ - setCurrentDirectory fp - shell "git remote add upstream https://github.com/NixOS/nixpkgs" - & runProcess_ - shell "git fetch upstream" & runProcess_ + setCurrentDirectory fp + shell "git remote add upstream https://github.com/NixOS/nixpkgs" + & runProcess_ + shell "git fetch upstream" & runProcess_ setCurrentDirectory fp System.Posix.Env.setEnv "NIX_PATH" ("nixpkgs=" <> fp) True diff --git a/test/RewriteSpec.hs b/test/RewriteSpec.hs index 54641ef..b3d4a12 100644 --- a/test/RewriteSpec.hs +++ b/test/RewriteSpec.hs @@ -23,13 +23,13 @@ spec = do it "quotes an unquoted meta.homepage URL" do nixQuotedHomepageBad <- T.readFile "test_data/quoted_homepage_bad.nix" nixQuotedHomepageGood <- T.readFile "test_data/quoted_homepage_good.nix" - let options = Utils.Options False False "" False False False + let options = Utils.Options False False "" False False False False let updateEnv = Utils.UpdateEnv "inadyn" "2.5" "2.6" Nothing options -- TODO test correct file is being read let rwArgs = Rewrite.Args updateEnv "inadyn" undefined undefined (logs, (newContents, result)) <- ( runFinal - . embedToFinal @IO + . embedToFinal . Output.runOutputList . File.runPure [nixQuotedHomepageBad] . Process.runPure ["\"http://troglobit.com/project/inadyn/\""]