diff --git a/README.md b/README.md index 26c3c65..1b4f7fb 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,19 @@ necessary for fetching and updating the packages. `nix/sources.nix` file that returns the sources as a Nix object. * [Show](#show): shows the packages' information. +### Configuration + +The following environment variables are read by `niv`: + +| Name | Note | +| --------------- | ---- | +| GITHUB_TOKEN | When set, the value is used to authenticate GitHub API requests. | +| GITHUB_HOST | The GitHub host to use when fetching packages. Port may be appended here. | +| GITHUB_API_HOST | The host used when performing GitHub API requests. Use `GITHUB_API_PORT` for specifying the port. | +| GITHUB_API_PORT | The port used when performing GitHub API requests. Defaults to `443` for secure requests. Defaults to `80` for insecure requests. See also: `GITHUB_INSECURE`. | +| GITHUB_INSECURE | When set to anything but the empty string, requests are performed over `http` instead of `https`. | +| GITHUB_PATH | The base path used when performing GitHub API requests. | + The next two sections cover [common use cases](#getting-started) and [full command description](#commands). diff --git a/README.tpl.md b/README.tpl.md index 672392b..951b1ee 100644 --- a/README.tpl.md +++ b/README.tpl.md @@ -46,6 +46,19 @@ necessary for fetching and updating the packages. `nix/sources.nix` file that returns the sources as a Nix object. * [Show](#show): shows the packages' information. +### Configuration + +The following environment variables are read by `niv`: + +| Name | Note | +| --------------- | ---- | +| GITHUB_TOKEN | When set, the value is used to authenticate GitHub API requests. | +| GITHUB_HOST | The GitHub host to use when fetching packages. Port may be appended here. | +| GITHUB_API_HOST | The host used when performing GitHub API requests. Use `GITHUB_API_PORT` for specifying the port. | +| GITHUB_API_PORT | The port used when performing GitHub API requests. Defaults to `443` for secure requests. Defaults to `80` for insecure requests. See also: `GITHUB_INSECURE`. | +| GITHUB_INSECURE | When set to anything but the empty string, requests are performed over `http` instead of `https`. | +| GITHUB_PATH | The base path used when performing GitHub API requests. | + The next two sections cover [common use cases](#getting-started) and [full command description](#commands). diff --git a/package.yaml b/package.yaml index 9a6bd79..aa9e610 100644 --- a/package.yaml +++ b/package.yaml @@ -27,8 +27,8 @@ dependencies: - directory - file-embed - filepath - - github - hashable + - http-conduit - mtl - optparse-applicative - process @@ -43,7 +43,6 @@ library: - src dependencies: - aeson - - github - tasty - tasty-hunit - unordered-containers diff --git a/script/gen b/script/gen index b0b8db4..287a8cc 100755 --- a/script/gen +++ b/script/gen @@ -11,4 +11,10 @@ set -euo pipefail echo "Updating README" cat $(nix-build -A readme) > README.md -$(nix-build -A niv-svg-gen) + +if [ $# -gt 0 ] && [ $1 == "svg" ]; then + echo "Updating niv.svg" + $(nix-build -A niv-svg-gen) +fi + +echo done diff --git a/src/Niv/Cli.hs b/src/Niv/Cli.hs index df519ef..72d43fc 100644 --- a/src/Niv/Cli.hs +++ b/src/Niv/Cli.hs @@ -20,7 +20,7 @@ import Data.String.QQ (s) import Niv.Logger import Niv.GitHub import Niv.Update -import System.Exit (exitFailure, ExitCode(ExitSuccess)) +import System.Exit (ExitCode(ExitSuccess)) import System.FilePath ((), takeDirectory) import System.Process (readProcessWithExitCode) import UnliftIO @@ -520,11 +520,6 @@ mapWithKeyM_ f m = do forM_ (HMS.toList m) $ \(k, v) -> HMS.singleton k <$> f k v -abort :: T.Text -> IO a -abort msg = do - T.putStrLn msg - exitFailure - nixPrefetchURL :: Bool -> T.Text -> IO T.Text nixPrefetchURL unpack (T.unpack -> url) = do (exitCode, sout, serr) <- runNixPrefetch @@ -689,6 +684,3 @@ ticket: Thanks! I'll buy you a beer. |] <> T.unlines ["stdout: ", sout, "stderr: ", serr] - -tshow :: Show a => a -> T.Text -tshow = T.pack . show diff --git a/src/Niv/GitHub.hs b/src/Niv/GitHub.hs index 80b75fe..53a2be5 100644 --- a/src/Niv/GitHub.hs +++ b/src/Niv/GitHub.hs @@ -10,68 +10,23 @@ module Niv.GitHub where import Control.Arrow import Data.Bool +import Data.Functor import Data.Maybe import Data.String.QQ (s) -import GHC.Exts (toList) import Niv.Update -import Data.Text.Encoding (encodeUtf8) -import qualified Data.Aeson as Aeson -import qualified Data.Text as T -import qualified GitHub as GH -import qualified GitHub.Data.Name as GH import System.Environment (lookupEnv) +import System.Exit (exitFailure) +import System.IO.Unsafe (unsafePerformIO) +import Text.Read (readMaybe) +import qualified Data.Aeson as Aeson +import qualified Data.ByteString.Char8 as BS8 +import qualified Data.HashMap.Strict as HMS +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import qualified Data.Text.IO as T +import qualified Network.HTTP.Simple as HTTP -data GithubRepo = GithubRepo - { repoDescription :: Maybe T.Text - , repoHomepage :: Maybe T.Text - , repoDefaultBranch :: Maybe T.Text - } - -executeRequest - :: Aeson.FromJSON a => GH.Request 'GH.RO a -> IO (Either GH.Error a) -executeRequest req = do - token <- fmap (GH.OAuth . encodeUtf8 . T.pack) <$> lookupEnv "GITHUB_TOKEN" - GH.executeRequestMaybe token req - -githubRepo :: T.Text -> T.Text -> IO GithubRepo -githubRepo owner repo = request >>= pickResponse >>= return . translate - where - pickResponse :: Either GH.Error GH.Repo -> IO GH.Repo - pickResponse = \case - Left e -> do - warnCouldNotFetchGitHubRepo e (owner, repo) - error (show e) - Right x -> return x - request :: IO (Either GH.Error GH.Repo) - request = executeRequest (GH.repositoryR (GH.N owner) (GH.N repo)) - translate :: GH.Repo -> GithubRepo - translate r = GithubRepo - { repoDescription = GH.repoDescription r - , repoHomepage = GH.repoHomepage r - , repoDefaultBranch = GH.repoDefaultBranch r - } - -warnCouldNotFetchGitHubRepo :: GH.Error -> (T.Text, T.Text) -> IO () -warnCouldNotFetchGitHubRepo e (T.unpack -> owner, T.unpack -> repo) = - putStrLn $ unlines [ line1, line2, line3 ] - where - line1 = "WARNING: Could not read from GitHub repo: " <> owner <> "/" <> repo - line2 = [s| -I assumed that your package was a GitHub repository. An error occurred while -gathering information from the repository. Check whether your package was added -correctly: - - niv show - -If not, try re-adding it: - - niv drop - niv add - -Make sure the repository exists. -|] - line3 = unwords [ "(Error was:", show e, ")" ] - +-- | The GitHub update function -- TODO: fetchers for: -- * npm -- * hackage @@ -111,7 +66,78 @@ githubUpdate prefetch latestRev ghRepo = proc () -> do githubURLTemplate :: T.Text githubURLTemplate = - "https://github.com///archive/.tar.gz" + (if githubSecure then "https://" else "http://") <> + githubHost <> githubPath <> "//archive/.tar.gz" + +-- Bunch of GitHub helpers + +data GithubRepo = GithubRepo + { repoDescription :: Maybe T.Text + , repoHomepage :: Maybe T.Text + , repoDefaultBranch :: Maybe T.Text + } + +githubRepo :: T.Text -> T.Text -> IO GithubRepo +githubRepo owner repo = do + request <- defaultRequest ["repos", owner, repo] + -- we don't use httpJSONEither because it adds an "Accept: + -- application/json" header that GitHub chokes on + resp0 <- HTTP.httpBS request + let resp = fmap Aeson.eitherDecodeStrict resp0 + case (HTTP.getResponseStatusCode resp, HTTP.getResponseBody resp) of + (200, Right (Aeson.Object m)) -> do + let lookupText k = case HMS.lookup k m of + Just (Aeson.String t) -> Just t + _ -> Nothing + pure GithubRepo + { repoDescription = lookupText "description" + , repoHomepage = lookupText "homepage" + , repoDefaultBranch = lookupText "default_branch" + } + (200, Right v) -> do + error $ "expected object, got " <> show v + (200, Left e) -> do + error $ "github didn't return JSON: " <> show e + _ -> abortCouldNotFetchGitHubRepo (tshow (request,resp0)) (owner, repo) + +-- | TODO: Error instead of T.Text? +abortCouldNotFetchGitHubRepo :: T.Text -> (T.Text, T.Text) -> IO a +abortCouldNotFetchGitHubRepo e (T.unpack -> owner, T.unpack -> repo) = do + putStrLn $ unlines [ line1, line2, T.unpack line3 ] + exitFailure + where + line1 = "WARNING: Could not read from GitHub repo: " <> owner <> "/" <> repo + line2 = [s| +I assumed that your package was a GitHub repository. An error occurred while +gathering information from the repository. Check whether your package was added +correctly: + + niv show + +If not, try re-adding it: + + niv drop + niv add + +Make sure the repository exists. +|] + line3 = T.unwords [ "(Error was:", e, ")" ] + +defaultRequest :: [T.Text] -> IO HTTP.Request +defaultRequest (map T.encodeUtf8 -> parts) = do + let path = T.encodeUtf8 githubPath <> BS8.intercalate "/" (parts) + mtoken <- lookupEnv "GITHUB_TOKEN" + pure $ + (flip (maybe id) mtoken $ \token -> + HTTP.addRequestHeader "authorization" ("token " <> BS8.pack token) + ) $ + HTTP.setRequestPath path $ + HTTP.addRequestHeader "user-agent" "niv" $ + HTTP.addRequestHeader "accept" "application/vnd.github.v3+json" $ + HTTP.setRequestSecure githubSecure $ + HTTP.setRequestHost (T.encodeUtf8 githubApiHost) $ + HTTP.setRequestPort githubApiPort $ + HTTP.defaultRequest -- | Get the latest revision for owner, repo and branch. -- TODO: explain no error handling @@ -123,15 +149,69 @@ githubLatestRev -> T.Text -- ^ branch -> IO T.Text -githubLatestRev owner repo branch = - executeRequest ( - GH.commitsWithOptionsForR (GH.N owner) (GH.N repo) (GH.FetchAtLeast 1) - [GH.CommitQuerySha branch] - ) >>= \case - Right (toList -> (commit:_)) -> do - let GH.N rev = GH.commitSha commit - pure $ rev - Right (toList -> []) -> do - error "No rev: no commits" - Left e -> error $ "No rev: " <> show e - _ -> error $ "No rev: impossible" +githubLatestRev owner repo branch = do + request <- defaultRequest [ "repos", owner, repo, "commits", branch ] <&> + HTTP.addRequestHeader "accept" "application/vnd.github.v3.sha" + resp <- HTTP.httpBS request + case HTTP.getResponseStatusCode resp of + 200 -> pure $ T.decodeUtf8 $ HTTP.getResponseBody resp + _ -> abortCouldNotGetRev owner repo branch resp + +abortCouldNotGetRev :: T.Text -> T.Text -> T.Text -> HTTP.Response BS8.ByteString -> IO a +abortCouldNotGetRev owner repo branch resp = abort $ T.unlines [ line1, line2, line3 ] + where + line1 = T.unwords + [ "Cannot get latest revision for branch" + , "'" <> branch <> "'" + , "(" <> owner <> "/" <> repo <> ")" + ] + line2 = "The request failed: " <> tshow resp + line3 = [s| +NOTE: You may want to retry with an authentication token: + + GITHUB_TOKEN=... niv + +For more information on rate-limiting, see + + https://developer.github.com/v3/#rate-limiting + +|] + +githubHost :: T.Text +githubHost = unsafePerformIO $ do + lookupEnv "GITHUB_HOST" >>= \case + Just (T.pack -> x) -> pure x + Nothing -> pure "github.com" + +githubApiPort :: Int +githubApiPort = unsafePerformIO $ do + lookupEnv "GITHUB_API_PORT" >>= \case + Just (readMaybe -> Just x) -> pure x + _ -> pure $ if githubSecure then 443 else 80 + +githubApiHost :: T.Text +githubApiHost = unsafePerformIO $ do + lookupEnv "GITHUB_API_HOST" >>= \case + Just (T.pack -> x) -> pure x + Nothing -> pure "api.github.com" + +githubSecure :: Bool +githubSecure = unsafePerformIO $ do + lookupEnv "GITHUB_INSECURE" >>= \case + Just "" -> pure True + Just _ -> pure False + Nothing -> pure True + +githubPath :: T.Text +githubPath = unsafePerformIO $ do + lookupEnv "GITHUB_PATH" >>= \case + Just (T.pack -> x) -> pure $ fromMaybe x (T.stripSuffix "/" x) <> "/" + Nothing -> pure "/" + +abort :: T.Text -> IO a +abort msg = do + T.putStrLn msg + exitFailure + +tshow :: Show a => a -> T.Text +tshow = T.pack . show diff --git a/src/Niv/Logger.hs b/src/Niv/Logger.hs index 9cd283e..5199ba4 100644 --- a/src/Niv/Logger.hs +++ b/src/Niv/Logger.hs @@ -7,6 +7,7 @@ module Niv.Logger where import Control.Monad import Data.Profunctor +import System.Exit (exitFailure) import System.IO.Unsafe (unsafePerformIO) import qualified Data.Text as T import UnliftIO @@ -19,7 +20,13 @@ job str act = do indent tryAny act <* deindent >>= \case Right () -> say $ green "Done" <> ": " <> str - Left e -> say $ red "ERROR" <> ":\n" <> show e + Left e -> do + -- don't wrap if the error ain't too long + let showErr = do + let se = show e + (if length se > 40 then ":\n" else ": ") <> se + say $ red "ERROR" <> showErr + exitFailure where indent = void $ atomicModifyIORef jobStack (\x -> (x + 1, undefined)) deindent = void $ atomicModifyIORef jobStack (\x -> (x - 1, undefined)) diff --git a/tests/default.nix b/tests/default.nix index d76d7b9..ba31e81 100644 --- a/tests/default.nix +++ b/tests/default.nix @@ -18,20 +18,10 @@ let niv_HEAD = "a489b65a5c3a29983701069d1ce395b23d9bde64"; niv_HEAD- = "abc51449406ba3279c466b4d356b4ae8522ceb58"; nixpkgs-channels_HEAD = "571b40d3f50466d3e91c1e609d372de96d782793"; - nivForTest = niv.overrideDerivation(old: { - # TODO: Remove this patch by adding an argument to the github - # subcommand to support GitHub entreprise. - prePatch = '' - sed "/import Data.Text.Encoding (encodeUtf8)/d" -i src/Niv/GitHub.hs - sed "/import System.Environment (lookupEnv)/d" -i src/Niv/GitHub.hs - sed "s|token <- fmap (GH.OAuth . encodeUtf8 . T.pack) <$> lookupEnv \"GITHUB_TOKEN\"|let token = Just (GH.EnterpriseOAuth \"http://localhost:3333\" \"\")|" -i src/Niv/GitHub.hs - sed "s|https://github.com|http://localhost:3333|" -i src/Niv/GitHub.hs - ''; - }); in pkgs.runCommand "test" { buildInputs = [ pkgs.haskellPackages.wai-app-static - nivForTest + niv pkgs.nix pkgs.jq pkgs.netcat-gnu @@ -40,6 +30,11 @@ in pkgs.runCommand "test" '' set -euo pipefail + export GITHUB_HOST="localhost:3333" + export GITHUB_API_HOST="localhost" + export GITHUB_API_PORT="3333" + export GITHUB_INSECURE="true" + echo *** Starting the webserver... mkdir -p mock @@ -63,15 +58,18 @@ in pkgs.runCommand "test" mkdir -p mock/repos/nmattia/niv/ cp ${./data/repos/nmattia/niv/repository.json} mock/repos/nmattia/niv/index.html + + mkdir -p mock/repos/nmattia/niv/commits + cat ${./data/repos/nmattia/niv/commits.json} | jq -j '.[0] | .sha' > mock/repos/nmattia/niv/commits/master # XXX: cat so we don't inherit the read-only permissions - cat ${./data/repos/nmattia/niv/commits.json} > mock/repos/nmattia/niv/commits mkdir -p mock/nmattia/niv/archive cp ${./data/archives + "/${niv_HEAD}.tar.gz"} \ mock/nmattia/niv/archive/${niv_HEAD}.tar.gz mkdir -p mock/repos/NixOS/nixpkgs-channels cp ${./data/repos/NixOS/nixpkgs-channels/repository.json} mock/repos/NixOS/nixpkgs-channels/index.html - cat ${./data/repos/NixOS/nixpkgs-channels/commits.json} > mock/repos/NixOS/nixpkgs-channels/commits + mkdir -p mock/repos/NixOS/nixpkgs-channels/commits + cat ${./data/repos/NixOS/nixpkgs-channels/commits.json} | jq -j '.[0] | .sha' > mock/repos/NixOS/nixpkgs-channels/commits/nixos-19.03 mkdir -p mock/NixOS/nixpkgs-channels/archive cp ${./data/archives + "/${nixpkgs-channels_HEAD}.tar.gz"} \ mock/NixOS/nixpkgs-channels/archive/${nixpkgs-channels_HEAD}.tar.gz @@ -97,8 +95,8 @@ in pkgs.runCommand "test" echo -e "\n*** niv add nmattia/niv" # We use the HEAD~1 commit to update it in the next step - # (e.g. we drop the first element of the commit array) - cat ${./data/repos/nmattia/niv/commits.json} | jq 'del(.[0])' > mock/repos/nmattia/niv/commits + # (i.e. we use the second element of the commit array) + cat ${./data/repos/nmattia/niv/commits.json} | jq -j '.[1] | .sha' > mock/repos/nmattia/niv/commits/master cp ${./data/archives + "/${niv_HEAD-}.tar.gz"} \ mock/nmattia/niv/archive/${niv_HEAD-}.tar.gz niv add nmattia/niv @@ -111,7 +109,7 @@ in pkgs.runCommand "test" ## - nixpkgs-channels points to HEAD echo -e "\n*** niv update niv" - cat ${./data/repos/nmattia/niv/commits.json} | jq '.[0] | [.]' > mock/repos/nmattia/niv/commits + cat ${./data/repos/nmattia/niv/commits.json} | jq -j '.[0] | .sha' > mock/repos/nmattia/niv/commits/master niv update niv echo -n "niv.rev == ${niv_HEAD} (HEAD): " cat nix/sources.json | jq -e '.niv | .rev == "${niv_HEAD}"'