From 04cb2aca07bd02bca9eb9ba50d642a88cd3b629a Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 4 Nov 2022 13:41:25 +0800 Subject: [PATCH 01/43] Bump flake inputs --- flake.lock | 500 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 465 insertions(+), 35 deletions(-) diff --git a/flake.lock b/flake.lock index fb6135d..fbdde31 100644 --- a/flake.lock +++ b/flake.lock @@ -16,6 +16,21 @@ "type": "github" } }, + "blank": { + "locked": { + "lastModified": 1625557891, + "narHash": "sha256-O8/MWsPBGhhyPoPLHZAuoZiiHo9q6FLlEeIDEXuj6T4=", + "owner": "divnix", + "repo": "blank", + "rev": "5a5d2684073d9f563072ed07c871d577a6c614a8", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "blank", + "type": "github" + } + }, "cabal-32": { "flake": false, "locked": { @@ -83,13 +98,103 @@ "type": "github" } }, + "devshell": { + "inputs": { + "flake-utils": [ + "haskell-nix", + "tullia", + "std", + "flake-utils" + ], + "nixpkgs": [ + "haskell-nix", + "tullia", + "std", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1663445644, + "narHash": "sha256-+xVlcK60x7VY1vRJbNUEAHi17ZuoQxAIH4S4iUFUGBA=", + "owner": "numtide", + "repo": "devshell", + "rev": "e3dc3e21594fe07bdb24bdf1c8657acaa4cb8f66", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "devshell", + "type": "github" + } + }, + "dmerge": { + "inputs": { + "nixlib": [ + "haskell-nix", + "tullia", + "std", + "nixpkgs" + ], + "yants": [ + "haskell-nix", + "tullia", + "std", + "yants" + ] + }, + "locked": { + "lastModified": 1659548052, + "narHash": "sha256-fzI2gp1skGA8mQo/FBFrUAtY0GQkAIAaV/V127TJPyY=", + "owner": "divnix", + "repo": "data-merge", + "rev": "d160d18ce7b1a45b88344aa3f13ed1163954b497", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "data-merge", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1635892615, + "narHash": "sha256-harGbMZr4hzat2BWBU+Y5OYXlu+fVz7E4WeQzHi5o8A=", + "owner": "input-output-hk", + "repo": "flake-compat", + "rev": "eca47d3377946315596da653862d341ee5341318", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1650374568, + "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "b4a34015c698c7793d592d66adbab377907a2be8", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-utils": { "locked": { - "lastModified": 1659877975, - "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", "owner": "numtide", "repo": "flake-utils", - "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", "type": "github" }, "original": { @@ -113,6 +218,51 @@ "type": "github" } }, + "flake-utils_3": { + "locked": { + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_4": { + "locked": { + "lastModified": 1659877975, + "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_5": { + "locked": { + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "ghc-8.6.5-iohk": { "flake": false, "locked": { @@ -130,14 +280,33 @@ "type": "github" } }, + "gomod2nix": { + "inputs": { + "nixpkgs": "nixpkgs_2", + "utils": "utils" + }, + "locked": { + "lastModified": 1655245309, + "narHash": "sha256-d/YPoQ/vFn1+GTmSdvbSBSTOai61FONxB4+Lt6w/IVI=", + "owner": "tweag", + "repo": "gomod2nix", + "rev": "40d32f82fc60d66402eb0972e6e368aeab3faf58", + "type": "github" + }, + "original": { + "owner": "tweag", + "repo": "gomod2nix", + "type": "github" + } + }, "hackage": { "flake": false, "locked": { - "lastModified": 1661735830, - "narHash": "sha256-xJpfFmpzJVci+fM9mjTKyIbkUpJf5XkZMygY+YRKzwU=", + "lastModified": 1667524657, + "narHash": "sha256-Hsp8KdtPWDNteVXRj0X9rEs3I1EJmpG4nYxHE4s/r8s=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "9b22f61f50b19bf401ab104ff0e239f8ada87547", + "rev": "bf170e96de8ed70c445fa759aaa29e081dacab86", "type": "github" }, "original": { @@ -153,12 +322,12 @@ "cabal-34": "cabal-34", "cabal-36": "cabal-36", "cardano-shell": "cardano-shell", + "flake-compat": "flake-compat", "flake-utils": "flake-utils_2", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk", "hackage": "hackage", "hpc-coveralls": "hpc-coveralls", "hydra": "hydra", - "nix-tools": "nix-tools", "nixpkgs": [ "haskell-nix", "nixpkgs-unstable" @@ -169,14 +338,15 @@ "nixpkgs-2205": "nixpkgs-2205", "nixpkgs-unstable": "nixpkgs-unstable", "old-ghc-nix": "old-ghc-nix", - "stackage": "stackage" + "stackage": "stackage", + "tullia": "tullia" }, "locked": { - "lastModified": 1661940924, - "narHash": "sha256-aXmYSdeDjSWpJPtNAtvAKabKUyOOiJXzZ7EEYYKNrYI=", + "lastModified": 1667533063, + "narHash": "sha256-EHZOvFNdfgA8U3xNKZkyGZT7Ydnr/3pihMDyGEwLlag=", "owner": "input-output-hk", "repo": "haskell.nix", - "rev": "c83b33f4198fd8ba62748feb5024ece2b5329a8e", + "rev": "e601c9ce609af07a78edf1c57f5985931788aeb2", "type": "github" }, "original": { @@ -240,6 +410,46 @@ "type": "github" } }, + "mdbook-kroki-preprocessor": { + "flake": false, + "locked": { + "lastModified": 1661755005, + "narHash": "sha256-1TJuUzfyMycWlOQH67LR63/ll2GDZz25I3JfScy/Jnw=", + "owner": "JoelCourtney", + "repo": "mdbook-kroki-preprocessor", + "rev": "93adb5716d035829efed27f65f2f0833a7d3e76f", + "type": "github" + }, + "original": { + "owner": "JoelCourtney", + "repo": "mdbook-kroki-preprocessor", + "type": "github" + } + }, + "n2c": { + "inputs": { + "flake-utils": "flake-utils_5", + "nixpkgs": [ + "haskell-nix", + "tullia", + "std", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1665039323, + "narHash": "sha256-SAh3ZjFGsaCI8FRzXQyp56qcGdAqgKEfJWPCQ0Sr7tQ=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "b008fe329ffb59b67bf9e7b08ede6ee792f2741a", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, "nix": { "inputs": { "lowdown-src": "lowdown-src", @@ -261,19 +471,92 @@ "type": "github" } }, - "nix-tools": { - "flake": false, + "nix-nomad": { + "inputs": { + "flake-compat": "flake-compat_2", + "flake-utils": [ + "haskell-nix", + "tullia", + "nix2container", + "flake-utils" + ], + "gomod2nix": "gomod2nix", + "nixpkgs": [ + "haskell-nix", + "tullia", + "nixpkgs" + ], + "nixpkgs-lib": [ + "haskell-nix", + "tullia", + "nixpkgs" + ] + }, "locked": { - "lastModified": 1659569011, - "narHash": "sha256-wHS0H5+TERmDnPCfzH4A+rSR5TvjYMWus9BNeNAMyUM=", - "owner": "input-output-hk", - "repo": "nix-tools", - "rev": "555d57e1ea81b79945f2608aa261df20f6b602a5", + "lastModified": 1658277770, + "narHash": "sha256-T/PgG3wUn8Z2rnzfxf2VqlR1CBjInPE0l1yVzXxPnt0=", + "owner": "tristanpemble", + "repo": "nix-nomad", + "rev": "054adcbdd0a836ae1c20951b67ed549131fd2d70", "type": "github" }, "original": { - "owner": "input-output-hk", - "repo": "nix-tools", + "owner": "tristanpemble", + "repo": "nix-nomad", + "type": "github" + } + }, + "nix2container": { + "inputs": { + "flake-utils": "flake-utils_3", + "nixpkgs": "nixpkgs_3" + }, + "locked": { + "lastModified": 1658567952, + "narHash": "sha256-XZ4ETYAMU7XcpEeAFP3NOl9yDXNuZAen/aIJ84G+VgA=", + "owner": "nlewo", + "repo": "nix2container", + "rev": "60bb43d405991c1378baf15a40b5811a53e32ffa", + "type": "github" + }, + "original": { + "owner": "nlewo", + "repo": "nix2container", + "type": "github" + } + }, + "nixago": { + "inputs": { + "flake-utils": [ + "haskell-nix", + "tullia", + "std", + "flake-utils" + ], + "nixago-exts": [ + "haskell-nix", + "tullia", + "std", + "blank" + ], + "nixpkgs": [ + "haskell-nix", + "tullia", + "std", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1661824785, + "narHash": "sha256-/PnwdWoO/JugJZHtDUioQp3uRiWeXHUdgvoyNbXesz8=", + "owner": "nix-community", + "repo": "nixago", + "rev": "8c1f9e5f1578d4b2ea989f618588d62a335083c3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixago", "type": "github" } }, @@ -310,11 +593,11 @@ }, "nixpkgs-2105": { "locked": { - "lastModified": 1655034179, - "narHash": "sha256-rf1/7AbzuYDw6+8Xvvf3PtEOygymLBrFsFxvext5ZjI=", + "lastModified": 1659914493, + "narHash": "sha256-lkA5X3VNMKirvA+SUzvEhfA7XquWLci+CGi505YFAIs=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "046ee4af7a9f016a364f8f78eeaa356ba524ac31", + "rev": "022caabb5f2265ad4006c1fa5b1ebe69fb0c3faf", "type": "github" }, "original": { @@ -326,11 +609,11 @@ }, "nixpkgs-2111": { "locked": { - "lastModified": 1656782578, - "narHash": "sha256-1eMCBEqJplPotTo/SZ/t5HU6Sf2I8qKlZi9MX7jv9fw=", + "lastModified": 1659446231, + "narHash": "sha256-hekabNdTdgR/iLsgce5TGWmfIDZ86qjPhxDg/8TlzhE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "573603b7fdb9feb0eb8efc16ee18a015c667ab1b", + "rev": "eabc38219184cc3e04a974fe31857d8e0eac098d", "type": "github" }, "original": { @@ -342,11 +625,11 @@ }, "nixpkgs-2205": { "locked": { - "lastModified": 1657876628, - "narHash": "sha256-URmf0O2cQ/3heg2DJOeLyU/JmfVMqG4X5t9crQXMaeY=", + "lastModified": 1663981975, + "narHash": "sha256-TKaxWAVJR+a5JJauKZqibmaM5e/Pi5tBDx9s8fl/kSE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "549d82bdd40f760a438c3c3497c1c61160f3de55", + "rev": "309faedb8338d3ae8ad8f1043b3ccf48c9cc2970", "type": "github" }, "original": { @@ -373,11 +656,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1657888067, - "narHash": "sha256-GnwJoFBTPfW3+mz7QEeJEEQ9OMHZOiIJ/qDhZxrlKh8=", + "lastModified": 1663905476, + "narHash": "sha256-0CSwRKaYravh9v6qSlBpM0gNg0UhKT2lL7Yn6Zbx7UM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "65fae659e31098ca4ac825a6fef26d890aaf3f4e", + "rev": "e14f9fb57315f0d4abde222364f19f88c77d2b79", "type": "github" }, "original": { @@ -387,6 +670,53 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1653581809, + "narHash": "sha256-Uvka0V5MTGbeOfWte25+tfRL3moECDh1VwokWSZUdoY=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "83658b28fe638a170a19b8933aa008b30640fbd1", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1654807842, + "narHash": "sha256-ADymZpr6LuTEBXcy6RtFHcUZdjKTBRTMYwu19WOx17E=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "fc909087cc3386955f21b4665731dbdaceefb1d8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1665087388, + "narHash": "sha256-FZFPuW9NWHJteATOf79rZfwfRn5fE0wi9kRzvGfDHPA=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "95fda953f6db2e9496d2682c4fc7b82f959878f7", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "old-ghc-nix": { "flake": false, "locked": { @@ -417,11 +747,11 @@ "stackage": { "flake": false, "locked": { - "lastModified": 1661925696, - "narHash": "sha256-eMM1dhdSv0AlLtD2cULJsLhJF4ZWdvpqRcCOw9DbXLI=", + "lastModified": 1667524765, + "narHash": "sha256-rY58ROG9paYqqhUPFxZArU59qOIatIFHrurhVo7JXX4=", "owner": "input-output-hk", "repo": "stackage.nix", - "rev": "575039e5410ef653e302d53938aa6df403e5a8b1", + "rev": "ed1ec5f81f9eb32eb627fd447088eb782e7ff71b", "type": "github" }, "original": { @@ -429,6 +759,106 @@ "repo": "stackage.nix", "type": "github" } + }, + "std": { + "inputs": { + "blank": "blank", + "devshell": "devshell", + "dmerge": "dmerge", + "flake-utils": "flake-utils_4", + "makes": [ + "haskell-nix", + "tullia", + "std", + "blank" + ], + "mdbook-kroki-preprocessor": "mdbook-kroki-preprocessor", + "microvm": [ + "haskell-nix", + "tullia", + "std", + "blank" + ], + "n2c": "n2c", + "nixago": "nixago", + "nixpkgs": "nixpkgs_4", + "yants": "yants" + }, + "locked": { + "lastModified": 1665513321, + "narHash": "sha256-D6Pacw9yf/HMs84KYuCxHXnNDL7v43gtcka5URagFqE=", + "owner": "divnix", + "repo": "std", + "rev": "94a90eedb9cfc115b12ae8f6622d9904788559e4", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "std", + "type": "github" + } + }, + "tullia": { + "inputs": { + "nix-nomad": "nix-nomad", + "nix2container": "nix2container", + "nixpkgs": [ + "haskell-nix", + "nixpkgs" + ], + "std": "std" + }, + "locked": { + "lastModified": 1666200256, + "narHash": "sha256-cJPS8zBu30SMhxMe7I8DWutwqMuhPsEez87y9gxMKc4=", + "owner": "input-output-hk", + "repo": "tullia", + "rev": "575362c2244498e8d2c97f72861510fa72e75d44", + "type": "github" + }, + "original": { + "owner": "input-output-hk", + "repo": "tullia", + "type": "github" + } + }, + "utils": { + "locked": { + "lastModified": 1653893745, + "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "yants": { + "inputs": { + "nixpkgs": [ + "haskell-nix", + "tullia", + "std", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1660507851, + "narHash": "sha256-BKjq7JnVuUR/xDtcv6Vm9GYGKAblisXrAgybor9hT/s=", + "owner": "divnix", + "repo": "yants", + "rev": "0b895ca02a8fa72bad50b454cb3e7d8a66407c96", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "yants", + "type": "github" + } } }, "root": "root", From 83405a1467da23029e53bd7e89ebf3456351e807 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Wed, 30 Nov 2022 11:22:32 +0800 Subject: [PATCH 02/43] Formatting --- flake.nix | 59 ++++++++++++++++++++++++++----------------------------- 1 file changed, 28 insertions(+), 31 deletions(-) diff --git a/flake.nix b/flake.nix index fba06a9..db368d6 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,6 @@ { - description = "Foliage is a tool to create custom Haskell package repositories, in a fully reproducible way."; + description = + "Foliage is a tool to create custom Haskell package repositories, in a fully reproducible way."; inputs = { nixpkgs.follows = "haskell-nix/nixpkgs-unstable"; @@ -9,39 +10,35 @@ outputs = { self, nixpkgs, flake-utils, haskell-nix }: flake-utils.lib.eachDefaultSystem (system: - let - pkgs = import nixpkgs { inherit system; inherit (haskell-nix) config; overlays = [haskell-nix.overlay]; }; - project = pkgs.haskell-nix.cabalProject { - src = ./.; - compiler-nix-name = "ghc8107"; - shell.tools = { - cabal = {}; - hlint = {}; - haskell-language-server = {}; + let + pkgs = import nixpkgs { + inherit system; + inherit (haskell-nix) config; + overlays = [ haskell-nix.overlay ]; }; - shell.buildInputs = with pkgs; [ - nixpkgs-fmt - fsatrace - ]; - modules = [{ - packages.foliage.components.exes.foliage.dontStrip = false; - }]; - }; - in { - packages.default = project.foliage.components.exes.foliage; - devShell = pkgs.mkShell { - name = "foliage-dev-shell"; - buildInputs = with pkgs; [ - fsatrace - ]; - }; - }); + project = pkgs.haskell-nix.cabalProject { + src = ./.; + compiler-nix-name = "ghc8107"; + shell.tools = { + cabal = { }; + hlint = { }; + haskell-language-server = { }; + }; + shell.buildInputs = with pkgs; [ nixpkgs-fmt fsatrace ]; + modules = + [{ packages.foliage.components.exes.foliage.dontStrip = false; }]; + }; + in { + packages.default = project.foliage.components.exes.foliage; + devShell = pkgs.mkShell { + name = "foliage-dev-shell"; + buildInputs = with pkgs; [ fsatrace ]; + }; + }); nixConfig = { - extra-substituters = [ - "https://cache.iog.io" - "https://foliage.cachix.org" - ]; + extra-substituters = + [ "https://cache.iog.io" "https://foliage.cachix.org" ]; extra-trusted-public-keys = [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" "foliage.cachix.org-1:kAFyYLnk8JcRURWReWZCatM9v3Rk24F5wNMpEj14Q/g=" From d6419b2fbd2132797a7dccc0ce890dd1db241d8e Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Wed, 30 Nov 2022 11:32:17 +0800 Subject: [PATCH 03/43] Remove devShell --- flake.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index db368d6..8a68921 100644 --- a/flake.nix +++ b/flake.nix @@ -24,16 +24,12 @@ hlint = { }; haskell-language-server = { }; }; - shell.buildInputs = with pkgs; [ nixpkgs-fmt fsatrace ]; modules = [{ packages.foliage.components.exes.foliage.dontStrip = false; }]; + }; in { packages.default = project.foliage.components.exes.foliage; - devShell = pkgs.mkShell { - name = "foliage-dev-shell"; - buildInputs = with pkgs; [ fsatrace ]; - }; }); nixConfig = { From 31f81d3e45bc215e3adfa0802d57173bac57edbd Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Wed, 30 Nov 2022 11:50:38 +0800 Subject: [PATCH 04/43] Static executable --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 8a68921..64d8105 100644 --- a/flake.nix +++ b/flake.nix @@ -16,7 +16,7 @@ inherit (haskell-nix) config; overlays = [ haskell-nix.overlay ]; }; - project = pkgs.haskell-nix.cabalProject { + project = pkgs.pkgsCross.musl64.haskell-nix.cabalProject { src = ./.; compiler-nix-name = "ghc8107"; shell.tools = { From ae66e36f048c92e5f76116c2dcef225077a51ea9 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Tue, 13 Dec 2022 09:37:01 +0800 Subject: [PATCH 05/43] Update flake.nix --- flake.nix | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 64d8105..b33c848 100644 --- a/flake.nix +++ b/flake.nix @@ -30,14 +30,19 @@ }; in { packages.default = project.foliage.components.exes.foliage; + hydraJobs.foliage = project.foliage.components.exes.foliage; }); nixConfig = { - extra-substituters = - [ "https://cache.iog.io" "https://foliage.cachix.org" ]; + extra-substituters = [ + "https://cache.iog.io" + "https://foliage.cachix.org" + "https://ci.zw3rk.com" + ]; extra-trusted-public-keys = [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" "foliage.cachix.org-1:kAFyYLnk8JcRURWReWZCatM9v3Rk24F5wNMpEj14Q/g=" + "loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=" ]; }; } From 1ef0e3ad3b8d6ec1d82e5c897d6e8886b2e085ea Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Tue, 13 Dec 2022 09:52:23 +0800 Subject: [PATCH 06/43] Update flake.nix --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index b33c848..53d7c18 100644 --- a/flake.nix +++ b/flake.nix @@ -37,7 +37,7 @@ extra-substituters = [ "https://cache.iog.io" "https://foliage.cachix.org" - "https://ci.zw3rk.com" + "https://cache.zw3rk.com" ]; extra-trusted-public-keys = [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" From 05bd69f5c3491562958654682aab3f3dafb9e6f7 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Tue, 13 Dec 2022 10:26:26 +0800 Subject: [PATCH 07/43] Build statically only on linux --- flake.nix | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/flake.nix b/flake.nix index 53d7c18..9e70ed1 100644 --- a/flake.nix +++ b/flake.nix @@ -16,21 +16,34 @@ inherit (haskell-nix) config; overlays = [ haskell-nix.overlay ]; }; - project = pkgs.pkgsCross.musl64.haskell-nix.cabalProject { - src = ./.; - compiler-nix-name = "ghc8107"; - shell.tools = { - cabal = { }; - hlint = { }; - haskell-language-server = { }; - }; - modules = - [{ packages.foliage.components.exes.foliage.dontStrip = false; }]; + static-pkgs = pkgs.pkgsCross.musl64; + + mkFoliage = haskell-nix: + let + project = haskell-nix.cabalProject { + src = ./.; + compiler-nix-name = "ghc8107"; + shell.tools = { + cabal = { }; + hlint = { }; + haskell-language-server = { }; + }; + modules = [{ + packages.foliage.components.exes.foliage.dontStrip = false; + }]; + }; + in project.foliage.components.exes.foliage; + + in rec { + packages = rec { + default = foliage; + foliage = mkFoliage pkgs.haskell-nix; + } // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { + foliage-static = mkFoliage static-pkgs.haskell-nix; }; - in { - packages.default = project.foliage.components.exes.foliage; - hydraJobs.foliage = project.foliage.components.exes.foliage; + + hydraJobs = packages; }); nixConfig = { @@ -38,7 +51,7 @@ "https://cache.iog.io" "https://foliage.cachix.org" "https://cache.zw3rk.com" - ]; + ]; extra-trusted-public-keys = [ "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ=" "foliage.cachix.org-1:kAFyYLnk8JcRURWReWZCatM9v3Rk24F5wNMpEj14Q/g=" From 3e12822144efedcecfa77c92281dc42214b471a4 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Tue, 13 Dec 2022 10:50:11 +0800 Subject: [PATCH 08/43] aarch64-linux needs something different --- flake.nix | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 9e70ed1..f0bc53a 100644 --- a/flake.nix +++ b/flake.nix @@ -9,6 +9,7 @@ }; outputs = { self, nixpkgs, flake-utils, haskell-nix }: + flake-utils.lib.eachDefaultSystem (system: let pkgs = import nixpkgs { @@ -17,7 +18,13 @@ overlays = [ haskell-nix.overlay ]; }; - static-pkgs = pkgs.pkgsCross.musl64; + static-pkgs = if pkgs.stdenv.hostPlatform.isLinux then + if pkgs.stdenv.hostPlatform.isAarch64 then + pkgs.pkgsCross.aarch64-multiplatform-musl + else + pkgs.pkgsCross.musl64 + else + pkgs; mkFoliage = haskell-nix: let @@ -43,7 +50,7 @@ foliage-static = mkFoliage static-pkgs.haskell-nix; }; - hydraJobs = packages; + hydraJobs = packages; }); nixConfig = { From 33a645bc3b8014e798bb6cd9569678ee90b26e1a Mon Sep 17 00:00:00 2001 From: Michael Peyton Jones Date: Fri, 16 Dec 2022 15:15:40 +0000 Subject: [PATCH 09/43] Build in parallel This cuts the time to build `cardano-haskell-packages` down to 30s from >2mins. --- app/Foliage/CmdBuild.hs | 14 +++++++------- app/Foliage/Options.hs | 12 +++++++++++- foliage.cabal | 1 + 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index f3a206f..2955f3c 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -6,10 +6,9 @@ module Foliage.CmdBuild (cmdBuild) where import Codec.Archive.Tar qualified as Tar import Codec.Archive.Tar.Entry qualified as Tar import Codec.Compression.GZip qualified as GZip -import Control.Monad (unless, when) +import Control.Monad (unless, when, void) import Data.ByteString.Lazy qualified as BL import Data.ByteString.Lazy qualified as BSL -import Data.Foldable (for_) import Data.List (sortOn) import Data.Maybe (fromMaybe) import Data.Traversable (for) @@ -45,7 +44,8 @@ cmdBuild buildOptions = do opts = shakeOptions { shakeFiles = cacheDir, - shakeVerbosity = Verbose + shakeVerbosity = Verbose, + shakeThreads = buildOptsNumThreads buildOptions } buildAction :: BuildOptions -> Action () @@ -91,9 +91,9 @@ buildAction makeAllPackageVersionsPage currentTime outputDir packageVersions - for_ packageVersions $ makePackageVersionPage inputDir outputDir + void $ forP packageVersions $ makePackageVersionPage inputDir outputDir - for_ packageVersions $ \pkgMeta@PackageVersionMeta {pkgId} -> do + void $ forP packageVersions $ \pkgMeta@PackageVersionMeta {pkgId} -> do let PackageIdentifier {pkgName, pkgVersion} = pkgId cabalFilePath <- maybe (originalCabalFile pkgMeta) pure (revisedCabalFile inputDir pkgMeta) copyFileChanged cabalFilePath (outputDir "index" prettyShow pkgName prettyShow pkgVersion prettyShow pkgName <.> "cabal") @@ -123,7 +123,7 @@ buildAction targetKeys <- maybeReadKeysAt "target" metadataEntries <- - for packageVersions $ \pkg@PackageVersionMeta {pkgId, pkgSpec} -> do + forP packageVersions $ \pkg@PackageVersionMeta {pkgId, pkgSpec} -> do let PackageIdentifier {pkgName, pkgVersion} = pkgId let PackageVersionSpec {packageVersionTimestamp} = pkgSpec targets <- prepareIndexPkgMetadata expiryTime pkg @@ -235,7 +235,7 @@ getPackageVersions inputDir = do ] fail "no package metadata found" - for metaFiles $ \metaFile -> do + forP metaFiles $ \metaFile -> do let [pkgName, pkgVersion, _] = splitDirectories metaFile let Just name = simpleParsec pkgName let Just version = simpleParsec pkgVersion diff --git a/app/Foliage/Options.hs b/app/Foliage/Options.hs index 010da20..9271d7f 100644 --- a/app/Foliage/Options.hs +++ b/app/Foliage/Options.hs @@ -51,7 +51,8 @@ data BuildOptions = BuildOptions buildOptsCurrentTime :: Maybe UTCTime, buildOptsExpireSignaturesOn :: Maybe UTCTime, buildOptsInputDir :: FilePath, - buildOptsOutputDir :: FilePath + buildOptsOutputDir :: FilePath, + buildOptsNumThreads :: Int } buildCommand :: Parser Command @@ -90,6 +91,15 @@ buildCommand = <> showDefault <> value "_repo" ) + <*> option + auto + ( long "num-jobs" + <> short 'j' + <> metavar "JOBS" + <> help "Number of jobs to run in parallel, 0 is 'all available cores'" + <> showDefault + <> value 1 + ) ) where signOpts = diff --git a/foliage.cabal b/foliage.cabal index 7715b4f..9424729 100644 --- a/foliage.cabal +++ b/foliage.cabal @@ -41,6 +41,7 @@ executable foliage ImportQualifiedPost LambdaCase NamedFieldPuns ViewPatterns ghc-options: -Wall + ghc-options: -threaded build-depends: base >=4.14.3.0 && <4.18, aeson >=2.0.3.0 && <2.2, From 4bb1b290a988e70385f230a5bdd37a14ea536351 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Wed, 21 Dec 2022 16:36:13 +0800 Subject: [PATCH 10/43] Bump compiler --- cabal.project | 5 ++++- flake.nix | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cabal.project b/cabal.project index b16303c..e678df4 100644 --- a/cabal.project +++ b/cabal.project @@ -1,3 +1,6 @@ packages: . index-state: 2022-08-29T00:00:00Z -with-compiler: ghc-8.10.7 +with-compiler: ghc-9.2.4 +allow-newer: + , tomland:base + , tomland:text diff --git a/flake.nix b/flake.nix index f0bc53a..07f7873 100644 --- a/flake.nix +++ b/flake.nix @@ -30,7 +30,7 @@ let project = haskell-nix.cabalProject { src = ./.; - compiler-nix-name = "ghc8107"; + compiler-nix-name = "ghc924"; shell.tools = { cabal = { }; hlint = { }; From 1c861b3b6c86d6c200d7e7dba891ed8c8545cdcb Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Mon, 9 Jan 2023 11:01:02 +0800 Subject: [PATCH 11/43] Update flake --- flake.lock | 34 +++++++++++----------------------- flake.nix | 32 ++++++++++++-------------------- 2 files changed, 23 insertions(+), 43 deletions(-) diff --git a/flake.lock b/flake.lock index fbdde31..ce62c78 100644 --- a/flake.lock +++ b/flake.lock @@ -189,21 +189,6 @@ } }, "flake-utils": { - "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_2": { "locked": { "lastModified": 1644229661, "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", @@ -218,7 +203,7 @@ "type": "github" } }, - "flake-utils_3": { + "flake-utils_2": { "locked": { "lastModified": 1653893745, "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", @@ -233,7 +218,7 @@ "type": "github" } }, - "flake-utils_4": { + "flake-utils_3": { "locked": { "lastModified": 1659877975, "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", @@ -248,7 +233,7 @@ "type": "github" } }, - "flake-utils_5": { + "flake-utils_4": { "locked": { "lastModified": 1653893745, "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", @@ -323,7 +308,7 @@ "cabal-36": "cabal-36", "cardano-shell": "cardano-shell", "flake-compat": "flake-compat", - "flake-utils": "flake-utils_2", + "flake-utils": "flake-utils", "ghc-8.6.5-iohk": "ghc-8.6.5-iohk", "hackage": "hackage", "hpc-coveralls": "hpc-coveralls", @@ -428,7 +413,7 @@ }, "n2c": { "inputs": { - "flake-utils": "flake-utils_5", + "flake-utils": "flake-utils_4", "nixpkgs": [ "haskell-nix", "tullia", @@ -508,7 +493,7 @@ }, "nix2container": { "inputs": { - "flake-utils": "flake-utils_3", + "flake-utils": "flake-utils_2", "nixpkgs": "nixpkgs_3" }, "locked": { @@ -736,7 +721,10 @@ }, "root": { "inputs": { - "flake-utils": "flake-utils", + "flake-utils": [ + "haskell-nix", + "flake-utils" + ], "haskell-nix": "haskell-nix", "nixpkgs": [ "haskell-nix", @@ -765,7 +753,7 @@ "blank": "blank", "devshell": "devshell", "dmerge": "dmerge", - "flake-utils": "flake-utils_4", + "flake-utils": "flake-utils_3", "makes": [ "haskell-nix", "tullia", diff --git a/flake.nix b/flake.nix index 07f7873..40912d1 100644 --- a/flake.nix +++ b/flake.nix @@ -5,7 +5,7 @@ inputs = { nixpkgs.follows = "haskell-nix/nixpkgs-unstable"; haskell-nix.url = "github:input-output-hk/haskell.nix"; - flake-utils.url = "github:numtide/flake-utils"; + flake-utils.follows = "haskell-nix/flake-utils"; }; outputs = { self, nixpkgs, flake-utils, haskell-nix }: @@ -18,7 +18,7 @@ overlays = [ haskell-nix.overlay ]; }; - static-pkgs = if pkgs.stdenv.hostPlatform.isLinux then + pkgs-static-where-possible = if pkgs.stdenv.hostPlatform.isLinux then if pkgs.stdenv.hostPlatform.isAarch64 then pkgs.pkgsCross.aarch64-multiplatform-musl else @@ -26,28 +26,20 @@ else pkgs; - mkFoliage = haskell-nix: - let - project = haskell-nix.cabalProject { - src = ./.; - compiler-nix-name = "ghc924"; - shell.tools = { - cabal = { }; - hlint = { }; - haskell-language-server = { }; - }; - modules = [{ - packages.foliage.components.exes.foliage.dontStrip = false; - }]; - }; - in project.foliage.components.exes.foliage; + project = pkgs-static-where-possible.haskell-nix.cabalProject { + src = ./.; + compiler-nix-name = "ghc924"; + shell.tools = { + cabal = { }; + hlint = { }; + haskell-language-server = { }; + }; + }; in rec { packages = rec { default = foliage; - foliage = mkFoliage pkgs.haskell-nix; - } // pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { - foliage-static = mkFoliage static-pkgs.haskell-nix; + foliage = project.foliage.components.exes.foliage; }; hydraJobs = packages; From e3f5c7b45e4bc0fdd95df790fccf6a69dc257336 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Mon, 9 Jan 2023 11:01:05 +0800 Subject: [PATCH 12/43] Document how to access private repositories Closes #31 --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 2a1a12f..5cd1eed 100644 --- a/README.md +++ b/README.md @@ -273,6 +273,18 @@ metadata file `//meta.toml` includes the option `force-version = true`, then the version in the package cabal file will be overwritten with ``. Notice this is done after applying patches. +# Accessing private repositories + +Foliage uses curl to fetch remote assets. If you need to access private +repositories you can use one of the following methods. + +- You can instruct curl to use a netrc file. + - Add netrc-optional to ~/.curlrc (see [here](https://curl.se/docs/manpage.html#-K) for the places curl looks in). + - Add the relevant credentials to ~/.netrc +- You can also add the credentials directly to curl's configuration, e.g. + - Add `user = "USERNAME:PASSWORD"` to .curlrc in the current directly + - Set `CURL_HOME=$PWD` + ## Author - Andrea Bedini (@andreabedini) From 9ba14bdb787d7a3546fc312a1360890b7626fbf4 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Mon, 9 Jan 2023 11:17:06 +0800 Subject: [PATCH 13/43] Bump some actions --- .github/workflows/nix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index e4f2d82..b0c1cb1 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -14,14 +14,14 @@ jobs: steps: - uses: actions/checkout@v2 - - uses: cachix/install-nix-action@v17 + - uses: cachix/install-nix-action@v18 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= substituters = https://cache.iog.io https://cache.nixos.org - - uses: cachix/cachix-action@v10 + - uses: cachix/cachix-action@v12 with: name: foliage authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' From a0c7067fb429b168097e3b0bfdd36569328bc4d8 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Mon, 9 Jan 2023 14:11:17 +0800 Subject: [PATCH 14/43] back to 8.10.7 for the flake --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 40912d1..40d80fa 100644 --- a/flake.nix +++ b/flake.nix @@ -28,7 +28,7 @@ project = pkgs-static-where-possible.haskell-nix.cabalProject { src = ./.; - compiler-nix-name = "ghc924"; + compiler-nix-name = "ghc8107"; shell.tools = { cabal = { }; hlint = { }; From fd7829e72f72584676bc14bfc0b2147c6e037b2c Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Mon, 9 Jan 2023 15:47:08 +0800 Subject: [PATCH 15/43] Update flake.lock --- flake.lock | 58 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index ce62c78..23e58fd 100644 --- a/flake.lock +++ b/flake.lock @@ -287,11 +287,11 @@ "hackage": { "flake": false, "locked": { - "lastModified": 1667524657, - "narHash": "sha256-Hsp8KdtPWDNteVXRj0X9rEs3I1EJmpG4nYxHE4s/r8s=", + "lastModified": 1673137629, + "narHash": "sha256-HJfuXQ6NkhdZIMo4/S/Y9QX1xmuBS7XSB4xhmEVZSWc=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "bf170e96de8ed70c445fa759aaa29e081dacab86", + "rev": "5cc584651a4895c7abad166cf695a3ee8b873386", "type": "github" }, "original": { @@ -313,6 +313,7 @@ "hackage": "hackage", "hpc-coveralls": "hpc-coveralls", "hydra": "hydra", + "iserv-proxy": "iserv-proxy", "nixpkgs": [ "haskell-nix", "nixpkgs-unstable" @@ -321,17 +322,18 @@ "nixpkgs-2105": "nixpkgs-2105", "nixpkgs-2111": "nixpkgs-2111", "nixpkgs-2205": "nixpkgs-2205", + "nixpkgs-2211": "nixpkgs-2211", "nixpkgs-unstable": "nixpkgs-unstable", "old-ghc-nix": "old-ghc-nix", "stackage": "stackage", "tullia": "tullia" }, "locked": { - "lastModified": 1667533063, - "narHash": "sha256-EHZOvFNdfgA8U3xNKZkyGZT7Ydnr/3pihMDyGEwLlag=", + "lastModified": 1673139096, + "narHash": "sha256-EzMIx2nxXoHbMFHIKFiXVE6AC+PcCm4NjKChk9MAgM8=", "owner": "input-output-hk", "repo": "haskell.nix", - "rev": "e601c9ce609af07a78edf1c57f5985931788aeb2", + "rev": "f5da0a34b5dbbeb57e6d7595f46a930cd62bcd77", "type": "github" }, "original": { @@ -379,6 +381,22 @@ "type": "indirect" } }, + "iserv-proxy": { + "flake": false, + "locked": { + "lastModified": 1639165170, + "narHash": "sha256-QsWL/sBDL5GM8IXd/dE/ORiL4RvteEN+aok23tXgAoc=", + "rev": "6e95df7be6dd29680f983db07a057fc2f34f81f6", + "revCount": 7, + "type": "git", + "url": "https://gitlab.haskell.org/ghc/iserv-proxy.git" + }, + "original": { + "rev": "6e95df7be6dd29680f983db07a057fc2f34f81f6", + "type": "git", + "url": "https://gitlab.haskell.org/ghc/iserv-proxy.git" + } + }, "lowdown-src": { "flake": false, "locked": { @@ -624,6 +642,22 @@ "type": "github" } }, + "nixpkgs-2211": { + "locked": { + "lastModified": 1669997163, + "narHash": "sha256-vhjC0kZMFoN6jzK0GR+tBzKi5KgBXgehadfidW8+Va4=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "6f87491a54d8d64d30af6663cb3bf5d2ee7db958", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-22.11-darwin", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -735,11 +769,11 @@ "stackage": { "flake": false, "locked": { - "lastModified": 1667524765, - "narHash": "sha256-rY58ROG9paYqqhUPFxZArU59qOIatIFHrurhVo7JXX4=", + "lastModified": 1673050158, + "narHash": "sha256-/1WWFQDUisdEuSOUTvQTzkED9Z6riX3XuVMuaMIbyq4=", "owner": "input-output-hk", "repo": "stackage.nix", - "rev": "ed1ec5f81f9eb32eb627fd447088eb782e7ff71b", + "rev": "46c3d80f024585ddf22a54f3c505799dca778865", "type": "github" }, "original": { @@ -797,11 +831,11 @@ "std": "std" }, "locked": { - "lastModified": 1666200256, - "narHash": "sha256-cJPS8zBu30SMhxMe7I8DWutwqMuhPsEez87y9gxMKc4=", + "lastModified": 1668711738, + "narHash": "sha256-CBjky16o9pqsGE1bWu6nRlRajgSXMEk+yaFQLibqXcE=", "owner": "input-output-hk", "repo": "tullia", - "rev": "575362c2244498e8d2c97f72861510fa72e75d44", + "rev": "ead1f515c251f0e060060ef0e2356a51d3dfe4b0", "type": "github" }, "original": { From 57796e38b082c22d5f4f0fa586b8a631f365ec72 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 20 Jan 2023 11:56:17 +0800 Subject: [PATCH 16/43] Prefer cabalProject' to cabalProject cabalProject is too strict and accidentally evaulates IFD meant for other systems. --- flake.nix | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/flake.nix b/flake.nix index 40d80fa..d2ad6a1 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,7 @@ else pkgs; - project = pkgs-static-where-possible.haskell-nix.cabalProject { + project = pkgs-static-where-possible.haskell-nix.cabalProject' { src = ./.; compiler-nix-name = "ghc8107"; shell.tools = { @@ -36,14 +36,9 @@ }; }; - in rec { - packages = rec { - default = foliage; - foliage = project.foliage.components.exes.foliage; - }; + flake = project.flake { }; - hydraJobs = packages; - }); + in flake // { packages.default = flake.packages."foliage:exe:foliage"; }); nixConfig = { extra-substituters = [ From 0993761f7fdc67e0a9fc2816bc4d06d5f4a4e3d0 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Sun, 29 Jan 2023 12:27:04 +0800 Subject: [PATCH 17/43] Clean up --- app/Foliage/CmdBuild.hs | 35 +++++++++++++++++------------------ cabal.project | 2 +- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index 2955f3c..0cd71dd 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -6,9 +6,8 @@ module Foliage.CmdBuild (cmdBuild) where import Codec.Archive.Tar qualified as Tar import Codec.Archive.Tar.Entry qualified as Tar import Codec.Compression.GZip qualified as GZip -import Control.Monad (unless, when, void) +import Control.Monad (unless, void, when) import Data.ByteString.Lazy qualified as BL -import Data.ByteString.Lazy qualified as BSL import Data.List (sortOn) import Data.Maybe (fromMaybe) import Data.Traversable (for) @@ -137,8 +136,8 @@ buildAction let tarContents = Tar.write $ sortOn Tar.entryTime (cabalEntries ++ metadataEntries) traced "Writing index" $ do - BSL.writeFile (anchorPath outputDirRoot repoLayoutIndexTar) tarContents - BSL.writeFile (anchorPath outputDirRoot repoLayoutIndexTarGz) $ GZip.compress tarContents + BL.writeFile (anchorPath outputDirRoot repoLayoutIndexTar) tarContents + BL.writeFile (anchorPath outputDirRoot repoLayoutIndexTarGz) $ GZip.compress tarContents privateKeysRoot <- maybeReadKeysAt "root" privateKeysTarget <- maybeReadKeysAt "target" @@ -245,20 +244,20 @@ getPackageVersions inputDir = do readPackageVersionSpec' (inputDir metaFile) >>= \case PackageVersionSpec {packageVersionRevisions, packageVersionTimestamp = Nothing} | not (null packageVersionRevisions) -> do - putError $ - unlines - [ inputDir metaFile <> " has cabal file revisions but the original package has no timestamp.", - "This combination doesn't make sense. Either add a timestamp on the original package or remove the revisions" - ] - fail "invalid package metadata" + putError $ + unlines + [ inputDir metaFile <> " has cabal file revisions but the original package has no timestamp.", + "This combination doesn't make sense. Either add a timestamp on the original package or remove the revisions" + ] + fail "invalid package metadata" PackageVersionSpec {packageVersionRevisions, packageVersionTimestamp = Just pkgTs} | any ((< pkgTs) . revisionTimestamp) packageVersionRevisions -> do - putError $ - unlines - [ inputDir metaFile <> " has a revision with timestamp earlier than the package itself.", - "Adjust the timestamps so that all revisions come after the original package" - ] - fail "invalid package metadata" + putError $ + unlines + [ inputDir metaFile <> " has a revision with timestamp earlier than the package itself.", + "Adjust the timestamps so that all revisions come after the original package" + ] + fail "invalid package metadata" meta -> return meta @@ -267,7 +266,7 @@ getPackageVersions inputDir = do prepareIndexPkgCabal :: PackageId -> UTCTime -> FilePath -> Action Tar.Entry prepareIndexPkgCabal pkgId timestamp filePath = do need [filePath] - contents <- liftIO $ BSL.readFile filePath + contents <- liftIO $ BL.readFile filePath return $ mkTarEntry contents (IndexPkgCabal pkgId) timestamp prepareIndexPkgMetadata :: Maybe UTCTime -> PackageVersionMeta -> Action Targets @@ -284,7 +283,7 @@ prepareIndexPkgMetadata expiryTime PackageVersionMeta {pkgId, pkgSpec} = do targetsDelegations = Nothing } -mkTarEntry :: BSL.ByteString -> IndexFile dec -> UTCTime -> Tar.Entry +mkTarEntry :: BL.ByteString -> IndexFile dec -> UTCTime -> Tar.Entry mkTarEntry contents indexFile timestamp = (Tar.fileEntry tarPath contents) { Tar.entryTime = floor $ Time.utcTimeToPOSIXSeconds timestamp, diff --git a/cabal.project b/cabal.project index e678df4..77fbcdb 100644 --- a/cabal.project +++ b/cabal.project @@ -1,6 +1,6 @@ packages: . index-state: 2022-08-29T00:00:00Z -with-compiler: ghc-9.2.4 +with-compiler: ghc-9.2.5 allow-newer: , tomland:base , tomland:text From b374dfde9f6930c0b3604e6622c8e68435893059 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 10 Feb 2023 11:16:04 +0800 Subject: [PATCH 18/43] Add update-flake-lock.yml --- .github/workflows/update-flake-lock.yml | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/update-flake-lock.yml diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml new file mode 100644 index 0000000..6194e12 --- /dev/null +++ b/.github/workflows/update-flake-lock.yml @@ -0,0 +1,27 @@ +name: update-flake-lock + +on: + workflow_dispatch: # allows manual triggering + schedule: + - cron: '0 0 * * 0' # runs weekly on Sunday at 00:00 + +jobs: + lockfile: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Install Nix + uses: cachix/install-nix-action@v16 + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - name: Update flake.lock + uses: DeterminateSystems/update-flake-lock@v16 + with: + pr-title: "Update flake.lock" # Title of PR to be created + pr-labels: | # Labels to be set on the PR + dependencies + automated + - name: Print PR number + run: echo Opened pull request ${{ steps.update.outputs.pull-request-number }}. From 9288fdf587f8938f4d839d19b2289814bbfcee18 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 10 Feb 2023 11:20:05 +0800 Subject: [PATCH 19/43] Update nix workflow --- .github/workflows/nix.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index b0c1cb1..eb875c8 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -18,12 +18,12 @@ jobs: with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - substituters = https://cache.iog.io https://cache.nixos.org - uses: cachix/cachix-action@v12 with: name: foliage authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - - run: nix build + - run: nix build --accept-flake-config + + - run: nix flake check --accept-flake-config --allow-import-from-derivation From f76eb918451ac275367a340fee86f4e7189314ee Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 10 Feb 2023 11:21:28 +0800 Subject: [PATCH 20/43] Update workflows --- .github/workflows/nix.yml | 4 ++-- .github/workflows/update-flake-lock.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index eb875c8..91f9bac 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -12,9 +12,9 @@ jobs: - ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v18 + - uses: cachix/install-nix-action@v19 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/update-flake-lock.yml b/.github/workflows/update-flake-lock.yml index 6194e12..d51e30e 100644 --- a/.github/workflows/update-flake-lock.yml +++ b/.github/workflows/update-flake-lock.yml @@ -10,9 +10,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Install Nix - uses: cachix/install-nix-action@v16 + uses: cachix/install-nix-action@v19 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} From 9995734db5f58e0231049b547ccec5772759781b Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Mon, 13 Feb 2023 16:54:30 +0800 Subject: [PATCH 21/43] Write json metedata to output dir --- app/Foliage/CmdBuild.hs | 41 ++++++++++++++++++++++++++++++++++++ app/Foliage/Pages.hs | 26 +++++++++++------------ app/Foliage/PrepareSource.hs | 10 ++------- app/Foliage/Utils/GitHub.hs | 14 ++++++++++++ foliage.cabal | 1 + 5 files changed, 71 insertions(+), 21 deletions(-) create mode 100644 app/Foliage/Utils/GitHub.hs diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index 0cd71dd..2ec5444 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -7,8 +7,10 @@ import Codec.Archive.Tar qualified as Tar import Codec.Archive.Tar.Entry qualified as Tar import Codec.Compression.GZip qualified as GZip import Control.Monad (unless, void, when) +import Data.Aeson qualified as Aeson import Data.ByteString.Lazy qualified as BL import Data.List (sortOn) +import Data.Map.Strict qualified as Map import Data.Maybe (fromMaybe) import Data.Traversable (for) import Development.Shake @@ -26,6 +28,7 @@ import Foliage.PrepareSource (addPrepareSourceRule, prepareSource) import Foliage.RemoteAsset (addFetchRemoteAssetRule) import Foliage.Shake import Foliage.Time qualified as Time +import Foliage.Utils.GitHub (githubRepoUrl) import Hackage.Security.Util.Path (castRoot, toFilePath) cmdBuild :: BuildOptions -> IO () @@ -90,6 +93,8 @@ buildAction makeAllPackageVersionsPage currentTime outputDir packageVersions + makeMetadataFile outputDir packageVersions + void $ forP packageVersions $ makePackageVersionPage inputDir outputDir void $ forP packageVersions $ \pkgMeta@PackageVersionMeta {pkgId} -> do @@ -222,6 +227,42 @@ buildAction timestampInfoSnapshot = snapshotInfo } +makeMetadataFile :: FilePath -> [PackageVersionMeta] -> Action () +makeMetadataFile outputDir packageVersions = + liftIO $ + Aeson.encodeFile + (outputDir "metadata.json") + ( Map.fromList + [ (prettyShow pkgId, encodeMetadataSpec pkgSpec) + | PackageVersionMeta pkgId pkgSpec <- packageVersions + ] + ) + +encodeMetadataSpec :: PackageVersionSpec -> Aeson.Value +encodeMetadataSpec + PackageVersionSpec + { packageVersionSource, + packageVersionForce, + packageVersionTimestamp + } = + Aeson.object + ( ("source" Aeson..= source) + : ["forced-version" Aeson..= True | packageVersionForce] + ++ (case packageVersionTimestamp of Nothing -> []; Just t -> ["timestamp" Aeson..= t]) + ) + where + source = case packageVersionSource of + TarballSource uri mSubdir -> + Aeson.object + ( ("url" Aeson..= show uri) + : case mSubdir of Nothing -> []; Just s -> ["subdir" Aeson..= s] + ) + GitHubSource repo rev mSubdir -> + Aeson.object + ( ("url" Aeson..= show (githubRepoUrl repo rev)) + : case mSubdir of Nothing -> []; Just s -> ["subdir" Aeson..= s] + ) + getPackageVersions :: FilePath -> Action [PackageVersionMeta] getPackageVersions inputDir = do metaFiles <- getDirectoryFiles inputDir ["*/*/meta.toml"] diff --git a/app/Foliage/Pages.hs b/app/Foliage/Pages.hs index 7b79dfe..67c8084 100644 --- a/app/Foliage/Pages.hs +++ b/app/Foliage/Pages.hs @@ -58,8 +58,8 @@ data AllPackagesPageEntry = AllPackagesPageEntry makeAllPackagesPage :: UTCTime -> FilePath -> [PackageVersionMeta] -> Action () makeAllPackagesPage currentTime outputDir packageVersions = do let packages = - sortOn allPackagesPageEntryPkgId $ - map + sortOn allPackagesPageEntryPkgId + $ map ( ( \PackageVersionMeta {pkgId, pkgSpec = PackageVersionSpec {packageVersionTimestamp, packageVersionRevisions, packageVersionSource}} -> AllPackagesPageEntry { allPackagesPageEntryPkgId = pkgId, @@ -72,7 +72,7 @@ makeAllPackagesPage currentTime outputDir packageVersions = do . head . sortOn (Down . pkgVersion . pkgId) ) - $ groupBy ((==) `on` (pkgName . pkgId)) packageVersions + $ groupBy ((==) `on` (pkgName . pkgId)) packageVersions traced "webpages / all-packages" $ do IO.createDirectoryIfMissing True (outputDir "all-packages") TL.writeFile (outputDir "all-packages" "index.html") $ @@ -105,16 +105,16 @@ makeAllPackageVersionsPage currentTime outputDir packageVersions = do allPackageVersionsPageEntryTimestamp = fromMaybe currentTime packageVersionTimestamp, allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds (fromMaybe currentTime packageVersionTimestamp), allPackageVersionsPageEntrySource = packageVersionSource - } : - map - ( \RevisionSpec {revisionTimestamp} -> - AllPackageVersionsPageEntryRevision - { allPackageVersionsPageEntryPkgId = pkgId, - allPackageVersionsPageEntryTimestamp = revisionTimestamp, - allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds revisionTimestamp - } - ) - packageVersionRevisions + } + : map + ( \RevisionSpec {revisionTimestamp} -> + AllPackageVersionsPageEntryRevision + { allPackageVersionsPageEntryPkgId = pkgId, + allPackageVersionsPageEntryTimestamp = revisionTimestamp, + allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds revisionTimestamp + } + ) + packageVersionRevisions ) packageVersions diff --git a/app/Foliage/PrepareSource.hs b/app/Foliage/PrepareSource.hs index dfa989e..b855604 100644 --- a/app/Foliage/PrepareSource.hs +++ b/app/Foliage/PrepareSource.hs @@ -8,7 +8,6 @@ module Foliage.PrepareSource where import Control.Monad (when) import Data.ByteString qualified as BS import Data.Foldable (for_) -import Data.Text qualified as T import Development.Shake import Development.Shake.Classes import Development.Shake.Rule @@ -18,8 +17,8 @@ import Distribution.Types.PackageName (unPackageName) import Foliage.Meta import Foliage.RemoteAsset (fetchRemoteAsset) import Foliage.UpdateCabalFile (rewritePackageVersion) +import Foliage.Utils.GitHub (githubRepoUrl) import GHC.Generics -import Network.URI (URI (..), URIAuth (..), nullURI, nullURIAuth) import System.Directory qualified as IO import System.FilePath ((<.>), ()) @@ -79,12 +78,7 @@ addPrepareSourceRule inputDir cacheDir = addBuiltinRule noLint noIdentity run -- metadata. -- GitHubSource repo rev mSubdir -> do - let url = - nullURI - { uriScheme = "https:", - uriAuthority = Just nullURIAuth {uriRegName = "github.com"}, - uriPath = "/" T.unpack (unGitHubRepo repo) "tarball" T.unpack (unGitHubRev rev) - } + let url = githubRepoUrl repo rev tarballPath <- fetchRemoteAsset url diff --git a/app/Foliage/Utils/GitHub.hs b/app/Foliage/Utils/GitHub.hs new file mode 100644 index 0000000..0a9490e --- /dev/null +++ b/app/Foliage/Utils/GitHub.hs @@ -0,0 +1,14 @@ +module Foliage.Utils.GitHub where + +import Data.Text qualified as T +import Foliage.Meta (GitHubRepo (unGitHubRepo), GitHubRev (unGitHubRev)) +import Network.URI (URI (..), URIAuth (..), nullURI, nullURIAuth) +import System.FilePath (()) + +githubRepoUrl :: GitHubRepo -> GitHubRev -> URI +githubRepoUrl repo rev = + nullURI + { uriScheme = "https:", + uriAuthority = Just nullURIAuth {uriRegName = "github.com"}, + uriPath = "/" T.unpack (unGitHubRepo repo) "tarball" T.unpack (unGitHubRev rev) + } diff --git a/foliage.cabal b/foliage.cabal index 9424729..71a3eed 100644 --- a/foliage.cabal +++ b/foliage.cabal @@ -34,6 +34,7 @@ executable foliage Foliage.Time Foliage.UpdateCabalFile Foliage.Utils.Aeson + Foliage.Utils.GitHub Network.URI.Orphans default-language: Haskell2010 From 25b2afaa7acee595c6fa5773d655a9713d5d85c3 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Tue, 14 Feb 2023 10:28:04 +0800 Subject: [PATCH 22/43] Drop nix flake check It evaluates check jobs for all systems, which require to be run on that system given haskell.nix use of IFD. --- .github/workflows/nix.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 91f9bac..4266c3d 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -25,5 +25,3 @@ jobs: authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix build --accept-flake-config - - - run: nix flake check --accept-flake-config --allow-import-from-derivation From 2631bf606794bebe217e3c3b3f6fd250538f7315 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Tue, 14 Feb 2023 10:29:16 +0800 Subject: [PATCH 23/43] Implement metadata output. Implement a new flag "--write-metadata" which will create, in the output directory, a file "foliage/packages.json" with metadata about the package provenance. The file package.json file looks like this ``` { "Win32-network-0.1.0.0": { "source": { "url": "github:input-output-hk/Win32-network/3825d3abf75f83f406c1f7161883c438dac7277d" }, "timestamp": "2022-10-17T00:00:00Z" }, "Win32-network-0.1.1.0": { ... ``` The idea is that tools fetching the repo can also fetch this file to obtain information about the package provenance. --- app/Foliage/CmdBuild.hs | 27 +++++++++++++++------------ app/Foliage/Options.hs | 8 +++++++- app/Foliage/PrepareSource.hs | 4 ++-- app/Foliage/Utils/GitHub.hs | 17 ++++++++++++++--- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index 2ec5444..66fa510 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -30,6 +30,7 @@ import Foliage.Shake import Foliage.Time qualified as Time import Foliage.Utils.GitHub (githubRepoUrl) import Hackage.Security.Util.Path (castRoot, toFilePath) +import System.Directory (createDirectoryIfMissing) cmdBuild :: BuildOptions -> IO () cmdBuild buildOptions = do @@ -57,7 +58,8 @@ buildAction buildOptsCurrentTime = mCurrentTime, buildOptsExpireSignaturesOn = mExpireSignaturesOn, buildOptsInputDir = inputDir, - buildOptsOutputDir = outputDir + buildOptsOutputDir = outputDir, + buildOptsWriteMetadata = doWritePackageMeta } = do outputDirRoot <- liftIO $ makeAbsolute (fromFilePath outputDir) @@ -93,7 +95,8 @@ buildAction makeAllPackageVersionsPage currentTime outputDir packageVersions - makeMetadataFile outputDir packageVersions + when doWritePackageMeta $ + makeMetadataFile outputDir packageVersions void $ forP packageVersions $ makePackageVersionPage inputDir outputDir @@ -228,15 +231,15 @@ buildAction } makeMetadataFile :: FilePath -> [PackageVersionMeta] -> Action () -makeMetadataFile outputDir packageVersions = - liftIO $ - Aeson.encodeFile - (outputDir "metadata.json") - ( Map.fromList - [ (prettyShow pkgId, encodeMetadataSpec pkgSpec) - | PackageVersionMeta pkgId pkgSpec <- packageVersions - ] - ) +makeMetadataFile outputDir packageVersions = traced "writing metadata" $ do + createDirectoryIfMissing True (outputDir "foliage") + Aeson.encodeFile + (outputDir "foliage" "packages.json") + ( Map.fromList + [ (prettyShow pkgId, encodeMetadataSpec pkgSpec) + | PackageVersionMeta pkgId pkgSpec <- packageVersions + ] + ) encodeMetadataSpec :: PackageVersionSpec -> Aeson.Value encodeMetadataSpec @@ -259,7 +262,7 @@ encodeMetadataSpec ) GitHubSource repo rev mSubdir -> Aeson.object - ( ("url" Aeson..= show (githubRepoUrl repo rev)) + ( ("url" Aeson..= githubRepoUrl repo rev) : case mSubdir of Nothing -> []; Just s -> ["subdir" Aeson..= s] ) diff --git a/app/Foliage/Options.hs b/app/Foliage/Options.hs index 9271d7f..285edee 100644 --- a/app/Foliage/Options.hs +++ b/app/Foliage/Options.hs @@ -52,7 +52,8 @@ data BuildOptions = BuildOptions buildOptsExpireSignaturesOn :: Maybe UTCTime, buildOptsInputDir :: FilePath, buildOptsOutputDir :: FilePath, - buildOptsNumThreads :: Int + buildOptsNumThreads :: Int, + buildOptsWriteMetadata :: Bool } buildCommand :: Parser Command @@ -100,6 +101,11 @@ buildCommand = <> showDefault <> value 1 ) + <*> switch + ( long "write-metadata" + <> help "Write metadata in the output-directory" + <> showDefault + ) ) where signOpts = diff --git a/app/Foliage/PrepareSource.hs b/app/Foliage/PrepareSource.hs index b855604..34443a6 100644 --- a/app/Foliage/PrepareSource.hs +++ b/app/Foliage/PrepareSource.hs @@ -17,7 +17,7 @@ import Distribution.Types.PackageName (unPackageName) import Foliage.Meta import Foliage.RemoteAsset (fetchRemoteAsset) import Foliage.UpdateCabalFile (rewritePackageVersion) -import Foliage.Utils.GitHub (githubRepoUrl) +import Foliage.Utils.GitHub (githubRepoTarballUrl) import GHC.Generics import System.Directory qualified as IO import System.FilePath ((<.>), ()) @@ -78,7 +78,7 @@ addPrepareSourceRule inputDir cacheDir = addBuiltinRule noLint noIdentity run -- metadata. -- GitHubSource repo rev mSubdir -> do - let url = githubRepoUrl repo rev + let url = githubRepoTarballUrl repo rev tarballPath <- fetchRemoteAsset url diff --git a/app/Foliage/Utils/GitHub.hs b/app/Foliage/Utils/GitHub.hs index 0a9490e..3030658 100644 --- a/app/Foliage/Utils/GitHub.hs +++ b/app/Foliage/Utils/GitHub.hs @@ -1,14 +1,25 @@ -module Foliage.Utils.GitHub where +module Foliage.Utils.GitHub + ( githubRepoTarballUrl, + githubRepoUrl, + ) +where import Data.Text qualified as T import Foliage.Meta (GitHubRepo (unGitHubRepo), GitHubRev (unGitHubRev)) import Network.URI (URI (..), URIAuth (..), nullURI, nullURIAuth) import System.FilePath (()) -githubRepoUrl :: GitHubRepo -> GitHubRev -> URI -githubRepoUrl repo rev = +githubRepoTarballUrl :: GitHubRepo -> GitHubRev -> URI +githubRepoTarballUrl repo rev = nullURI { uriScheme = "https:", uriAuthority = Just nullURIAuth {uriRegName = "github.com"}, uriPath = "/" T.unpack (unGitHubRepo repo) "tarball" T.unpack (unGitHubRev rev) } + +githubRepoUrl :: GitHubRepo -> GitHubRev -> URI +githubRepoUrl repo rev = + nullURI + { uriScheme = "github:", + uriPath = T.unpack (unGitHubRepo repo) T.unpack (unGitHubRev rev) + } From 1e45bfc0fac537468dce838c4309f2ccaaf0395b Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Mon, 20 Feb 2023 12:15:08 +0800 Subject: [PATCH 24/43] List package name and version separately in metadata --- app/Foliage/CmdBuild.hs | 62 +++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index 66fa510..f523703 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -235,36 +235,38 @@ makeMetadataFile outputDir packageVersions = traced "writing metadata" $ do createDirectoryIfMissing True (outputDir "foliage") Aeson.encodeFile (outputDir "foliage" "packages.json") - ( Map.fromList - [ (prettyShow pkgId, encodeMetadataSpec pkgSpec) - | PackageVersionMeta pkgId pkgSpec <- packageVersions - ] - ) - -encodeMetadataSpec :: PackageVersionSpec -> Aeson.Value -encodeMetadataSpec - PackageVersionSpec - { packageVersionSource, - packageVersionForce, - packageVersionTimestamp - } = - Aeson.object - ( ("source" Aeson..= source) - : ["forced-version" Aeson..= True | packageVersionForce] - ++ (case packageVersionTimestamp of Nothing -> []; Just t -> ["timestamp" Aeson..= t]) - ) - where - source = case packageVersionSource of - TarballSource uri mSubdir -> - Aeson.object - ( ("url" Aeson..= show uri) - : case mSubdir of Nothing -> []; Just s -> ["subdir" Aeson..= s] - ) - GitHubSource repo rev mSubdir -> - Aeson.object - ( ("url" Aeson..= githubRepoUrl repo rev) - : case mSubdir of Nothing -> []; Just s -> ["subdir" Aeson..= s] - ) + (map encodePackageVersionMeta packageVersions) + where + encodePackageVersionMeta + PackageVersionMeta + { pkgId = PackageIdentifier {pkgName, pkgVersion}, + pkgSpec = + PackageVersionSpec + { packageVersionSource, + packageVersionForce, + packageVersionTimestamp + } + } = + Aeson.object + ( [ "pkg-name" Aeson..= pkgName, + "pkg-version" Aeson..= pkgVersion, + "source" Aeson..= source + ] + ++ ["forced-version" Aeson..= True | packageVersionForce] + ++ (case packageVersionTimestamp of Nothing -> []; Just t -> ["timestamp" Aeson..= t]) + ) + where + source = case packageVersionSource of + TarballSource uri mSubdir -> + Aeson.object + ( ("url" Aeson..= show uri) + : case mSubdir of Nothing -> []; Just s -> ["subdir" Aeson..= s] + ) + GitHubSource repo rev mSubdir -> + Aeson.object + ( ("url" Aeson..= githubRepoUrl repo rev) + : case mSubdir of Nothing -> []; Just s -> ["subdir" Aeson..= s] + ) getPackageVersions :: FilePath -> Action [PackageVersionMeta] getPackageVersions inputDir = do From b12a27a9844f765a368047ee0172ee672c373804 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Mon, 20 Feb 2023 13:42:28 +0800 Subject: [PATCH 25/43] Use dir query parameter to include subdir into url --- app/Foliage/CmdBuild.hs | 33 ++++++++++++++++++--------------- app/Foliage/Utils/GitHub.hs | 8 -------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index f523703..60521d5 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -10,8 +10,8 @@ import Control.Monad (unless, void, when) import Data.Aeson qualified as Aeson import Data.ByteString.Lazy qualified as BL import Data.List (sortOn) -import Data.Map.Strict qualified as Map import Data.Maybe (fromMaybe) +import Data.Text qualified as T import Data.Traversable (for) import Development.Shake import Development.Shake.FilePath @@ -28,8 +28,8 @@ import Foliage.PrepareSource (addPrepareSourceRule, prepareSource) import Foliage.RemoteAsset (addFetchRemoteAssetRule) import Foliage.Shake import Foliage.Time qualified as Time -import Foliage.Utils.GitHub (githubRepoUrl) import Hackage.Security.Util.Path (castRoot, toFilePath) +import Network.URI (URI (uriPath, uriQuery, uriScheme), nullURI) import System.Directory (createDirectoryIfMissing) cmdBuild :: BuildOptions -> IO () @@ -250,23 +250,26 @@ makeMetadataFile outputDir packageVersions = traced "writing metadata" $ do Aeson.object ( [ "pkg-name" Aeson..= pkgName, "pkg-version" Aeson..= pkgVersion, - "source" Aeson..= source + "url" Aeson..= sourceUrl packageVersionSource ] ++ ["forced-version" Aeson..= True | packageVersionForce] ++ (case packageVersionTimestamp of Nothing -> []; Just t -> ["timestamp" Aeson..= t]) ) - where - source = case packageVersionSource of - TarballSource uri mSubdir -> - Aeson.object - ( ("url" Aeson..= show uri) - : case mSubdir of Nothing -> []; Just s -> ["subdir" Aeson..= s] - ) - GitHubSource repo rev mSubdir -> - Aeson.object - ( ("url" Aeson..= githubRepoUrl repo rev) - : case mSubdir of Nothing -> []; Just s -> ["subdir" Aeson..= s] - ) + + sourceUrl :: PackageVersionSource -> URI + sourceUrl (TarballSource uri Nothing) = uri + sourceUrl (TarballSource uri (Just subdir)) = uri {uriQuery = "?dir=" ++ subdir} + sourceUrl (GitHubSource repo rev Nothing) = + nullURI + { uriScheme = "github:", + uriPath = T.unpack (unGitHubRepo repo) T.unpack (unGitHubRev rev) + } + sourceUrl (GitHubSource repo rev (Just subdir)) = + nullURI + { uriScheme = "github:", + uriPath = T.unpack (unGitHubRepo repo) T.unpack (unGitHubRev rev), + uriQuery = "?dir=" ++ subdir + } getPackageVersions :: FilePath -> Action [PackageVersionMeta] getPackageVersions inputDir = do diff --git a/app/Foliage/Utils/GitHub.hs b/app/Foliage/Utils/GitHub.hs index 3030658..96c1461 100644 --- a/app/Foliage/Utils/GitHub.hs +++ b/app/Foliage/Utils/GitHub.hs @@ -1,6 +1,5 @@ module Foliage.Utils.GitHub ( githubRepoTarballUrl, - githubRepoUrl, ) where @@ -16,10 +15,3 @@ githubRepoTarballUrl repo rev = uriAuthority = Just nullURIAuth {uriRegName = "github.com"}, uriPath = "/" T.unpack (unGitHubRepo repo) "tarball" T.unpack (unGitHubRev rev) } - -githubRepoUrl :: GitHubRepo -> GitHubRev -> URI -githubRepoUrl repo rev = - nullURI - { uriScheme = "github:", - uriPath = T.unpack (unGitHubRepo repo) T.unpack (unGitHubRev rev) - } From e8fa66a8e95be947348de4a8357d14981732e5e1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 26 Feb 2023 01:56:15 +0000 Subject: [PATCH 26/43] flake.lock: Update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flake lock file updates: • Updated input 'haskell-nix': 'github:input-output-hk/haskell.nix/f5da0a34b5dbbeb57e6d7595f46a930cd62bcd77' (2023-01-08) → 'github:input-output-hk/haskell.nix/986ebab4075d4c1ecac1270a9304e65e7ef715bc' (2023-02-26) • Updated input 'haskell-nix/cabal-34': 'github:haskell/cabal/942639c18c0cd8ec53e0a6f8d120091af35312cd' (2021-12-24) → 'github:haskell/cabal/5ff598c67f53f7c4f48e31d722ba37172230c462' (2022-02-26) • Updated input 'haskell-nix/cabal-36': 'github:haskell/cabal/f27667f8ec360c475027dcaee0138c937477b070' (2022-01-08) → 'github:haskell/cabal/8fd619e33d34924a94e691c5fea2c42f0fc7f144' (2022-11-22) • Updated input 'haskell-nix/flake-compat': 'github:input-output-hk/flake-compat/eca47d3377946315596da653862d341ee5341318' (2021-11-02) → 'github:input-output-hk/flake-compat/45f2638735f8cdc40fe302742b79f248d23eb368' (2023-01-04) • Updated input 'haskell-nix/flake-utils': 'github:numtide/flake-utils/3cecb5b042f7f209c56ffd8371b2711a290ec797' (2022-02-07) → 'github:numtide/flake-utils/5aed5285a952e0b949eb3ba02c12fa4fcfef535f' (2022-11-02) • Updated input 'haskell-nix/hackage': 'github:input-output-hk/hackage.nix/5cc584651a4895c7abad166cf695a3ee8b873386' (2023-01-08) → 'github:input-output-hk/hackage.nix/6ee340fbe61f03ab2a440d83f29c802d7e494ba8' (2023-02-26) • Updated input 'haskell-nix/hydra': 'github:NixOS/hydra/28b682b85b7efc5cf7974065792a1f22203a5927' (2022-03-10) → 'github:NixOS/hydra/f48f00ee6d5727ae3e488cbf9ce157460853fea8' (2022-12-23) • Updated input 'haskell-nix/hydra/nix': 'github:NixOS/nix/a1cd7e58606a41fcf62bf8637804cf8306f17f62' (2022-01-24) → 'github:NixOS/nix/11e45768b34fdafdcf019ddbd337afa16127ff0f' (2022-08-27) • Updated input 'haskell-nix/hydra/nix/nixpkgs': 'github:NixOS/nixpkgs/82891b5e2c2359d7e58d08849e4c89511ab94234' (2021-09-28) → 'github:NixOS/nixpkgs/365e1b3a859281cf11b94f87231adeabbdd878a2' (2022-07-13) • Updated input 'haskell-nix/iserv-proxy': 'git+https://gitlab.haskell.org/ghc/iserv-proxy.git?rev=6e95df7be6dd29680f983db07a057fc2f34f81f6' (2021-12-10) → 'git+https://gitlab.haskell.org/hamishmack/iserv-proxy.git?ref=hkm%2fremote-iserv&rev=50d0abb3317ac439a4e7495b185a64af9b7b9300' (2022-12-14) • Updated input 'haskell-nix/nixpkgs-2205': 'github:NixOS/nixpkgs/309faedb8338d3ae8ad8f1043b3ccf48c9cc2970' (2022-09-24) → 'github:NixOS/nixpkgs/0874168639713f547c05947c76124f78441ea46c' (2023-01-01) • Updated input 'haskell-nix/nixpkgs-2211': 'github:NixOS/nixpkgs/6f87491a54d8d64d30af6663cb3bf5d2ee7db958' (2022-12-02) → 'github:NixOS/nixpkgs/b7ce17b1ebf600a72178f6302c77b6382d09323f' (2023-02-07) • Updated input 'haskell-nix/nixpkgs-unstable': 'github:NixOS/nixpkgs/e14f9fb57315f0d4abde222364f19f88c77d2b79' (2022-09-23) → 'github:NixOS/nixpkgs/747927516efcb5e31ba03b7ff32f61f6d47e7d87' (2023-02-07) • Updated input 'haskell-nix/stackage': 'github:input-output-hk/stackage.nix/46c3d80f024585ddf22a54f3c505799dca778865' (2023-01-07) → 'github:input-output-hk/stackage.nix/f6e0966f3dad73156035963a7d16c48521d8e4f2' (2023-02-26) • Updated input 'haskell-nix/tullia': 'github:input-output-hk/tullia/ead1f515c251f0e060060ef0e2356a51d3dfe4b0' (2022-11-17) → 'github:input-output-hk/tullia/621365f2c725608f381b3ad5b57afef389fd4c31' (2023-02-06) • Updated input 'haskell-nix/tullia/std': 'github:divnix/std/94a90eedb9cfc115b12ae8f6622d9904788559e4' (2022-10-11) → 'github:divnix/std/516387e3d8d059b50e742a2ff1909ed3c8f82826' (2023-01-24) • Added input 'haskell-nix/tullia/std/arion': follows 'haskell-nix/tullia/std/blank' • Added input 'haskell-nix/tullia/std/incl': 'github:divnix/incl/ce7bebaee048e4cd7ebdb4cee7885e00c4e2abca' (2022-11-24) • Added input 'haskell-nix/tullia/std/incl/nixlib': follows 'haskell-nix/tullia/std/nixpkgs' • Removed input 'haskell-nix/tullia/std/mdbook-kroki-preprocessor' • Updated input 'haskell-nix/tullia/std/n2c/flake-utils': 'github:numtide/flake-utils/1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1' (2022-05-30) → follows 'haskell-nix/tullia/std/flake-utils' • Added input 'haskell-nix/tullia/std/nosys': 'github:divnix/nosys/2d0d5207f6a230e9d0f660903f8db9807b54814f' (2022-11-08) • Updated input 'haskell-nix/tullia/std/yants': 'github:divnix/yants/0b895ca02a8fa72bad50b454cb3e7d8a66407c96' (2022-08-14) → 'github:divnix/yants/d18f356ec25cb94dc9c275870c3a7927a10f8c3c' (2022-10-30) --- flake.lock | 211 +++++++++++++++++++++++++++++------------------------ 1 file changed, 117 insertions(+), 94 deletions(-) diff --git a/flake.lock b/flake.lock index 23e58fd..b209a71 100644 --- a/flake.lock +++ b/flake.lock @@ -51,11 +51,11 @@ "cabal-34": { "flake": false, "locked": { - "lastModified": 1640353650, - "narHash": "sha256-N1t6M3/wqj90AEdRkeC8i923gQYUpzSr8b40qVOZ1Rk=", + "lastModified": 1645834128, + "narHash": "sha256-wG3d+dOt14z8+ydz4SL7pwGfe7SiimxcD/LOuPCV6xM=", "owner": "haskell", "repo": "cabal", - "rev": "942639c18c0cd8ec53e0a6f8d120091af35312cd", + "rev": "5ff598c67f53f7c4f48e31d722ba37172230c462", "type": "github" }, "original": { @@ -68,11 +68,11 @@ "cabal-36": { "flake": false, "locked": { - "lastModified": 1641652457, - "narHash": "sha256-BlFPKP4C4HRUJeAbdembX1Rms1LD380q9s0qVDeoAak=", + "lastModified": 1669081697, + "narHash": "sha256-I5or+V7LZvMxfbYgZATU4awzkicBwwok4mVoje+sGmU=", "owner": "haskell", "repo": "cabal", - "rev": "f27667f8ec360c475027dcaee0138c937477b070", + "rev": "8fd619e33d34924a94e691c5fea2c42f0fc7f144", "type": "github" }, "original": { @@ -159,15 +159,16 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1635892615, - "narHash": "sha256-harGbMZr4hzat2BWBU+Y5OYXlu+fVz7E4WeQzHi5o8A=", + "lastModified": 1672831974, + "narHash": "sha256-z9k3MfslLjWQfnjBtEtJZdq3H7kyi2kQtUThfTgdRk0=", "owner": "input-output-hk", "repo": "flake-compat", - "rev": "eca47d3377946315596da653862d341ee5341318", + "rev": "45f2638735f8cdc40fe302742b79f248d23eb368", "type": "github" }, "original": { "owner": "input-output-hk", + "ref": "hkm/gitlab-fix", "repo": "flake-compat", "type": "github" } @@ -190,11 +191,11 @@ }, "flake-utils": { "locked": { - "lastModified": 1644229661, - "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", "owner": "numtide", "repo": "flake-utils", - "rev": "3cecb5b042f7f209c56ffd8371b2711a290ec797", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", "type": "github" }, "original": { @@ -233,21 +234,6 @@ "type": "github" } }, - "flake-utils_4": { - "locked": { - "lastModified": 1653893745, - "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, "ghc-8.6.5-iohk": { "flake": false, "locked": { @@ -287,11 +273,11 @@ "hackage": { "flake": false, "locked": { - "lastModified": 1673137629, - "narHash": "sha256-HJfuXQ6NkhdZIMo4/S/Y9QX1xmuBS7XSB4xhmEVZSWc=", + "lastModified": 1677371292, + "narHash": "sha256-8ky5ewzq+7/RJLwZTbuGwlkej83H9IvoLwMh0iLyZ/0=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "5cc584651a4895c7abad166cf695a3ee8b873386", + "rev": "6ee340fbe61f03ab2a440d83f29c802d7e494ba8", "type": "github" }, "original": { @@ -329,11 +315,11 @@ "tullia": "tullia" }, "locked": { - "lastModified": 1673139096, - "narHash": "sha256-EzMIx2nxXoHbMFHIKFiXVE6AC+PcCm4NjKChk9MAgM8=", + "lastModified": 1677372667, + "narHash": "sha256-9FQa8yvbkmq1UUhW0h1w4C37X90SzplqQFaQzf/D+Gs=", "owner": "input-output-hk", "repo": "haskell.nix", - "rev": "f5da0a34b5dbbeb57e6d7595f46a930cd62bcd77", + "rev": "986ebab4075d4c1ecac1270a9304e65e7ef715bc", "type": "github" }, "original": { @@ -369,11 +355,11 @@ ] }, "locked": { - "lastModified": 1646878427, - "narHash": "sha256-KtbrofMtN8GlM7D+n90kixr7QpSlVmdN+vK5CA/aRzc=", + "lastModified": 1671755331, + "narHash": "sha256-hXsgJj0Cy0ZiCiYdW2OdBz5WmFyOMKuw4zyxKpgUKm4=", "owner": "NixOS", "repo": "hydra", - "rev": "28b682b85b7efc5cf7974065792a1f22203a5927", + "rev": "f48f00ee6d5727ae3e488cbf9ce157460853fea8", "type": "github" }, "original": { @@ -381,20 +367,44 @@ "type": "indirect" } }, + "incl": { + "inputs": { + "nixlib": [ + "haskell-nix", + "tullia", + "std", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1669263024, + "narHash": "sha256-E/+23NKtxAqYG/0ydYgxlgarKnxmDbg6rCMWnOBqn9Q=", + "owner": "divnix", + "repo": "incl", + "rev": "ce7bebaee048e4cd7ebdb4cee7885e00c4e2abca", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "incl", + "type": "github" + } + }, "iserv-proxy": { "flake": false, "locked": { - "lastModified": 1639165170, - "narHash": "sha256-QsWL/sBDL5GM8IXd/dE/ORiL4RvteEN+aok23tXgAoc=", - "rev": "6e95df7be6dd29680f983db07a057fc2f34f81f6", - "revCount": 7, + "lastModified": 1670983692, + "narHash": "sha256-avLo34JnI9HNyOuauK5R69usJm+GfW3MlyGlYxZhTgY=", + "ref": "hkm/remote-iserv", + "rev": "50d0abb3317ac439a4e7495b185a64af9b7b9300", + "revCount": 10, "type": "git", - "url": "https://gitlab.haskell.org/ghc/iserv-proxy.git" + "url": "https://gitlab.haskell.org/hamishmack/iserv-proxy.git" }, "original": { - "rev": "6e95df7be6dd29680f983db07a057fc2f34f81f6", + "ref": "hkm/remote-iserv", "type": "git", - "url": "https://gitlab.haskell.org/ghc/iserv-proxy.git" + "url": "https://gitlab.haskell.org/hamishmack/iserv-proxy.git" } }, "lowdown-src": { @@ -413,25 +423,14 @@ "type": "github" } }, - "mdbook-kroki-preprocessor": { - "flake": false, - "locked": { - "lastModified": 1661755005, - "narHash": "sha256-1TJuUzfyMycWlOQH67LR63/ll2GDZz25I3JfScy/Jnw=", - "owner": "JoelCourtney", - "repo": "mdbook-kroki-preprocessor", - "rev": "93adb5716d035829efed27f65f2f0833a7d3e76f", - "type": "github" - }, - "original": { - "owner": "JoelCourtney", - "repo": "mdbook-kroki-preprocessor", - "type": "github" - } - }, "n2c": { "inputs": { - "flake-utils": "flake-utils_4", + "flake-utils": [ + "haskell-nix", + "tullia", + "std", + "flake-utils" + ], "nixpkgs": [ "haskell-nix", "tullia", @@ -460,16 +459,16 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1643066034, - "narHash": "sha256-xEPeMcNJVOeZtoN+d+aRwolpW8mFSEQx76HTRdlhPhg=", + "lastModified": 1661606874, + "narHash": "sha256-9+rpYzI+SmxJn+EbYxjGv68Ucp22bdFUSy/4LkHkkDQ=", "owner": "NixOS", "repo": "nix", - "rev": "a1cd7e58606a41fcf62bf8637804cf8306f17f62", + "rev": "11e45768b34fdafdcf019ddbd337afa16127ff0f", "type": "github" }, "original": { "owner": "NixOS", - "ref": "2.6.0", + "ref": "2.11.0", "repo": "nix", "type": "github" } @@ -565,17 +564,18 @@ }, "nixpkgs": { "locked": { - "lastModified": 1632864508, - "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=", + "lastModified": 1657693803, + "narHash": "sha256-G++2CJ9u0E7NNTAi9n5G8TdDmGJXcIjkJ3NF8cetQB8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234", + "rev": "365e1b3a859281cf11b94f87231adeabbdd878a2", "type": "github" }, "original": { - "id": "nixpkgs", - "ref": "nixos-21.05-small", - "type": "indirect" + "owner": "NixOS", + "ref": "nixos-22.05-small", + "repo": "nixpkgs", + "type": "github" } }, "nixpkgs-2003": { @@ -628,11 +628,11 @@ }, "nixpkgs-2205": { "locked": { - "lastModified": 1663981975, - "narHash": "sha256-TKaxWAVJR+a5JJauKZqibmaM5e/Pi5tBDx9s8fl/kSE=", + "lastModified": 1672580127, + "narHash": "sha256-3lW3xZslREhJogoOkjeZtlBtvFMyxHku7I/9IVehhT8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "309faedb8338d3ae8ad8f1043b3ccf48c9cc2970", + "rev": "0874168639713f547c05947c76124f78441ea46c", "type": "github" }, "original": { @@ -644,11 +644,11 @@ }, "nixpkgs-2211": { "locked": { - "lastModified": 1669997163, - "narHash": "sha256-vhjC0kZMFoN6jzK0GR+tBzKi5KgBXgehadfidW8+Va4=", + "lastModified": 1675730325, + "narHash": "sha256-uNvD7fzO5hNlltNQUAFBPlcEjNG5Gkbhl/ROiX+GZU4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "6f87491a54d8d64d30af6663cb3bf5d2ee7db958", + "rev": "b7ce17b1ebf600a72178f6302c77b6382d09323f", "type": "github" }, "original": { @@ -668,18 +668,19 @@ "type": "github" }, "original": { - "id": "nixpkgs", + "owner": "NixOS", + "repo": "nixpkgs", "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "indirect" + "type": "github" } }, "nixpkgs-unstable": { "locked": { - "lastModified": 1663905476, - "narHash": "sha256-0CSwRKaYravh9v6qSlBpM0gNg0UhKT2lL7Yn6Zbx7UM=", + "lastModified": 1675758091, + "narHash": "sha256-7gFSQbSVAFUHtGCNHPF7mPc5CcqDk9M2+inlVPZSneg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "e14f9fb57315f0d4abde222364f19f88c77d2b79", + "rev": "747927516efcb5e31ba03b7ff32f61f6d47e7d87", "type": "github" }, "original": { @@ -736,6 +737,21 @@ "type": "github" } }, + "nosys": { + "locked": { + "lastModified": 1667881534, + "narHash": "sha256-FhwJ15uPLRsvaxtt/bNuqE/ykMpNAPF0upozFKhTtXM=", + "owner": "divnix", + "repo": "nosys", + "rev": "2d0d5207f6a230e9d0f660903f8db9807b54814f", + "type": "github" + }, + "original": { + "owner": "divnix", + "repo": "nosys", + "type": "github" + } + }, "old-ghc-nix": { "flake": false, "locked": { @@ -769,11 +785,11 @@ "stackage": { "flake": false, "locked": { - "lastModified": 1673050158, - "narHash": "sha256-/1WWFQDUisdEuSOUTvQTzkED9Z6riX3XuVMuaMIbyq4=", + "lastModified": 1677370244, + "narHash": "sha256-JSh7/wauT+jn3FXxC6pCiYfE19uoy6/tpKsXeUGlg0Q=", "owner": "input-output-hk", "repo": "stackage.nix", - "rev": "46c3d80f024585ddf22a54f3c505799dca778865", + "rev": "f6e0966f3dad73156035963a7d16c48521d8e4f2", "type": "github" }, "original": { @@ -784,17 +800,23 @@ }, "std": { "inputs": { + "arion": [ + "haskell-nix", + "tullia", + "std", + "blank" + ], "blank": "blank", "devshell": "devshell", "dmerge": "dmerge", "flake-utils": "flake-utils_3", + "incl": "incl", "makes": [ "haskell-nix", "tullia", "std", "blank" ], - "mdbook-kroki-preprocessor": "mdbook-kroki-preprocessor", "microvm": [ "haskell-nix", "tullia", @@ -804,14 +826,15 @@ "n2c": "n2c", "nixago": "nixago", "nixpkgs": "nixpkgs_4", + "nosys": "nosys", "yants": "yants" }, "locked": { - "lastModified": 1665513321, - "narHash": "sha256-D6Pacw9yf/HMs84KYuCxHXnNDL7v43gtcka5URagFqE=", + "lastModified": 1674526466, + "narHash": "sha256-tMTaS0bqLx6VJ+K+ZT6xqsXNpzvSXJTmogkraBGzymg=", "owner": "divnix", "repo": "std", - "rev": "94a90eedb9cfc115b12ae8f6622d9904788559e4", + "rev": "516387e3d8d059b50e742a2ff1909ed3c8f82826", "type": "github" }, "original": { @@ -831,11 +854,11 @@ "std": "std" }, "locked": { - "lastModified": 1668711738, - "narHash": "sha256-CBjky16o9pqsGE1bWu6nRlRajgSXMEk+yaFQLibqXcE=", + "lastModified": 1675695930, + "narHash": "sha256-B7rEZ/DBUMlK1AcJ9ajnAPPxqXY6zW2SBX+51bZV0Ac=", "owner": "input-output-hk", "repo": "tullia", - "rev": "ead1f515c251f0e060060ef0e2356a51d3dfe4b0", + "rev": "621365f2c725608f381b3ad5b57afef389fd4c31", "type": "github" }, "original": { @@ -869,11 +892,11 @@ ] }, "locked": { - "lastModified": 1660507851, - "narHash": "sha256-BKjq7JnVuUR/xDtcv6Vm9GYGKAblisXrAgybor9hT/s=", + "lastModified": 1667096281, + "narHash": "sha256-wRRec6ze0gJHmGn6m57/zhz/Kdvp9HS4Nl5fkQ+uIuA=", "owner": "divnix", "repo": "yants", - "rev": "0b895ca02a8fa72bad50b454cb3e7d8a66407c96", + "rev": "d18f356ec25cb94dc9c275870c3a7927a10f8c3c", "type": "github" }, "original": { From e3e34324ff52d366f9226969162f83b258b776a7 Mon Sep 17 00:00:00 2001 From: Ben Gamari Date: Wed, 1 Mar 2023 13:42:52 -0500 Subject: [PATCH 27/43] Don't use lazy IO when reading .cabal files Lazy IO tends to keep file handles open until the entire file is read, which in this case may never happen. This leads to the process exceeding the system's file descriptor limit, resulting in #35. --- app/Foliage/CmdBuild.hs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index 60521d5..dcdec9d 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -8,6 +8,7 @@ import Codec.Archive.Tar.Entry qualified as Tar import Codec.Compression.GZip qualified as GZip import Control.Monad (unless, void, when) import Data.Aeson qualified as Aeson +import Data.ByteString qualified as BS import Data.ByteString.Lazy qualified as BL import Data.List (sortOn) import Data.Maybe (fromMaybe) @@ -315,8 +316,8 @@ getPackageVersions inputDir = do prepareIndexPkgCabal :: PackageId -> UTCTime -> FilePath -> Action Tar.Entry prepareIndexPkgCabal pkgId timestamp filePath = do need [filePath] - contents <- liftIO $ BL.readFile filePath - return $ mkTarEntry contents (IndexPkgCabal pkgId) timestamp + contents <- liftIO $ BS.readFile filePath + return $ mkTarEntry (BL.fromStrict contents) (IndexPkgCabal pkgId) timestamp prepareIndexPkgMetadata :: Maybe UTCTime -> PackageVersionMeta -> Action Targets prepareIndexPkgMetadata expiryTime PackageVersionMeta {pkgId, pkgSpec} = do From 945d48176b519da6c34e41cce2725ea2e4544b3b Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 2 Mar 2023 15:26:44 +0800 Subject: [PATCH 28/43] Fix CI --- .github/workflows/nix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index 4266c3d..ef627f8 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -14,7 +14,7 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: cachix/install-nix-action@v19 + - uses: cachix/install-nix-action@v20 with: extra_nix_config: | access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} From b35bbd26c997dcabe0939ab89f2cbe403f60f7d6 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 2 Mar 2023 10:48:31 +0800 Subject: [PATCH 29/43] More resiliant code --- app/Foliage/CmdBuild.hs | 53 ++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index dcdec9d..5bff9b7 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -137,11 +137,10 @@ buildAction targets <- prepareIndexPkgMetadata expiryTime pkg let path = outputDir "index" prettyShow pkgName prettyShow pkgVersion "package.json" liftIO $ BL.writeFile path $ renderSignedJSON targetKeys targets - return $ - mkTarEntry - (renderSignedJSON targetKeys targets) - (IndexPkgMetadata pkgId) - (fromMaybe currentTime packageVersionTimestamp) + mkTarEntry + (renderSignedJSON targetKeys targets) + (IndexPkgMetadata pkgId) + (fromMaybe currentTime packageVersionTimestamp) let tarContents = Tar.write $ sortOn Tar.entryTime (cabalEntries ++ metadataEntries) traced "Writing index" $ do @@ -285,9 +284,15 @@ getPackageVersions inputDir = do fail "no package metadata found" forP metaFiles $ \metaFile -> do - let [pkgName, pkgVersion, _] = splitDirectories metaFile - let Just name = simpleParsec pkgName - let Just version = simpleParsec pkgVersion + (pkgName, pkgVersion) <- case splitDirectories metaFile of + [pkgName, pkgVersion, _] -> pure (pkgName, pkgVersion) + _else -> fail $ "internal error: I should not be looking at " ++ metaFile + name <- case simpleParsec pkgName of + Nothing -> fail $ "invalid package name: " ++ pkgName + Just name -> pure name + version <- case simpleParsec pkgVersion of + Nothing -> fail $ "invalid package version: " ++ pkgVersion + Just version -> pure version let pkgId = PackageIdentifier name version pkgSpec <- @@ -317,7 +322,7 @@ prepareIndexPkgCabal :: PackageId -> UTCTime -> FilePath -> Action Tar.Entry prepareIndexPkgCabal pkgId timestamp filePath = do need [filePath] contents <- liftIO $ BS.readFile filePath - return $ mkTarEntry (BL.fromStrict contents) (IndexPkgCabal pkgId) timestamp + mkTarEntry (BL.fromStrict contents) (IndexPkgCabal pkgId) timestamp prepareIndexPkgMetadata :: Maybe UTCTime -> PackageVersionMeta -> Action Targets prepareIndexPkgMetadata expiryTime PackageVersionMeta {pkgId, pkgSpec} = do @@ -333,21 +338,25 @@ prepareIndexPkgMetadata expiryTime PackageVersionMeta {pkgId, pkgSpec} = do targetsDelegations = Nothing } -mkTarEntry :: BL.ByteString -> IndexFile dec -> UTCTime -> Tar.Entry -mkTarEntry contents indexFile timestamp = - (Tar.fileEntry tarPath contents) - { Tar.entryTime = floor $ Time.utcTimeToPOSIXSeconds timestamp, - Tar.entryOwnership = - Tar.Ownership - { Tar.ownerName = "foliage", - Tar.groupName = "foliage", - Tar.ownerId = 0, - Tar.groupId = 0 - } - } +mkTarEntry :: BL.ByteString -> IndexFile dec -> UTCTime -> Action Tar.Entry +mkTarEntry contents indexFile timestamp = do + tarPath <- case Tar.toTarPath False indexPath of + Left e -> fail $ "Invalid tar path " ++ indexPath ++ "(" ++ e ++ ")" + Right tarPath -> pure tarPath + + pure + (Tar.fileEntry tarPath contents) + { Tar.entryTime = floor $ Time.utcTimeToPOSIXSeconds timestamp, + Tar.entryOwnership = + Tar.Ownership + { Tar.ownerName = "foliage", + Tar.groupName = "foliage", + Tar.ownerId = 0, + Tar.groupId = 0 + } + } where indexPath = toFilePath $ castRoot $ indexFileToPath hackageIndexLayout indexFile - Right tarPath = Tar.toTarPath False indexPath anchorPath :: Path Absolute -> (RepoLayout -> RepoPath) -> FilePath anchorPath outputDirRoot p = From eaff46e671e7e3ba656253da945b5a8ab2560a86 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 2 Mar 2023 11:33:53 +0800 Subject: [PATCH 30/43] Whitespace --- .../PackageDescription/FieldGrammar/Copy.hs | 6 +- app/Distribution/Types/Orphans.hs | 4 +- app/Foliage/CmdImportIndex.hs | 60 +++++++++---------- app/Foliage/Meta.hs | 30 ++++++---- app/Foliage/Meta/Aeson.hs | 8 +-- app/Foliage/PrepareSdist.hs | 8 ++- app/Foliage/RemoteAsset.hs | 6 +- app/Foliage/Utils/Aeson.hs | 11 ++-- 8 files changed, 74 insertions(+), 59 deletions(-) diff --git a/app/Distribution/PackageDescription/FieldGrammar/Copy.hs b/app/Distribution/PackageDescription/FieldGrammar/Copy.hs index 212484c..6f6eba3 100644 --- a/app/Distribution/PackageDescription/FieldGrammar/Copy.hs +++ b/app/Distribution/PackageDescription/FieldGrammar/Copy.hs @@ -18,7 +18,7 @@ module Distribution.PackageDescription.FieldGrammar.Copy where import Distribution.CabalSpecVersion -import qualified Distribution.Compat.CharParsing as P +import Distribution.Compat.CharParsing qualified as P import Distribution.Compat.Newtype (Newtype, pack', unpack') import Distribution.Compat.Prelude import Distribution.Compiler (CompilerFlavor (..)) @@ -29,8 +29,8 @@ import Distribution.PackageDescription import Distribution.PackageDescription.FieldGrammar hiding (packageDescriptionFieldGrammar) import Distribution.Parsec import Distribution.Pretty (Pretty (..), showToken) -import qualified Distribution.SPDX as SPDX -import qualified Distribution.Types.Lens as L +import Distribution.SPDX qualified as SPDX +import Distribution.Types.Lens qualified as L import Distribution.Utils.Path import Distribution.Version (Version, VersionRange) import Prelude () diff --git a/app/Distribution/Types/Orphans.hs b/app/Distribution/Types/Orphans.hs index 3af7a11..ba3a784 100644 --- a/app/Distribution/Types/Orphans.hs +++ b/app/Distribution/Types/Orphans.hs @@ -2,14 +2,14 @@ module Distribution.Types.Orphans where +import Data.Aeson (ToJSON (toJSON)) import Development.Shake.Classes (Hashable) +import Distribution.Pretty (prettyShow) import Distribution.Types.PackageId (PackageIdentifier) import Distribution.Types.PackageName (PackageName) import Distribution.Types.Version (Version) import Distribution.Types.VersionRange (VersionRange) import Distribution.Utils.ShortText (ShortText) -import Data.Aeson (ToJSON (toJSON)) -import Distribution.Pretty (prettyShow) instance ToJSON PackageIdentifier where toJSON = toJSON . prettyShow diff --git a/app/Foliage/CmdImportIndex.hs b/app/Foliage/CmdImportIndex.hs index e4d601c..f387dca 100644 --- a/app/Foliage/CmdImportIndex.hs +++ b/app/Foliage/CmdImportIndex.hs @@ -48,31 +48,31 @@ importIndex f (Tar.Next e es) m = case isCabalFile e of Just (pkgId, contents, time) | f pkgId -> - do - putStrLn $ "Found cabal file " ++ prettyShow pkgId ++ " with timestamp " ++ show time - let -- new package - go Nothing = - pure $ - Just $ - PackageVersionSpec - { packageVersionSource = TarballSource (pkgIdToHackageUrl pkgId) Nothing, - packageVersionTimestamp = Just time, - packageVersionRevisions = [], - packageVersionForce = False - } - -- Existing package, new revision - go (Just sm) = do - let revnum = 1 + fromMaybe 0 (latestRevisionNumber sm) - newRevision = RevisionSpec {revisionNumber = revnum, revisionTimestamp = time} - -- Repeatedly adding at the end of a list is bad performance but good for the moment. - let sm' = sm {packageVersionRevisions = packageVersionRevisions sm ++ [newRevision]} - let PackageIdentifier pkgName pkgVersion = pkgId - let outDir = "_sources" unPackageName pkgName prettyShow pkgVersion "revisions" - createDirectoryIfMissing True outDir - BSL.writeFile (outDir show revnum <.> "cabal") contents - return $ Just sm' - m' <- M.alterF go pkgId m - importIndex f es m' + do + putStrLn $ "Found cabal file " ++ prettyShow pkgId ++ " with timestamp " ++ show time + let -- new package + go Nothing = + pure $ + Just $ + PackageVersionSpec + { packageVersionSource = TarballSource (pkgIdToHackageUrl pkgId) Nothing, + packageVersionTimestamp = Just time, + packageVersionRevisions = [], + packageVersionForce = False + } + -- Existing package, new revision + go (Just sm) = do + let revnum = 1 + fromMaybe 0 (latestRevisionNumber sm) + newRevision = RevisionSpec {revisionNumber = revnum, revisionTimestamp = time} + -- Repeatedly adding at the end of a list is bad performance but good for the moment. + let sm' = sm {packageVersionRevisions = packageVersionRevisions sm ++ [newRevision]} + let PackageIdentifier pkgName pkgVersion = pkgId + let outDir = "_sources" unPackageName pkgName prettyShow pkgVersion "revisions" + createDirectoryIfMissing True outDir + BSL.writeFile (outDir show revnum <.> "cabal") contents + return $ Just sm' + m' <- M.alterF go pkgId m + importIndex f es m' _ -> importIndex f es m importIndex _f Tar.Done m = return m @@ -106,9 +106,9 @@ isCabalFile Tar.entryTime = posixSecondsToUTCTime . fromIntegral -> time } | ".cabal" `isSuffixOf` path = - let [pkgName, pkgVersion, _] = splitDirectories path - Just name = simpleParsec pkgName - Just version = simpleParsec pkgVersion - packageId = PackageIdentifier name version - in Just (packageId, contents, time) + let [pkgName, pkgVersion, _] = splitDirectories path + Just name = simpleParsec pkgName + Just version = simpleParsec pkgVersion + packageId = PackageIdentifier name version + in Just (packageId, contents, time) isCabalFile _ = Nothing diff --git a/app/Foliage/Meta.hs b/app/Foliage/Meta.hs index 12662b8..3ea928a 100644 --- a/app/Foliage/Meta.hs +++ b/app/Foliage/Meta.hs @@ -86,14 +86,18 @@ writePackageMeta fp a = void $ Toml.encodeToFile packageMetaCodec fp a packageMetaCodec :: TomlCodec PackageMeta packageMetaCodec = PackageMeta - <$> Toml.list packageMetaEntryCodec "entries" .= packageMetaEntries + <$> Toml.list packageMetaEntryCodec "entries" + .= packageMetaEntries packageMetaEntryCodec :: TomlCodec PackageMetaEntry packageMetaEntryCodec = PackageMetaEntry - <$> timeCodec "timestamp" .= packageMetaEntryTimestamp - <*> Toml.arrayOf _VersionRange "preferred-versions" .= packageMetaEntryPreferred - <*> Toml.arrayOf _Version "deprecated-versions" .= packageMetaEntryDeprecated + <$> timeCodec "timestamp" + .= packageMetaEntryTimestamp + <*> Toml.arrayOf _VersionRange "preferred-versions" + .= packageMetaEntryPreferred + <*> Toml.arrayOf _Version "deprecated-versions" + .= packageMetaEntryDeprecated _Version :: Toml.TomlBiMap Version Toml.AnyValue _Version = Toml._TextBy showVersion parseVersion @@ -185,10 +189,14 @@ data PackageVersionSpec = PackageVersionSpec sourceMetaCodec :: TomlCodec PackageVersionSpec sourceMetaCodec = PackageVersionSpec - <$> Toml.dioptional (timeCodec "timestamp") .= packageVersionTimestamp - <*> packageSourceCodec .= packageVersionSource - <*> Toml.list revisionMetaCodec "revisions" .= packageVersionRevisions - <*> withDefault False (Toml.bool "force-version") .= packageVersionForce + <$> Toml.dioptional (timeCodec "timestamp") + .= packageVersionTimestamp + <*> packageSourceCodec + .= packageVersionSource + <*> Toml.list revisionMetaCodec "revisions" + .= packageVersionRevisions + <*> withDefault False (Toml.bool "force-version") + .= packageVersionForce readPackageVersionSpec :: FilePath -> IO PackageVersionSpec readPackageVersionSpec = Toml.decodeFile sourceMetaCodec @@ -206,8 +214,10 @@ data RevisionSpec = RevisionSpec revisionMetaCodec :: TomlCodec RevisionSpec revisionMetaCodec = RevisionSpec - <$> timeCodec "timestamp" .= revisionTimestamp - <*> Toml.int "number" .= revisionNumber + <$> timeCodec "timestamp" + .= revisionTimestamp + <*> Toml.int "number" + .= revisionNumber timeCodec :: Toml.Key -> TomlCodec UTCTime timeCodec key = Toml.dimap (utcToZonedTime utc) zonedTimeToUTC $ Toml.zonedTime key diff --git a/app/Foliage/Meta/Aeson.hs b/app/Foliage/Meta/Aeson.hs index 54f1aa6..31b5fb3 100644 --- a/app/Foliage/Meta/Aeson.hs +++ b/app/Foliage/Meta/Aeson.hs @@ -1,13 +1,13 @@ {-# LANGUAGE DerivingVia #-} +{-# LANGUAGE InstanceSigs #-} {-# LANGUAGE StandaloneDeriving #-} {-# OPTIONS_GHC -Wno-orphans #-} -{-# LANGUAGE InstanceSigs #-} module Foliage.Meta.Aeson where -import Distribution.Types.Orphans () import Data.Aeson import Data.Text +import Distribution.Types.Orphans () import Foliage.Meta import Foliage.Utils.Aeson import Network.URI (URI) @@ -30,8 +30,8 @@ instance ToJSON PackageVersionSource where toJSON = genericToJSON defaultOptions - { sumEncoding = ObjectWithSingleField - , omitNothingFields = True + { sumEncoding = ObjectWithSingleField, + omitNothingFields = True } instance ToJSON URI where diff --git a/app/Foliage/PrepareSdist.hs b/app/Foliage/PrepareSdist.hs index 6ccaca6..681d9ec 100644 --- a/app/Foliage/PrepareSdist.hs +++ b/app/Foliage/PrepareSdist.hs @@ -53,7 +53,7 @@ addPrepareSdistRule outputDirRoot = addBuiltinRule noLint noIdentity run case ehvExisting of Right hvExisting | hvExisting == hvExpected -> - return RunResult {runChanged = ChangedNothing, runStore = old, runValue = path} + return RunResult {runChanged = ChangedNothing, runStore = old, runValue = path} Right hvExisting -> do putWarn $ "Changed " ++ path ++ " (expecting hash " ++ showHashValue hvExpected ++ " found " ++ showHashValue hvExisting ++ "). I will rebuild it." run (PrepareSdistRule srcDir) (Just old) RunDependenciesChanged @@ -71,10 +71,12 @@ addPrepareSdistRule outputDirRoot = addBuiltinRule noLint noIdentity run _differentOrMissing -> ChangedRecomputeDiff when (changed == ChangedRecomputeSame) $ - putInfo $ "Wrote " ++ path ++ " (same hash " ++ showHashValue hv ++ ")" + putInfo $ + "Wrote " ++ path ++ " (same hash " ++ showHashValue hv ++ ")" when (changed == ChangedRecomputeDiff) $ - putInfo $ "Wrote " ++ path ++ " (new hash " ++ showHashValue hv ++ ")" + putInfo $ + "Wrote " ++ path ++ " (new hash " ++ showHashValue hv ++ ")" return $ RunResult {runChanged = changed, runStore = new, runValue = path} diff --git a/app/Foliage/RemoteAsset.hs b/app/Foliage/RemoteAsset.hs index 90fee7d..801c2e6 100644 --- a/app/Foliage/RemoteAsset.hs +++ b/app/Foliage/RemoteAsset.hs @@ -35,10 +35,12 @@ addFetchRemoteAssetRule cacheDir = addBuiltinRule noLint noIdentity run run :: BuiltinRun RemoteAsset FilePath run (RemoteAsset uri) old _mode = do unless (uriQuery uri == "") $ - fail $ "Query elements in URI are not supported: " <> show uri + fail $ + "Query elements in URI are not supported: " <> show uri unless (uriFragment uri == "") $ - fail $ "Fragments in URI are not supported: " <> show uri + fail $ + "Fragments in URI are not supported: " <> show uri let scheme = dropWhileEnd (not . isAlpha) $ uriScheme uri Just host = uriRegName <$> uriAuthority uri diff --git a/app/Foliage/Utils/Aeson.hs b/app/Foliage/Utils/Aeson.hs index dfbf42d..220cb25 100644 --- a/app/Foliage/Utils/Aeson.hs +++ b/app/Foliage/Utils/Aeson.hs @@ -2,22 +2,23 @@ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE UndecidableInstances #-} + module Foliage.Utils.Aeson where import Data.Aeson -import GHC.Generics import Data.Coerce +import GHC.Generics newtype MyAesonEncoding a = MyAesonEncoding a - deriving Generic + deriving (Generic) myOptions :: Options myOptions = defaultOptions - { sumEncoding = ObjectWithSingleField - , omitNothingFields = True + { sumEncoding = ObjectWithSingleField, + omitNothingFields = True } instance (Generic a, GToJSON' Value Zero (Rep a), GToJSON' Encoding Zero (Rep a)) => ToJSON (MyAesonEncoding a) where - toJSON = coerce (genericToJSON myOptions :: a -> Value) + toJSON = coerce (genericToJSON myOptions :: a -> Value) toEncoding = coerce (genericToEncoding myOptions :: a -> Encoding) From 4b6fed5d6ad5721d10377e9582ee16f3baaa13d4 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 2 Mar 2023 11:40:16 +0800 Subject: [PATCH 31/43] Drop PackageVersionMeta It's not the right abstraction --- app/Foliage/CmdBuild.hs | 44 ++++++++++++++++++-------------------- app/Foliage/Meta.hs | 14 +++--------- app/Foliage/Meta/Aeson.hs | 2 -- app/Foliage/Pages.hs | 22 +++++++++---------- app/Foliage/RemoteAsset.hs | 6 ++++-- app/Foliage/Shake.hs | 4 ++-- 6 files changed, 41 insertions(+), 51 deletions(-) diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index 5bff9b7..298ec3c 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -99,20 +99,20 @@ buildAction when doWritePackageMeta $ makeMetadataFile outputDir packageVersions - void $ forP packageVersions $ makePackageVersionPage inputDir outputDir + void $ forP packageVersions $ uncurry $ makePackageVersionPage inputDir outputDir - void $ forP packageVersions $ \pkgMeta@PackageVersionMeta {pkgId} -> do + void $ forP packageVersions $ \(pkgId, pkgSpec) -> do let PackageIdentifier {pkgName, pkgVersion} = pkgId - cabalFilePath <- maybe (originalCabalFile pkgMeta) pure (revisedCabalFile inputDir pkgMeta) + cabalFilePath <- maybe (originalCabalFile pkgId pkgSpec) pure (revisedCabalFile inputDir pkgId pkgSpec) copyFileChanged cabalFilePath (outputDir "index" prettyShow pkgName prettyShow pkgVersion prettyShow pkgName <.> "cabal") cabalEntries <- foldMap - ( \pkgMeta@PackageVersionMeta {pkgId, pkgSpec} -> do + ( \(pkgId, pkgSpec) -> do let PackageVersionSpec {packageVersionTimestamp, packageVersionRevisions} = pkgSpec -- original cabal file, with its timestamp (if specified) - cabalFilePath <- originalCabalFile pkgMeta + cabalFilePath <- originalCabalFile pkgId pkgSpec let cabalFileTimestamp = fromMaybe currentTime packageVersionTimestamp cf <- prepareIndexPkgCabal pkgId cabalFileTimestamp cabalFilePath @@ -131,10 +131,10 @@ buildAction targetKeys <- maybeReadKeysAt "target" metadataEntries <- - forP packageVersions $ \pkg@PackageVersionMeta {pkgId, pkgSpec} -> do + forP packageVersions $ \(pkgId, pkgSpec) -> do let PackageIdentifier {pkgName, pkgVersion} = pkgId let PackageVersionSpec {packageVersionTimestamp} = pkgSpec - targets <- prepareIndexPkgMetadata expiryTime pkg + targets <- prepareIndexPkgMetadata expiryTime pkgId pkgSpec let path = outputDir "index" prettyShow pkgName prettyShow pkgVersion "package.json" liftIO $ BL.writeFile path $ renderSignedJSON targetKeys targets mkTarEntry @@ -230,23 +230,21 @@ buildAction timestampInfoSnapshot = snapshotInfo } -makeMetadataFile :: FilePath -> [PackageVersionMeta] -> Action () +makeMetadataFile :: FilePath -> [(PackageId, PackageVersionSpec)] -> Action () makeMetadataFile outputDir packageVersions = traced "writing metadata" $ do createDirectoryIfMissing True (outputDir "foliage") Aeson.encodeFile (outputDir "foliage" "packages.json") - (map encodePackageVersionMeta packageVersions) + (map encodePackageVersion packageVersions) where - encodePackageVersionMeta - PackageVersionMeta - { pkgId = PackageIdentifier {pkgName, pkgVersion}, - pkgSpec = - PackageVersionSpec - { packageVersionSource, - packageVersionForce, - packageVersionTimestamp - } - } = + encodePackageVersion + ( PackageIdentifier {pkgName, pkgVersion}, + PackageVersionSpec + { packageVersionSource, + packageVersionForce, + packageVersionTimestamp + } + ) = Aeson.object ( [ "pkg-name" Aeson..= pkgName, "pkg-version" Aeson..= pkgVersion, @@ -271,7 +269,7 @@ makeMetadataFile outputDir packageVersions = traced "writing metadata" $ do uriQuery = "?dir=" ++ subdir } -getPackageVersions :: FilePath -> Action [PackageVersionMeta] +getPackageVersions :: FilePath -> Action [(PackageId, PackageVersionSpec)] getPackageVersions inputDir = do metaFiles <- getDirectoryFiles inputDir ["*/*/meta.toml"] @@ -316,7 +314,7 @@ getPackageVersions inputDir = do meta -> return meta - return $ PackageVersionMeta pkgId pkgSpec + return (pkgId, pkgSpec) prepareIndexPkgCabal :: PackageId -> UTCTime -> FilePath -> Action Tar.Entry prepareIndexPkgCabal pkgId timestamp filePath = do @@ -324,8 +322,8 @@ prepareIndexPkgCabal pkgId timestamp filePath = do contents <- liftIO $ BS.readFile filePath mkTarEntry (BL.fromStrict contents) (IndexPkgCabal pkgId) timestamp -prepareIndexPkgMetadata :: Maybe UTCTime -> PackageVersionMeta -> Action Targets -prepareIndexPkgMetadata expiryTime PackageVersionMeta {pkgId, pkgSpec} = do +prepareIndexPkgMetadata :: Maybe UTCTime -> PackageId -> PackageVersionSpec -> Action Targets +prepareIndexPkgMetadata expiryTime pkgId pkgSpec = do srcDir <- prepareSource pkgId pkgSpec sdist <- prepareSdist srcDir targetFileInfo <- computeFileInfoSimple' sdist diff --git a/app/Foliage/Meta.hs b/app/Foliage/Meta.hs index 3ea928a..429f404 100644 --- a/app/Foliage/Meta.hs +++ b/app/Foliage/Meta.hs @@ -13,7 +13,6 @@ module Foliage.Meta packageMetaEntryTimestamp, readPackageMeta, writePackageMeta, - PackageVersionMeta (PackageVersionMeta, pkgId, pkgSpec), packageVersionTimestamp, packageVersionSource, packageVersionRevisions, @@ -50,7 +49,7 @@ import Distribution.Aeson () import Distribution.Parsec (simpleParsec) import Distribution.Pretty (prettyShow) import Distribution.Types.Orphans () -import Distribution.Types.PackageId (PackageIdentifier (..)) +import Distribution.Types.PackageId (PackageId, PackageIdentifier (..)) import Distribution.Types.PackageName (unPackageName) import Distribution.Types.Version (Version) import Distribution.Types.VersionRange (VersionRange, anyVersion, intersectVersionRanges, notThisVersion) @@ -242,17 +241,10 @@ consolidateRanges PackageMetaEntry {packageMetaEntryPreferred, packageMetaEntryD simplifyVersionRange $ foldr intersectVersionRanges anyVersion (map notThisVersion packageMetaEntryDeprecated ++ packageMetaEntryPreferred) -data PackageVersionMeta = PackageVersionMeta - { pkgId :: PackageIdentifier, - pkgSpec :: PackageVersionSpec - } - deriving (Show, Eq) - deriving (Generic) - cabalFileRevisionPath :: FilePath -> PackageIdentifier -> Int -> FilePath cabalFileRevisionPath inputDir PackageIdentifier {pkgName, pkgVersion} revisionNumber = inputDir unPackageName pkgName prettyShow pkgVersion "revisions" show revisionNumber <.> "cabal" -revisedCabalFile :: FilePath -> PackageVersionMeta -> Maybe FilePath -revisedCabalFile inputDir PackageVersionMeta {pkgId, pkgSpec} = do +revisedCabalFile :: FilePath -> PackageId -> PackageVersionSpec -> Maybe FilePath +revisedCabalFile inputDir pkgId pkgSpec = do cabalFileRevisionPath inputDir pkgId <$> latestRevisionNumber pkgSpec diff --git a/app/Foliage/Meta/Aeson.hs b/app/Foliage/Meta/Aeson.hs index 31b5fb3..339b39c 100644 --- a/app/Foliage/Meta/Aeson.hs +++ b/app/Foliage/Meta/Aeson.hs @@ -20,8 +20,6 @@ deriving via MyAesonEncoding RevisionSpec instance ToJSON RevisionSpec deriving via MyAesonEncoding PackageVersionSpec instance ToJSON PackageVersionSpec -deriving via MyAesonEncoding PackageVersionMeta instance ToJSON PackageVersionMeta - deriving via Text instance ToJSON GitHubRepo deriving via Text instance ToJSON GitHubRev diff --git a/app/Foliage/Pages.hs b/app/Foliage/Pages.hs index 67c8084..9bd714c 100644 --- a/app/Foliage/Pages.hs +++ b/app/Foliage/Pages.hs @@ -24,9 +24,9 @@ import Data.Time (UTCTime) import Data.Time.Clock.POSIX (POSIXTime, utcTimeToPOSIXSeconds) import Development.Shake (Action, traced) import Distribution.Aeson (jsonGenericPackageDescription) -import Distribution.Package (PackageIdentifier (pkgName, pkgVersion)) +import Distribution.Package (PackageId, PackageIdentifier (pkgName, pkgVersion)) import Distribution.Pretty (prettyShow) -import Foliage.Meta (PackageVersionMeta (..), PackageVersionSource, PackageVersionSpec (PackageVersionSpec, packageVersionRevisions, packageVersionSource, packageVersionTimestamp), RevisionSpec (RevisionSpec, revisionTimestamp), revisedCabalFile) +import Foliage.Meta (PackageVersionSource, PackageVersionSpec (PackageVersionSpec, packageVersionRevisions, packageVersionSource, packageVersionTimestamp), RevisionSpec (RevisionSpec, revisionTimestamp), revisedCabalFile) import Foliage.Meta.Aeson () import Foliage.Shake (originalCabalFile, readGenericPackageDescription') import Foliage.Utils.Aeson (MyAesonEncoding (..)) @@ -55,12 +55,12 @@ data AllPackagesPageEntry = AllPackagesPageEntry deriving stock (Generic) deriving (ToJSON) via MyAesonEncoding AllPackagesPageEntry -makeAllPackagesPage :: UTCTime -> FilePath -> [PackageVersionMeta] -> Action () +makeAllPackagesPage :: UTCTime -> FilePath -> [(PackageId, PackageVersionSpec)] -> Action () makeAllPackagesPage currentTime outputDir packageVersions = do let packages = sortOn allPackagesPageEntryPkgId $ map - ( ( \PackageVersionMeta {pkgId, pkgSpec = PackageVersionSpec {packageVersionTimestamp, packageVersionRevisions, packageVersionSource}} -> + ( ( \(pkgId, PackageVersionSpec {packageVersionTimestamp, packageVersionRevisions, packageVersionSource}) -> AllPackagesPageEntry { allPackagesPageEntryPkgId = pkgId, allPackagesPageEntryTimestamp = fromMaybe currentTime packageVersionTimestamp, @@ -70,9 +70,9 @@ makeAllPackagesPage currentTime outputDir packageVersions = do } ) . head - . sortOn (Down . pkgVersion . pkgId) + . sortOn (Down . pkgVersion . fst) ) - $ groupBy ((==) `on` (pkgName . pkgId)) packageVersions + $ groupBy ((==) `on` (pkgName . fst)) packageVersions traced "webpages / all-packages" $ do IO.createDirectoryIfMissing True (outputDir "all-packages") TL.writeFile (outputDir "all-packages" "index.html") $ @@ -94,12 +94,12 @@ data AllPackageVersionsPageEntry deriving stock (Generic) deriving (ToJSON) via MyAesonEncoding AllPackageVersionsPageEntry -makeAllPackageVersionsPage :: UTCTime -> FilePath -> [PackageVersionMeta] -> Action () +makeAllPackageVersionsPage :: UTCTime -> FilePath -> [(PackageId, PackageVersionSpec)] -> Action () makeAllPackageVersionsPage currentTime outputDir packageVersions = do let entries = sortOn (Down . allPackageVersionsPageEntryTimestamp) $ foldMap - ( \PackageVersionMeta {pkgId, pkgSpec = PackageVersionSpec {packageVersionTimestamp, packageVersionRevisions, packageVersionSource}} -> + ( \(pkgId, PackageVersionSpec {packageVersionTimestamp, packageVersionRevisions, packageVersionSource}) -> AllPackageVersionsPageEntryPackage { allPackageVersionsPageEntryPkgId = pkgId, allPackageVersionsPageEntryTimestamp = fromMaybe currentTime packageVersionTimestamp, @@ -124,9 +124,9 @@ makeAllPackageVersionsPage currentTime outputDir packageVersions = do renderMustache allPackageVersionsPageTemplate $ object ["entries" .= entries] -makePackageVersionPage :: FilePath -> FilePath -> PackageVersionMeta -> Action () -makePackageVersionPage inputDir outputDir pkgMeta@PackageVersionMeta {pkgId, pkgSpec} = do - cabalFilePath <- maybe (originalCabalFile pkgMeta) pure (revisedCabalFile inputDir pkgMeta) +makePackageVersionPage :: FilePath -> FilePath -> PackageId -> PackageVersionSpec -> Action () +makePackageVersionPage inputDir outputDir pkgId pkgSpec = do + cabalFilePath <- maybe (originalCabalFile pkgId pkgSpec) pure (revisedCabalFile inputDir pkgId pkgSpec) pkgDesc <- readGenericPackageDescription' cabalFilePath traced ("webpages / package / " ++ prettyShow pkgId) $ do IO.createDirectoryIfMissing True (outputDir "package" prettyShow pkgId) diff --git a/app/Foliage/RemoteAsset.hs b/app/Foliage/RemoteAsset.hs index 801c2e6..e5f1806 100644 --- a/app/Foliage/RemoteAsset.hs +++ b/app/Foliage/RemoteAsset.hs @@ -43,8 +43,10 @@ addFetchRemoteAssetRule cacheDir = addBuiltinRule noLint noIdentity run "Fragments in URI are not supported: " <> show uri let scheme = dropWhileEnd (not . isAlpha) $ uriScheme uri - Just host = uriRegName <$> uriAuthority uri - path = cacheDir joinPath (scheme : host : pathSegments uri) + host <- case uriRegName <$> uriAuthority uri of + Nothing -> fail $ "invalid uri " ++ show uri + Just host -> pure host + let path = cacheDir joinPath (scheme : host : pathSegments uri) -- parse etag from store let oldETag = fromMaybe BS.empty old diff --git a/app/Foliage/Shake.hs b/app/Foliage/Shake.hs index 0885b7e..6f92937 100644 --- a/app/Foliage/Shake.hs +++ b/app/Foliage/Shake.hs @@ -42,7 +42,7 @@ readGenericPackageDescription' fp = do need [fp] liftIO $ readGenericPackageDescription Verbosity.silent fp -originalCabalFile :: PackageVersionMeta -> Action FilePath -originalCabalFile PackageVersionMeta {pkgId, pkgSpec} = do +originalCabalFile :: PackageId -> PackageVersionSpec -> Action FilePath +originalCabalFile pkgId pkgSpec = do srcDir <- prepareSource pkgId pkgSpec return $ srcDir unPackageName (pkgName pkgId) <.> "cabal" From 9a4d097cde69e42f3905252f3797c745b5a3b70d Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 2 Mar 2023 15:16:10 +0800 Subject: [PATCH 32/43] Rework internals and add consistency check The function preparePackageVersion is now responsible for doing everything we need to do to be able to include the package in the index. The function also returns a denormalised view of the package information which can be taken as a proof that everything is consistent. --- app/Foliage/CmdBuild.hs | 68 ++++++++----------- app/Foliage/CmdImportIndex.hs | 2 + app/Foliage/Meta.hs | 13 ---- app/Foliage/Pages.hs | 97 +++++++++++++++------------- app/Foliage/PreparePackageVersion.hs | 84 ++++++++++++++++++++++++ app/Foliage/PrepareSdist.hs | 3 +- app/Foliage/Shake.hs | 9 --- app/Main.hs | 2 + foliage.cabal | 5 +- templates/packageVersion.mustache | 16 ++--- 10 files changed, 177 insertions(+), 122 deletions(-) create mode 100644 app/Foliage/PreparePackageVersion.hs diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index 298ec3c..fc45c91 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -1,4 +1,5 @@ {-# LANGUAGE DerivingVia #-} +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} module Foliage.CmdBuild (cmdBuild) where @@ -24,8 +25,9 @@ import Foliage.Meta import Foliage.Meta.Aeson () import Foliage.Options import Foliage.Pages -import Foliage.PrepareSdist -import Foliage.PrepareSource (addPrepareSourceRule, prepareSource) +import Foliage.PreparePackageVersion (PreparedPackageVersion (..), preparePackageVersion) +import Foliage.PrepareSdist (addPrepareSdistRule) +import Foliage.PrepareSource (addPrepareSourceRule) import Foliage.RemoteAsset (addFetchRemoteAssetRule) import Foliage.Shake import Foliage.Time qualified as Time @@ -99,31 +101,21 @@ buildAction when doWritePackageMeta $ makeMetadataFile outputDir packageVersions - void $ forP packageVersions $ uncurry $ makePackageVersionPage inputDir outputDir + void $ forP packageVersions $ makePackageVersionPage outputDir - void $ forP packageVersions $ \(pkgId, pkgSpec) -> do + void $ forP packageVersions $ \PreparedPackageVersion {pkgId, cabalFilePath} -> do let PackageIdentifier {pkgName, pkgVersion} = pkgId - cabalFilePath <- maybe (originalCabalFile pkgId pkgSpec) pure (revisedCabalFile inputDir pkgId pkgSpec) copyFileChanged cabalFilePath (outputDir "index" prettyShow pkgName prettyShow pkgVersion prettyShow pkgName <.> "cabal") cabalEntries <- foldMap - ( \(pkgId, pkgSpec) -> do - let PackageVersionSpec {packageVersionTimestamp, packageVersionRevisions} = pkgSpec - + ( \PreparedPackageVersion {pkgId, pkgTimestamp, originalCabalFilePath, cabalFileRevisions} -> do -- original cabal file, with its timestamp (if specified) - cabalFilePath <- originalCabalFile pkgId pkgSpec - let cabalFileTimestamp = fromMaybe currentTime packageVersionTimestamp - cf <- prepareIndexPkgCabal pkgId cabalFileTimestamp cabalFilePath + let cabalFileTimestamp = fromMaybe currentTime pkgTimestamp + cf <- prepareIndexPkgCabal pkgId cabalFileTimestamp originalCabalFilePath -- all revised cabal files, with their timestamp - revcf <- - for packageVersionRevisions $ - \RevisionSpec {revisionTimestamp, revisionNumber} -> - prepareIndexPkgCabal - pkgId - revisionTimestamp - (cabalFileRevisionPath inputDir pkgId revisionNumber) + revcf <- for cabalFileRevisions $ uncurry (prepareIndexPkgCabal pkgId) return $ cf : revcf ) @@ -131,16 +123,15 @@ buildAction targetKeys <- maybeReadKeysAt "target" metadataEntries <- - forP packageVersions $ \(pkgId, pkgSpec) -> do + forP packageVersions $ \ppv@PreparedPackageVersion {pkgId, pkgTimestamp} -> do let PackageIdentifier {pkgName, pkgVersion} = pkgId - let PackageVersionSpec {packageVersionTimestamp} = pkgSpec - targets <- prepareIndexPkgMetadata expiryTime pkgId pkgSpec + targets <- prepareIndexPkgMetadata expiryTime ppv let path = outputDir "index" prettyShow pkgName prettyShow pkgVersion "package.json" liftIO $ BL.writeFile path $ renderSignedJSON targetKeys targets mkTarEntry (renderSignedJSON targetKeys targets) (IndexPkgMetadata pkgId) - (fromMaybe currentTime packageVersionTimestamp) + (fromMaybe currentTime pkgTimestamp) let tarContents = Tar.write $ sortOn Tar.entryTime (cabalEntries ++ metadataEntries) traced "Writing index" $ do @@ -230,7 +221,7 @@ buildAction timestampInfoSnapshot = snapshotInfo } -makeMetadataFile :: FilePath -> [(PackageId, PackageVersionSpec)] -> Action () +makeMetadataFile :: FilePath -> [PreparedPackageVersion] -> Action () makeMetadataFile outputDir packageVersions = traced "writing metadata" $ do createDirectoryIfMissing True (outputDir "foliage") Aeson.encodeFile @@ -238,20 +229,19 @@ makeMetadataFile outputDir packageVersions = traced "writing metadata" $ do (map encodePackageVersion packageVersions) where encodePackageVersion - ( PackageIdentifier {pkgName, pkgVersion}, - PackageVersionSpec - { packageVersionSource, - packageVersionForce, - packageVersionTimestamp - } - ) = + PreparedPackageVersion + { pkgId = PackageIdentifier {pkgName, pkgVersion}, + pkgTimestamp, + pkgVersionForce, + pkgVersionSource + } = Aeson.object ( [ "pkg-name" Aeson..= pkgName, "pkg-version" Aeson..= pkgVersion, - "url" Aeson..= sourceUrl packageVersionSource + "url" Aeson..= sourceUrl pkgVersionSource ] - ++ ["forced-version" Aeson..= True | packageVersionForce] - ++ (case packageVersionTimestamp of Nothing -> []; Just t -> ["timestamp" Aeson..= t]) + ++ ["forced-version" Aeson..= True | pkgVersionForce] + ++ (case pkgTimestamp of Nothing -> []; Just t -> ["timestamp" Aeson..= t]) ) sourceUrl :: PackageVersionSource -> URI @@ -269,7 +259,7 @@ makeMetadataFile outputDir packageVersions = traced "writing metadata" $ do uriQuery = "?dir=" ++ subdir } -getPackageVersions :: FilePath -> Action [(PackageId, PackageVersionSpec)] +getPackageVersions :: FilePath -> Action [PreparedPackageVersion] getPackageVersions inputDir = do metaFiles <- getDirectoryFiles inputDir ["*/*/meta.toml"] @@ -314,7 +304,7 @@ getPackageVersions inputDir = do meta -> return meta - return (pkgId, pkgSpec) + preparePackageVersion inputDir pkgId pkgSpec prepareIndexPkgCabal :: PackageId -> UTCTime -> FilePath -> Action Tar.Entry prepareIndexPkgCabal pkgId timestamp filePath = do @@ -322,11 +312,9 @@ prepareIndexPkgCabal pkgId timestamp filePath = do contents <- liftIO $ BS.readFile filePath mkTarEntry (BL.fromStrict contents) (IndexPkgCabal pkgId) timestamp -prepareIndexPkgMetadata :: Maybe UTCTime -> PackageId -> PackageVersionSpec -> Action Targets -prepareIndexPkgMetadata expiryTime pkgId pkgSpec = do - srcDir <- prepareSource pkgId pkgSpec - sdist <- prepareSdist srcDir - targetFileInfo <- computeFileInfoSimple' sdist +prepareIndexPkgMetadata :: Maybe UTCTime -> PreparedPackageVersion -> Action Targets +prepareIndexPkgMetadata expiryTime PreparedPackageVersion {pkgId, sdistPath} = do + targetFileInfo <- computeFileInfoSimple' sdistPath let packagePath = repoLayoutPkgTarGz hackageRepoLayout pkgId return Targets diff --git a/app/Foliage/CmdImportIndex.hs b/app/Foliage/CmdImportIndex.hs index f387dca..25ec972 100644 --- a/app/Foliage/CmdImportIndex.hs +++ b/app/Foliage/CmdImportIndex.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE ViewPatterns #-} + module Foliage.CmdImportIndex ( cmdImportIndex, ) diff --git a/app/Foliage/Meta.hs b/app/Foliage/Meta.hs index 429f404..7130367 100644 --- a/app/Foliage/Meta.hs +++ b/app/Foliage/Meta.hs @@ -31,8 +31,6 @@ module Foliage.Meta UTCTime, latestRevisionNumber, consolidateRanges, - cabalFileRevisionPath, - revisedCabalFile, ) where @@ -49,8 +47,6 @@ import Distribution.Aeson () import Distribution.Parsec (simpleParsec) import Distribution.Pretty (prettyShow) import Distribution.Types.Orphans () -import Distribution.Types.PackageId (PackageId, PackageIdentifier (..)) -import Distribution.Types.PackageName (unPackageName) import Distribution.Types.Version (Version) import Distribution.Types.VersionRange (VersionRange, anyVersion, intersectVersionRanges, notThisVersion) import Distribution.Version (isAnyVersion, isNoVersion, simplifyVersionRange) @@ -58,7 +54,6 @@ import Foliage.Time (UTCTime) import GHC.Generics (Generic) import Network.URI (URI, parseURI) import Network.URI.Orphans () -import System.FilePath ((<.>), ()) import Toml (TomlCodec, (.=)) import Toml qualified @@ -240,11 +235,3 @@ consolidateRanges PackageMetaEntry {packageMetaEntryPreferred, packageMetaEntryD range = simplifyVersionRange $ foldr intersectVersionRanges anyVersion (map notThisVersion packageMetaEntryDeprecated ++ packageMetaEntryPreferred) - -cabalFileRevisionPath :: FilePath -> PackageIdentifier -> Int -> FilePath -cabalFileRevisionPath inputDir PackageIdentifier {pkgName, pkgVersion} revisionNumber = - inputDir unPackageName pkgName prettyShow pkgVersion "revisions" show revisionNumber <.> "cabal" - -revisedCabalFile :: FilePath -> PackageId -> PackageVersionSpec -> Maybe FilePath -revisedCabalFile inputDir pkgId pkgSpec = do - cabalFileRevisionPath inputDir pkgId <$> latestRevisionNumber pkgSpec diff --git a/app/Foliage/Pages.hs b/app/Foliage/Pages.hs index 9bd714c..523dff0 100644 --- a/app/Foliage/Pages.hs +++ b/app/Foliage/Pages.hs @@ -15,7 +15,7 @@ module Foliage.Pages where import Data.Aeson (KeyValue ((.=)), ToJSON, object) -import Data.Function (on) +import Data.Function (on, (&)) import Data.List (groupBy, sortOn) import Data.Maybe (fromMaybe, listToMaybe) import Data.Ord (Down (Down)) @@ -24,11 +24,12 @@ import Data.Time (UTCTime) import Data.Time.Clock.POSIX (POSIXTime, utcTimeToPOSIXSeconds) import Development.Shake (Action, traced) import Distribution.Aeson (jsonGenericPackageDescription) -import Distribution.Package (PackageId, PackageIdentifier (pkgName, pkgVersion)) +import Distribution.Package (PackageIdentifier (pkgName, pkgVersion)) import Distribution.Pretty (prettyShow) -import Foliage.Meta (PackageVersionSource, PackageVersionSpec (PackageVersionSpec, packageVersionRevisions, packageVersionSource, packageVersionTimestamp), RevisionSpec (RevisionSpec, revisionTimestamp), revisedCabalFile) +import Foliage.Meta (PackageVersionSource) import Foliage.Meta.Aeson () -import Foliage.Shake (originalCabalFile, readGenericPackageDescription') +import Foliage.PreparePackageVersion (PreparedPackageVersion (..)) +import Foliage.Shake (readGenericPackageDescription') import Foliage.Utils.Aeson (MyAesonEncoding (..)) import GHC.Generics (Generic) import System.Directory qualified as IO @@ -50,29 +51,32 @@ data AllPackagesPageEntry = AllPackagesPageEntry allPackagesPageEntryTimestamp :: UTCTime, allPackagesPageEntryTimestampPosix :: POSIXTime, allPackagesPageEntrySource :: PackageVersionSource, - allPackagesPageEntryRevision :: Maybe RevisionSpec + allPackagesPageEntryLatestRevisionTimestamp :: Maybe UTCTime } deriving stock (Generic) deriving (ToJSON) via MyAesonEncoding AllPackagesPageEntry -makeAllPackagesPage :: UTCTime -> FilePath -> [(PackageId, PackageVersionSpec)] -> Action () +makeAllPackagesPage :: UTCTime -> FilePath -> [PreparedPackageVersion] -> Action () makeAllPackagesPage currentTime outputDir packageVersions = do let packages = - sortOn allPackagesPageEntryPkgId - $ map - ( ( \(pkgId, PackageVersionSpec {packageVersionTimestamp, packageVersionRevisions, packageVersionSource}) -> - AllPackagesPageEntry - { allPackagesPageEntryPkgId = pkgId, - allPackagesPageEntryTimestamp = fromMaybe currentTime packageVersionTimestamp, - allPackagesPageEntryTimestampPosix = utcTimeToPOSIXSeconds (fromMaybe currentTime packageVersionTimestamp), - allPackagesPageEntrySource = packageVersionSource, - allPackagesPageEntryRevision = listToMaybe packageVersionRevisions - } - ) - . head - . sortOn (Down . pkgVersion . fst) + packageVersions + & groupBy ((==) `on` (pkgName . pkgId)) + & map + ( \group -> + group + & sortOn (Down . pkgVersion . pkgId) + & head + & ( \(PreparedPackageVersion {pkgId, pkgTimestamp, cabalFileRevisions, pkgVersionSource}) -> + AllPackagesPageEntry + { allPackagesPageEntryPkgId = pkgId, + allPackagesPageEntryTimestamp = fromMaybe currentTime pkgTimestamp, + allPackagesPageEntryTimestampPosix = utcTimeToPOSIXSeconds (fromMaybe currentTime pkgTimestamp), + allPackagesPageEntrySource = pkgVersionSource, + allPackagesPageEntryLatestRevisionTimestamp = fst <$> listToMaybe cabalFileRevisions + } + ) ) - $ groupBy ((==) `on` (pkgName . fst)) packageVersions + & sortOn allPackagesPageEntryPkgId traced "webpages / all-packages" $ do IO.createDirectoryIfMissing True (outputDir "all-packages") TL.writeFile (outputDir "all-packages" "index.html") $ @@ -94,29 +98,29 @@ data AllPackageVersionsPageEntry deriving stock (Generic) deriving (ToJSON) via MyAesonEncoding AllPackageVersionsPageEntry -makeAllPackageVersionsPage :: UTCTime -> FilePath -> [(PackageId, PackageVersionSpec)] -> Action () +makeAllPackageVersionsPage :: UTCTime -> FilePath -> [PreparedPackageVersion] -> Action () makeAllPackageVersionsPage currentTime outputDir packageVersions = do let entries = - sortOn (Down . allPackageVersionsPageEntryTimestamp) $ - foldMap - ( \(pkgId, PackageVersionSpec {packageVersionTimestamp, packageVersionRevisions, packageVersionSource}) -> - AllPackageVersionsPageEntryPackage - { allPackageVersionsPageEntryPkgId = pkgId, - allPackageVersionsPageEntryTimestamp = fromMaybe currentTime packageVersionTimestamp, - allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds (fromMaybe currentTime packageVersionTimestamp), - allPackageVersionsPageEntrySource = packageVersionSource - } - : map - ( \RevisionSpec {revisionTimestamp} -> - AllPackageVersionsPageEntryRevision - { allPackageVersionsPageEntryPkgId = pkgId, - allPackageVersionsPageEntryTimestamp = revisionTimestamp, - allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds revisionTimestamp - } - ) - packageVersionRevisions - ) - packageVersions + foldMap + ( \PreparedPackageVersion {pkgId, pkgTimestamp, pkgVersionSource, cabalFileRevisions} -> + AllPackageVersionsPageEntryPackage + { allPackageVersionsPageEntryPkgId = pkgId, + allPackageVersionsPageEntryTimestamp = fromMaybe currentTime pkgTimestamp, + allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds (fromMaybe currentTime pkgTimestamp), + allPackageVersionsPageEntrySource = pkgVersionSource + } + : map + ( \(revisionTimestamp, _) -> + AllPackageVersionsPageEntryRevision + { allPackageVersionsPageEntryPkgId = pkgId, + allPackageVersionsPageEntryTimestamp = revisionTimestamp, + allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds revisionTimestamp + } + ) + cabalFileRevisions + ) + packageVersions + & sortOn (Down . allPackageVersionsPageEntryTimestamp) traced "webpages / all-package-versions" $ do IO.createDirectoryIfMissing True (outputDir "all-package-versions") @@ -124,17 +128,18 @@ makeAllPackageVersionsPage currentTime outputDir packageVersions = do renderMustache allPackageVersionsPageTemplate $ object ["entries" .= entries] -makePackageVersionPage :: FilePath -> FilePath -> PackageId -> PackageVersionSpec -> Action () -makePackageVersionPage inputDir outputDir pkgId pkgSpec = do - cabalFilePath <- maybe (originalCabalFile pkgId pkgSpec) pure (revisedCabalFile inputDir pkgId pkgSpec) +makePackageVersionPage :: FilePath -> PreparedPackageVersion -> Action () +makePackageVersionPage outputDir PreparedPackageVersion {pkgId, pkgTimestamp, pkgVersionSource, cabalFilePath, cabalFileRevisions} = do pkgDesc <- readGenericPackageDescription' cabalFilePath traced ("webpages / package / " ++ prettyShow pkgId) $ do IO.createDirectoryIfMissing True (outputDir "package" prettyShow pkgId) TL.writeFile (outputDir "package" prettyShow pkgId "index.html") $ renderMustache packageVersionPageTemplate $ object - [ "pkgSpec" .= pkgSpec, - "pkgDesc" .= jsonGenericPackageDescription pkgDesc + [ "pkgVersionSource" .= pkgVersionSource, + "cabalFileRevisions" .= cabalFileRevisions, + "pkgDesc" .= jsonGenericPackageDescription pkgDesc, + "pkgTimestamp" .= pkgTimestamp ] indexPageTemplate :: Template diff --git a/app/Foliage/PreparePackageVersion.hs b/app/Foliage/PreparePackageVersion.hs new file mode 100644 index 0000000..6175e53 --- /dev/null +++ b/app/Foliage/PreparePackageVersion.hs @@ -0,0 +1,84 @@ +{-# LANGUAGE PatternSynonyms #-} + +module Foliage.PreparePackageVersion + ( PreparedPackageVersion + ( pkgId, + pkgTimestamp, + pkgVersionSource, + pkgVersionForce, + sdistPath, + cabalFilePath, + originalCabalFilePath, + cabalFileRevisions + ), + pattern PreparedPackageVersion, + preparePackageVersion, + ) +where + +import Control.Monad (unless) +import Data.List (sortOn) +import Data.Ord (Down (Down)) +import Development.Shake (Action) +import Distribution.Client.Compat.Prelude (prettyShow) +import Distribution.Types.PackageId +import Foliage.Meta (PackageVersionSource, PackageVersionSpec (..), RevisionSpec (..), UTCTime, latestRevisionNumber) +import Foliage.PrepareSdist (prepareSdist) +import Foliage.PrepareSource (prepareSource) +import System.FilePath (takeBaseName, takeFileName, (<.>), ()) + +data PreparedPackageVersion = PreparedPackageVersion + { pkgId :: PackageId, + pkgTimestamp :: Maybe UTCTime, + pkgVersionSource :: PackageVersionSource, + pkgVersionForce :: Bool, + sdistPath :: FilePath, + cabalFilePath :: FilePath, + originalCabalFilePath :: FilePath, + cabalFileRevisions :: [(UTCTime, FilePath)] + } + +preparePackageVersion :: FilePath -> PackageId -> PackageVersionSpec -> Action PreparedPackageVersion +preparePackageVersion inputDir pkgId pkgSpec = do + srcDir <- prepareSource pkgId pkgSpec + + let PackageIdentifier pkgName pkgVersion = pkgId + originalCabalFilePath = srcDir prettyShow pkgName <.> "cabal" + cabalFileRevisionPath revisionNumber = inputDir prettyShow pkgName prettyShow pkgVersion "revisions" show revisionNumber <.> "cabal" + + let cabalFilePath = + maybe + originalCabalFilePath + cabalFileRevisionPath + (latestRevisionNumber pkgSpec) + + sdistPath <- prepareSdist srcDir + + let expectedSdistName = prettyShow pkgId <.> "tar.gz" + unless (takeFileName sdistPath == expectedSdistName) $ + fail $ + unlines + [ "creating a source distribution for " ++ prettyShow pkgId ++ " has failed because", + "cabal has produced a source distribtion that does not match the expected file name:", + "actual: " ++ takeBaseName sdistPath, + "expected: " ++ expectedSdistName + ] + + let cabalFileRevisions = + sortOn + (Down . fst) + [ (revisionTimestamp, cabalFileRevisionPath revisionNumber) + | RevisionSpec {revisionTimestamp, revisionNumber} <- packageVersionRevisions pkgSpec + ] + + return + PreparedPackageVersion + { pkgId, + pkgTimestamp = packageVersionTimestamp pkgSpec, + pkgVersionSource = packageVersionSource pkgSpec, + pkgVersionForce = packageVersionForce pkgSpec, + sdistPath, + cabalFilePath, + originalCabalFilePath, + cabalFileRevisions + } diff --git a/app/Foliage/PrepareSdist.hs b/app/Foliage/PrepareSdist.hs index 681d9ec..acb329b 100644 --- a/app/Foliage/PrepareSdist.hs +++ b/app/Foliage/PrepareSdist.hs @@ -1,7 +1,6 @@ {-# LANGUAGE DataKinds #-} -{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DerivingStrategies #-} -{-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE TypeFamilies #-} module Foliage.PrepareSdist diff --git a/app/Foliage/Shake.hs b/app/Foliage/Shake.hs index 6f92937..a28883a 100644 --- a/app/Foliage/Shake.hs +++ b/app/Foliage/Shake.hs @@ -3,7 +3,6 @@ module Foliage.Shake readKeysAt, readPackageVersionSpec', readGenericPackageDescription', - originalCabalFile, ) where @@ -12,12 +11,9 @@ import Development.Shake import Development.Shake.FilePath import Distribution.Simple.PackageDescription import Distribution.Types.GenericPackageDescription -import Distribution.Types.PackageId -import Distribution.Types.PackageName import Distribution.Verbosity qualified as Verbosity import Foliage.HackageSecurity import Foliage.Meta -import Foliage.PrepareSource computeFileInfoSimple' :: FilePath -> Action FileInfo computeFileInfoSimple' fp = do @@ -41,8 +37,3 @@ readGenericPackageDescription' :: FilePath -> Action GenericPackageDescription readGenericPackageDescription' fp = do need [fp] liftIO $ readGenericPackageDescription Verbosity.silent fp - -originalCabalFile :: PackageId -> PackageVersionSpec -> Action FilePath -originalCabalFile pkgId pkgSpec = do - srcDir <- prepareSource pkgId pkgSpec - return $ srcDir unPackageName (pkgName pkgId) <.> "cabal" diff --git a/app/Main.hs b/app/Main.hs index 323d271..baca0ad 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE LambdaCase #-} + module Main where import Foliage.CmdBuild diff --git a/foliage.cabal b/foliage.cabal index 71a3eed..c89401c 100644 --- a/foliage.cabal +++ b/foliage.cabal @@ -27,6 +27,7 @@ executable foliage Foliage.Meta.Aeson Foliage.Options Foliage.Pages + Foliage.PreparePackageVersion Foliage.PrepareSource Foliage.PrepareSdist Foliage.RemoteAsset @@ -37,9 +38,7 @@ executable foliage Foliage.Utils.GitHub Network.URI.Orphans - default-language: Haskell2010 - default-extensions: - ImportQualifiedPost LambdaCase NamedFieldPuns ViewPatterns + default-language: GHC2021 ghc-options: -Wall ghc-options: -threaded diff --git a/templates/packageVersion.mustache b/templates/packageVersion.mustache index a25ea9a..cc13b7a 100644 --- a/templates/packageVersion.mustache +++ b/templates/packageVersion.mustache @@ -42,28 +42,26 @@
License

{{license}}

{{/pkgDesc}} - {{#pkgSpec}} - {{#packageVersionSource}} + {{#pkgVersionSource}}
Source
{{> packageVersionSource}}
- {{/packageVersionSource}} + {{/pkgVersionSource}}
Timestamp
-

{{packageVersionTimestamp}}

+

{{pkgTimestamp}}

Revisions
- {{#packageVersionRevisions}} + {{#cabalFileRevisions}}

{{revisionTimestamp}}

- {{/packageVersionRevisions}} - {{^packageVersionRevisions}} + {{/cabalFileRevisions}} + {{^cabalFileRevisions}}

None

- {{/packageVersionRevisions}} + {{/cabalFileRevisions}}
- {{/pkgSpec}} From 7ec1fb6ef271151a4ba832eae29ee9f03042ac7f Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 3 Mar 2023 08:23:01 +0800 Subject: [PATCH 33/43] Use NonEmpty to avoid partiality --- app/Foliage/Pages.hs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/Foliage/Pages.hs b/app/Foliage/Pages.hs index 523dff0..24347eb 100644 --- a/app/Foliage/Pages.hs +++ b/app/Foliage/Pages.hs @@ -16,9 +16,10 @@ where import Data.Aeson (KeyValue ((.=)), ToJSON, object) import Data.Function (on, (&)) -import Data.List (groupBy, sortOn) +import Data.List (sortOn) +import Data.List.NonEmpty qualified as NE import Data.Maybe (fromMaybe, listToMaybe) -import Data.Ord (Down (Down)) +import Data.Ord (Down (Down), comparing) import Data.Text.Lazy.IO.Utf8 qualified as TL import Data.Time (UTCTime) import Data.Time.Clock.POSIX (POSIXTime, utcTimeToPOSIXSeconds) @@ -60,12 +61,12 @@ makeAllPackagesPage :: UTCTime -> FilePath -> [PreparedPackageVersion] -> Action makeAllPackagesPage currentTime outputDir packageVersions = do let packages = packageVersions - & groupBy ((==) `on` (pkgName . pkgId)) + & NE.groupBy ((==) `on` (pkgName . pkgId)) & map ( \group -> group - & sortOn (Down . pkgVersion . pkgId) - & head + & NE.sortBy (comparing $ Down . pkgVersion . pkgId) + & NE.head & ( \(PreparedPackageVersion {pkgId, pkgTimestamp, cabalFileRevisions, pkgVersionSource}) -> AllPackagesPageEntry { allPackagesPageEntryPkgId = pkgId, From 5a3d546bdc3686b174c6ba6cef629222bb6ec492 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 3 Mar 2023 08:46:07 +0800 Subject: [PATCH 34/43] Add more comments --- app/Foliage/Pages.hs | 95 ++++++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 43 deletions(-) diff --git a/app/Foliage/Pages.hs b/app/Foliage/Pages.hs index 24347eb..351613a 100644 --- a/app/Foliage/Pages.hs +++ b/app/Foliage/Pages.hs @@ -58,31 +58,38 @@ data AllPackagesPageEntry = AllPackagesPageEntry deriving (ToJSON) via MyAesonEncoding AllPackagesPageEntry makeAllPackagesPage :: UTCTime -> FilePath -> [PreparedPackageVersion] -> Action () -makeAllPackagesPage currentTime outputDir packageVersions = do - let packages = - packageVersions - & NE.groupBy ((==) `on` (pkgName . pkgId)) - & map - ( \group -> - group - & NE.sortBy (comparing $ Down . pkgVersion . pkgId) - & NE.head - & ( \(PreparedPackageVersion {pkgId, pkgTimestamp, cabalFileRevisions, pkgVersionSource}) -> - AllPackagesPageEntry - { allPackagesPageEntryPkgId = pkgId, - allPackagesPageEntryTimestamp = fromMaybe currentTime pkgTimestamp, - allPackagesPageEntryTimestampPosix = utcTimeToPOSIXSeconds (fromMaybe currentTime pkgTimestamp), - allPackagesPageEntrySource = pkgVersionSource, - allPackagesPageEntryLatestRevisionTimestamp = fst <$> listToMaybe cabalFileRevisions - } - ) - ) - & sortOn allPackagesPageEntryPkgId +makeAllPackagesPage currentTime outputDir packageVersions = traced "webpages / all-packages" $ do IO.createDirectoryIfMissing True (outputDir "all-packages") TL.writeFile (outputDir "all-packages" "index.html") $ renderMustache allPackagesPageTemplate $ object ["packages" .= packages] + where + packages = + packageVersions + -- group package versions by package name + & NE.groupBy ((==) `on` (pkgName . pkgId)) + -- for each package name pick the most recent version + & map + ( \group -> + group + -- sort them from the most recent version to the least recent + & NE.sortBy (comparing $ Down . pkgVersion . pkgId) + -- pick the most recent version + & NE.head + -- turn it into the template data + & ( \(PreparedPackageVersion {pkgId, pkgTimestamp, cabalFileRevisions, pkgVersionSource}) -> + AllPackagesPageEntry + { allPackagesPageEntryPkgId = pkgId, + allPackagesPageEntryTimestamp = fromMaybe currentTime pkgTimestamp, + allPackagesPageEntryTimestampPosix = utcTimeToPOSIXSeconds (fromMaybe currentTime pkgTimestamp), + allPackagesPageEntrySource = pkgVersionSource, + allPackagesPageEntryLatestRevisionTimestamp = fst <$> listToMaybe cabalFileRevisions + } + ) + ) + -- sort packages by pkgId + & sortOn allPackagesPageEntryPkgId data AllPackageVersionsPageEntry = AllPackageVersionsPageEntryPackage @@ -100,34 +107,36 @@ data AllPackageVersionsPageEntry deriving (ToJSON) via MyAesonEncoding AllPackageVersionsPageEntry makeAllPackageVersionsPage :: UTCTime -> FilePath -> [PreparedPackageVersion] -> Action () -makeAllPackageVersionsPage currentTime outputDir packageVersions = do - let entries = - foldMap - ( \PreparedPackageVersion {pkgId, pkgTimestamp, pkgVersionSource, cabalFileRevisions} -> - AllPackageVersionsPageEntryPackage - { allPackageVersionsPageEntryPkgId = pkgId, - allPackageVersionsPageEntryTimestamp = fromMaybe currentTime pkgTimestamp, - allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds (fromMaybe currentTime pkgTimestamp), - allPackageVersionsPageEntrySource = pkgVersionSource - } - : map - ( \(revisionTimestamp, _) -> - AllPackageVersionsPageEntryRevision - { allPackageVersionsPageEntryPkgId = pkgId, - allPackageVersionsPageEntryTimestamp = revisionTimestamp, - allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds revisionTimestamp - } - ) - cabalFileRevisions - ) - packageVersions - & sortOn (Down . allPackageVersionsPageEntryTimestamp) - +makeAllPackageVersionsPage currentTime outputDir packageVersions = traced "webpages / all-package-versions" $ do IO.createDirectoryIfMissing True (outputDir "all-package-versions") TL.writeFile (outputDir "all-package-versions" "index.html") $ renderMustache allPackageVersionsPageTemplate $ object ["entries" .= entries] + where + entries = + -- collect all cabal file revisions including the original cabal file + foldMap + ( \PreparedPackageVersion {pkgId, pkgTimestamp, pkgVersionSource, cabalFileRevisions} -> + -- original cabal file + AllPackageVersionsPageEntryPackage + { allPackageVersionsPageEntryPkgId = pkgId, + allPackageVersionsPageEntryTimestamp = fromMaybe currentTime pkgTimestamp, + allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds (fromMaybe currentTime pkgTimestamp), + allPackageVersionsPageEntrySource = pkgVersionSource + } + -- list of revisions + : [ AllPackageVersionsPageEntryRevision + { allPackageVersionsPageEntryPkgId = pkgId, + allPackageVersionsPageEntryTimestamp = revisionTimestamp, + allPackageVersionsPageEntryTimestampPosix = utcTimeToPOSIXSeconds revisionTimestamp + } + | (revisionTimestamp, _) <- cabalFileRevisions + ] + ) + packageVersions + -- sort them by timestamp + & sortOn (Down . allPackageVersionsPageEntryTimestamp) makePackageVersionPage :: FilePath -> PreparedPackageVersion -> Action () makePackageVersionPage outputDir PreparedPackageVersion {pkgId, pkgTimestamp, pkgVersionSource, cabalFilePath, cabalFileRevisions} = do From 899a79cd5351dae7f524d1a8e10b0a1596125592 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 3 Mar 2023 08:51:33 +0800 Subject: [PATCH 35/43] Parsing the cabal file should go in preparePackageVersion --- app/Foliage/Pages.hs | 4 +--- app/Foliage/PreparePackageVersion.hs | 7 +++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/app/Foliage/Pages.hs b/app/Foliage/Pages.hs index 351613a..301180b 100644 --- a/app/Foliage/Pages.hs +++ b/app/Foliage/Pages.hs @@ -30,7 +30,6 @@ import Distribution.Pretty (prettyShow) import Foliage.Meta (PackageVersionSource) import Foliage.Meta.Aeson () import Foliage.PreparePackageVersion (PreparedPackageVersion (..)) -import Foliage.Shake (readGenericPackageDescription') import Foliage.Utils.Aeson (MyAesonEncoding (..)) import GHC.Generics (Generic) import System.Directory qualified as IO @@ -139,8 +138,7 @@ makeAllPackageVersionsPage currentTime outputDir packageVersions = & sortOn (Down . allPackageVersionsPageEntryTimestamp) makePackageVersionPage :: FilePath -> PreparedPackageVersion -> Action () -makePackageVersionPage outputDir PreparedPackageVersion {pkgId, pkgTimestamp, pkgVersionSource, cabalFilePath, cabalFileRevisions} = do - pkgDesc <- readGenericPackageDescription' cabalFilePath +makePackageVersionPage outputDir PreparedPackageVersion {pkgId, pkgTimestamp, pkgVersionSource, pkgDesc, cabalFileRevisions} = do traced ("webpages / package / " ++ prettyShow pkgId) $ do IO.createDirectoryIfMissing True (outputDir "package" prettyShow pkgId) TL.writeFile (outputDir "package" prettyShow pkgId "index.html") $ diff --git a/app/Foliage/PreparePackageVersion.hs b/app/Foliage/PreparePackageVersion.hs index 6175e53..dc4547d 100644 --- a/app/Foliage/PreparePackageVersion.hs +++ b/app/Foliage/PreparePackageVersion.hs @@ -6,6 +6,7 @@ module Foliage.PreparePackageVersion pkgTimestamp, pkgVersionSource, pkgVersionForce, + pkgDesc, sdistPath, cabalFilePath, originalCabalFilePath, @@ -21,10 +22,12 @@ import Data.List (sortOn) import Data.Ord (Down (Down)) import Development.Shake (Action) import Distribution.Client.Compat.Prelude (prettyShow) +import Distribution.PackageDescription (GenericPackageDescription) import Distribution.Types.PackageId import Foliage.Meta (PackageVersionSource, PackageVersionSpec (..), RevisionSpec (..), UTCTime, latestRevisionNumber) import Foliage.PrepareSdist (prepareSdist) import Foliage.PrepareSource (prepareSource) +import Foliage.Shake (readGenericPackageDescription') import System.FilePath (takeBaseName, takeFileName, (<.>), ()) data PreparedPackageVersion = PreparedPackageVersion @@ -32,6 +35,7 @@ data PreparedPackageVersion = PreparedPackageVersion pkgTimestamp :: Maybe UTCTime, pkgVersionSource :: PackageVersionSource, pkgVersionForce :: Bool, + pkgDesc :: GenericPackageDescription, sdistPath :: FilePath, cabalFilePath :: FilePath, originalCabalFilePath :: FilePath, @@ -52,6 +56,8 @@ preparePackageVersion inputDir pkgId pkgSpec = do cabalFileRevisionPath (latestRevisionNumber pkgSpec) + pkgDesc <- readGenericPackageDescription' cabalFilePath + sdistPath <- prepareSdist srcDir let expectedSdistName = prettyShow pkgId <.> "tar.gz" @@ -77,6 +83,7 @@ preparePackageVersion inputDir pkgId pkgSpec = do pkgTimestamp = packageVersionTimestamp pkgSpec, pkgVersionSource = packageVersionSource pkgSpec, pkgVersionForce = packageVersionForce pkgSpec, + pkgDesc, sdistPath, cabalFilePath, originalCabalFilePath, From 3e99be2211999e6e57e9cd5493f3b39970ea23fa Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 3 Mar 2023 11:51:11 +0800 Subject: [PATCH 36/43] Better diagnostics - Fold most of getPackageVersions into preparePackageVersion We pass to preparePackageVersion the metadata file we are dealing with so it can be mentioned in case of errors. - Replace `fail` by `error`. Shake uses exceptions to signal build failures. It turns out `error` gets displayed slightly better when caught by shake. Also we can use it in pure code. --- app/Foliage/CmdBuild.hs | 84 ++++++++-------------------- app/Foliage/PreparePackageVersion.hs | 63 +++++++++++++++++---- app/Foliage/PrepareSdist.hs | 20 ++++--- app/Foliage/RemoteAsset.hs | 12 ++-- 4 files changed, 93 insertions(+), 86 deletions(-) diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index fc45c91..06b91e7 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -1,5 +1,4 @@ {-# LANGUAGE DerivingVia #-} -{-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} module Foliage.CmdBuild (cmdBuild) where @@ -18,7 +17,6 @@ import Data.Traversable (for) import Development.Shake import Development.Shake.FilePath import Distribution.Package -import Distribution.Parsec (simpleParsec) import Distribution.Pretty (prettyShow) import Foliage.HackageSecurity hiding (ToJSON, toJSON) import Foliage.Meta @@ -128,10 +126,11 @@ buildAction targets <- prepareIndexPkgMetadata expiryTime ppv let path = outputDir "index" prettyShow pkgName prettyShow pkgVersion "package.json" liftIO $ BL.writeFile path $ renderSignedJSON targetKeys targets - mkTarEntry - (renderSignedJSON targetKeys targets) - (IndexPkgMetadata pkgId) - (fromMaybe currentTime pkgTimestamp) + pure $ + mkTarEntry + (renderSignedJSON targetKeys targets) + (IndexPkgMetadata pkgId) + (fromMaybe currentTime pkgTimestamp) let tarContents = Tar.write $ sortOn Tar.entryTime (cabalEntries ++ metadataEntries) traced "Writing index" $ do @@ -264,53 +263,19 @@ getPackageVersions inputDir = do metaFiles <- getDirectoryFiles inputDir ["*/*/meta.toml"] when (null metaFiles) $ do - putError $ + error $ unlines [ "We could not find any package metadata file (i.e. _sources///meta.toml)", "Make sure you are passing the right input directory. The default input directory is _sources" ] - fail "no package metadata found" - forP metaFiles $ \metaFile -> do - (pkgName, pkgVersion) <- case splitDirectories metaFile of - [pkgName, pkgVersion, _] -> pure (pkgName, pkgVersion) - _else -> fail $ "internal error: I should not be looking at " ++ metaFile - name <- case simpleParsec pkgName of - Nothing -> fail $ "invalid package name: " ++ pkgName - Just name -> pure name - version <- case simpleParsec pkgVersion of - Nothing -> fail $ "invalid package version: " ++ pkgVersion - Just version -> pure version - let pkgId = PackageIdentifier name version - - pkgSpec <- - readPackageVersionSpec' (inputDir metaFile) >>= \case - PackageVersionSpec {packageVersionRevisions, packageVersionTimestamp = Nothing} - | not (null packageVersionRevisions) -> do - putError $ - unlines - [ inputDir metaFile <> " has cabal file revisions but the original package has no timestamp.", - "This combination doesn't make sense. Either add a timestamp on the original package or remove the revisions" - ] - fail "invalid package metadata" - PackageVersionSpec {packageVersionRevisions, packageVersionTimestamp = Just pkgTs} - | any ((< pkgTs) . revisionTimestamp) packageVersionRevisions -> do - putError $ - unlines - [ inputDir metaFile <> " has a revision with timestamp earlier than the package itself.", - "Adjust the timestamps so that all revisions come after the original package" - ] - fail "invalid package metadata" - meta -> - return meta - - preparePackageVersion inputDir pkgId pkgSpec + forP metaFiles $ preparePackageVersion inputDir prepareIndexPkgCabal :: PackageId -> UTCTime -> FilePath -> Action Tar.Entry prepareIndexPkgCabal pkgId timestamp filePath = do need [filePath] contents <- liftIO $ BS.readFile filePath - mkTarEntry (BL.fromStrict contents) (IndexPkgCabal pkgId) timestamp + pure $ mkTarEntry (BL.fromStrict contents) (IndexPkgCabal pkgId) timestamp prepareIndexPkgMetadata :: Maybe UTCTime -> PreparedPackageVersion -> Action Targets prepareIndexPkgMetadata expiryTime PreparedPackageVersion {pkgId, sdistPath} = do @@ -324,24 +289,23 @@ prepareIndexPkgMetadata expiryTime PreparedPackageVersion {pkgId, sdistPath} = d targetsDelegations = Nothing } -mkTarEntry :: BL.ByteString -> IndexFile dec -> UTCTime -> Action Tar.Entry -mkTarEntry contents indexFile timestamp = do - tarPath <- case Tar.toTarPath False indexPath of - Left e -> fail $ "Invalid tar path " ++ indexPath ++ "(" ++ e ++ ")" - Right tarPath -> pure tarPath - - pure - (Tar.fileEntry tarPath contents) - { Tar.entryTime = floor $ Time.utcTimeToPOSIXSeconds timestamp, - Tar.entryOwnership = - Tar.Ownership - { Tar.ownerName = "foliage", - Tar.groupName = "foliage", - Tar.ownerId = 0, - Tar.groupId = 0 - } - } +mkTarEntry :: BL.ByteString -> IndexFile dec -> UTCTime -> Tar.Entry +mkTarEntry contents indexFile timestamp = + (Tar.fileEntry tarPath contents) + { Tar.entryTime = floor $ Time.utcTimeToPOSIXSeconds timestamp, + Tar.entryOwnership = + Tar.Ownership + { Tar.ownerName = "foliage", + Tar.groupName = "foliage", + Tar.ownerId = 0, + Tar.groupId = 0 + } + } where + tarPath = case Tar.toTarPath False indexPath of + Left e -> error $ "Invalid tar path " ++ indexPath ++ "(" ++ e ++ ")" + Right tp -> tp + indexPath = toFilePath $ castRoot $ indexFileToPath hackageIndexLayout indexFile anchorPath :: Path Absolute -> (RepoLayout -> RepoPath) -> FilePath diff --git a/app/Foliage/PreparePackageVersion.hs b/app/Foliage/PreparePackageVersion.hs index dc4547d..854bdda 100644 --- a/app/Foliage/PreparePackageVersion.hs +++ b/app/Foliage/PreparePackageVersion.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE LambdaCase #-} {-# LANGUAGE PatternSynonyms #-} module Foliage.PreparePackageVersion @@ -21,13 +22,16 @@ import Control.Monad (unless) import Data.List (sortOn) import Data.Ord (Down (Down)) import Development.Shake (Action) -import Distribution.Client.Compat.Prelude (prettyShow) -import Distribution.PackageDescription (GenericPackageDescription) +import Development.Shake.FilePath (joinPath, splitDirectories) +import Distribution.Client.Compat.Prelude (fromMaybe, prettyShow) +import Distribution.Parsec (simpleParsec) +import Distribution.Types.GenericPackageDescription (GenericPackageDescription (packageDescription)) +import Distribution.Types.PackageDescription (PackageDescription (package)) import Distribution.Types.PackageId import Foliage.Meta (PackageVersionSource, PackageVersionSpec (..), RevisionSpec (..), UTCTime, latestRevisionNumber) import Foliage.PrepareSdist (prepareSdist) import Foliage.PrepareSource (prepareSource) -import Foliage.Shake (readGenericPackageDescription') +import Foliage.Shake (readGenericPackageDescription', readPackageVersionSpec') import System.FilePath (takeBaseName, takeFileName, (<.>), ()) data PreparedPackageVersion = PreparedPackageVersion @@ -42,13 +46,47 @@ data PreparedPackageVersion = PreparedPackageVersion cabalFileRevisions :: [(UTCTime, FilePath)] } -preparePackageVersion :: FilePath -> PackageId -> PackageVersionSpec -> Action PreparedPackageVersion -preparePackageVersion inputDir pkgId pkgSpec = do +preparePackageVersion :: FilePath -> FilePath -> Action PreparedPackageVersion +preparePackageVersion inputDir metaFile = do + let (name, version) = case splitDirectories metaFile of + [n, v, _] -> (n, v) + _else -> error $ "internal error: I should not be looking at " ++ metaFile + + let pkgName = fromMaybe (error $ "invalid package name: " ++ name) $ simpleParsec name + let pkgVersion = fromMaybe (error $ "invalid package version: " ++ version) $ simpleParsec version + let pkgId = PackageIdentifier pkgName pkgVersion + + pkgSpec <- + readPackageVersionSpec' (inputDir metaFile) >>= \case + PackageVersionSpec {packageVersionRevisions, packageVersionTimestamp = Nothing} + | not (null packageVersionRevisions) -> do + error $ + unlines + [ inputDir metaFile <> " has cabal file revisions but the original package has no timestamp.", + "This combination doesn't make sense. Either add a timestamp on the original package or remove the revisions" + ] + PackageVersionSpec {packageVersionRevisions, packageVersionTimestamp = Just pkgTs} + | any ((< pkgTs) . revisionTimestamp) packageVersionRevisions -> do + error $ + unlines + [ inputDir metaFile <> " has a revision with timestamp earlier than the package itself.", + "Adjust the timestamps so that all revisions come after the original package" + ] + meta -> + return meta + srcDir <- prepareSource pkgId pkgSpec - let PackageIdentifier pkgName pkgVersion = pkgId - originalCabalFilePath = srcDir prettyShow pkgName <.> "cabal" - cabalFileRevisionPath revisionNumber = inputDir prettyShow pkgName prettyShow pkgVersion "revisions" show revisionNumber <.> "cabal" + let originalCabalFilePath = srcDir prettyShow pkgName <.> "cabal" + cabalFileRevisionPath revisionNumber = + joinPath + [ inputDir, + prettyShow pkgName, + prettyShow pkgVersion, + "revisions", + show revisionNumber + ] + <.> "cabal" let cabalFilePath = maybe @@ -61,13 +99,16 @@ preparePackageVersion inputDir pkgId pkgSpec = do sdistPath <- prepareSdist srcDir let expectedSdistName = prettyShow pkgId <.> "tar.gz" - unless (takeFileName sdistPath == expectedSdistName) $ - fail $ + unless (takeFileName sdistPath == expectedSdistName) $ do + error $ unlines [ "creating a source distribution for " ++ prettyShow pkgId ++ " has failed because", "cabal has produced a source distribtion that does not match the expected file name:", "actual: " ++ takeBaseName sdistPath, - "expected: " ++ expectedSdistName + "expected: " ++ expectedSdistName, + "possible cause: the package name and/or version implied by the metadata file path does not match what is contained in the cabal file", + "metadata file: " ++ metaFile, + "version in cabal file: " ++ prettyShow (Distribution.Types.PackageId.pkgVersion $ package $ packageDescription pkgDesc) ] let cabalFileRevisions = diff --git a/app/Foliage/PrepareSdist.hs b/app/Foliage/PrepareSdist.hs index acb329b..e41a2e7 100644 --- a/app/Foliage/PrepareSdist.hs +++ b/app/Foliage/PrepareSdist.hs @@ -70,20 +70,24 @@ addPrepareSdistRule outputDirRoot = addBuiltinRule noLint noIdentity run _differentOrMissing -> ChangedRecomputeDiff when (changed == ChangedRecomputeSame) $ - putInfo $ - "Wrote " ++ path ++ " (same hash " ++ showHashValue hv ++ ")" + putInfo ("Wrote " ++ path ++ " (same hash " ++ showHashValue hv ++ ")") when (changed == ChangedRecomputeDiff) $ - putInfo $ - "Wrote " ++ path ++ " (new hash " ++ showHashValue hv ++ ")" + putInfo ("Wrote " ++ path ++ " (new hash " ++ showHashValue hv ++ ")") return $ RunResult {runChanged = changed, runStore = new, runValue = path} makeSdist srcDir = do - cabalFile <- do - getDirectoryFiles srcDir ["*.cabal"] >>= \case - [f] -> pure f - fs -> fail $ "Invalid srcDir: " ++ srcDir ++ ". It contains multiple cabal files: " ++ unwords fs + cabalFiles <- getDirectoryFiles srcDir ["*.cabal"] + let cabalFile = case cabalFiles of + [f] -> f + fs -> + error $ + unlines + [ "Invalid source directory: " ++ srcDir, + "It contains multiple cabal files, while only one is allowed", + unwords fs + ] traced "cabal sdist" $ do gpd <- readGenericPackageDescription Verbosity.normal (srcDir cabalFile) diff --git a/app/Foliage/RemoteAsset.hs b/app/Foliage/RemoteAsset.hs index e5f1806..f054256 100644 --- a/app/Foliage/RemoteAsset.hs +++ b/app/Foliage/RemoteAsset.hs @@ -35,17 +35,15 @@ addFetchRemoteAssetRule cacheDir = addBuiltinRule noLint noIdentity run run :: BuiltinRun RemoteAsset FilePath run (RemoteAsset uri) old _mode = do unless (uriQuery uri == "") $ - fail $ - "Query elements in URI are not supported: " <> show uri + error ("Query elements in URI are not supported: " <> show uri) unless (uriFragment uri == "") $ - fail $ - "Fragments in URI are not supported: " <> show uri + error ("Fragments in URI are not supported: " <> show uri) let scheme = dropWhileEnd (not . isAlpha) $ uriScheme uri - host <- case uriRegName <$> uriAuthority uri of - Nothing -> fail $ "invalid uri " ++ show uri - Just host -> pure host + + let host = maybe (error $ "invalid uri " ++ show uri) uriRegName (uriAuthority uri) + let path = cacheDir joinPath (scheme : host : pathSegments uri) -- parse etag from store From 164217b7fb46796f8aba13ced2e6e118ecb5232c Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 3 Mar 2023 12:31:29 +0800 Subject: [PATCH 37/43] Dance around compiler versions --- cabal.project | 2 +- flake.nix | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cabal.project b/cabal.project index 77fbcdb..dd43461 100644 --- a/cabal.project +++ b/cabal.project @@ -1,5 +1,5 @@ packages: . -index-state: 2022-08-29T00:00:00Z +index-state: 2023-01-08T00:00:00Z with-compiler: ghc-9.2.5 allow-newer: , tomland:base diff --git a/flake.nix b/flake.nix index d2ad6a1..0a3dabd 100644 --- a/flake.nix +++ b/flake.nix @@ -18,17 +18,18 @@ overlays = [ haskell-nix.overlay ]; }; - pkgs-static-where-possible = if pkgs.stdenv.hostPlatform.isLinux then - if pkgs.stdenv.hostPlatform.isAarch64 then - pkgs.pkgsCross.aarch64-multiplatform-musl + pkgs-static-where-possible = + if pkgs.stdenv.hostPlatform.isLinux then + if pkgs.stdenv.hostPlatform.isAarch64 then + pkgs.pkgsCross.aarch64-multiplatform-musl + else + pkgs.pkgsCross.musl64 else - pkgs.pkgsCross.musl64 - else - pkgs; + pkgs; project = pkgs-static-where-possible.haskell-nix.cabalProject' { src = ./.; - compiler-nix-name = "ghc8107"; + compiler-nix-name = "ghc925"; shell.tools = { cabal = { }; hlint = { }; @@ -38,7 +39,8 @@ flake = project.flake { }; - in flake // { packages.default = flake.packages."foliage:exe:foliage"; }); + in + flake // { packages.default = flake.packages."foliage:exe:foliage"; }); nixConfig = { extra-substituters = [ From 2c7342d3d0dc57e35dd985b00c27c927d6b6c2e0 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 3 Mar 2023 14:48:11 +0800 Subject: [PATCH 38/43] Fixes --- cabal.project | 4 ++-- flake.nix | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cabal.project b/cabal.project index 77fbcdb..5ccaf65 100644 --- a/cabal.project +++ b/cabal.project @@ -1,6 +1,6 @@ packages: . -index-state: 2022-08-29T00:00:00Z -with-compiler: ghc-9.2.5 +index-state: 2023-03-01T00:00:00Z +with-compiler: ghc-9.2.6 allow-newer: , tomland:base , tomland:text diff --git a/flake.nix b/flake.nix index d2ad6a1..5540372 100644 --- a/flake.nix +++ b/flake.nix @@ -18,17 +18,18 @@ overlays = [ haskell-nix.overlay ]; }; - pkgs-static-where-possible = if pkgs.stdenv.hostPlatform.isLinux then - if pkgs.stdenv.hostPlatform.isAarch64 then - pkgs.pkgsCross.aarch64-multiplatform-musl + pkgs-static-where-possible = + if pkgs.stdenv.hostPlatform.isLinux then + if pkgs.stdenv.hostPlatform.isAarch64 then + pkgs.pkgsCross.aarch64-multiplatform-musl + else + pkgs.pkgsCross.musl64 else - pkgs.pkgsCross.musl64 - else - pkgs; + pkgs; project = pkgs-static-where-possible.haskell-nix.cabalProject' { src = ./.; - compiler-nix-name = "ghc8107"; + compiler-nix-name = "ghc926"; shell.tools = { cabal = { }; hlint = { }; @@ -38,7 +39,8 @@ flake = project.flake { }; - in flake // { packages.default = flake.packages."foliage:exe:foliage"; }); + in + flake // { packages.default = flake.packages."foliage:exe:foliage"; }); nixConfig = { extra-substituters = [ From 2e5acb44df76f2d6e8bcdba267fbea7e02a2ba1e Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 3 Mar 2023 14:48:40 +0800 Subject: [PATCH 39/43] Bump again --- flake.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.lock b/flake.lock index b209a71..de46b90 100644 --- a/flake.lock +++ b/flake.lock @@ -273,11 +273,11 @@ "hackage": { "flake": false, "locked": { - "lastModified": 1677371292, - "narHash": "sha256-8ky5ewzq+7/RJLwZTbuGwlkej83H9IvoLwMh0iLyZ/0=", + "lastModified": 1677809131, + "narHash": "sha256-MQ+ybbu5VFA6ERvh8mXcYEVlBCTte4BDxRaILSwBdAk=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "6ee340fbe61f03ab2a440d83f29c802d7e494ba8", + "rev": "4ea4b12737ea42e47eb5c6a7e4a3eafe9da2fc52", "type": "github" }, "original": { @@ -315,11 +315,11 @@ "tullia": "tullia" }, "locked": { - "lastModified": 1677372667, - "narHash": "sha256-9FQa8yvbkmq1UUhW0h1w4C37X90SzplqQFaQzf/D+Gs=", + "lastModified": 1677809260, + "narHash": "sha256-RSiv9/LP8/+sf15B305z4DYaj7IiAVjPBkqPfByxAp4=", "owner": "input-output-hk", "repo": "haskell.nix", - "rev": "986ebab4075d4c1ecac1270a9304e65e7ef715bc", + "rev": "02a5acdfc937129e51e41de0eafd0c44f29896b4", "type": "github" }, "original": { @@ -785,11 +785,11 @@ "stackage": { "flake": false, "locked": { - "lastModified": 1677370244, - "narHash": "sha256-JSh7/wauT+jn3FXxC6pCiYfE19uoy6/tpKsXeUGlg0Q=", + "lastModified": 1677543037, + "narHash": "sha256-7CPQZZafTQiw7YL+a2KCamb9a3PU0do9iZKy1AaRkBo=", "owner": "input-output-hk", "repo": "stackage.nix", - "rev": "f6e0966f3dad73156035963a7d16c48521d8e4f2", + "rev": "f4d29fa4403f45541d9f3993523df6027c21fe90", "type": "github" }, "original": { From 3b0acac2c3eb930cd1709bbed4a7d14860f5fbc2 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 3 Mar 2023 16:40:42 +0800 Subject: [PATCH 40/43] Drop the shell tools because nothing works --- flake.nix | 5 ----- 1 file changed, 5 deletions(-) diff --git a/flake.nix b/flake.nix index 0a3dabd..af34b8d 100644 --- a/flake.nix +++ b/flake.nix @@ -30,11 +30,6 @@ project = pkgs-static-where-possible.haskell-nix.cabalProject' { src = ./.; compiler-nix-name = "ghc925"; - shell.tools = { - cabal = { }; - hlint = { }; - haskell-language-server = { }; - }; }; flake = project.flake { }; From ae45f208a2b13e554150408a00662918261172c9 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 3 Mar 2023 16:55:03 +0800 Subject: [PATCH 41/43] Restrict flake to x86_64-linux --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e306afe..9a29da5 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ outputs = { self, nixpkgs, flake-utils, haskell-nix }: - flake-utils.lib.eachDefaultSystem (system: + flake-utils.lib.eachSystem [ "x86_64-linux" ] (system: let pkgs = import nixpkgs { inherit system; From b5e4f43dd27d16a83f002b82deeb6d3ac595129f Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 3 Mar 2023 18:48:14 +0800 Subject: [PATCH 42/43] Fix revisions timestamp on package page --- app/Foliage/Pages.hs | 2 +- templates/packageVersion.mustache | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Foliage/Pages.hs b/app/Foliage/Pages.hs index 301180b..9df7777 100644 --- a/app/Foliage/Pages.hs +++ b/app/Foliage/Pages.hs @@ -145,7 +145,7 @@ makePackageVersionPage outputDir PreparedPackageVersion {pkgId, pkgTimestamp, pk renderMustache packageVersionPageTemplate $ object [ "pkgVersionSource" .= pkgVersionSource, - "cabalFileRevisions" .= cabalFileRevisions, + "cabalFileRevisions" .= map fst cabalFileRevisions, "pkgDesc" .= jsonGenericPackageDescription pkgDesc, "pkgTimestamp" .= pkgTimestamp ] diff --git a/templates/packageVersion.mustache b/templates/packageVersion.mustache index cc13b7a..91b95e6 100644 --- a/templates/packageVersion.mustache +++ b/templates/packageVersion.mustache @@ -55,7 +55,7 @@
Revisions
{{#cabalFileRevisions}} -

{{revisionTimestamp}}

+

{{.}}

{{/cabalFileRevisions}} {{^cabalFileRevisions}}

None

From 234f8fcbf1f953e93c56fff8fb1691c4d6b28c55 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 10 Mar 2023 13:29:22 +0800 Subject: [PATCH 43/43] Maybe fix issue --- app/Foliage/CmdBuild.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/Foliage/CmdBuild.hs b/app/Foliage/CmdBuild.hs index 06b91e7..067895a 100644 --- a/app/Foliage/CmdBuild.hs +++ b/app/Foliage/CmdBuild.hs @@ -120,6 +120,7 @@ buildAction packageVersions targetKeys <- maybeReadKeysAt "target" + metadataEntries <- forP packageVersions $ \ppv@PreparedPackageVersion {pkgId, pkgTimestamp} -> do let PackageIdentifier {pkgName, pkgVersion} = pkgId @@ -279,7 +280,7 @@ prepareIndexPkgCabal pkgId timestamp filePath = do prepareIndexPkgMetadata :: Maybe UTCTime -> PreparedPackageVersion -> Action Targets prepareIndexPkgMetadata expiryTime PreparedPackageVersion {pkgId, sdistPath} = do - targetFileInfo <- computeFileInfoSimple' sdistPath + targetFileInfo <- liftIO $ computeFileInfoSimple sdistPath let packagePath = repoLayoutPkgTarGz hackageRepoLayout pkgId return Targets