Use ghcr.io to store closure (#39)

---------

Co-authored-by: Yvan Sraka <yvan@sraka.xyz>
Co-authored-by: Moritz Angermann <moritz.angermann@gmail.com>
Co-authored-by: Andrea Bedini <andrea.bedini@tweag.io>

.github/workflows/hello.yml

@@ -0,0 +1,89 @@
+name: Test that we can build the hello world example form hackage
+on:
+  push:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: devx {0}
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - x86_64-darwin
+          - x86_64-linux
+          # Skipping because we do not have runners for these set up.
+          #- aarch64-darwin
+          #- aarch64-linux
+        compiler-nix-name:
+          - ghc8107
+          - ghc927
+          - ghc961
+        target-platform:
+          - ""
+          - "-static"
+          - "-js"
+          # Comming soon.
+          #- "-windows"
+        variant:
+          - ""
+          - "-minimal"
+        iog:
+          - ""
+          - "-iog"
+        exclude:
+          # Just cross compiling javascript with ghc 9.6.1 for now
+          - compiler-nix-name: ghc8107
+            target-platform: "-js"
+          - compiler-nix-name: ghc927
+            target-platform: "-js"
+          # Static builds not working for darwin yet
+          - platform: x86_64-darwin
+            target-platform: "-static"
+          # Static tools not working right now (so just building "-static-minimal" for now)
+          - target-platform: "-static"
+            variant: ""
+          # hlint and hls not working on ghc 9.6.1 yet
+          - target-platform: ghc961
+            variant: ""
+          # Windows cross compilation only works on x86_64 right now.
+          - platform: aarch64-darwin
+            target-platform: "-windows"
+          - platform: aarch64-linux
+            target-platform: "-windows"
+
+    runs-on: ${{ matrix.platform == 'x86_64-linux' && 'ubuntu-latest' || 'macos-latest' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Install Nix with good defaults
+        uses: cachix/install-nix-action@v17
+        with:
+          extra_nix_config: |
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+            substituters = https://cache.iog.io/ https://cache.zw3rk.com/ https://cache.nixos.org/
+          nix_path: nixpkgs=channel:nixos-unstable
+      - name: Install jq
+        shell: bash
+        run: |
+          curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o jq
+          chmod +x jq
+      - name: Download and import closure
+        shell: bash
+        run: |
+          ./fetch-docker.sh input-output-hk/devx ${{ matrix.platform }}.${{ matrix.compiler-nix-name }}${{ matrix.target-platform }}${{ matrix.variant }}${{ matrix.iog }}-env | zstd -d | nix-store --import | tee store-paths.txt
+          sudo ln -s $(tail -n 1 store-paths.txt) /usr/local/bin/devx-env.sh
+          sudo cp extra/devx /usr/local/bin
+      - name: cabal update
+        run: cabal update
+      - name: unapck hello
+        run: cabal unpack hello
+      - name: build hello
+        working-directory: hello-1.0.0.2
+        run: cabal build
+      - name: test hello
+        working-directory: hello-1.0.0.2
+        run: cabal test

.github/workflows/main.yml

@@ -0,0 +1,92 @@
+name: Prebuild DevX closures
+on:
+  push:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - x86_64-darwin
+          - x86_64-linux
+          - aarch64-darwin
+          # Currently broken
+          #- aarch64-linux
+        compiler-nix-name:
+          - ghc8107
+          - ghc927
+          - ghc961
+        target-platform:
+          - ""
+          - "-static"
+          - "-js"
+          # Coming soon.
+          #- "-windows"
+        variant:
+          - ""
+          - "-minimal"
+        iog:
+          - ""
+          - "-iog"
+        exclude:
+          # Just cross compiling javascript with ghc 9.6.1 for now
+          - compiler-nix-name: ghc8107
+            target-platform: "-js"
+          - compiler-nix-name: ghc927
+            target-platform: "-js"
+          # Static builds not working for darwin yet
+          - platform: x86_64-darwin
+            target-platform: "-static"
+          # Static tools not working right now (so just building "-static-minimal" for now)
+          - target-platform: "-static"
+            variant: ""
+          # hlint and hls not working on ghc 9.6.1 yet
+          - target-platform: ghc961
+            variant: ""
+          # Windows cross compilation only works on x86_64 right now.
+          - platform: aarch64-darwin
+            target-platform: "-windows"
+          - platform: aarch64-linux
+            target-platform: "-windows"
+        
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Install Nix with good defaults
+        uses: cachix/install-nix-action@v17
+        with:
+          extra_nix_config: |
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=
+            substituters = https://cache.iog.io/ https://cache.zw3rk.com/ https://cache.nixos.org/
+          nix_path: nixpkgs=channel:nixos-unstable
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Log in to the Container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Work around issue fetching cabal revision files
+        run: |
+          # This will cause the revised `.cabal` files used whan building alex
+          # for building GHC to be fetched using the `x86_64-linux` `.drv` for `fetchurl`.
+          # Later when we want the non `x86_64-linux` version later it will find the
+          # fixed output of the derivation is already in the `/nix/store` and will
+          # not try to fetch it using the platform we do not have a builder for.
+          if [[ "${{ matrix.platform }}" != "x86_64-linux" && "${{ matrix.target-platform }}" = "-js" ]]; then
+            nix build ".#hydraJobs.x86_64-linux.${{ matrix.compiler-nix-name }}-js-minimal" --show-trace
+          fi
+      - name: Compute and upload closure and developer environment to ghcr.io
+        env:
+          NIX_STORE_SECRET_KEY: ${{ secrets.SECRET_KEY }}
+          DEV_SHELL: ${{ matrix.platform }}.${{ matrix.compiler-nix-name }}${{ matrix.target-platform }}${{ matrix.variant }}${{ matrix.iog }}-env
+        run: ./extra/ghcr-upload.sh

cross-js.nix

@@ -1,6 +1,6 @@
-{ pkgs, compiler, compiler-nix-name, withHLS ? true, withHlint ? true, withIOG ? true  }:
+{ pkgs, compiler, compiler-nix-name, toolsModule, withHLS ? true, withHlint ? true, withIOG ? true  }:
 let tool-version-map = import ./tool-map.nix;
-    tool = tool-name: pkgs.haskell-nix.tool compiler-nix-name tool-name (tool-version-map compiler-nix-name tool-name);
+    tool = tool-name: pkgs.haskell-nix.tool compiler-nix-name tool-name [(tool-version-map compiler-nix-name tool-name) toolsModule];
     cabal-install = tool "cabal";
     # add a trace helper. This will trace a message about disabling a component despite requesting it, if it's not supported in that compiler.
     compiler-not-in = compiler-list: name: (if __elem compiler-nix-name compiler-list then __trace "No ${name}. Not yet compatible with ${compiler-nix-name}" false else true);    
@@ -20,7 +20,7 @@ let tool-version-map = import ./tool-map.nix;
                 "$@" \
                 $NIX_CABAL_FLAGS 
             ;;
-            clean)
+            clean|unpack)
             cabal "$@"
             ;;
             *)
@@ -29,6 +29,12 @@ let tool-version-map = import ./tool-map.nix;
         esac        
         '';
     };    
+    wrapped-hsc2hs = pkgs.pkgsBuildBuild.writeShellApplication {
+        name = "${compiler.targetPrefix}hsc2hs";
+        text = ''
+          ${compiler}/bin/${compiler.targetPrefix}hsc2hs --cross-compile "$@"
+        '';
+    };
 in
 pkgs.mkShell ({
     # Note [cabal override]:
@@ -43,10 +49,10 @@ pkgs.mkShell ({
     # open source, and licenses accordingly.  Otherwise we'd have to link gmp
     # dynamically.  This requirement will be gone with gmp-bignum.
     #
-    NIX_CABAL_FLAGS = pkgs.lib.optionals pkgs.stdenv.hostPlatform.isGhcjs [
-    "--with-ghc=javascript-unknown-linux-ghcjs-ghc"
-    "--with-ghc-pkg=javascript-unknown-linux-ghcjs-pkg"
-    "--with-hsc2hs=javascript-unknown-linux-ghcjs-hsc2hs"
+    NIX_CABAL_FLAGS = [
+    "--with-ghc=javascript-unknown-ghcjs-ghc"
+    "--with-ghc-pkg=javascript-unknown-ghcjs-ghc-pkg"
+    "--with-hsc2hs=javascript-unknown-ghcjs-hsc2hs"
     # ensure that the linker knows we want a static build product
     # "--enable-executable-static"
     ];
@@ -69,9 +75,11 @@ pkgs.mkShell ({
     '';
     buildInputs = [];
 
-    nativeBuildInputs = [ wrapped-cabal compiler ] ++ (with pkgs; [
+    nativeBuildInputs = [ wrapped-hsc2hs wrapped-cabal compiler ] ++ (with pkgs; [
         haskell-nix.cabal-install.${compiler-nix-name}
         pkgconfig
+        (tool "happy")
+        (tool "alex")
         stdenv.cc.cc.lib ]) ++ (with pkgs.buildPackages; [
     ])
     ++ pkgs.lib.optional (withHLS && (compiler-not-in ["ghc961"] "Haskell Language Server")) (tool "haskell-language-server")

dynamic.nix

@@ -1,7 +1,7 @@
 # define a development shell for dynamically linked applications (default)
-{ pkgs, compiler, compiler-nix-name, withHLS ? true, withHlint ? true, withIOG ? true }:
+{ pkgs, compiler, compiler-nix-name, toolsModule, withHLS ? true, withHlint ? true, withIOG ? true }:
 let tool-version-map = import ./tool-map.nix;
-    tool = tool-name: pkgs.haskell-nix.tool compiler-nix-name tool-name (tool-version-map compiler-nix-name tool-name);
+    tool = tool-name: pkgs.haskell-nix.tool compiler-nix-name tool-name [(tool-version-map compiler-nix-name tool-name) toolsModule];
     cabal-install = tool "cabal";
     # add a trace helper. This will trace a message about disabling a component despite requesting it, if it's not supported in that compiler.
     compiler-not-in = compiler-list: name: (if __elem compiler-nix-name compiler-list then __trace "No ${name}. Not yet compatible with ${compiler-nix-name}" false else true);
@@ -18,7 +18,7 @@ let tool-version-map = import ./tool-map.nix;
         case "$1" in
             build) cabal "$@"
             ;;
-            clean) cabal "$@"
+            clean|unpack) cabal "$@"
             ;;
             *) cabal "$@"
             ;;
@@ -33,6 +33,8 @@ pkgs.mkShell {
     shellHook = with pkgs; ''
         export PS1="\[\033[01;33m\][\w]$\[\033[00m\] "
         ${figlet}/bin/figlet -f rectangles 'IOG Haskell Shell'
+        export CABAL_DIR=$HOME/.cabal
+        echo "CABAL_DIR set to $CABAL_DIR"
     ''
     # this one is only needed on macOS right now, due to a bug in loading libcrypto.
     + lib.optionalString stdenv.hostPlatform.isMacOS
@@ -57,8 +59,8 @@ pkgs.mkShell {
         ]
         ++ pkgs.lib.optional pkgs.stdenv.hostPlatform.isLinux systemd
     )
-    ++ pkgs.lib.optional (withHLS && (compiler-not-in ["ghc961"] "Haskell Language Server")) (tool "haskell-language-server")
-    ++ pkgs.lib.optional (withHlint && (compiler-not-in ["ghc961"] "HLint")) (tool "hlint")
+    ++ pkgs.lib.optional (withHLS && (compiler-not-in (["ghc961"] ++ pkgs.lib.optional (pkgs.stdenv.hostPlatform.isDarwin && pkgs.stdenv.hostPlatform.isAarch64) "ghc902") "Haskell Language Server")) (tool "haskell-language-server")
+    ++ pkgs.lib.optional (withHlint && (compiler-not-in (["ghc961"] ++ pkgs.lib.optional (pkgs.stdenv.hostPlatform.isDarwin && pkgs.stdenv.hostPlatform.isAarch64) "ghc902") "HLint")) (tool "hlint")
     ++ pkgs.lib.optional withIOG
         (with pkgs; [ cddl cbor-diag ]
         ++ map pkgs.lib.getDev (with pkgs; [ libsodium-vrf secp256k1 R_4_1_3]))

extra/devx

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source /usr/local/bin/devx-env.sh
+source "$1"

extra/ghcr-upload.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p zstd -p oras
+set -euox pipefail
+
+nix build ".#hydraJobs.${DEV_SHELL}" --show-trace
+nix-store --export $(nix-store -qR result) | zstd -z8T8 >${DEV_SHELL}
+oras push ghcr.io/input-output-hk/devx:${DEV_SHELL} ${DEV_SHELL}

fetch-docker.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+TOKEN=$(curl --silent https://ghcr.io/token\?scope\=repository:$1:pull | jq -r .token)
+
+BLOB=$(curl \
+--silent \
+--request 'GET' \
+--header "Authorization: Bearer $TOKEN" \
+--header "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
+--header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
+--header "Accept: application/vnd.oci.image.manifest.v1+json" \
+"https://ghcr.io/v2/$1/manifests/$2" | tee manifest.json | jq -r '.layers[0].digest')
+
+curl \
+--location \
+--request GET \
+--header "Authorization: Bearer ${TOKEN}" \
+"https://ghcr.io/v2/$1/blobs/${BLOB}"

flake.lock

@@ -190,16 +190,20 @@
       }
     },
     "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
       "locked": {
-        "lastModified": 1659877975,
-        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
-        "owner": "numtide",
+        "lastModified": 1681378341,
+        "narHash": "sha256-2qUN04W6X9cHHytEsJTM41CmusifPTC0bgTtYsHSNY8=",
+        "owner": "hamishmack",
         "repo": "flake-utils",
-        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "rev": "2767bafdb189cd623354620c2dacbeca8fd58b17",
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
+        "owner": "hamishmack",
+        "ref": "hkm/nested-hydraJobs",
         "repo": "flake-utils",
         "type": "github"
       }
@@ -289,11 +293,11 @@
     "hackage": {
       "flake": false,
       "locked": {
-        "lastModified": 1681950405,
-        "narHash": "sha256-JQJz2rGneoPb+uNLBqi8SgKahyDnRd+gxBySLUjxwYw=",
+        "lastModified": 1682209574,
+        "narHash": "sha256-zpg1PQjrPUGKpdO/MyLR9vBWwzmiKwwUrPXrq+pqyQc=",
         "owner": "input-output-hk",
         "repo": "hackage.nix",
-        "rev": "0a68428cdf1744e891ceaa735c31d3f82ff59745",
+        "rev": "619e219b49e679ba59fec8507d201422d548191b",
         "type": "github"
       },
       "original": {
@@ -332,11 +336,11 @@
         "tullia": "tullia"
       },
       "locked": {
-        "lastModified": 1681951859,
-        "narHash": "sha256-0LJh64BLfuEH0pkp4hQdE1+Dp4VDBJhidqrPawhEYfw=",
+        "lastModified": 1682295813,
+        "narHash": "sha256-CxteV5uhCP0xxAY3gr3sh2any2AlxYg2aAIMc4cuVDw=",
         "owner": "input-output-hk",
         "repo": "haskell.nix",
-        "rev": "d844bb8fe89b2dbbd6a46d1c74a89b47e5040999",
+        "rev": "e5eab9e936deb58165ca617ec8c45b6b9421458c",
         "type": "github"
       },
       "original": {
@@ -849,11 +853,11 @@
     "stackage": {
       "flake": false,
       "locked": {
-        "lastModified": 1681949335,
-        "narHash": "sha256-ZOw4jJ34XWIpmAyZC7PvQW4FTyAKisEiENoGQ7C/7/M=",
+        "lastModified": 1682122158,
+        "narHash": "sha256-ja3xcKc/KZmGw5bC0w/J51eJyzhmqIy1b7SMvku51tg=",
         "owner": "input-output-hk",
         "repo": "stackage.nix",
-        "rev": "e4028bb56b58abdd4a040ec38debb5c38ba731ad",
+        "rev": "ac8c5de753931eaf9a8cc1ef310cf8d8db0430fa",
         "type": "github"
       },
       "original": {
@@ -907,6 +911,21 @@
         "type": "github"
       }
     },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
     "tullia": {
       "inputs": {
         "nix-nomad": "nix-nomad",

flake.nix

@@ -3,7 +3,7 @@
 
     inputs.haskellNix.url = "github:input-output-hk/haskell.nix";
     inputs.nixpkgs.follows = "haskellNix/nixpkgs-unstable";
-    inputs.flake-utils.url = "github:numtide/flake-utils";
+    inputs.flake-utils.url = "github:hamishmack/flake-utils/hkm/nested-hydraJobs";
     inputs.iohk-nix.url = "github:input-output-hk/iohk-nix";
 
     outputs = { self, nixpkgs, flake-utils, haskellNix, iohk-nix }:
@@ -54,7 +54,6 @@
              inherit system;
              inherit (haskellNix) config;
            };
-         in rec {
            # These are for checking IOG projects build in an environment
            # without haskell packages built by haskell.nix.
            #
@@ -62,7 +61,7 @@
            #
            # nix develop github:input-output-hk/devx#ghc924 --no-write-lock-file -c cabal build
            #
-           devShells =
+           devShellsWithToolsModule = toolsModule:
              let compilers = pkgs: builtins.removeAttrs pkgs.haskell-nix.compiler
                 # Exclude old versions of GHC to speed up `nix flake check`
                 [ "ghc844"
@@ -71,14 +70,14 @@
                   "ghc8101" "ghc8102" "ghc8103" "ghc8104" "ghc8105" "ghc8106" "ghc810420210212"
                   "ghc901"
                   "ghc921" "ghc922" "ghc923" "ghc924" "ghc925" "ghc926"
-                  "ghc941" "ghc942" "ghc943"
+                  "ghc941" "ghc942" "ghc943" "ghc944"
                   "ghc96020230302"
                  ];
                  js-compilers = pkgs: builtins.removeAttrs (compilers pkgs)
                  [
                   "ghc902"
                   "ghc927"
-                  "ghc944"
+                  "ghc945"
                  ];
                  static-pkgs = if pkgs.stdenv.hostPlatform.isLinux
                                then if pkgs.stdenv.hostPlatform.isAarch64
@@ -87,54 +86,71 @@
                                else pkgs;
                  js-pkgs = pkgs.pkgsCross.ghcjs;
              in (builtins.mapAttrs (compiler-nix-name: compiler:
-                  import ./dynamic.nix { inherit pkgs compiler compiler-nix-name; withIOG = false; }
+                  import ./dynamic.nix { inherit pkgs compiler compiler-nix-name toolsModule; withIOG = false; }
                   ) (compilers pkgs)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-minimal" (
-                    import ./dynamic.nix { inherit pkgs compiler compiler-nix-name; withHLS = false; withHlint = false; withIOG = false; }
+                    import ./dynamic.nix { inherit pkgs compiler compiler-nix-name toolsModule; withHLS = false; withHlint = false; withIOG = false; }
                   )) (compilers pkgs)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-static" (
-                    import ./static.nix { pkgs = static-pkgs; inherit compiler compiler-nix-name; withIOG = false; }
+                    import ./static.nix { pkgs = static-pkgs; inherit compiler compiler-nix-name toolsModule; withIOG = false; }
                   )) (compilers static-pkgs.buildPackages)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-static-minimal" (
-                    import ./static.nix { pkgs = static-pkgs; inherit compiler compiler-nix-name; withHLS = false; withHlint = false; withIOG = false; }
+                    import ./static.nix { pkgs = static-pkgs; inherit compiler compiler-nix-name toolsModule; withHLS = false; withHlint = false; withIOG = false; }
                   )) (compilers static-pkgs.buildPackages)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-js" (
-                    import ./cross-js.nix { pkgs = js-pkgs.buildPackages; inherit compiler compiler-nix-name; }
+                    import ./cross-js.nix { pkgs = js-pkgs.buildPackages; inherit compiler compiler-nix-name toolsModule; }
                   )) (js-compilers js-pkgs.buildPackages)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-js-minimal" (
-                    import ./cross-js.nix { pkgs = js-pkgs.buildPackages; inherit compiler compiler-nix-name; withHLS = false; withHlint = false; }
+                    import ./cross-js.nix { pkgs = js-pkgs.buildPackages; inherit compiler compiler-nix-name toolsModule; withHLS = false; withHlint = false; }
                   )) (js-compilers js-pkgs.buildPackages)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-iog" (
-                    import ./dynamic.nix { inherit pkgs compiler compiler-nix-name; withIOG = true; }
+                    import ./dynamic.nix { inherit pkgs compiler compiler-nix-name toolsModule; withIOG = true; }
                   )) (compilers pkgs)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-minimal-iog" (
-                    import ./dynamic.nix { inherit pkgs compiler compiler-nix-name; withHLS = false; withHlint = false; withIOG = true; }
+                    import ./dynamic.nix { inherit pkgs compiler compiler-nix-name toolsModule; withHLS = false; withHlint = false; withIOG = true; }
                   )) (compilers pkgs)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-static-iog" (
-                    import ./static.nix { pkgs = static-pkgs; inherit compiler compiler-nix-name; withIOG = true; }
+                    import ./static.nix { pkgs = static-pkgs; inherit compiler compiler-nix-name toolsModule; withIOG = true; }
                   )) (compilers static-pkgs.buildPackages)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-static-minimal-iog" (
-                    import ./static.nix { pkgs = static-pkgs; inherit compiler compiler-nix-name; withHLS = false; withHlint = false; withIOG = true; }
+                    import ./static.nix { pkgs = static-pkgs; inherit compiler compiler-nix-name toolsModule; withHLS = false; withHlint = false; withIOG = true; }
                   )) (compilers static-pkgs.buildPackages)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-js-iog" (
-                    import ./cross-js.nix { pkgs = js-pkgs.buildPackages; inherit compiler compiler-nix-name; withIOG = true; }
+                    import ./cross-js.nix { pkgs = js-pkgs.buildPackages; inherit compiler compiler-nix-name toolsModule; withIOG = true; }
                   )) (js-compilers js-pkgs.buildPackages)
               // pkgs.lib.mapAttrs' (compiler-nix-name: compiler:
                   pkgs.lib.nameValuePair "${compiler-nix-name}-js-minimal-iog" (
-                    import ./cross-js.nix { pkgs = js-pkgs.buildPackages; inherit compiler compiler-nix-name; withHLS = false; withHlint = false; withIOG = true; }
+                    import ./cross-js.nix { pkgs = js-pkgs.buildPackages; inherit compiler compiler-nix-name toolsModule; withHLS = false; withHlint = false; withIOG = true; }
                   )) (js-compilers js-pkgs.buildPackages)
              );
-        hydraJobs = devShells;
+        devShells = devShellsWithToolsModule {};
+        # Eval must be done on linux when we use hydra to build environment
+        # scripts for other platforms.  That way a linux GHA can download the
+        # cached files without needing access to the actual build platform.
+        devShellsWithEvalOnLinux = devShellsWithToolsModule { evalSystem = "x86_64-linux"; };
+      in {
+        inherit devShells;
+        hydraJobs = devShells //
+          (pkgs.lib.mapAttrs' (name: drv:
+            pkgs.lib.nameValuePair "${name}-env" (
+            pkgs.runCommand "${name}-env.sh" {
+                requiredSystemFeatures = [ "recursive-nix" ];
+                nativeBuildInputs = [ pkgs.nix ];
+              } ''
+              nix --offline --extra-experimental-features "nix-command flakes" \
+                print-dev-env ${drv.drvPath} >> $out
+            '')) devShellsWithEvalOnLinux) // {
+          };
        });
 
     # --- Flake Local Nix Configuration ----------------------------

static.nix

@@ -1,6 +1,6 @@
-{ pkgs, compiler, compiler-nix-name, withHLS ? true, withHlint ? true, withIOG ? true }:
+{ pkgs, compiler, compiler-nix-name, toolsModule, withHLS ? true, withHlint ? true, withIOG ? true }:
 let tool-version-map = import ./tool-map.nix;
-    tool = tool-name: pkgs.haskell-nix.tool compiler-nix-name tool-name (tool-version-map compiler-nix-name tool-name);
+    tool = tool-name: pkgs.pkgsBuildBuild.haskell-nix.tool compiler-nix-name tool-name [(tool-version-map compiler-nix-name tool-name) toolsModule];
     cabal-install = tool "cabal";
     # add a trace helper. This will trace a message about disabling a component despite requesting it, if it's not supported in that compiler.
     compiler-not-in = compiler-list: name: (if __elem compiler-nix-name compiler-list then __trace "No ${name}. Not yet compatible with ${compiler-nix-name}" false else true);
@@ -40,7 +40,7 @@ let tool-version-map = import ./tool-map.nix;
                 --ghc-option=-L${lib.getLib static-secp256k1}/lib \
                 --ghc-option=-L${lib.getLib static-openssl}/lib
             ;;
-            clean)
+            clean|unpack)
             cabal "$@"
             ;;
             *)
@@ -114,8 +114,8 @@ pkgs.mkShell ({
         pkgconfig
         stdenv.cc.cc.lib ]) ++ (with pkgs.buildPackages; [
     ])
-    ++ pkgs.lib.optional (withHLS && (compiler-not-in ["ghc961"] "Haskell Language Server")) (tool "haskell-language-server")
-    ++ pkgs.lib.optional (withHlint && (compiler-not-in ["ghc961"] "HLint")) (tool "hlint")
+    ++ pkgs.lib.optional (withHLS && (compiler-not-in (["ghc961"] ++ pkgs.lib.optional (pkgs.stdenv.hostPlatform.isDarwin && pkgs.stdenv.hostPlatform.isAarch64) "ghc902") "Haskell Language Server")) (tool "haskell-language-server")
+    ++ pkgs.lib.optional (withHlint && (compiler-not-in (["ghc961"] ++ pkgs.lib.optional (pkgs.stdenv.hostPlatform.isDarwin && pkgs.stdenv.hostPlatform.isAarch64) "ghc902") "HLint")) (tool "hlint")
     ++ pkgs.lib.optional withIOG
         (with pkgs; [ cddl cbor-diag ]
         ++ map pkgs.lib.getDev (with pkgs; [

tool-map.nix

@@ -1,13 +1,12 @@
 # see https://haskell-language-server.readthedocs.io/en/latest/support/ghc-version-support.html
 # so we assume "latest" for all hls. 
 # for hlint however, we need hlint-3.3 for ghc-8.10.7.
-let fixed-versions = { "hlint" = { "ghc8107" = "3.3"; }; }; in
-compiler-nix-name: tool:
+let fixed-versions = { "hlint" = { "ghc8107" = { version = "3.3"; }; }; }; in
+compiler-nix-name: tool: {
   # for HLS, we rely on the cabal.project configuration from the upstream project to have the correct configuration.
   # Building HLS from hackage requires setting all those constraints as well, and just isn't practical to do for each
   # HLS release. Therefore we rely on the HLS upstream repository to provide the proper configuration information.
-  if tool == "haskell-language-server"
-    then {pkgs, ...}: rec {
+  haskell-language-server = {pkgs, ...}: rec {
       # Use the github source of HLS that is tested with haskell.nix CI
       src = pkgs.haskell-nix.sources."hls-1.10";
       # `tool` normally ignores the `cabal.project` (if there is one in the hackage source).
@@ -15,5 +14,7 @@ compiler-nix-name: tool:
       cabalProject = __readFile (src + "/cabal.project");
       # sha256 for a `source-repository-package` in the `cabal.project` file.
       sha256map."https://github.com/pepeiborra/ekg-json"."7a0af7a8fd38045fd15fb13445bdcc7085325460" = "sha256-fVwKxGgM0S4Kv/4egVAAiAjV7QB5PBqMVMCfsv7otIQ=";
-    }
-  else fixed-versions.${tool}.${compiler-nix-name} or "latest"
+  };
+  happy = { version = "1.20.1.1"; };
+  alex = { version = "3.2.7.3"; };
+}.${tool} or fixed-versions.${tool}.${compiler-nix-name} or {}