Merge pull request #498 from nix-community/lock

Add module `lock`: Generic lock file interactions

tests/default.nix

@@ -43,7 +43,7 @@
         b.toString
         (config.d2n.utils.writePureShellScript
           [
-            inputs.alejandra.defaultPackage.${system}
+            pkgs.alejandra
             pkgs.coreutils
             pkgs.gitMinimal
             pkgs.nix

v1/nix/modules/drv-parts/lock/default.nix

@@ -0,0 +1,139 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  l = lib // builtins;
+  cfg = config.lock;
+
+  # LOAD
+  file = cfg.repoRoot + cfg.lockFileRel;
+  data = l.fromJSON (l.readFile file);
+  fileExist = l.pathExists file;
+
+  refresh = config.deps.writePython3Bin "refresh" {} ''
+    import tempfile
+    import subprocess
+    import json
+    from pathlib import Path
+
+    refresh_scripts = json.loads('${l.toJSON cfg.fields}')  # noqa: E501
+    repo_path = Path(subprocess.run(
+        ['git', 'rev-parse', '--show-toplevel'],
+        check=True, text=True, capture_output=True)
+        .stdout.strip())
+    lock_path_rel = Path('${cfg.lockFileRel}')
+    lock_path = repo_path / lock_path_rel.relative_to(lock_path_rel.anchor)
+
+
+    def run_refresh_script(script):
+        with tempfile.NamedTemporaryFile() as out_file:
+            subprocess.run(
+                [script],
+                check=True, shell=True, env={"out": out_file.name})
+            return json.load(out_file)
+
+
+    def run_refresh_scripts(refresh_scripts):
+        """
+          recursively iterate over a nested dict and replace all values,
+          executable scripts, with the content of their $out files.
+        """
+        for name, value in refresh_scripts.items():
+            if isinstance(value, dict):
+                refresh_scripts[name] = run_refresh_scripts(value)
+            else:
+                refresh_scripts[name] = run_refresh_script(value)
+        return refresh_scripts
+
+
+    lock_data = run_refresh_scripts(refresh_scripts)
+    with open(lock_path, 'w') as out_file:
+        json.dump(lock_data, out_file, indent=2)
+  '';
+
+  computeFODHash = fod: let
+    unhashedFOD = fod.overrideAttrs (old: {
+      outputHash = l.fakeSha256;
+      name = "${old.name}-UNHASHED_FOD";
+    });
+    drvPath = l.unsafeDiscardStringContext unhashedFOD.drvPath;
+  in
+    config.deps.writePython3 "update-FOD-hash-${config.name}" {} ''
+      import json
+      import os
+      import re
+      import subprocess
+      import sys
+
+      out_path = os.getenv("out")
+      drv_path = "${drvPath}"  # noqa: E501
+      nix_build = ["${config.deps.nix}/bin/nix", "build", "-L", drv_path]  # noqa: E501
+      with subprocess.Popen(nix_build, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) as process:  # noqa: E501
+          for line in process.stdout:
+              line = line.strip()
+              print(line)
+              search = r"error: hash mismatch in fixed-output derivation '.*UNHASHED_FOD.*':"  # noqa: E501
+              if re.match(search, line):
+                  print("line matched")
+                  specified = next(process.stdout).strip().split(" ", 1)
+                  got = next(process.stdout).strip().split(" ", 1)
+                  assert specified[0].strip() == "specified:"
+                  assert got[0].strip() == "got:"
+                  checksum = got[1].strip()
+                  print(f"Found hash: {checksum}")
+                  with open(out_path, 'w') as f:
+                      json.dump(checksum, f, indent=2)
+                  exit(0)
+      print("Could not determine hash", file=sys.stdout)
+      exit(1)
+    '';
+
+  errorMissingFile = ''
+    The lock file ${cfg.lockFileRel} for drv-parts module '${config.name}' is missing, please update it.
+    To create the lock file, execute:
+      bash -c $(nix-build ${config.lock.refresh.drvPath})/bin/refresh
+  '';
+
+  errorOutdated = path': let
+    path = l.concatStringsSep "." path';
+  in ''
+    The lock file ${cfg.lockFileRel} for drv-parts module '${config.name}' misses expected attribute `${path}`.
+    To update the lock file, execute:
+      bash -c $(nix-build ${config.lock.refresh.drvPath})/bin/refresh
+  '';
+
+  fileContent =
+    if ! fileExist
+    then throw errorMissingFile
+    else data;
+
+  loadField = path: val:
+    if l.hasAttrByPath path fileContent
+    then (l.getAttrFromPath path fileContent)
+    else throw (errorOutdated path);
+
+  loadedContent =
+    l.mapAttrsRecursiveCond
+    (val: ! l.isDerivation val)
+    loadField
+    cfg.fields;
+in {
+  imports = [
+    ./interface.nix
+  ];
+
+  config = {
+    lock.refresh = refresh;
+
+    lock.content = loadedContent;
+
+    lock.lib = {inherit computeFODHash;};
+
+    deps = {nixpkgs, ...}:
+      l.mapAttrs (_: l.mkDefault) {
+        inherit (nixpkgs) nix;
+        inherit (nixpkgs.writers) writePython3Bin;
+      };
+  };
+}

v1/nix/modules/drv-parts/lock/interface.nix

@@ -0,0 +1,57 @@
+{
+  config,
+  lib,
+  ...
+}: let
+  l = lib // builtins;
+  t = l.types;
+in {
+  options.lock = {
+    # GLOBAL OPTIONS
+    repoRoot = l.mkOption {
+      type = t.path;
+      description = "The root of the current repo. Eg. 'self' in a flake";
+      example = lib.literalExpression ''
+        self
+      '';
+    };
+
+    lockFileRel = l.mkOption {
+      type = t.str;
+      description = "Location of the cache file relative to the repoRoot";
+      example = lib.literalExpression ''
+        /rel/path/to/my/package/cache.json
+      '';
+    };
+
+    content = l.mkOption {
+      type = t.submodule {
+        freeformType = t.anything;
+      };
+    };
+
+    fields = l.mkOption {
+      type = t.attrs;
+      description = "Fields to manage via a lock file";
+      default = {};
+      example = {
+        pname = true;
+        version = true;
+      };
+    };
+
+    refresh = l.mkOption {
+      type = t.package;
+      description = "Script to refresh the cache file of this package";
+      readOnly = true;
+    };
+
+    lib.computeFODHash = l.mkOption {
+      type = t.functionTo t.path;
+      description = ''
+        Helper function to write the hash of a given FOD to $out.
+      '';
+      readOnly = true;
+    };
+  };
+}

v1/nix/modules/drvs/ansible/default.nix

@@ -10,8 +10,13 @@ in {
     ../../drv-parts/mach-nix-xs
   ];
 
+  # use lock file to manage hash for fetchPip
+  lock.fields.fetchPipHash =
+    config.lock.lib.computeFODHash config.mach-nix.pythonSources;
+
   deps = {nixpkgs, ...}: {
     python = nixpkgs.python39;
+    inherit (nixpkgs.writers) writePython3;
   };
 
   name = "ansible";
@@ -35,7 +40,7 @@ in {
     inherit python;
     name = config.name;
     requirementsList = ["${config.name}==${config.version}"];
-    hash = "sha256-dCo1llHcCiFrBOEd6mWhwqwVglsN2grSbcdBj8OzKDY=";
+    hash = config.lock.content.fetchPipHash;
     maxDate = "2023-01-01";
   };
 }

v1/nix/modules/drvs/ansible/lock-x86_64-linux.json

@@ -0,0 +1,3 @@
+{
+  "fetchPipHash": "sha256-dCo1llHcCiFrBOEd6mWhwqwVglsN2grSbcdBj8OzKDY="
+}

v1/nix/modules/flake-parts/packages.nix

@@ -7,7 +7,9 @@
 }: let
   system = "x86_64-linux";
   # A module imported into every package setting up the eval cache
-  evalCacheSetup = {config, ...}: {
+  setup = {config, ...}: {
+    lock.lockFileRel = "/v1/nix/modules/drvs/${config.name}/lock-${system}.json";
+    lock.repoRoot = self;
     eval-cache.cacheFileRel = "/v1/nix/modules/drvs/${config.name}/cache-${system}.json";
     eval-cache.repoRoot = self;
     eval-cache.enable = true;
@@ -21,7 +23,8 @@
         inputs.drv-parts.modules.drv-parts.docs
         module
         ../drv-parts/eval-cache
-        evalCacheSetup
+        ../drv-parts/lock
+        setup
       ];
       specialArgs.dependencySets = {
         nixpkgs = inputs.nixpkgsV1.legacyPackages.${system};
@@ -29,7 +32,7 @@
       specialArgs.drv-parts = inputs.drv-parts;
     };
   in
-    evaled // evaled.config.public;
+    evaled.config.public;
 in {
   # map all modules in ../drvs to a package output in the flake.
   flake.packages.${system} = lib.mapAttrs (_: drvModule: makeDrv drvModule) self.modules.drvs;