Add fileset helpers (#731)

---------

Co-authored-by: Ivan Petkov <ivanppetkov@gmail.com>

CHANGELOG.md

@@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+### Added
+* Added a number of fileset helpers to more easily compose source filtering:
+   * `fileset.cargoTomlAndLock`: for `Cargo.toml` and `Cargo.lock` files
+   * `fileset.commonCargoSources`: for files commonly used by cargo projects
+   * `fileset.configToml`: for `config.toml` files
+   * `fileset.rust`: for `*.rs` files
+   * `fileset.toml`: for `*.toml` files
+
 ### Fixed
 * `buildTrunkPackage` will pass in `--release=true` (instead of just
   `--release`) for trunk versions 0.21 or higher to avoid argument ambiguities

checks/default.nix

@@ -346,6 +346,42 @@ in
 
   features = callPackage ./features { };
 
+  fileset = myLib.buildPackage {
+    src = lib.fileset.toSource {
+      root = ./simple;
+      fileset = myLib.fileset.commonCargoSources ./simple;
+    };
+  };
+
+  filesetBuildScript = myLib.buildPackage {
+    src = lib.fileset.toSource {
+      root = ./with-build-script;
+      fileset = myLib.fileset.commonCargoSources ./with-build-script;
+    };
+  };
+
+  filesetWorkspace = myLib.buildPackage {
+    src = lib.fileset.toSource {
+      root = ./workspace;
+      fileset = myLib.fileset.commonCargoSources ./workspace;
+    };
+  };
+
+  filesetWorkspaceInheritance = myLib.buildPackage {
+    src = lib.fileset.toSource {
+      root = ./workspace-inheritance;
+      fileset = myLib.fileset.commonCargoSources ./workspace-inheritance;
+    };
+  };
+
+  filesetWorkspaceRoot = myLib.buildPackage {
+    src = lib.fileset.toSource {
+      root = ./workspace-root;
+      fileset = myLib.fileset.commonCargoSources ./workspace-root;
+    };
+    pname = "workspace-root";
+  };
+
   gitOverlappingRepo = myLib.buildPackage {
     src = ./git-overlapping;
   };

docs/API.md

@@ -1076,7 +1076,6 @@ cleanSourceWith {
   name = "source"; # Be reproducible, regardless of the directory name
 }
 ```
-
 Note that it is possible to compose source filters, especially if
 `filterCargoSources` omits files which are relevant to the build. For example:
 
@@ -1094,6 +1093,47 @@ cleanSourceWith {
 }
 ```
 
+### `craneLib.fileset.cargoTomlAndLock`
+
+`cargoTomlAndLock :: path -> fileset`
+
+A [fileset] helper which will only include any `Cargo.toml` and `Cargo.lock`
+files from the specified path.
+
+### `craneLib.fileset.commonCargoSources`
+
+`commonCargoSources :: path -> fileset`
+
+A [fileset] helper which will only include any files commonly used by cargo
+projects from the specified path. Essentially a union of:
+
+* `craneLib.fileset.cargoTomlAndLock`
+* `craneLib.fileset.rust`
+* `craneLib.fileset.toml`
+
+### `craneLib.fileset.configToml`
+
+`configToml :: path -> fileset`
+
+A [fileset] helper which will only include `config.toml` files from the
+specified path.
+
+Note that cargo usually only pays attention to `config.toml` files if they are
+present inside of a directory named `.cargo`. This fileset will contain any
+`config.toml` file, even if its parent directory is _not_ named `.cargo`.
+
+### `craneLib.fileset.rust`
+
+`rust :: path -> fileset`
+
+A [fileset] helper which will only include `*.rs` files from the specified path.
+
+### `craneLib.fileset.toml`
+
+`toml :: path -> fileset`
+
+A [fileset] helper which will only include `*.toml` files from the specified path.
+
 ### `craneLib.mkCargoDerivation`
 
 `mkCargoDerivation :: set -> drv`
@@ -1866,3 +1906,5 @@ Defines `replaceCargoLock()` which handles replacing or inserting a specified
 **Automatic behavior:** if `cargoLock` is set and
 `doNotReplaceCargoLock` is not set, then `replaceCargoLock "$cargoLock"` will be
 run as a pre patch hook.
+
+[fileset]: https://nixos.org/manual/nixpkgs/unstable/#sec-functions-library-fileset

docs/SUMMARY.md

@@ -21,7 +21,7 @@
   * [Workspace with Trunk](./examples/trunk-workspace.md)
   * [End-to-End Testing](./examples/end-to-end-testing.md)
   * [Building with SQLx](./examples/sqlx.md)
-* [Source filtering](./source-filtering.md)
+* [Source filtering and filesets](./source-filtering.md)
 * [Local development](./local_development.md)
 * [Custom cargo commands](./custom_cargo_commands.md)
 * [Customizing builds](./customizing_builds.md)

docs/source-filtering.md

@@ -43,3 +43,30 @@ craneLib.buildPackage {
   };
 }
 ```
+
+## Fileset filtering
+
+A more composable alternative to source filtering is using [filesets]:
+
+```nix
+let
+  unfilteredRoot = ./.; # The original, unfiltered source
+  src = lib.fileset.toSource {
+    root = unfilteredRoot;
+    fileset = lib.fileset.unions [
+      # Default files from crane (Rust and cargo files)
+      (craneLib.fileset.commonCargoSources unfilteredRoot)
+      # Also keep any markdown files
+      (lib.fileset.fileFilter (file: file.hasExt == "md") unfilteredRoot)
+      # Example of a folder for images, icons, etc
+      (lib.fileset.maybeMissing ./assets)
+    ];
+  };
+in
+craneLib.buildPackage {
+  # other attributes omitted
+  inherit src;
+}
+```
+
+[filesets]: https://nixos.org/manual/nixpkgs/unstable/#sec-functions-library-fileset

examples/quick-start-workspace/flake.nix

@@ -71,9 +71,9 @@
           fileset = lib.fileset.unions [
             ./Cargo.toml
             ./Cargo.lock
-            ./crates/my-common
-            ./crates/my-workspace-hack
-            crate
+            (craneLib.fileset.commonCargoSources ./crates/my-common)
+            (craneLib.fileset.commonCargoSources ./crates/my-workspace-hack)
+            (craneLib.fileset.commonCargoSources crate)
           ];
         };
 

examples/sqlx/flake.nix

@@ -18,13 +18,15 @@
 
         craneLib = crane.mkLib pkgs;
 
-        sqlFilter = path: _type: null != builtins.match ".*sql$" path;
-        sqlOrCargo = path: type: (sqlFilter path type) || (craneLib.filterCargoSources path type);
-
-        src = lib.cleanSourceWith {
-          src = ./.; # The original, unfiltered source
-          filter = sqlOrCargo;
-          name = "source"; # Be reproducible, regardless of the directory name
+        unfilteredRoot = ./.; # The original, unfiltered source
+        src = lib.fileset.toSource {
+          root = unfilteredRoot;
+          fileset = lib.fileset.unions [
+            # Default files from crane (Rust and cargo files)
+            (craneLib.fileset.commonCargoSources unfilteredRoot)
+            # Include all the .sql migrations as well
+            ./migrations
+          ];
         };
 
         # Common arguments can be set here to avoid repeating them later

examples/trunk-workspace/flake.nix

@@ -40,19 +40,21 @@
         });
 
         # When filtering sources, we want to allow assets other than .rs files
-        src = lib.cleanSourceWith {
-          src = ./.; # The original, unfiltered source
-          filter = path: type:
-            (lib.hasSuffix "\.html" path) ||
-            (lib.hasSuffix "\.scss" path) ||
+        unfilteredRoot = ./.; # The original, unfiltered source
+        src = lib.fileset.toSource {
+          root = unfilteredRoot;
+          fileset = lib.fileset.unions [
+            # Default files from crane (Rust and cargo files)
+            (craneLib.fileset.commonCargoSources unfilteredRoot)
+            (lib.fileset.fileFilter
+              (file: lib.any file.hasExt [ "html" "scss" ])
+              unfilteredRoot
+            )
             # Example of a folder for images, icons, etc
-            (lib.hasInfix "/assets/" path) ||
-            # Default filter from crane (allow .rs files)
-            (craneLib.filterCargoSources path type)
-          ;
+            (lib.fileset.maybeMissing ./assets)
+          ];
         };
 
-
         # Arguments to be used by both the client and the server
         # When building a workspace with crane, it's a good idea
         # to set "pname" and "version".

examples/trunk/flake.nix

@@ -40,16 +40,19 @@
         });
 
         # When filtering sources, we want to allow assets other than .rs files
-        src = lib.cleanSourceWith {
-          src = ./.; # The original, unfiltered source
-          filter = path: type:
-            (lib.hasSuffix "\.html" path) ||
-            (lib.hasSuffix "\.scss" path) ||
+        unfilteredRoot = ./.; # The original, unfiltered source
+        src = lib.fileset.toSource {
+          root = unfilteredRoot;
+          fileset = lib.fileset.unions [
+            # Default files from crane (Rust and cargo files)
+            (craneLib.fileset.commonCargoSources unfilteredRoot)
+            (lib.fileset.fileFilter
+              (file: lib.any file.hasExt [ "html" "scss" ])
+              unfilteredRoot
+            )
             # Example of a folder for images, icons, etc
-            (lib.hasInfix "/assets/" path) ||
-            # Default filter from crane (allow .rs files)
-            (craneLib.filterCargoSources path type)
-          ;
+            (lib.fileset.maybeMissing ./assets)
+          ];
         };
 
         # Common arguments can be set here to avoid repeating them later

lib/default.nix

@@ -87,6 +87,15 @@ let
       downloadCargoPackage = callPackage ./downloadCargoPackage.nix { };
       downloadCargoPackageFromGit = callPackage ./downloadCargoPackageFromGit.nix { };
       filterCargoSources = callPackage ./filterCargoSources.nix { };
+
+      fileset = {
+        cargoTomlAndLock = callPackage ./fileset/cargoTomlAndLock.nix { };
+        commonCargoSources = callPackage ./fileset/commonCargoSources.nix { };
+        configToml = callPackage ./fileset/configToml.nix { };
+        rust = callPackage ./fileset/rust.nix { };
+        toml = callPackage ./fileset/toml.nix { };
+      };
+
       findCargoFiles = callPackage ./findCargoFiles.nix { };
       inheritCargoArtifactsHook = callPackage ./setupHooks/inheritCargoArtifacts.nix { };
       installCargoArtifactsHook = callPackage ./setupHooks/installCargoArtifacts.nix { };

lib/fileset/cargoTomlAndLock.nix

@@ -0,0 +1,7 @@
+{ lib
+}:
+
+path:
+lib.fileset.fileFilter
+  (file: lib.elem file.name [ "Cargo.toml" "Cargo.lock" ])
+  path

lib/fileset/commonCargoSources.nix

@@ -0,0 +1,12 @@
+{ fileset
+, lib
+}:
+
+path:
+lib.fileset.unions [
+  (fileset.cargoTomlAndLock path)
+  (fileset.rust path)
+  # Keep all toml files as they are commonly used to configure other
+  # cargo-based tools
+  (fileset.toml path)
+]

lib/fileset/configToml.nix

@@ -0,0 +1,9 @@
+{ lib
+}:
+
+path:
+lib.fileset.fileFilter
+  # Technically this should be scoped to `.cargo/config.toml` but (currently)
+  # there is no way to do this with file sets in a generic manner
+  (file: file.name == "config.toml")
+  path

lib/fileset/rust.nix

@@ -0,0 +1,7 @@
+{ lib
+}:
+
+path:
+lib.fileset.fileFilter
+  (file: file.hasExt "rs")
+  path

lib/fileset/toml.nix

@@ -0,0 +1,7 @@
+{ lib
+}:
+
+path:
+lib.fileset.fileFilter
+  (file: file.hasExt "toml")
+  path

pkgs/default.nix

@@ -4,20 +4,14 @@
   book =
     let
       inherit (pkgs) lib;
-      root = myLib.path ./..;
-      rootPrefix = toString root;
-      cleanedSrc = lib.cleanSourceWith {
-        src = root;
-        filter = path: _:
-          let
-            relativePath = lib.removePrefix rootPrefix path;
-          in
-          lib.any (prefix: lib.hasPrefix prefix relativePath) [
-            "/docs" # Build the docs directory
-            "/examples" # But also include examples as we cross-reference them
-            "/README.md"
-            "/CHANGELOG.md"
-          ];
+      cleanedSrc = lib.fileset.toSource {
+        root = ./..;
+        fileset = lib.fileset.unions [
+          ./../docs
+          ./../examples
+          ./../README.md
+          ./../CHANGELOG.md
+        ];
       };
     in
     pkgs.runCommand "crane-book" { } ''