From ef80ead953c1b28316cc3f8613904edc2eb90c28 Mon Sep 17 00:00:00 2001 From: Rijk van Putten Date: Fri, 8 Nov 2024 21:39:11 +0100 Subject: [PATCH] Add fileset helpers (#731) --------- Co-authored-by: Ivan Petkov --- CHANGELOG.md | 8 +++++ checks/default.nix | 36 +++++++++++++++++++ docs/API.md | 44 +++++++++++++++++++++++- docs/SUMMARY.md | 2 +- docs/source-filtering.md | 27 +++++++++++++++ examples/quick-start-workspace/flake.nix | 6 ++-- examples/sqlx/flake.nix | 16 +++++---- examples/trunk-workspace/flake.nix | 22 ++++++------ examples/trunk/flake.nix | 21 ++++++----- lib/default.nix | 9 +++++ lib/fileset/cargoTomlAndLock.nix | 7 ++++ lib/fileset/commonCargoSources.nix | 12 +++++++ lib/fileset/configToml.nix | 9 +++++ lib/fileset/rust.nix | 7 ++++ lib/fileset/toml.nix | 7 ++++ pkgs/default.nix | 22 +++++------- 16 files changed, 210 insertions(+), 45 deletions(-) create mode 100644 lib/fileset/cargoTomlAndLock.nix create mode 100644 lib/fileset/commonCargoSources.nix create mode 100644 lib/fileset/configToml.nix create mode 100644 lib/fileset/rust.nix create mode 100644 lib/fileset/toml.nix diff --git a/CHANGELOG.md b/CHANGELOG.md index ff1eec2..e728802 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## Unreleased +### Added +* Added a number of fileset helpers to more easily compose source filtering: + * `fileset.cargoTomlAndLock`: for `Cargo.toml` and `Cargo.lock` files + * `fileset.commonCargoSources`: for files commonly used by cargo projects + * `fileset.configToml`: for `config.toml` files + * `fileset.rust`: for `*.rs` files + * `fileset.toml`: for `*.toml` files + ### Fixed * `buildTrunkPackage` will pass in `--release=true` (instead of just `--release`) for trunk versions 0.21 or higher to avoid argument ambiguities diff --git a/checks/default.nix b/checks/default.nix index 39e1d4b..6d4e2fa 100644 --- a/checks/default.nix +++ b/checks/default.nix @@ -346,6 +346,42 @@ in features = callPackage ./features { }; + fileset = myLib.buildPackage { + src = lib.fileset.toSource { + root = ./simple; + fileset = myLib.fileset.commonCargoSources ./simple; + }; + }; + + filesetBuildScript = myLib.buildPackage { + src = lib.fileset.toSource { + root = ./with-build-script; + fileset = myLib.fileset.commonCargoSources ./with-build-script; + }; + }; + + filesetWorkspace = myLib.buildPackage { + src = lib.fileset.toSource { + root = ./workspace; + fileset = myLib.fileset.commonCargoSources ./workspace; + }; + }; + + filesetWorkspaceInheritance = myLib.buildPackage { + src = lib.fileset.toSource { + root = ./workspace-inheritance; + fileset = myLib.fileset.commonCargoSources ./workspace-inheritance; + }; + }; + + filesetWorkspaceRoot = myLib.buildPackage { + src = lib.fileset.toSource { + root = ./workspace-root; + fileset = myLib.fileset.commonCargoSources ./workspace-root; + }; + pname = "workspace-root"; + }; + gitOverlappingRepo = myLib.buildPackage { src = ./git-overlapping; }; diff --git a/docs/API.md b/docs/API.md index 6447ef5..761a2d6 100644 --- a/docs/API.md +++ b/docs/API.md @@ -1076,7 +1076,6 @@ cleanSourceWith { name = "source"; # Be reproducible, regardless of the directory name } ``` - Note that it is possible to compose source filters, especially if `filterCargoSources` omits files which are relevant to the build. For example: @@ -1094,6 +1093,47 @@ cleanSourceWith { } ``` +### `craneLib.fileset.cargoTomlAndLock` + +`cargoTomlAndLock :: path -> fileset` + +A [fileset] helper which will only include any `Cargo.toml` and `Cargo.lock` +files from the specified path. + +### `craneLib.fileset.commonCargoSources` + +`commonCargoSources :: path -> fileset` + +A [fileset] helper which will only include any files commonly used by cargo +projects from the specified path. Essentially a union of: + +* `craneLib.fileset.cargoTomlAndLock` +* `craneLib.fileset.rust` +* `craneLib.fileset.toml` + +### `craneLib.fileset.configToml` + +`configToml :: path -> fileset` + +A [fileset] helper which will only include `config.toml` files from the +specified path. + +Note that cargo usually only pays attention to `config.toml` files if they are +present inside of a directory named `.cargo`. This fileset will contain any +`config.toml` file, even if its parent directory is _not_ named `.cargo`. + +### `craneLib.fileset.rust` + +`rust :: path -> fileset` + +A [fileset] helper which will only include `*.rs` files from the specified path. + +### `craneLib.fileset.toml` + +`toml :: path -> fileset` + +A [fileset] helper which will only include `*.toml` files from the specified path. + ### `craneLib.mkCargoDerivation` `mkCargoDerivation :: set -> drv` @@ -1866,3 +1906,5 @@ Defines `replaceCargoLock()` which handles replacing or inserting a specified **Automatic behavior:** if `cargoLock` is set and `doNotReplaceCargoLock` is not set, then `replaceCargoLock "$cargoLock"` will be run as a pre patch hook. + +[fileset]: https://nixos.org/manual/nixpkgs/unstable/#sec-functions-library-fileset diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index cb1cbf6..3ab047d 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -21,7 +21,7 @@ * [Workspace with Trunk](./examples/trunk-workspace.md) * [End-to-End Testing](./examples/end-to-end-testing.md) * [Building with SQLx](./examples/sqlx.md) -* [Source filtering](./source-filtering.md) +* [Source filtering and filesets](./source-filtering.md) * [Local development](./local_development.md) * [Custom cargo commands](./custom_cargo_commands.md) * [Customizing builds](./customizing_builds.md) diff --git a/docs/source-filtering.md b/docs/source-filtering.md index 4c99367..ecc6f14 100644 --- a/docs/source-filtering.md +++ b/docs/source-filtering.md @@ -43,3 +43,30 @@ craneLib.buildPackage { }; } ``` + +## Fileset filtering + +A more composable alternative to source filtering is using [filesets]: + +```nix +let + unfilteredRoot = ./.; # The original, unfiltered source + src = lib.fileset.toSource { + root = unfilteredRoot; + fileset = lib.fileset.unions [ + # Default files from crane (Rust and cargo files) + (craneLib.fileset.commonCargoSources unfilteredRoot) + # Also keep any markdown files + (lib.fileset.fileFilter (file: file.hasExt == "md") unfilteredRoot) + # Example of a folder for images, icons, etc + (lib.fileset.maybeMissing ./assets) + ]; + }; +in +craneLib.buildPackage { + # other attributes omitted + inherit src; +} +``` + +[filesets]: https://nixos.org/manual/nixpkgs/unstable/#sec-functions-library-fileset diff --git a/examples/quick-start-workspace/flake.nix b/examples/quick-start-workspace/flake.nix index 1989ebb..894068e 100644 --- a/examples/quick-start-workspace/flake.nix +++ b/examples/quick-start-workspace/flake.nix @@ -71,9 +71,9 @@ fileset = lib.fileset.unions [ ./Cargo.toml ./Cargo.lock - ./crates/my-common - ./crates/my-workspace-hack - crate + (craneLib.fileset.commonCargoSources ./crates/my-common) + (craneLib.fileset.commonCargoSources ./crates/my-workspace-hack) + (craneLib.fileset.commonCargoSources crate) ]; }; diff --git a/examples/sqlx/flake.nix b/examples/sqlx/flake.nix index ea9eccc..f4b999a 100644 --- a/examples/sqlx/flake.nix +++ b/examples/sqlx/flake.nix @@ -18,13 +18,15 @@ craneLib = crane.mkLib pkgs; - sqlFilter = path: _type: null != builtins.match ".*sql$" path; - sqlOrCargo = path: type: (sqlFilter path type) || (craneLib.filterCargoSources path type); - - src = lib.cleanSourceWith { - src = ./.; # The original, unfiltered source - filter = sqlOrCargo; - name = "source"; # Be reproducible, regardless of the directory name + unfilteredRoot = ./.; # The original, unfiltered source + src = lib.fileset.toSource { + root = unfilteredRoot; + fileset = lib.fileset.unions [ + # Default files from crane (Rust and cargo files) + (craneLib.fileset.commonCargoSources unfilteredRoot) + # Include all the .sql migrations as well + ./migrations + ]; }; # Common arguments can be set here to avoid repeating them later diff --git a/examples/trunk-workspace/flake.nix b/examples/trunk-workspace/flake.nix index d28f2aa..cea6939 100644 --- a/examples/trunk-workspace/flake.nix +++ b/examples/trunk-workspace/flake.nix @@ -40,19 +40,21 @@ }); # When filtering sources, we want to allow assets other than .rs files - src = lib.cleanSourceWith { - src = ./.; # The original, unfiltered source - filter = path: type: - (lib.hasSuffix "\.html" path) || - (lib.hasSuffix "\.scss" path) || + unfilteredRoot = ./.; # The original, unfiltered source + src = lib.fileset.toSource { + root = unfilteredRoot; + fileset = lib.fileset.unions [ + # Default files from crane (Rust and cargo files) + (craneLib.fileset.commonCargoSources unfilteredRoot) + (lib.fileset.fileFilter + (file: lib.any file.hasExt [ "html" "scss" ]) + unfilteredRoot + ) # Example of a folder for images, icons, etc - (lib.hasInfix "/assets/" path) || - # Default filter from crane (allow .rs files) - (craneLib.filterCargoSources path type) - ; + (lib.fileset.maybeMissing ./assets) + ]; }; - # Arguments to be used by both the client and the server # When building a workspace with crane, it's a good idea # to set "pname" and "version". diff --git a/examples/trunk/flake.nix b/examples/trunk/flake.nix index cdf0eda..2b2de6e 100644 --- a/examples/trunk/flake.nix +++ b/examples/trunk/flake.nix @@ -40,16 +40,19 @@ }); # When filtering sources, we want to allow assets other than .rs files - src = lib.cleanSourceWith { - src = ./.; # The original, unfiltered source - filter = path: type: - (lib.hasSuffix "\.html" path) || - (lib.hasSuffix "\.scss" path) || + unfilteredRoot = ./.; # The original, unfiltered source + src = lib.fileset.toSource { + root = unfilteredRoot; + fileset = lib.fileset.unions [ + # Default files from crane (Rust and cargo files) + (craneLib.fileset.commonCargoSources unfilteredRoot) + (lib.fileset.fileFilter + (file: lib.any file.hasExt [ "html" "scss" ]) + unfilteredRoot + ) # Example of a folder for images, icons, etc - (lib.hasInfix "/assets/" path) || - # Default filter from crane (allow .rs files) - (craneLib.filterCargoSources path type) - ; + (lib.fileset.maybeMissing ./assets) + ]; }; # Common arguments can be set here to avoid repeating them later diff --git a/lib/default.nix b/lib/default.nix index 114dfe4..9b5deb1 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -87,6 +87,15 @@ let downloadCargoPackage = callPackage ./downloadCargoPackage.nix { }; downloadCargoPackageFromGit = callPackage ./downloadCargoPackageFromGit.nix { }; filterCargoSources = callPackage ./filterCargoSources.nix { }; + + fileset = { + cargoTomlAndLock = callPackage ./fileset/cargoTomlAndLock.nix { }; + commonCargoSources = callPackage ./fileset/commonCargoSources.nix { }; + configToml = callPackage ./fileset/configToml.nix { }; + rust = callPackage ./fileset/rust.nix { }; + toml = callPackage ./fileset/toml.nix { }; + }; + findCargoFiles = callPackage ./findCargoFiles.nix { }; inheritCargoArtifactsHook = callPackage ./setupHooks/inheritCargoArtifacts.nix { }; installCargoArtifactsHook = callPackage ./setupHooks/installCargoArtifacts.nix { }; diff --git a/lib/fileset/cargoTomlAndLock.nix b/lib/fileset/cargoTomlAndLock.nix new file mode 100644 index 0000000..abc810e --- /dev/null +++ b/lib/fileset/cargoTomlAndLock.nix @@ -0,0 +1,7 @@ +{ lib +}: + +path: +lib.fileset.fileFilter + (file: lib.elem file.name [ "Cargo.toml" "Cargo.lock" ]) + path diff --git a/lib/fileset/commonCargoSources.nix b/lib/fileset/commonCargoSources.nix new file mode 100644 index 0000000..481f49c --- /dev/null +++ b/lib/fileset/commonCargoSources.nix @@ -0,0 +1,12 @@ +{ fileset +, lib +}: + +path: +lib.fileset.unions [ + (fileset.cargoTomlAndLock path) + (fileset.rust path) + # Keep all toml files as they are commonly used to configure other + # cargo-based tools + (fileset.toml path) +] diff --git a/lib/fileset/configToml.nix b/lib/fileset/configToml.nix new file mode 100644 index 0000000..0bae90e --- /dev/null +++ b/lib/fileset/configToml.nix @@ -0,0 +1,9 @@ +{ lib +}: + +path: +lib.fileset.fileFilter + # Technically this should be scoped to `.cargo/config.toml` but (currently) + # there is no way to do this with file sets in a generic manner + (file: file.name == "config.toml") + path diff --git a/lib/fileset/rust.nix b/lib/fileset/rust.nix new file mode 100644 index 0000000..a8c9906 --- /dev/null +++ b/lib/fileset/rust.nix @@ -0,0 +1,7 @@ +{ lib +}: + +path: +lib.fileset.fileFilter + (file: file.hasExt "rs") + path diff --git a/lib/fileset/toml.nix b/lib/fileset/toml.nix new file mode 100644 index 0000000..f244831 --- /dev/null +++ b/lib/fileset/toml.nix @@ -0,0 +1,7 @@ +{ lib +}: + +path: +lib.fileset.fileFilter + (file: file.hasExt "toml") + path diff --git a/pkgs/default.nix b/pkgs/default.nix index ea68d9b..68b7130 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -4,20 +4,14 @@ book = let inherit (pkgs) lib; - root = myLib.path ./..; - rootPrefix = toString root; - cleanedSrc = lib.cleanSourceWith { - src = root; - filter = path: _: - let - relativePath = lib.removePrefix rootPrefix path; - in - lib.any (prefix: lib.hasPrefix prefix relativePath) [ - "/docs" # Build the docs directory - "/examples" # But also include examples as we cross-reference them - "/README.md" - "/CHANGELOG.md" - ]; + cleanedSrc = lib.fileset.toSource { + root = ./..; + fileset = lib.fileset.unions [ + ./../docs + ./../examples + ./../README.md + ./../CHANGELOG.md + ]; }; in pkgs.runCommand "crane-book" { } ''