From 8f73eb3cd5cf6ebfd5f540190a2c08169ebb934d Mon Sep 17 00:00:00 2001 From: Sergey Ivanov Date: Mon, 11 Oct 2021 21:07:57 +0400 Subject: [PATCH] Initial fork of https://github.com/numtide/flake-utils --- .github/dependabot.yml | 6 ++ .github/workflows/nix.yml | 16 +++ .gitignore | 6 ++ LICENSE | 21 ++++ README.md | 169 ++++++++++++++++++++++++++++++ check-utils.nix | 48 +++++++++ default.nix | 156 +++++++++++++++++++++++++++ examples/checks-utils/flake.nix | 30 ++++++ examples/each-system/flake.nix | 19 ++++ examples/simple-flake/flake.nix | 13 +++ examples/simple-flake/overlay.nix | 11 ++ examples/simple-flake/shell.nix | 4 + filterPackages.nix | 31 ++++++ flake.nix | 6 ++ flattenTree.nix | 35 +++++++ simpleFlake.nix | 80 ++++++++++++++ 16 files changed, 651 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/nix.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 check-utils.nix create mode 100644 default.nix create mode 100644 examples/checks-utils/flake.nix create mode 100644 examples/each-system/flake.nix create mode 100644 examples/simple-flake/flake.nix create mode 100644 examples/simple-flake/overlay.nix create mode 100644 examples/simple-flake/shell.nix create mode 100644 filterPackages.nix create mode 100644 flake.nix create mode 100644 flattenTree.nix create mode 100644 simpleFlake.nix diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..5ace460 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml new file mode 100644 index 0000000..f5190ff --- /dev/null +++ b/.github/workflows/nix.yml @@ -0,0 +1,16 @@ +name: Nix +on: [ push ] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + # Nix Flakes doesn't work on shallow clones + fetch-depth: 0 + - uses: cachix/install-nix-action@v14 + with: + install_url: https://github.com/numtide/nix-flakes-installer/releases/download/nix-2.4pre20200618_377345e/install + - run: echo "experimental-features = nix-command flakes" | sudo tee -a /etc/nix/nix.conf + - run: nix flake check diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c61512f --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# Nix +result +result-* + +# Don't keep the example lockfile around +/example/flake.lock diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..4cc9fe1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 zimbatm + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..ccacc3d --- /dev/null +++ b/README.md @@ -0,0 +1,169 @@ +# flake-utils + +[![Support room on Matrix](https://img.shields.io/matrix/flake-utils:numtide.com.svg?label=%23flake-utils%3Anumtide.com&logo=matrix&server_fqdn=matrix.numtide.com)](https://matrix.to/#/#flake-utils:numtide.com) + +**STATUS: stable** + +Pure Nix flake utility functions. + +The goal of this project is to build a collection of pure Nix functions that don't +depend on nixpkgs, and that are useful in the context of writing other Nix +flakes. + +## Usage + +### `allSystems -> []` + +A list of all systems defined in nixpkgs. For a smaller list see `defaultSystems` + +### `defaultSystems -> []` + +The list of systems supported by nixpkgs and built by hydra. +Useful if you want add additional platforms: + +```nix +eachSystem (defaultSystems ++ ["armv7l-linux"]) (system: { hello = 42; }) +``` + +### `eachSystem -> [] -> ( -> attrs)` + +A common case is to build the same structure for each system. Instead of +building the hierarchy manually or per prefix, iterate over each systems and +then re-build the hierarchy. + +Eg: + +```nix +eachSystem ["x86_64-linux"] (system: { hello = 42; }) +# => { hello = { x86_64-linux = 42; }; } +eachSystem allSystems (system: { hello = 42; }) +# => { + hello.aarch64-darwin = 42, + hello.aarch64-genode = 42, + hello.aarch64-linux = 42, + ... + hello.x86_64-redox = 42, + hello.x86_64-solaris = 42, + hello.x86_64-windows = 42 +} +``` + +### `eachDefaultSystem -> ( -> attrs)` + +`eachSystem` pre-populated with `defaultSystems`. + +#### Example + +[$ examples/each-system/flake.nix](examples/each-system/flake.nix) as nix +```nix +{ + description = "Flake utils demo"; + + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let pkgs = nixpkgs.legacyPackages.${system}; in + rec { + packages = flake-utils.lib.flattenTree { + hello = pkgs.hello; + gitAndTools = pkgs.gitAndTools; + }; + defaultPackage = packages.hello; + apps.hello = flake-utils.lib.mkApp { drv = packages.hello; }; + defaultApp = apps.hello; + } + ); +} +``` + +### `mkApp { drv, name ? drv.pname or drv.name, exePath ? drv.passthru.exePath or "/bin/${name}"` + +A small utility that builds the structure expected by the special `apps` and `defaultApp` prefixes. + + +### `flattenTree -> attrs -> attrs` + +Nix flakes insists on having a flat attribute set of derivations in +various places like the `packages` and `checks` attributes. + +This function traverses a tree of attributes (by respecting +recurseIntoAttrs) and only returns their derivations, with a flattened +key-space. + +Eg: +```nix +flattenTree { hello = pkgs.hello; gitAndTools = pkgs.gitAndTools } +``` +Returns: + +```nix +{ + hello = «derivation»; + "gitAndTools/git" = «derivation»; + "gitAndTools/hub" = «derivation»; + # ... +} +``` + +### `simpleFlake -> attrs -> attrs` + +This function should be useful for most common use-cases where you have a +simple flake that builds a package. It takes nixpkgs and a bunch of other +parameters and outputs a value that is compatible as a flake output. + +Input: +```nix +{ + # pass an instance of self + self +, # pass an instance of the nixpkgs flake + nixpkgs +, # we assume that the name maps to the project name, and also that the + # overlay has an attribute with the `name` prefix that contains all of the + # project's packages. + name +, # nixpkgs config + config ? { } +, # pass either a function or a file + overlay ? null +, # use this to load other flakes overlays to supplement nixpkgs + preOverlays ? [ ] +, # maps to the devShell output. Pass in a shell.nix file or function. + shell ? null +, # pass the list of supported systems + systems ? [ "x86_64-linux" ] +}: null +``` + +#### Example + +Here is how it looks like in practice: + +[$ examples/simple-flake/flake.nix](examples/simple-flake/flake.nix) as nix +```nix +{ + description = "Flake utils demo"; + + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.simpleFlake { + inherit self nixpkgs; + name = "simple-flake"; + overlay = ./overlay.nix; + shell = ./shell.nix; + }; +} +``` + +## Known issues + +``` +$ nix flake check +warning: unknown flake output 'lib' +``` + +nixpkgs is currently having the same issue so I assume that it will be +eventually standardized. + diff --git a/check-utils.nix b/check-utils.nix new file mode 100644 index 0000000..cd2cb76 --- /dev/null +++ b/check-utils.nix @@ -0,0 +1,48 @@ +systemOrPkgs: +let + inherit (builtins) foldl' unsafeDiscardStringContext elemAt match split concatStringsSep isList substring stringLength length attrNames; + system = systemOrPkgs.system or systemOrPkgs; + pipe = val: functions: foldl' (x: f: f x) val functions; + max = x: y: if x > y then x else y; + + # Minimized copy-paste https://github.com/NixOS/nixpkgs/blob/master/lib/strings.nix#L746-L762 + sanitizeDerivationName = string: pipe (toString string) [ + # Get rid of string context. This is safe under the assumption that the + # resulting string is only used as a derivation name + unsafeDiscardStringContext + # Strip all leading "." + (x: elemAt (match "\\.*(.*)" x) 0) + # Split out all invalid characters + # https://github.com/NixOS/nix/blob/2.3.2/src/libstore/store-api.cc#L85-L112 + # https://github.com/NixOS/nix/blob/2242be83c61788b9c0736a92bb0b5c7bbfc40803/nix-rust/src/store/path.rs#L100-L125 + (split "[^[:alnum:]+._?=-]+") + # Replace invalid character ranges with a "-" + (map (s: if isList s then "-" else s)) + (concatStringsSep "") + # Limit to 211 characters (minus 4 chars for ".drv") + (x: substring (max (stringLength x - 207) 0) (-1) x) + # If the result is empty, replace it with "?EMPTY?" + (x: if stringLength x == 0 then "?EMPTY?" else x) + ]; + + # Minimized version of 'sanitizeDerivationName' function + str = it: if it == null then "null" else (sanitizeDerivationName it); + + test = name: command: derivation { + inherit name system; + builder = "/bin/sh"; + args = [ "-c" command ]; + }; +in +{ + + isEqual = a: b: + if a == b + then test "SUCCESS__${str a}__IS_EQUAL__${str b}" "echo success > $out" + else test "FAILURE__${str a}__NOT_EQUAL__${str b}" "exit 1"; + + hasKey = attrset: key: + if attrset ? ${str key} + then test "SUCCESS__${str key}__EXISTS_IN_ATTRSET" "echo success > $out" + else test "FAILURE__${str key}__DOES_NOT_EXISTS_IN_ATTRSET_SIZE_${str(length (attrNames attrset))}" "exit 1"; +} diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..fdce02f --- /dev/null +++ b/default.nix @@ -0,0 +1,156 @@ +let + # The list of systems supported by nixpkgs and hydra + defaultSystems = [ + "aarch64-linux" + "aarch64-darwin" + "i686-linux" + "x86_64-darwin" + "x86_64-linux" + ]; + + # List of all systems defined in nixpkgs + # Keep in sync with nixpkgs wit the following command: + # $ nix-instantiate --json --eval --expr "with import {}; lib.platforms.all" | jq + allSystems = [ + "aarch64-linux" + "armv5tel-linux" + "armv6l-linux" + "armv7a-linux" + "armv7l-linux" + "mipsel-linux" + "i686-cygwin" + "i686-freebsd" + "i686-linux" + "i686-netbsd" + "i686-openbsd" + "x86_64-cygwin" + "x86_64-freebsd" + "x86_64-linux" + "x86_64-netbsd" + "x86_64-openbsd" + "x86_64-solaris" + "x86_64-darwin" + "i686-darwin" + "aarch64-darwin" + "armv7a-darwin" + "x86_64-windows" + "i686-windows" + "wasm64-wasi" + "wasm32-wasi" + "x86_64-redox" + "powerpc64le-linux" + "riscv32-linux" + "riscv64-linux" + "arm-none" + "armv6l-none" + "aarch64-none" + "avr-none" + "i686-none" + "x86_64-none" + "powerpc-none" + "msp430-none" + "riscv64-none" + "riscv32-none" + "vc4-none" + "js-ghcjs" + "aarch64-genode" + "x86_64-genode" + ]; + + # eachSystem using defaultSystems + eachDefaultSystem = eachSystem defaultSystems; + + # Builds a map from =value to .=value for each system. + # + # + eachSystem = systems: f: + let + op = attrs: system: + let + ret = f system; + op = attrs: key: + attrs // + { + ${key} = (attrs.${key} or { }) // { ${system} = ret.${key}; }; + } + ; + in + builtins.foldl' op attrs (builtins.attrNames ret); + in + builtins.foldl' op { } systems + ; + + # Nix flakes insists on having a flat attribute set of derivations in + # various places like the `packages` and `checks` attributes. + # + # This function traverses a tree of attributes (by respecting + # recurseIntoAttrs) and only returns their derivations, with a flattened + # key-space. + # + # Eg: + # + # flattenTree { hello = pkgs.hello; gitAndTools = pkgs.gitAndTools }; + # + # Returns: + # + # { + # hello = «derivation»; + # "gitAndTools/git" = «derivation»; + # "gitAndTools/hub" = «derivation»; + # # ... + # } + flattenTree = tree: import ./flattenTree.nix tree; + + # Nix check functionality validates packages for various conditions, like if + # they build for any given platform or if they are marked broken. + # + # This function filters a flattend package set for conditinos that + # would *trivially* break `nix flake check`. It does not flatten a tree and it + # does not implement advanced package validation checks. + # + # Eg: + # + # filterPackages "x86_64-linux" { + # hello = pkgs.hello; + # "gitAndTools/git" = pkgs.gitAndTools // {meta.broken = true;}; + # }; + # + # Returns: + # + # { + # hello = «derivation»; + # } + filterPackages = import ./filterPackages.nix { inherit allSystems; }; + + # Returns the structure used by `nix app` + mkApp = + { drv + , name ? drv.pname or drv.name + , exePath ? drv.passthru.exePath or "/bin/${name}" + }: + { + type = "app"; + program = "${drv}${exePath}"; + }; + + # This function tries to capture a common flake pattern. + simpleFlake = import ./simpleFlake.nix { inherit lib; }; + + # Helper functions for Nix evaluation + check-utils = import ./check-utils.nix; + + lib = { + inherit + allSystems + check-utils + defaultSystems + eachDefaultSystem + eachSystem + filterPackages + flattenTree + mkApp + simpleFlake + ; + }; +in +lib diff --git a/examples/checks-utils/flake.nix b/examples/checks-utils/flake.nix new file mode 100644 index 0000000..dbaa4f6 --- /dev/null +++ b/examples/checks-utils/flake.nix @@ -0,0 +1,30 @@ +{ + description = "Flake utils demo"; + + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + inherit (flake-utils.lib.check-utils system) isEqual hasKey; + testDataset = { key1 = "value1"; key2 = 123; key3 = "some>value with^invalid&characters"; }; + in + rec { + checks = { + # Successful cases + success_isEqual = isEqual testDataset.key1 "value1"; + success_hasKey = hasKey testDataset "key2"; + + # Failing cases + failure_isEqual = isEqual testDataset.key1 "failing-data"; + failure_hasKey = hasKey testDataset "failing-data"; + + # Formatting + formatting_number = isEqual testDataset.key2 123; + formatting_null = isEqual null null; + formatting_invalid_chars = isEqual testDataset.key3 "some>value with^invalid&characters"; + + }; + } + ); +} diff --git a/examples/each-system/flake.nix b/examples/each-system/flake.nix new file mode 100644 index 0000000..28bd85f --- /dev/null +++ b/examples/each-system/flake.nix @@ -0,0 +1,19 @@ +{ + description = "Flake utils demo"; + + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let pkgs = nixpkgs.legacyPackages.${system}; in + rec { + packages = flake-utils.lib.flattenTree { + hello = pkgs.hello; + gitAndTools = pkgs.gitAndTools; + }; + defaultPackage = packages.hello; + apps.hello = flake-utils.lib.mkApp { drv = packages.hello; }; + defaultApp = apps.hello; + } + ); +} diff --git a/examples/simple-flake/flake.nix b/examples/simple-flake/flake.nix new file mode 100644 index 0000000..a027b51 --- /dev/null +++ b/examples/simple-flake/flake.nix @@ -0,0 +1,13 @@ +{ + description = "Flake utils demo"; + + inputs.flake-utils.url = "github:numtide/flake-utils"; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.simpleFlake { + inherit self nixpkgs; + name = "simple-flake"; + overlay = ./overlay.nix; + shell = ./shell.nix; + }; +} diff --git a/examples/simple-flake/overlay.nix b/examples/simple-flake/overlay.nix new file mode 100644 index 0000000..772e417 --- /dev/null +++ b/examples/simple-flake/overlay.nix @@ -0,0 +1,11 @@ +final: prev: +{ + # this key should be the same as the simpleFlake name attribute. + simple-flake = { + # assuming that hello is a project-specific package; + hello = prev.hello; + + # demonstrating recursive packages + terraform-providers = prev.terraform-providers; + }; +} diff --git a/examples/simple-flake/shell.nix b/examples/simple-flake/shell.nix new file mode 100644 index 0000000..b96bfda --- /dev/null +++ b/examples/simple-flake/shell.nix @@ -0,0 +1,4 @@ +{ pkgs ? import }: +pkgs.mkShell { + buildInputs = [ pkgs.jq ]; +} diff --git a/filterPackages.nix b/filterPackages.nix new file mode 100644 index 0000000..5bd4eba --- /dev/null +++ b/filterPackages.nix @@ -0,0 +1,31 @@ +{ allSystems }: +system: packages: +let + # Adopted from nixpkgs.lib + inherit (builtins) listToAttrs concatMap attrNames; + nameValuePair = name: value: { inherit name value; }; + filterAttrs = pred: set: + listToAttrs ( + concatMap + (name: + let v = set.${name}; in + if pred name v then [ (nameValuePair name v) ] else [ ] + ) + (attrNames set) + ); + + # Everything that nix flake check requires for the packages output + sieve = n: v: + with v; + let + inherit (builtins) isAttrs; + isDerivation = x: isAttrs x && x ? type && x.type == "derivation"; + isBroken = meta.broken or false; + platforms = meta.hydraPlatforms or meta.platforms or allSystems; + in + # check for isDerivation, so this is independently useful of + # flattenTree, which also does filter on derviations + isDerivation v && !isBroken && builtins.elem system platforms + ; +in +filterAttrs sieve packages diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..2d1646b --- /dev/null +++ b/flake.nix @@ -0,0 +1,6 @@ +{ + description = "Pure Nix flake utility functions"; + outputs = { self }: { + lib = import ./.; + }; +} diff --git a/flattenTree.nix b/flattenTree.nix new file mode 100644 index 0000000..3ad30d0 --- /dev/null +++ b/flattenTree.nix @@ -0,0 +1,35 @@ +tree: +let + op = sum: path: val: + let + pathStr = builtins.concatStringsSep "/" path; + in + if (builtins.typeOf val) != "set" then + # ignore that value + # builtins.trace "${pathStr} is not of type set" + sum + else if val ? type && val.type == "derivation" then + # builtins.trace "${pathStr} is a derivation" + # we used to use the derivation outPath as the key, but that crashes Nix + # so fallback on constructing a static key + (sum // { + "${pathStr}" = val; + }) + else if val ? recurseForDerivations && val.recurseForDerivations == true then + # builtins.trace "${pathStr} is a recursive" + # recurse into that attribute set + (recurse sum path val) + else + # ignore that value + # builtins.trace "${pathStr} is something else" + sum + ; + + recurse = sum: path: val: + builtins.foldl' + (sum: key: op sum (path ++ [ key ]) val.${key}) + sum + (builtins.attrNames val) + ; +in +recurse { } [ ] tree diff --git a/simpleFlake.nix b/simpleFlake.nix new file mode 100644 index 0000000..c26bd08 --- /dev/null +++ b/simpleFlake.nix @@ -0,0 +1,80 @@ +{ lib }: +# This function returns a flake outputs-compatible schema. +{ + # pass an instance of self + self +, # pass an instance of the nixpkgs flake + nixpkgs +, # we assume that the name maps to the project name, and also that the + # overlay has an attribute with the `name` prefix that contains all of the + # project's packages. + name +, # nixpkgs config + config ? { } +, # pass either a function or a file + overlay ? null +, # use this to load other flakes overlays to supplement nixpkgs + preOverlays ? [ ] +, # maps to the devShell output. Pass in a shell.nix file or function. + shell ? null +, # pass the list of supported systems + systems ? [ "x86_64-linux" ] +}: +let + loadOverlay = obj: + if obj == null then + [ ] + else + [ (maybeImport obj) ] + ; + + maybeImport = obj: + if (builtins.typeOf obj == "path") || (builtins.typeOf obj == "string") then + import obj + else + obj + ; + + overlays = preOverlays ++ (loadOverlay overlay); + + shell_ = maybeImport shell; + + outputs = lib.eachSystem systems (system: + let + pkgs = import nixpkgs { + inherit + config + overlays + system + ; + }; + + packages = pkgs.${name} or { }; + in + { + # Use the legacy packages since it's more forgiving. + legacyPackages = packages; + } + // + ( + if packages ? defaultPackage then { + defaultPackage = packages.defaultPackage; + } else { } + ) + // + ( + if packages ? checks then { + checks = packages.checks; + } else { } + ) + // + ( + if shell != null then { + devShell = shell_ { inherit pkgs; }; + } else if packages ? devShell then { + devShell = packages.devShell; + } else { } + ) + ); +in +outputs