[Chore] Flakify xrefcheck

Problem: For historical reasons some Serokell projects use niv to
organize nix-related dependencies. However, the majority of other
projects in our company use nix flakes to organize these dependencies.

Solution: Add flake and provide ci attibutes as flake outputs.
This commit is contained in:
Savely Krendelhoff 2022-12-23 10:43:18 +03:00
parent d50a8e76ba
commit 015b12126a
No known key found for this signature in database
GPG Key ID: F70DFD34F40238DE
11 changed files with 1196 additions and 357 deletions

35
ci.nix
View File

@ -1,35 +0,0 @@
# SPDX-FileCopyrightText: 2021 Serokell <https://serokell.io>
#
# SPDX-License-Identifier: MPL-2.0
rec {
sources = import ./nix/sources.nix;
haskell-nix = import sources."haskell.nix" {
sourcesOverride = { hackage = sources."hackage.nix"; stackage = sources."stackage.nix"; };
};
serokell-nix = import sources."serokell.nix";
pkgs = import sources.nixpkgs (
haskell-nix.nixpkgsArgs // {
overlays =
haskell-nix.nixpkgsArgs.overlays
++ [ serokell-nix.overlay ]; # contains trailing whitespace check
}
);
project-src = pkgs.haskell-nix.haskellLib.cleanGit {
name = "xrefcheck";
src = pkgs.haskell-nix.haskellLib.cleanSourceWith {
filter = path: type: !(pkgs.lib.hasInfix "tests/golden/helpers" path);
src = ./.;
};
};
# TODO: drop this when `serokell/nixpkgs` acquires stylish-haskell >= 0.13.0.0.
pkgs-stylish = import sources.nixpkgs-stylish {};
xrefcheck-lib-and-tests = (import ./xrefcheck.nix { linux = true; });
xrefcheck-static = (import ./xrefcheck.nix { linux-static = true; }).components.exes.xrefcheck;
xrefcheck-windows = (import ./xrefcheck.nix { windows = true; }).components.exes.xrefcheck;
trailing-whitespace-check = pkgs.build.checkTrailingWhitespace project-src;
}

View File

@ -1,7 +1,14 @@
# SPDX-FileCopyrightText: 2019 Serokell <https://serokell.io>
#
# SPDX-FileCopyrightText: 2023 Serokell <https://serokell.io>
# SPDX-License-Identifier: MPL-2.0
{ platform ? "linux-static" }:
(import ./xrefcheck.nix { ${platform} = true; }).components.exes.xrefcheck
let
inherit (import
(let lock = builtins.fromJSON (builtins.readFile ./flake.lock); in
fetchTarball {
url = "https://github.com/edolstra/flake-compat/archive/${lock.nodes.flake-compat.locked.rev}.tar.gz";
sha256 = lock.nodes.flake-compat.locked.narHash;
})
{ src = ./.; }) defaultNix;
system = builtins.currentSystem;
in defaultNix //
(defaultNix.legacyPackages.${system}.lib.attrsets.mapAttrs (_: val: val.${system}) defaultNix)

View File

@ -1,18 +0,0 @@
# SPDX-FileCopyrightText: 2020 Serokell <https://serokell.io>
#
# SPDX-License-Identifier: MPL-2.0
{ pkgs ? import (import ../nix/sources.nix).nixpkgs { } }:
let
executable =
(import ../xrefcheck.nix { linux-static = true; }).components.exes.xrefcheck;
binOnly = pkgs.runCommand "xrefcheck-bin" { } ''
mkdir -p $out/bin
cp ${executable}/bin/xrefcheck $out/bin
${pkgs.nukeReferences}/bin/nuke-refs $out/bin/xrefcheck
'';
in pkgs.dockerTools.buildImage {
name = "xrefcheck";
contents = [ binOnly pkgs.cacert ];
config.Entrypoint = "xrefcheck";
}

1091
flake.lock Normal file

File diff suppressed because it is too large Load Diff

2
flake.lock.license Normal file
View File

@ -0,0 +1,2 @@
SPDX-FileCopyrightText: 2023 Serokell <https://serokell.io>
SPDX-License-Identifier: MPL-2.0

91
flake.nix Normal file
View File

@ -0,0 +1,91 @@
# SPDX-FileCopyrightText: 2023 Serokell <https://serokell.io>
# SPDX-License-Identifier: MPL-2.0
{
description = "The xrefcheck flake";
nixConfig.flake-registry = "https://github.com/serokell/flake-registry/raw/master/flake-registry.json";
outputs = { self, flake-utils, haskell-nix, serokell-nix, ... }:
flake-utils.lib.eachSystem [ "x86_64-linux" ] (system:
let
pkgs = haskell-nix.legacyPackages.${system}.extend serokell-nix.overlay;
flake = (pkgs.haskell-nix.stackProject {
src = ./.;
modules = [{
packages.xrefcheck = {
ghcOptions =
[ "-Werror" ];
components.tests = {
ftp-tests = {
build-tools = [ pkgs.vsftpd ];
preCheck = ''
echo "Starting vsftpd..."
touch /tmp/vsftpd.log
vsftpd \
-orun_as_launching_user=yes \
-olisten_port=2221 \
-olisten=yes \
-oftp_username=$(whoami) \
-oanon_root=${./ftp-tests/ftp_root} \
-opasv_min_port=2222 \
-ohide_file='{.*}' \
-odeny_file='{.*}' \
-oseccomp_sandbox=no \
-olog_ftp_protocol=yes \
-oxferlog_enable=yes \
-ovsftpd_log_file=/tmp/vsftpd.log &
sleep 1
tail -f /tmp/vsftpd.log &
'';
testFlags = [ "--ftp-host" "ftp://localhost:2221" ];
};
xrefcheck-tests.build-tools = [ pkgs.git ];
};
};
}];
}).flake { crossPlatforms = p: [ p.musl64 p.mingwW64 ]; };
in
pkgs.lib.lists.foldr pkgs.lib.recursiveUpdate {} [
{ inherit (flake) packages apps; }
{
legacyPackages = pkgs;
apps.default = self.apps.${system}."x86_64-unknown-linux-musl:xrefcheck:exe:xrefcheck";
packages = {
default = self.packages.${system}.xrefcheck;
xrefcheck = self.packages.${system}."xrefcheck:exe:xrefcheck";
xrefcheck-static = self.packages.${system}."x86_64-unknown-linux-musl:xrefcheck:exe:xrefcheck";
xrefcheck-windows = self.packages.${system}."x86_64-w64-mingw32:xrefcheck:exe:xrefcheck";
docker-image =
let
executable = self.packages.${system}.xrefcheck-static;
binOnly = pkgs.runCommand "xrefcheck-bin" {} ''
mkdir -p $out/bin
cp ${executable}/bin/xrefcheck $out/bin
${pkgs.nukeReferences}/bin/nuke-refs $out/bin/xrefcheck
'';
in pkgs.dockerTools.buildImage {
name = "xrefcheck";
contents = [ binOnly pkgs.cacert ];
config.Entrypoint = "xrefcheck";
};
};
checks = {
trailing-whitespace = pkgs.build.checkTrailingWhitespace ./.;
reuse-lint = pkgs.build.reuseLint ./.;
};
}
]
);
}

View File

@ -1,74 +0,0 @@
{
"hackage.nix": {
"branch": "master",
"description": "Automatically generated Nix expressions for Hackage",
"homepage": "",
"owner": "input-output-hk",
"repo": "hackage.nix",
"rev": "af1008186119c18e9f4f42e1c463d69d5b5fd288",
"sha256": "1569jk15y3xq8wnjnf151xb125r2drr0cpska6mgsbpiddlp0mr6",
"type": "tarball",
"url": "https://github.com/input-output-hk/hackage.nix/archive/af1008186119c18e9f4f42e1c463d69d5b5fd288.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"haskell.nix": {
"branch": "master",
"description": "Alternative Haskell Infrastructure for Nixpkgs",
"homepage": "https://input-output-hk.github.io/haskell.nix",
"owner": "input-output-hk",
"repo": "haskell.nix",
"rev": "b302c7a5b5fe1379f57abb81cbf1c0daabe44616",
"sha256": "0cqbm6xxkn5dmm21cxcrv7yvz8lz6kssp5cj4bf837kirbbg037j",
"type": "tarball",
"url": "https://github.com/input-output-hk/haskell.nix/archive/b302c7a5b5fe1379f57abb81cbf1c0daabe44616.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs": {
"branch": "master",
"description": "Pinned Nixpkgs tree (master follows nixos-unstable-small, only tags have stable history)",
"homepage": "",
"owner": "serokell",
"repo": "nixpkgs",
"rev": "7887e272ab2ad8376d54ca5f58df0ed7f67676e6",
"sha256": "0sx7ff2ps79izkaqnigyj85vs6jffvx8x9kgl0qf4wv0n3b2wwmj",
"type": "tarball",
"url": "https://github.com/serokell/nixpkgs/archive/7887e272ab2ad8376d54ca5f58df0ed7f67676e6.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"nixpkgs-stylish": {
"branch": "master",
"description": "Nix Packages collection",
"homepage": "",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "19574af0af3ffaf7c9e359744ed32556f34536bd",
"sha256": "0v3c4r8v40jimicdxqvxnzmdypnafm2baam7z131zk6ljhb8jpg9",
"type": "tarball",
"url": "https://github.com/NixOS/nixpkgs/archive/19574af0af3ffaf7c9e359744ed32556f34536bd.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"serokell.nix": {
"branch": "master",
"description": "Serokell Nix infrastructure library",
"homepage": null,
"owner": "serokell",
"repo": "serokell.nix",
"rev": "652105c5fc5564f5d8e682d61bdc0d51bbc1f939",
"sha256": "19zkjhyjbksf2fw32h7m3x0v8nvgdy2qkjkxg2qzzb25as245vmj",
"type": "tarball",
"url": "https://github.com/serokell/serokell.nix/archive/652105c5fc5564f5d8e682d61bdc0d51bbc1f939.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"stackage.nix": {
"branch": "master",
"description": "Automatically generated Nix expressions of Stackage snapshots",
"homepage": "",
"owner": "input-output-hk",
"repo": "stackage.nix",
"rev": "9ef56f70c138620b65ea42cade5e493418b0ae50",
"sha256": "0jrnv0w2yqkxkfnc64y7qrab7grr84mwp56y6r5x1npw174wpl8v",
"type": "tarball",
"url": "https://github.com/input-output-hk/stackage.nix/archive/9ef56f70c138620b65ea42cade5e493418b0ae50.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
}
}

View File

@ -1,2 +0,0 @@
# SPDX-FileCopyrightText: 2019 Serokell <https://serokell.io>
# SPDX-License-Identifier: MPL-2.0

View File

@ -1,174 +0,0 @@
# This file has been generated by Niv.
let
#
# The fetchers. fetch_<type> fetches specs of type <type>.
#
fetch_file = pkgs: name: spec:
let
name' = sanitizeName name + "-src";
in
if spec.builtin or true then
builtins_fetchurl { inherit (spec) url sha256; name = name'; }
else
pkgs.fetchurl { inherit (spec) url sha256; name = name'; };
fetch_tarball = pkgs: name: spec:
let
name' = sanitizeName name + "-src";
in
if spec.builtin or true then
builtins_fetchTarball { name = name'; inherit (spec) url sha256; }
else
pkgs.fetchzip { name = name'; inherit (spec) url sha256; };
fetch_git = name: spec:
let
ref =
if spec ? ref then spec.ref else
if spec ? branch then "refs/heads/${spec.branch}" else
if spec ? tag then "refs/tags/${spec.tag}" else
abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!";
in
builtins.fetchGit { url = spec.repo; inherit (spec) rev; inherit ref; };
fetch_local = spec: spec.path;
fetch_builtin-tarball = name: throw
''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`.
$ niv modify ${name} -a type=tarball -a builtin=true'';
fetch_builtin-url = name: throw
''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`.
$ niv modify ${name} -a type=file -a builtin=true'';
#
# Various helpers
#
# https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695
sanitizeName = name:
(
concatMapStrings (s: if builtins.isList s then "-" else s)
(
builtins.split "[^[:alnum:]+._?=-]+"
((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name)
)
);
# The set of packages used when specs are fetched using non-builtins.
mkPkgs = sources: system:
let
sourcesNixpkgs =
import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; };
hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath;
hasThisAsNixpkgsPath = <nixpkgs> == ./.;
in
if builtins.hasAttr "nixpkgs" sources
then sourcesNixpkgs
else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then
import <nixpkgs> {}
else
abort
''
Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
add a package called "nixpkgs" to your sources.json.
'';
# The actual fetching function.
fetch = pkgs: name: spec:
if ! builtins.hasAttr "type" spec then
abort "ERROR: niv spec ${name} does not have a 'type' attribute"
else if spec.type == "file" then fetch_file pkgs name spec
else if spec.type == "tarball" then fetch_tarball pkgs name spec
else if spec.type == "git" then fetch_git name spec
else if spec.type == "local" then fetch_local spec
else if spec.type == "builtin-tarball" then fetch_builtin-tarball name
else if spec.type == "builtin-url" then fetch_builtin-url name
else
abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}";
# If the environment variable NIV_OVERRIDE_${name} is set, then use
# the path directly as opposed to the fetched source.
replace = name: drv:
let
saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name;
ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}";
in
if ersatz == "" then drv else
# this turns the string into an actual Nix path (for both absolute and
# relative paths)
if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}";
# Ports of functions for older nix versions
# a Nix version of mapAttrs if the built-in doesn't exist
mapAttrs = builtins.mapAttrs or (
f: set: with builtins;
listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set))
);
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295
range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1);
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257
stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1));
# https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269
stringAsChars = f: s: concatStrings (map f (stringToCharacters s));
concatMapStrings = f: list: concatStrings (map f list);
concatStrings = builtins.concatStringsSep "";
# https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331
optionalAttrs = cond: as: if cond then as else {};
# fetchTarball version that is compatible between all the versions of Nix
builtins_fetchTarball = { url, name ? null, sha256 }@attrs:
let
inherit (builtins) lessThan nixVersion fetchTarball;
in
if lessThan nixVersion "1.12" then
fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
else
fetchTarball attrs;
# fetchurl version that is compatible between all the versions of Nix
builtins_fetchurl = { url, name ? null, sha256 }@attrs:
let
inherit (builtins) lessThan nixVersion fetchurl;
in
if lessThan nixVersion "1.12" then
fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; }))
else
fetchurl attrs;
# Create the final "sources" from the config
mkSources = config:
mapAttrs (
name: spec:
if builtins.hasAttr "outPath" spec
then abort
"The values in sources.json should not have an 'outPath' attribute"
else
spec // { outPath = replace name (fetch config.pkgs name spec); }
) config.sources;
# The "config" used by the fetchers
mkConfig =
{ sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null
, sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile)
, system ? builtins.currentSystem
, pkgs ? mkPkgs sources system
}: rec {
# The sources, i.e. the attribute set of spec name to spec
inherit sources;
# The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers
inherit pkgs;
};
in
mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }

View File

@ -1,2 +0,0 @@
# SPDX-FileCopyrightText: 2019 Nicolas Mattia
# SPDX-License-Identifier: LicenseRef-MIT-Mattia

View File

@ -1,47 +0,0 @@
# SPDX-FileCopyrightText: 2020 Serokell <https://serokell.io>
#
# SPDX-License-Identifier: MPL-2.0
{ linux ? false, linux-static ? false, windows ? false }:
let
nixpkgs = (import ./ci.nix).pkgs;
src = (import ./ci.nix).project-src;
pkgs = if linux-static then nixpkgs.pkgsCross.musl64 else if windows then nixpkgs.pkgsCross.mingwW64 else nixpkgs;
project = pkgs.haskell-nix.stackProject {
src = pkgs.haskell-nix.haskellLib.cleanGit { src = ./.; keepGitDir = true; };
modules = [{
packages.xrefcheck = {
ghcOptions =
[ "-Werror" ];
components.tests = {
ftp-tests = {
build-tools = [ pkgs.vsftpd ];
preCheck = ''
echo "Starting vsftpd..."
touch /tmp/vsftpd.log
vsftpd \
-orun_as_launching_user=yes \
-olisten_port=2221 \
-olisten=yes \
-oftp_username=$(whoami) \
-oanon_root=${./ftp-tests/ftp_root} \
-opasv_min_port=2222 \
-ohide_file='{.*}' \
-odeny_file='{.*}' \
-oseccomp_sandbox=no \
-olog_ftp_protocol=yes \
-oxferlog_enable=yes \
-ovsftpd_log_file=/tmp/vsftpd.log &
sleep 1
tail -f /tmp/vsftpd.log &
'';
testFlags = [ "--ftp-host" "ftp://localhost:2221" ];
};
xrefcheck-tests.build-tools = [ pkgs.git ];
};
};
}];
};
in
project.xrefcheck