[OPS-1384] Introduce NixOS VM tests

Problem: Currently, the only way to test deploy-rs deployments is to
actually do a deployment to an existing NixOS instance (either in VM,
or a real machine) manually. This is a bit inconvenient and one can
forget to test changes when developing/reviewing deploy-rs changes.

Solution: Add NixOS VM tests.

.github/workflows/check.yml

@@ -2,10 +2,26 @@ name: Nix flake check
 on: pull_request
 
 jobs:
-  check:
-    runs-on: self-hosted
+  get-matrix:
+    runs-on: [self-hosted, nix]
+    outputs:
+      check-matrix: ${{ steps.set-check-matrix.outputs.matrix }}
     steps:
       - uses: actions/checkout@v4
 
-      - name: check flake
-        run: nix flake check -L
+      - id: set-check-matrix
+        run: echo "matrix=$(nix eval --json .#check-matrix.x86_64-linux)" >> $GITHUB_OUTPUT
+
+  check:
+    needs: get-matrix
+    name: check ${{ matrix.check }}
+    runs-on: [self-hosted, nix]
+    strategy:
+      fail-fast: false
+      # this matrix consists of the names of all checks defined in flake.nix
+      matrix: ${{fromJson(needs.get-matrix.outputs.check-matrix)}}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: check
+        run: nix build -L .#checks.x86_64-linux.${{ matrix.check }}

flake.nix

@@ -15,10 +15,9 @@
     };
   };
 
-  outputs = { self, nixpkgs, utils, ... }:
+  outputs = { self, nixpkgs, utils, ... }@inputs:
   {
-    overlay = final: prev:
-    let
+    overlays.default = final: prev: let
       system = final.stdenv.hostPlatform.system;
       darwinOptions = final.lib.optionalAttrs final.stdenv.isDarwin {
         buildInputs = with final.darwin.apple_sdk.frameworks; [
@@ -34,7 +33,13 @@
           pname = "deploy-rs";
           version = "0.1.0";
 
-          src = ./.;
+          src = final.lib.sourceByRegex ./. [
+            "Cargo\.lock"
+            "Cargo\.toml"
+            "src"
+            "src/bin"
+            ".*\.rs$"
+          ];
 
           cargoLock.lockFile = ./Cargo.lock;
         }) // { meta.description = "A Simple multi-profile Nix-flake deploy tool"; };
@@ -145,7 +150,15 @@
   } //
     utils.lib.eachSystem (utils.lib.defaultSystems ++ ["aarch64-darwin"]) (system:
       let
-        pkgs = import nixpkgs { inherit system; overlays = [ self.overlay ]; };
+        pkgs = import nixpkgs {
+          inherit system;
+          overlays = [ self.overlays.default ];
+        };
+
+        # make a matrix to use in GitHub pipeline
+        mkMatrix = name: attrs: {
+          include = map (v: { ${name} = v; }) (pkgs.lib.attrNames attrs);
+        };
       in
       {
         defaultPackage = self.packages."${system}".deploy-rs;
@@ -176,8 +189,12 @@
 
         checks = {
           deploy-rs = self.packages.${system}.default.overrideAttrs (super: { doCheck = true; });
-        };
+        } // (pkgs.lib.optionalAttrs (pkgs.lib.elem system ["x86_64-linux"]) (import ./nix/tests {
+          inherit inputs pkgs;
+        }));
 
-        lib = pkgs.deploy-rs.lib;
+        inherit (pkgs.deploy-rs) lib;
+
+        check-matrix = mkMatrix "check" self.checks.${system};
       });
 }

nix/tests/common.nix

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2024 Serokell <https://serokell.io/>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+{inputs, pkgs, ...}: {
+  nix = {
+    registry.nixpkgs.flake = inputs.nixpkgs;
+    extraOptions = ''
+      experimental-features = nix-command flakes
+    '';
+    settings = {
+      trusted-users = [ "root" "@wheel" ];
+      substituters = pkgs.lib.mkForce [];
+    };
+  };
+
+  virtualisation.graphics = false;
+  virtualisation.memorySize = 1536;
+  boot.loader.grub.enable = false;
+  documentation.enable = false;
+}

nix/tests/default.nix

@@ -0,0 +1,134 @@
+# SPDX-FileCopyrightText: 2024 Serokell <https://serokell.io/>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+{ pkgs , inputs , ... }:
+let
+  inherit (pkgs) system lib;
+
+  inherit (import "${pkgs.path}/nixos/tests/ssh-keys.nix" pkgs) snakeOilPrivateKey;
+
+  # Include all build dependencies to be able to build profiles offline
+  allDrvOutputs = pkg: pkgs.runCommand "allDrvOutputs" { refs = pkgs.writeReferencesToFile pkg.drvPath; } ''
+    touch $out
+    while read ref; do
+      case $ref in
+        *.drv)
+          cat $ref >>$out
+          ;;
+      esac
+    done <$refs
+  '';
+
+  mkTest = { name ? "", user ? "root", isLocal ? true, deployArgs }: let
+    nodes = {
+      server = { nodes, ... }: {
+        imports = [
+         ./server.nix
+         (import ./common.nix { inherit inputs pkgs; })
+        ];
+        virtualisation.additionalPaths = lib.optionals (!isLocal) [
+          pkgs.hello
+          pkgs.figlet
+          (allDrvOutputs nodes.server.system.build.toplevel)
+          pkgs.deploy-rs.deploy-rs
+        ];
+      };
+      client = { nodes, ... }: {
+        imports = [ (import ./common.nix { inherit inputs pkgs; }) ];
+        environment.systemPackages = [ pkgs.deploy-rs.deploy-rs ];
+        virtualisation.additionalPaths = lib.optionals isLocal [
+          pkgs.hello
+          pkgs.figlet
+          (allDrvOutputs nodes.server.system.build.toplevel)
+        ];
+      };
+    };
+
+    flakeInputs = ''
+      deploy-rs.url = "${../..}";
+      deploy-rs.inputs.utils.follows = "utils";
+      deploy-rs.inputs.flake-compat.follows = "flake-compat";
+
+      nixpkgs.url = "${inputs.nixpkgs}";
+      utils.url = "${inputs.utils}";
+      utils.inputs.systems.follows = "systems";
+      systems.url = "${inputs.utils.inputs.systems}";
+      flake-compat.url = "${inputs.flake-compat}";
+      flake-compat.flake = false;
+    '';
+
+    flake = builtins.toFile "flake.nix"
+      (lib.replaceStrings [ "##inputs##" ] [ flakeInputs ] (builtins.readFile ./deploy-flake.nix));
+
+  in pkgs.nixosTest {
+    inherit nodes name;
+
+    testScript = { nodes }: let
+      serverNetworkJSON = pkgs.writeText "server-network.json"
+        (builtins.toJSON nodes.server.system.build.networkConfig);
+    in ''
+      start_all()
+
+      # Prepare
+      client.succeed("mkdir tmp && cd tmp")
+      client.succeed("cp ${flake} ./flake.nix")
+      client.succeed("cp ${./server.nix} ./server.nix")
+      client.succeed("cp ${./common.nix} ./common.nix")
+      client.succeed("cp ${serverNetworkJSON} ./network.json")
+      client.succeed("nix flake lock")
+
+
+      # Setup SSH key
+      client.succeed("mkdir -m 700 /root/.ssh")
+      client.succeed('cp --no-preserve=mode ${snakeOilPrivateKey} /root/.ssh/id_ed25519')
+      client.succeed("chmod 600 /root/.ssh/id_ed25519")
+
+      # Test SSH connection
+      server.wait_for_open_port(22)
+      client.wait_for_unit("network.target")
+      client.succeed(
+        "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no server 'echo hello world' >&2",
+        timeout=30
+      )
+
+      # Make sure the hello and figlet packages are missing
+      server.fail("su ${user} -l -c 'hello | figlet'")
+
+      # Deploy to the server
+      client.succeed("deploy ${deployArgs}")
+
+      # Make sure packages are present after deployment
+      server.succeed("su ${user} -l -c 'hello | figlet' >&2")
+    '';
+  };
+in {
+  # Deployment with client-side build
+  local-build = mkTest {
+    name = "local-build";
+    deployArgs = "-s .#server -- --offline";
+  };
+  # Deployment with server-side build
+  remote-build = mkTest {
+    name = "remote-build";
+    isLocal = false;
+    deployArgs = "-s .#server --remote-build -- --offline";
+  };
+  # Deployment with overridden options
+  options-overriding = mkTest {
+    name = "options-overriding";
+    deployArgs = lib.concatStrings [
+      "-s .#server-override"
+      " --hostname server --profile-user root --ssh-user root --sudo 'sudo -u'"
+      " --ssh-opts='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
+      " --confirm-timeout 30 --activation-timeout 30"
+      " -- --offline"
+    ];
+  };
+  # User profile deployment
+  profile = mkTest {
+    name = "profile";
+    user = "deploy";
+    deployArgs = "-s .#profile -- --offline";
+  };
+}

nix/tests/deploy-flake.nix

@@ -0,0 +1,72 @@
+# SPDX-FileCopyrightText: 2024 Serokell <https://serokell.io/>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+{
+  inputs = {
+    # real inputs are substituted in ./default.nix
+##inputs##
+  };
+
+  outputs = { self, nixpkgs, deploy-rs, ... }@inputs: let
+    system = "x86_64-linux";
+    pkgs = inputs.nixpkgs.legacyPackages.${system};
+    user = "deploy";
+  in {
+    nixosConfigurations.server = nixpkgs.lib.nixosSystem {
+      inherit system pkgs;
+      specialArgs = { inherit inputs; };
+      modules = [
+        ./server.nix
+        ./common.nix
+        # Import the base config used by nixos tests
+        (pkgs.path + "/nixos/lib/testing/nixos-test-base.nix")
+        # Deployment breaks the network settings, so we need to restore them
+        (pkgs.lib.importJSON ./network.json)
+        # Deploy packages
+        { environment.systemPackages = [ pkgs.figlet pkgs.hello ]; }
+      ];
+    };
+
+    deploy.nodes = {
+      server = {
+        hostname = "server";
+        sshUser = "root";
+        sshOpts = [
+          "-o" "StrictHostKeyChecking=no"
+          "-o" "StrictHostKeyChecking=no"
+        ];
+        profiles.system.path = deploy-rs.lib."${system}".activate.nixos self.nixosConfigurations.server;
+      };
+      server-override = {
+        hostname = "override";
+        sshUser = "override";
+        user = "override";
+        sudo = "override";
+        sshOpts = [ ];
+        confirmTimeout = 0;
+        activationTimeout = 0;
+        profiles.system.path = deploy-rs.lib."${system}".activate.nixos self.nixosConfigurations.server;
+      };
+      profile = {
+        hostname = "server";
+        sshUser = "${user}";
+        sshOpts = [
+          "-o" "UserKnownHostsFile=/dev/null"
+          "-o" "StrictHostKeyChecking=no"
+        ];
+        profiles = {
+          "hello-world".path = let
+            activateProfile = pkgs.writeShellScriptBin "activate" ''
+              set -euo pipefail
+              mkdir -p /home/${user}/.nix-profile/bin
+              rm -f -- /home/${user}/.nix-profile/bin/hello /home/${user}/.nix-profile/bin/figlet
+              ln -s ${pkgs.hello}/bin/hello /home/${user}/.nix-profile/bin/hello
+              ln -s ${pkgs.figlet}/bin/figlet /home/${user}/.nix-profile/bin/figlet
+            '';
+          in deploy-rs.lib.${system}.activate.custom activateProfile "$PROFILE/bin/activate";
+        };
+      };
+    };
+  };
+}

nix/tests/server.nix

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: 2024 Serokell <https://serokell.io/>
+#
+# SPDX-License-Identifier: MPL-2.0
+{ pkgs, ... }:
+{
+  nix.settings.trusted-users = [ "deploy" ];
+  users = let
+    inherit (import "${pkgs.path}/nixos/tests/ssh-keys.nix" pkgs) snakeOilPublicKey;
+  in {
+    mutableUsers = false;
+    users = {
+      deploy = {
+        password = "";
+        isNormalUser = true;
+        createHome = true;
+        openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
+      };
+      root.openssh.authorizedKeys.keys = [ snakeOilPublicKey ];
+    };
+  };
+  services.openssh.enable = true;
+  virtualisation.writableStore = true;
+}