Merge pull request #313020 from jpds/nixos-test-vector-api+clickhouse

nixos/vector: Tests for API/Clickhouse

nixos/tests/all-tests.nix

@@ -1007,7 +1007,7 @@ in {
   vault-dev = handleTest ./vault-dev.nix {};
   vault-postgresql = handleTest ./vault-postgresql.nix {};
   vaultwarden = handleTest ./vaultwarden.nix {};
-  vector = handleTest ./vector.nix {};
+  vector = handleTest ./vector {};
   vengi-tools = handleTest ./vengi-tools.nix {};
   victoriametrics = handleTest ./victoriametrics.nix {};
   vikunja = handleTest ./vikunja.nix {};

nixos/tests/vector.nix

@@ -1,53 +0,0 @@
-{ system ? builtins.currentSystem, config ? { }
-, pkgs ? import ../.. { inherit system config; } }:
-
-with import ../lib/testing-python.nix { inherit system pkgs; };
-with pkgs.lib;
-
-{
-  test1 = makeTest {
-    name = "vector-test1";
-    meta.maintainers = [ pkgs.lib.maintainers.happysalada ];
-
-    nodes.machine = { config, pkgs, ... }: {
-      services.vector = {
-        enable = true;
-        journaldAccess = true;
-        settings = {
-          sources = {
-            journald.type = "journald";
-
-            vector_metrics.type = "internal_metrics";
-
-            vector_logs.type = "internal_logs";
-          };
-
-          sinks = {
-            file = {
-              type = "file";
-              inputs = [ "journald" "vector_logs" ];
-              path = "/var/lib/vector/logs.log";
-              encoding = { codec = "json"; };
-            };
-
-            prometheus_exporter = {
-              type = "prometheus_exporter";
-              inputs = [ "vector_metrics" ];
-              address = "[::]:9598";
-            };
-          };
-        };
-      };
-    };
-
-    # ensure vector is forwarding the messages appropriately
-    testScript = ''
-      machine.wait_for_unit("vector.service")
-      machine.wait_for_open_port(9598)
-      machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_build_info")
-      machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_component_received_bytes_total | grep journald")
-      machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_utilization | grep prometheus_exporter")
-      machine.wait_for_file("/var/lib/vector/logs.log")
-    '';
-  };
-}

nixos/tests/vector/api.nix

@@ -0,0 +1,39 @@
+import ../make-test-python.nix ({ lib, pkgs, ... }:
+
+{
+  name = "vector-api";
+  meta.maintainers = [ pkgs.lib.maintainers.happysalada ];
+
+  nodes.machineapi = { config, pkgs, ... }: {
+    services.vector = {
+      enable = true;
+      journaldAccess = false;
+      settings = {
+        api.enabled = true;
+
+        sources = {
+          demo_logs = {
+            type = "demo_logs";
+            format = "json";
+          };
+        };
+
+        sinks = {
+          file = {
+            type = "file";
+            inputs = [ "demo_logs" ];
+            path = "/var/lib/vector/logs.log";
+            encoding = { codec = "json"; };
+          };
+        };
+      };
+    };
+  };
+
+  testScript = ''
+    machineapi.wait_for_unit("vector")
+    machineapi.wait_for_open_port(8686)
+    machineapi.succeed("journalctl -o cat -u vector.service | grep 'API server running'")
+    machineapi.wait_until_succeeds("curl -sSf http://localhost:8686/health")
+  '';
+})

nixos/tests/vector/default.nix

@@ -0,0 +1,11 @@
+{ system ? builtins.currentSystem
+, config ? { }
+, pkgs ? import ../../.. { inherit system config; }
+}:
+
+{
+  file-sink = import ./file-sink.nix { inherit system pkgs; };
+  api = import ./api.nix { inherit system pkgs; };
+  dnstap = import ./dnstap.nix { inherit system pkgs; };
+  nginx-clickhouse = import ./nginx-clickhouse.nix { inherit system pkgs; };
+}

nixos/tests/vector/dnstap.nix

@@ -0,0 +1,118 @@
+import ../make-test-python.nix ({ lib, pkgs, ... }:
+
+let
+  dnstapSocket = "/var/run/vector/dnstap.sock";
+in
+{
+  name = "vector-dnstap";
+  meta.maintainers = [ pkgs.lib.maintainers.happysalada ];
+
+  nodes = {
+    unbound = { config, pkgs, ... }: {
+      networking.firewall.allowedUDPPorts = [ 53 ];
+
+      services.vector = {
+        enable = true;
+
+        settings = {
+          sources = {
+            dnstap = {
+              type = "dnstap";
+              multithreaded = true;
+              mode = "unix";
+              lowercase_hostnames = true;
+              socket_file_mode = 504;
+              socket_path = "${dnstapSocket}";
+            };
+          };
+
+          sinks = {
+            file = {
+              type = "file";
+              inputs = [ "dnstap" ];
+              path = "/var/lib/vector/logs.log";
+              encoding = { codec = "json"; };
+            };
+          };
+        };
+      };
+
+      systemd.services.vector.serviceConfig = {
+        RuntimeDirectory = "vector";
+        RuntimeDirectoryMode = "0770";
+      };
+
+      services.unbound = {
+        enable = true;
+        enableRootTrustAnchor = false;
+        package = pkgs.unbound-full;
+        settings = {
+          server = {
+            interface = [ "0.0.0.0" "::" ];
+            access-control = [ "192.168.1.0/24 allow" ];
+
+            domain-insecure = "local";
+            private-domain = "local";
+
+            local-zone = "local. static";
+            local-data = [
+              ''"test.local. 10800 IN A 192.168.123.5"''
+            ];
+          };
+
+          dnstap = {
+            dnstap-enable = "yes";
+            dnstap-socket-path = "${dnstapSocket}";
+            dnstap-send-identity = "yes";
+            dnstap-send-version = "yes";
+            dnstap-log-client-query-messages = "yes";
+            dnstap-log-client-response-messages = "yes";
+          };
+        };
+      };
+
+      systemd.services.unbound = {
+        after = [ "vector.service" ];
+        wants = [ "vector.service" ];
+        serviceConfig = {
+          # DNSTAP access
+          ReadWritePaths = [ "/var/run/vector" ];
+          SupplementaryGroups = [ "vector" ];
+        };
+      };
+    };
+
+    dnsclient = { config, pkgs, ... }: {
+      environment.systemPackages = [ pkgs.dig ];
+    };
+  };
+
+  testScript = ''
+    unbound.wait_for_unit("unbound")
+    unbound.wait_for_unit("vector")
+
+    unbound.wait_until_succeeds(
+      "journalctl -o cat -u vector.service | grep 'Socket permissions updated to 0o770'"
+    )
+    unbound.wait_until_succeeds(
+      "journalctl -o cat -u vector.service | grep 'component_type=dnstap' | grep 'Listening... path=\"${dnstapSocket}\"'"
+    )
+
+    unbound.wait_for_file("${dnstapSocket}")
+    unbound.succeed("test 770 -eq $(stat -c '%a' ${dnstapSocket})")
+
+    dnsclient.wait_for_unit("network-online.target")
+    dnsclient.succeed(
+      "dig @unbound test.local"
+    )
+
+    unbound.wait_for_file("/var/lib/vector/logs.log")
+
+    unbound.wait_until_succeeds(
+      "grep ClientQuery /var/lib/vector/logs.log | grep '\"domainName\":\"test.local.\"' | grep '\"rcodeName\":\"NoError\"'"
+    )
+    unbound.wait_until_succeeds(
+      "grep ClientResponse /var/lib/vector/logs.log | grep '\"domainName\":\"test.local.\"' | grep '\"rData\":\"192.168.123.5\"'"
+    )
+  '';
+})

nixos/tests/vector/file-sink.nix

@@ -0,0 +1,49 @@
+import ../make-test-python.nix ({ lib, pkgs, ... }:
+
+{
+  name = "vector-test1";
+  meta.maintainers = [ pkgs.lib.maintainers.happysalada ];
+
+  nodes.machine = { config, pkgs, ... }: {
+    services.vector = {
+      enable = true;
+      journaldAccess = true;
+      settings = {
+        sources = {
+          journald.type = "journald";
+
+          vector_metrics.type = "internal_metrics";
+
+          vector_logs.type = "internal_logs";
+        };
+
+        sinks = {
+          file = {
+            type = "file";
+            inputs = [ "journald" "vector_logs" ];
+            path = "/var/lib/vector/logs.log";
+            encoding = { codec = "json"; };
+          };
+
+          prometheus_exporter = {
+            type = "prometheus_exporter";
+            inputs = [ "vector_metrics" ];
+            address = "[::]:9598";
+          };
+        };
+      };
+    };
+  };
+
+  # ensure vector is forwarding the messages appropriately
+  testScript = ''
+    machine.wait_for_unit("vector.service")
+    machine.wait_for_open_port(9598)
+    machine.wait_until_succeeds("journalctl -o cat -u vector.service | grep 'version=\"${pkgs.vector.version}\"'")
+    machine.wait_until_succeeds("journalctl -o cat -u vector.service | grep 'API is disabled'")
+    machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_build_info")
+    machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_component_received_bytes_total | grep journald")
+    machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_utilization | grep prometheus_exporter")
+    machine.wait_for_file("/var/lib/vector/logs.log")
+  '';
+})

nixos/tests/vector/nginx-clickhouse.nix

@@ -0,0 +1,168 @@
+import ../make-test-python.nix ({ lib, pkgs, ... }:
+
+{
+  name = "vector-nginx-clickhouse";
+  meta.maintainers = [ pkgs.lib.maintainers.happysalada ];
+
+  nodes = {
+    clickhouse = { config, pkgs, ... }: {
+      virtualisation.memorySize = 4096;
+
+      # Clickhouse module can't listen on a non-loopback IP.
+      networking.firewall.allowedTCPPorts = [ 6000 ];
+      services.clickhouse.enable = true;
+
+      # Exercise Vector sink->source for now.
+      services.vector = {
+        enable = true;
+
+        settings = {
+          sources = {
+            vector_source = {
+              type = "vector";
+              address = "[::]:6000";
+            };
+          };
+
+          sinks = {
+            clickhouse = {
+              type = "clickhouse";
+              inputs = [ "vector_source" ];
+              endpoint = "http://localhost:8123";
+              database = "nginxdb";
+              table = "access_logs";
+              skip_unknown_fields = true;
+            };
+          };
+        };
+      };
+    };
+
+    nginx = { config, pkgs, ... }: {
+      services.nginx = {
+        enable = true;
+        virtualHosts.localhost = {};
+      };
+
+      services.vector = {
+        enable = true;
+
+        settings = {
+          sources = {
+            nginx_logs = {
+              type = "file";
+              include = [ "/var/log/nginx/access.log" ];
+              read_from = "end";
+            };
+          };
+
+          sinks = {
+            vector_sink = {
+              type = "vector";
+              inputs = [ "nginx_logs" ];
+              address = "clickhouse:6000";
+            };
+          };
+        };
+      };
+
+      systemd.services.vector.serviceConfig = {
+        SupplementaryGroups = [ "nginx" ];
+      };
+    };
+  };
+
+  testScript =
+  let
+    # work around quote/substitution complexity by Nix, Perl, bash and SQL.
+    databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS nginxdb";
+
+    tableDDL = pkgs.writeText "table.sql" ''
+      CREATE TABLE IF NOT EXISTS  nginxdb.access_logs (
+        message String
+      )
+      ENGINE = MergeTree()
+      ORDER BY tuple()
+    '';
+
+    # Graciously taken from https://clickhouse.com/docs/en/integrations/vector
+    tableView = pkgs.writeText "table-view.sql" ''
+      CREATE MATERIALIZED VIEW nginxdb.access_logs_view
+      (
+        RemoteAddr String,
+        Client String,
+        RemoteUser String,
+        TimeLocal DateTime,
+        RequestMethod String,
+        Request String,
+        HttpVersion String,
+        Status Int32,
+        BytesSent Int64,
+        UserAgent String
+      )
+      ENGINE = MergeTree()
+      ORDER BY RemoteAddr
+      POPULATE AS
+      WITH
+       splitByWhitespace(message) as split,
+       splitByRegexp('\S \d+ "([^"]*)"', message) as referer
+      SELECT
+        split[1] AS RemoteAddr,
+        split[2] AS Client,
+        split[3] AS RemoteUser,
+        parseDateTimeBestEffort(replaceOne(trim(LEADING '[' FROM split[4]), ':', ' ')) AS TimeLocal,
+        trim(LEADING '"' FROM split[6]) AS RequestMethod,
+        split[7] AS Request,
+        trim(TRAILING '"' FROM split[8]) AS HttpVersion,
+        split[9] AS Status,
+        split[10] AS BytesSent,
+        trim(BOTH '"' from referer[2]) AS UserAgent
+      FROM
+        (SELECT message FROM nginxdb.access_logs)
+    '';
+
+    selectQuery = pkgs.writeText "select.sql" "SELECT * from nginxdb.access_logs_view";
+  in
+  ''
+    clickhouse.wait_for_unit("clickhouse")
+    clickhouse.wait_for_open_port(8123)
+
+    clickhouse.wait_until_succeeds(
+      "journalctl -o cat -u clickhouse.service | grep 'Started ClickHouse server'"
+    )
+
+    clickhouse.wait_for_unit("vector")
+    clickhouse.wait_for_open_port(6000)
+
+    clickhouse.succeed(
+      "cat ${databaseDDL} | clickhouse-client"
+    )
+
+    clickhouse.succeed(
+      "cat ${tableDDL} | clickhouse-client"
+    )
+
+    clickhouse.succeed(
+      "cat ${tableView} | clickhouse-client"
+    )
+
+    nginx.wait_for_unit("nginx")
+    nginx.wait_for_open_port(80)
+    nginx.wait_for_unit("vector")
+    nginx.wait_until_succeeds(
+      "journalctl -o cat -u vector.service | grep 'Starting file server'"
+    )
+
+    nginx.succeed("curl http://localhost/")
+    nginx.succeed("curl http://localhost/")
+
+    nginx.wait_for_file("/var/log/nginx/access.log")
+    nginx.wait_until_succeeds(
+      "journalctl -o cat -u vector.service | grep 'Found new file to watch. file=/var/log/nginx/access.log'"
+    )
+
+    clickhouse.wait_until_succeeds(
+      "cat ${selectQuery} | clickhouse-client | grep 'curl'"
+    )
+  '';
+})