Merge pull request #77303 from tfc/port-elk-initdb-kafka

Port elk initdb kafka
2024-11-13 09:17:07 +03:00 · 2020-01-10 17:05:04 +01:00 · 2020-01-10 17:05:04 +01:00 · 47e0ce7f1a
commit 47e0ce7f1a
parent c940155c61 abebf8cdb3
4 changed files with 110 additions and 103 deletions
--- a/nixos/tests/elk.nix
+++ b/nixos/tests/elk.nix
@ -6,20 +6,12 @@
  # NIXPKGS_ALLOW_UNFREE=1 nix-build nixos/tests/elk.nix -A ELK-6 --arg enableUnfree true
 }:

-with import ../lib/testing.nix { inherit system pkgs; };
-with pkgs.lib;
-
 let
  esUrl = "http://localhost:9200";

-  totalHits = message :
-    "curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' " +
-    ''-d '{\"query\" : { \"match\" : { \"message\" : \"${message}\"}}}' '' +
-    "| jq .hits.total";
-
  mkElkTest = name : elk :
   let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
-   in makeTest {
+   in import ./make-test-python.nix ({
    inherit name;
    meta = with pkgs.stdenv.lib.maintainers; {
      maintainers = [ eelco offline basvandijk ];
@ -50,15 +42,15 @@ let
                                        elk.journalbeat.version "6" < 0; in {
                enable = true;
                package = elk.journalbeat;
-                extraConfig = mkOptionDefault (''
+                extraConfig = pkgs.lib.mkOptionDefault (''
                  logging:
                    to_syslog: true
                    level: warning
                    metrics.enabled: false
                  output.elasticsearch:
                    hosts: [ "127.0.0.1:9200" ]
-                    ${optionalString lt6 "template.enabled: false"}
-                '' + optionalString (!lt6) ''
+                    ${pkgs.lib.optionalString lt6 "template.enabled: false"}
+                '' + pkgs.lib.optionalString (!lt6) ''
                  journalbeat.inputs:
                  - paths: []
                    seek: cursor
@ -130,11 +122,23 @@ let
      };

    testScript = ''
-      startAll;
+      import json

-      # Wait until elasticsearch is listening for connections.
-      $one->waitForUnit("elasticsearch.service");
-      $one->waitForOpenPort(9200);
+
+      def total_hits(message):
+          dictionary = {"query": {"match": {"message": message}}}
+          return (
+              "curl --silent --show-error '${esUrl}/_search' "
+              + "-H 'Content-Type: application/json' "
+              + "-d '{}' ".format(json.dumps(dictionary))
+              + "| jq .hits.total"
+          )
+
+
+      start_all()
+
+      one.wait_for_unit("elasticsearch.service")
+      one.wait_for_open_port(9200)

      # Continue as long as the status is not "red". The status is probably
      # "yellow" instead of "green" because we are using a single elasticsearch
@ -142,42 +146,43 @@ let
      #
      # TODO: extend this test with multiple elasticsearch nodes
      #       and see if the status turns "green".
-      $one->waitUntilSucceeds(
-        "curl --silent --show-error '${esUrl}/_cluster/health' " .
-        "| jq .status | grep -v red");
+      one.wait_until_succeeds(
+          "curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"
+      )

-      # Perform some simple logstash tests.
-      $one->waitForUnit("logstash.service");
-      $one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers");
-      $one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons");
+      with subtest("Perform some simple logstash tests"):
+          one.wait_for_unit("logstash.service")
+          one.wait_until_succeeds("cat /tmp/logstash.out | grep flowers")
+          one.wait_until_succeeds("cat /tmp/logstash.out | grep -v dragons")

-      # See if kibana is healthy.
-      $one->waitForUnit("kibana.service");
-      $one->waitUntilSucceeds(
-        "curl --silent --show-error 'http://localhost:5601/api/status' " .
-        "| jq .status.overall.state | grep green");
+      with subtest("Kibana is healthy"):
+          one.wait_for_unit("kibana.service")
+          one.wait_until_succeeds(
+              "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"
+          )

-      # See if logstash messages arive in elasticsearch.
-      $one->waitUntilSucceeds("${totalHits "flowers"} | grep -v 0");
-      $one->waitUntilSucceeds("${totalHits "dragons"} | grep 0");
+      with subtest("Logstash messages arive in elasticsearch"):
+          one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
+          one.wait_until_succeeds(total_hits("dragons") + " | grep 0")

-      # Test if a message logged to the journal
-      # is ingested by elasticsearch via journalbeat.
-      $one->waitForUnit("journalbeat.service");
-      $one->execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat");
-      $one->waitUntilSucceeds(
-        "${totalHits "Supercalifragilisticexpialidocious"} | grep -v 0");
-
-    '' + optionalString (!elasticsearchGe7) ''
-      # Test elasticsearch-curator.
-      $one->systemctl("stop logstash");
-      $one->systemctl("start elasticsearch-curator");
-      $one->waitUntilSucceeds(
-        "! curl --silent --show-error '${esUrl}/_cat/indices' " .
-        "| grep logstash | grep -q ^$1");
+      with subtest(
+          "A message logged to the journal is ingested by elasticsearch via journalbeat"
+      ):
+          one.wait_for_unit("journalbeat.service")
+          one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat")
+          one.wait_until_succeeds(
+              total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
+          )
+    '' + pkgs.lib.optionalString (!elasticsearchGe7) ''
+      with subtest("Elasticsearch-curator works"):
+          one.systemctl("stop logstash")
+          one.systemctl("start elasticsearch-curator")
+          one.wait_until_succeeds(
+              '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep -q ^'
+          )
    '';
-  };
-in mapAttrs mkElkTest {
+  }) {};
+in pkgs.lib.mapAttrs mkElkTest {
  ELK-6 =
    if enableUnfree
    then {
--- a/nixos/tests/initdb.nix
+++ b/nixos/tests/initdb.nix
@ -1,26 +0,0 @@
-let
-  pkgs = import <nixpkgs> { };
-in
-with import <nixpkgs/nixos/lib/testing.nix> { inherit pkgs; system = builtins.currentSystem; };
-with pkgs.lib;
-
-makeTest {
-    name = "pg-initdb";
-
-    machine = {...}:
-      {
-        documentation.enable = false;
-        services.postgresql.enable = true;
-        services.postgresql.package = pkgs.postgresql_9_6;
-        environment.pathsToLink = [
-          "/share/postgresql"
-        ];
-      };
-
-    testScript = ''
-      $machine->start;
-      $machine->succeed("sudo -u postgres initdb -D /tmp/testpostgres2");
-      $machine->shutdown;
-    '';
-
-  }
--- a/nixos/tests/kafka.nix
+++ b/nixos/tests/kafka.nix
@ -3,11 +3,10 @@
  pkgs ? import ../.. { inherit system config; }
 }:

-with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;

 let
-  makeKafkaTest = name: kafkaPackage: (makeTest {
+  makeKafkaTest = name: kafkaPackage: (import ./make-test-python.nix ({
    inherit name;
    meta = with pkgs.stdenv.lib.maintainers; {
      maintainers = [ nequissimus ];
@ -45,24 +44,40 @@ let
    };

    testScript = ''
-      startAll;
+      start_all()

-      $zookeeper1->waitForUnit("default.target");
-      $zookeeper1->waitForUnit("zookeeper.service");
-      $zookeeper1->waitForOpenPort(2181);
+      zookeeper1.wait_for_unit("default.target")
+      zookeeper1.wait_for_unit("zookeeper.service")
+      zookeeper1.wait_for_open_port(2181)

-      $kafka->waitForUnit("default.target");
-      $kafka->waitForUnit("apache-kafka.service");
-      $kafka->waitForOpenPort(9092);
+      kafka.wait_for_unit("default.target")
+      kafka.wait_for_unit("apache-kafka.service")
+      kafka.wait_for_open_port(9092)

-      $kafka->waitUntilSucceeds("${kafkaPackage}/bin/kafka-topics.sh --create --zookeeper zookeeper1:2181 --partitions 1 --replication-factor 1 --topic testtopic");
-      $kafka->mustSucceed("echo 'test 1' | ${kafkaPackage}/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic testtopic");
+      kafka.wait_until_succeeds(
+          "${kafkaPackage}/bin/kafka-topics.sh --create "
+          + "--zookeeper zookeeper1:2181 --partitions 1 "
+          + "--replication-factor 1 --topic testtopic"
+      )
+      kafka.succeed(
+          "echo 'test 1' | "
+          + "${kafkaPackage}/bin/kafka-console-producer.sh "
+          + "--broker-list localhost:9092 --topic testtopic"
+      )
    '' + (if name == "kafka_0_9" then ''
-      $kafka->mustSucceed("${kafkaPackage}/bin/kafka-console-consumer.sh --zookeeper zookeeper1:2181 --topic testtopic --from-beginning --max-messages 1 | grep 'test 1'");
+      assert "test 1" in kafka.succeed(
+          "${kafkaPackage}/bin/kafka-console-consumer.sh "
+          + "--zookeeper zookeeper1:2181 --topic testtopic "
+          + "--from-beginning --max-messages 1"
+      )
    '' else ''
-      $kafka->mustSucceed("${kafkaPackage}/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic testtopic --from-beginning --max-messages 1 | grep 'test 1'");
+      assert "test 1" in kafka.succeed(
+          "${kafkaPackage}/bin/kafka-console-consumer.sh "
+          + "--bootstrap-server localhost:9092 --topic testtopic "
+          + "--from-beginning --max-messages 1"
+      )
    '');
-  });
+  }) {});

 in with pkgs; {
  kafka_0_9  = makeKafkaTest "kafka_0_9"  apacheKafka_0_9;
--- a/nixos/tests/postgresql.nix
+++ b/nixos/tests/postgresql.nix
@ -29,11 +29,15 @@ let

    machine = {...}:
      {
-        services.postgresql.enable = true;
-        services.postgresql.package = postgresql-package;
+        services.postgresql = {
+          enable = true;
+          package = postgresql-package;
+        };

-        services.postgresqlBackup.enable = true;
-        services.postgresqlBackup.databases = optional (!backup-all) "postgres";
+        services.postgresqlBackup = {
+          enable = true;
+          databases = optional (!backup-all) "postgres";
+        };
      };

    testScript = let
@ -49,23 +53,32 @@ let
      machine.start()
      machine.wait_for_unit("postgresql")

-      # postgresql should be available just after unit start
-      machine.succeed(
-          "cat ${test-sql} | sudo -u postgres psql"
-      )
-      machine.shutdown()  # make sure that postgresql survive restart (bug #1735)
-      time.sleep(2)
-      machine.start()
-      machine.wait_for_unit("postgresql")
+      with subtest("Postgresql is available just after unit start"):
+          machine.succeed(
+              "cat ${test-sql} | sudo -u postgres psql"
+          )
+
+      with subtest("Postgresql survives restart (bug #1735)"):
+          machine.shutdown()
+          time.sleep(2)
+          machine.start()
+          machine.wait_for_unit("postgresql")
+
      machine.fail(check_count("SELECT * FROM sth;", 3))
      machine.succeed(check_count("SELECT * FROM sth;", 5))
      machine.fail(check_count("SELECT * FROM sth;", 4))
      machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1))

-      # Check backup service
-      machine.succeed("systemctl start ${backupService}.service")
-      machine.succeed("zcat /var/backup/postgresql/${backupName}.sql.gz | grep '<test>ok</test>'")
-      machine.succeed("stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600")
+      with subtest("Backup service works"):
+          machine.succeed(
+              "systemctl start ${backupService}.service",
+              "zcat /var/backup/postgresql/${backupName}.sql.gz | grep '<test>ok</test>'",
+              "stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600",
+          )
+
+      with subtest("Initdb works"):
+          machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2")
+
      machine.shutdown()
    '';