Merge pull request #61237 from basvandijk/journalbeat-fixes

NixOS: support journalbeat >= 6

nixos/modules/services/logging/journalbeat.nix

@@ -5,11 +5,13 @@ with lib;
 let
   cfg = config.services.journalbeat;
 
+  lt6 = builtins.compareVersions cfg.package.version "6" < 0;
+
   journalbeatYml = pkgs.writeText "journalbeat.yml" ''
     name: ${cfg.name}
     tags: ${builtins.toJSON cfg.tags}
 
-    journalbeat.cursor_state_file: ${cfg.stateDir}/cursor-state
+    ${optionalString lt6 "journalbeat.cursor_state_file: /var/lib/${cfg.stateDir}/cursor-state"}
 
     ${cfg.extraConfig}
   '';
@@ -22,6 +24,16 @@ in
 
       enable = mkEnableOption "journalbeat";
 
+      package = mkOption {
+        type = types.package;
+        default = pkgs.journalbeat;
+        defaultText = "pkgs.journalbeat";
+        example = literalExample "pkgs.journalbeat7";
+        description = ''
+          The journalbeat package to use
+        '';
+      };
+
       name = mkOption {
         type = types.str;
         default = "journalbeat";
@@ -36,13 +48,17 @@ in
 
       stateDir = mkOption {
         type = types.str;
-        default = "/var/lib/journalbeat";
+        default = "journalbeat";
-        description = "The state directory. Journalbeat's own logs and other data are stored here.";
+        description = ''
+          Directory below <literal>/var/lib/</literal> to store journalbeat's
+          own logs and other data. This directory will be created automatically
+          using systemd's StateDirectory mechanism.
+        '';
       };
 
       extraConfig = mkOption {
         type = types.lines;
-        default = ''
+        default = optionalString lt6 ''
           journalbeat:
             seek_position: cursor
             cursor_seek_fallback: tail
@@ -61,7 +77,16 @@ in
 
   config = mkIf cfg.enable {
 
-    systemd.services.journalbeat = with pkgs; {
+    assertions = [
+      {
+        assertion = !hasPrefix "/" cfg.stateDir;
+        message =
+          "The option services.journalbeat.stateDir shouldn't be an absolute directory." +
+          " It should be a directory relative to /var/lib/.";
+      }
+    ];
+
+    systemd.services.journalbeat = {
       description = "Journalbeat log shipper";
       wantedBy = [ "multi-user.target" ];
       preStart = ''
@@ -69,7 +94,13 @@ in
         mkdir -p ${cfg.stateDir}/logs
       '';
       serviceConfig = {
-        ExecStart = "${pkgs.journalbeat}/bin/journalbeat -c ${journalbeatYml} -path.data ${cfg.stateDir}/data -path.logs ${cfg.stateDir}/logs";
+        StateDirectory = cfg.stateDir;
+        ExecStart = ''
+          ${cfg.package}/bin/journalbeat \
+            -c ${journalbeatYml} \
+            -path.data /var/lib/${cfg.stateDir}/data \
+            -path.logs /var/lib/${cfg.stateDir}/logs'';
+        Restart = "always";
       };
     };
   };

nixos/tests/elk.nix

@@ -12,6 +12,11 @@ with pkgs.lib;
 let
   esUrl = "http://localhost:9200";
 
+  totalHits = message :
+    "curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' " +
+    ''-d '{\"query\" : { \"match\" : { \"message\" : \"${message}\"}}}' '' +
+    "| jq .hits.total";
+
   mkElkTest = name : elk :
    let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
    in makeTest {
@@ -21,7 +26,7 @@ let
     };
     nodes = {
       one =
-        { pkgs, ... }: {
+        { pkgs, lib, ... }: {
             # Not giving the machine at least 2060MB results in elasticsearch failing with the following error:
             #
             #   OpenJDK 64-Bit Server VM warning:
@@ -40,6 +45,26 @@ let
             environment.systemPackages = [ pkgs.jq ];
 
             services = {
+
+              journalbeat = let lt6 = builtins.compareVersions
+                                        elk.journalbeat.version "6" < 0; in {
+                enable = true;
+                package = elk.journalbeat;
+                extraConfig = mkOptionDefault (''
+                  logging:
+                    to_syslog: true
+                    level: warning
+                    metrics.enabled: false
+                  output.elasticsearch:
+                    hosts: [ "127.0.0.1:9200" ]
+                    ${optionalString lt6 "template.enabled: false"}
+                '' + optionalString (!lt6) ''
+                  journalbeat.inputs:
+                  - paths: []
+                    seek: cursor
+                '');
+              };
+
               logstash = {
                 enable = true;
                 package = elk.logstash;
@@ -107,14 +132,19 @@ let
     testScript = ''
       startAll;
 
+      # Wait until elasticsearch is listening for connections.
       $one->waitForUnit("elasticsearch.service");
+      $one->waitForOpenPort(9200);
 
       # Continue as long as the status is not "red". The status is probably
       # "yellow" instead of "green" because we are using a single elasticsearch
       # node which elasticsearch considers risky.
       #
-      # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green".
+      # TODO: extend this test with multiple elasticsearch nodes
-      $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red");
+      #       and see if the status turns "green".
+      $one->waitUntilSucceeds(
+        "curl --silent --show-error '${esUrl}/_cluster/health' " .
+        "| jq .status | grep -v red");
 
       # Perform some simple logstash tests.
       $one->waitForUnit("logstash.service");
@@ -123,16 +153,28 @@ let
 
       # See if kibana is healthy.
       $one->waitForUnit("kibana.service");
-      $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green");
+      $one->waitUntilSucceeds(
+        "curl --silent --show-error 'http://localhost:5601/api/status' " .
+        "| jq .status.overall.state | grep green");
 
       # See if logstash messages arive in elasticsearch.
-      $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
+      $one->waitUntilSucceeds("${totalHits "flowers"} | grep -v 0");
-      $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
+      $one->waitUntilSucceeds("${totalHits "dragons"} | grep 0");
+
+      # Test if a message logged to the journal
+      # is ingested by elasticsearch via journalbeat.
+      $one->waitForUnit("journalbeat.service");
+      $one->execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat");
+      $one->waitUntilSucceeds(
+        "${totalHits "Supercalifragilisticexpialidocious"} | grep -v 0");
+
     '' + optionalString (!elasticsearchGe7) ''
       # Test elasticsearch-curator.
       $one->systemctl("stop logstash");
       $one->systemctl("start elasticsearch-curator");
-      $one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1");
+      $one->waitUntilSucceeds(
+        "! curl --silent --show-error '${esUrl}/_cat/indices' " .
+        "| grep logstash | grep -q ^$1");
     '';
   };
 in mapAttrs mkElkTest {
@@ -140,6 +182,7 @@ in mapAttrs mkElkTest {
     elasticsearch = pkgs.elasticsearch5;
     logstash      = pkgs.logstash5;
     kibana        = pkgs.kibana5;
+    journalbeat   = pkgs.journalbeat5;
   };
   "ELK-6" =
     if enableUnfree
@@ -147,11 +190,13 @@ in mapAttrs mkElkTest {
       elasticsearch = pkgs.elasticsearch6;
       logstash      = pkgs.logstash6;
       kibana        = pkgs.kibana6;
+      journalbeat   = pkgs.journalbeat6;
     }
     else {
       elasticsearch = pkgs.elasticsearch6-oss;
       logstash      = pkgs.logstash6-oss;
       kibana        = pkgs.kibana6-oss;
+      journalbeat   = pkgs.journalbeat6;
     };
   "ELK-7" =
     if enableUnfree
@@ -159,10 +204,12 @@ in mapAttrs mkElkTest {
       elasticsearch = pkgs.elasticsearch7;
       logstash      = pkgs.logstash7;
       kibana        = pkgs.kibana7;
+      journalbeat   = pkgs.journalbeat7;
     }
     else {
       elasticsearch = pkgs.elasticsearch7-oss;
       logstash      = pkgs.logstash7-oss;
       kibana        = pkgs.kibana7-oss;
+      journalbeat   = pkgs.journalbeat7;
     };
 }

pkgs/misc/logging/beats/6.x.nix

@@ -45,5 +45,8 @@ in {
       journal entries from Linuxes with systemd.
     '';
     buildInputs = [ systemd.dev ];
+    postFixup = let libPath = stdenv.lib.makeLibraryPath [ systemd.lib ]; in ''
+      patchelf --set-rpath ${libPath} "$bin/bin/journalbeat"
+    '';
   };
 }

pkgs/misc/logging/beats/7.x.nix

@@ -45,5 +45,8 @@ in {
       journal entries from Linuxes with systemd.
     '';
     buildInputs = [ systemd.dev ];
+    postFixup = let libPath = stdenv.lib.makeLibraryPath [ systemd.lib ]; in ''
+      patchelf --set-rpath ${libPath} "$bin/bin/journalbeat"
+    '';
   };
 }

pkgs/tools/system/journalbeat/default.nix

@@ -1,24 +1,9 @@
-{ lib, pkgs, buildGoPackage, fetchFromGitHub, makeWrapper }:
+{ lib, systemd, buildGoPackage, fetchFromGitHub, makeWrapper }:
-
-let
-
-  libPath = lib.makeLibraryPath [ pkgs.systemd.lib ];
-
-in buildGoPackage rec {
 
+buildGoPackage rec {
   name = "journalbeat-${version}";
   version = "5.6.8";
 
-  goPackagePath = "github.com/mheese/journalbeat";
-
-  buildInputs = [ makeWrapper pkgs.systemd ];
-
-  postInstall = ''
-    wrapProgram $bin/bin/journalbeat \
-      --argv0 journalbeat \
-      --prefix LD_LIBRARY_PATH : ${libPath}
-  '';
-
   src = fetchFromGitHub {
     owner = "mheese";
     repo = "journalbeat";
@@ -26,6 +11,14 @@ in buildGoPackage rec {
     sha256 = "1vgpwnwqjc93nvdpcd52748bwl3r371jb55l17bsgdzrmlcyfm8a";
   };
 
+  goPackagePath = "github.com/mheese/journalbeat";
+
+  buildInputs = [ systemd.dev ];
+
+  postFixup = let libPath = lib.makeLibraryPath [ systemd.lib ]; in ''
+    patchelf --set-rpath ${libPath} "$bin/bin/journalbeat"
+  '';
+
   meta = with lib; {
     homepage = https://github.com/mheese/journalbeat;
     description = "Journalbeat is a log shipper from systemd/journald to Logstash/Elasticsearch";

pkgs/top-level/all-packages.nix

@@ -976,6 +976,8 @@ in
     metricbeat5
     packetbeat5;
 
+  journalbeat5 = callPackage ../tools/system/journalbeat { };
+
   inherit (callPackages ../misc/logging/beats/6.x.nix {
     # XXX: this is failing with Go 1.12. Error is related to cgo, an
     # update to this package might fix it.
@@ -1002,6 +1004,7 @@ in
   heartbeat = heartbeat6;
   metricbeat = metricbeat6;
   packetbeat = packetbeat6;
+  journalbeat = journalbeat6;
 
   bfr = callPackage ../tools/misc/bfr { };
 
@@ -3787,8 +3790,6 @@ in
 
   joplin-desktop = callPackage ../applications/misc/joplin-desktop { };
 
-  journalbeat = callPackage ../tools/system/journalbeat { };
-
   journaldriver = callPackage ../tools/misc/journaldriver { };
 
   jp = callPackage ../development/tools/jp { };