git.iko.soy/typeable/nixpkgs
on master help sign in
Merge pull request #61237 from basvandijk/journalbeat-fixes

NixOS: support journalbeat >= 6

commit4b7aea9e8cd17bbabaea70400b50af4e8140c696 authorBas van Dijk <v.dijk.bas@gmail.com> dateFri 10 May 2019, 18:44 (8 years ago) parent38c093e477c552
6 files changed+110−32
nixos/modules/services/logging/journalbeat.nix+37−6 nixos/tests/elk.nix+54−7 pkgs/misc/logging/beats/6.x.nix+3 pkgs/misc/logging/beats/7.x.nix+3 pkgs/tools/system/journalbeat/default.nix+10−17 pkgs/top-level/all-packages.nix+3−2
nixos/modules/services/logging/journalbeat.nix+37−6
@@ -5,11 +5,13 @@ with lib;
55let
66 cfg = config.services.journalbeat;
77
8+ lt6 = builtins.compareVersions cfg.package.version "6" < 0;
9+
810 journalbeatYml = pkgs.writeText "journalbeat.yml" ''
911 name: ${cfg.name}
1012 tags: ${builtins.toJSON cfg.tags}
1113
12− journalbeat.cursor_state_file: ${cfg.stateDir}/cursor-state
14+ ${optionalString lt6 "journalbeat.cursor_state_file: /var/lib/${cfg.stateDir}/cursor-state"}
1315
1416 ${cfg.extraConfig}
1517 '';
@@ -22,6 +24,16 @@ in
2224
2325 enable = mkEnableOption "journalbeat";
2426
27+ package = mkOption {
28+ type = types.package;
29+ default = pkgs.journalbeat;
30+ defaultText = "pkgs.journalbeat";
31+ example = literalExample "pkgs.journalbeat7";
32+ description = ''
33+ The journalbeat package to use
34+ '';
35+ };
36+
2537 name = mkOption {
2638 type = types.str;
2739 default = "journalbeat";
@@ -36,13 +48,17 @@ in
3648
3749 stateDir = mkOption {
3850 type = types.str;
39− default = "/var/lib/journalbeat";
40− description = "The state directory. Journalbeat's own logs and other data are stored here.";
51+ default = "journalbeat";
52+ description = ''
53+ Directory below <literal>/var/lib/</literal> to store journalbeat's
54+ own logs and other data. This directory will be created automatically
55+ using systemd's StateDirectory mechanism.
56+ '';
4157 };
4258
4359 extraConfig = mkOption {
4460 type = types.lines;
45− default = ''
61+ default = optionalString lt6 ''
4662 journalbeat:
4763 seek_position: cursor
4864 cursor_seek_fallback: tail
@@ -61,7 +77,16 @@ in
6177
6278 config = mkIf cfg.enable {
6379
64− systemd.services.journalbeat = with pkgs; {
80+ assertions = [
81+ {
82+ assertion = !hasPrefix "/" cfg.stateDir;
83+ message =
84+ "The option services.journalbeat.stateDir shouldn't be an absolute directory." +
85+ " It should be a directory relative to /var/lib/.";
86+ }
87+ ];
88+
89+ systemd.services.journalbeat = {
6590 description = "Journalbeat log shipper";
6691 wantedBy = [ "multi-user.target" ];
6792 preStart = ''
@@ -69,7 +94,13 @@ in
6994 mkdir -p ${cfg.stateDir}/logs
7095 '';
7196 serviceConfig = {
72− ExecStart = "${pkgs.journalbeat}/bin/journalbeat -c ${journalbeatYml} -path.data ${cfg.stateDir}/data -path.logs ${cfg.stateDir}/logs";
97+ StateDirectory = cfg.stateDir;
98+ ExecStart = ''
99+ ${cfg.package}/bin/journalbeat \
100+ -c ${journalbeatYml} \
101+ -path.data /var/lib/${cfg.stateDir}/data \
102+ -path.logs /var/lib/${cfg.stateDir}/logs'';
103+ Restart = "always";
73104 };
74105 };
75106 };
nixos/tests/elk.nix+54−7
@@ -12,6 +12,11 @@ with pkgs.lib;
1212let
1313 esUrl = "http://localhost:9200";
1414
15+ totalHits = message :
16+ "curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' " +
17+ ''-d '{\"query\" : { \"match\" : { \"message\" : \"${message}\"}}}' '' +
18+ "| jq .hits.total";
19+
1520 mkElkTest = name : elk :
1621 let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
1722 in makeTest {
@@ -21,7 +26,7 @@ let
2126 };
2227 nodes = {
2328 one =
24− { pkgs, ... }: {
29+ { pkgs, lib, ... }: {
2530 # Not giving the machine at least 2060MB results in elasticsearch failing with the following error:
2631 #
2732 # OpenJDK 64-Bit Server VM warning:
@@ -40,6 +45,26 @@ let
4045 environment.systemPackages = [ pkgs.jq ];
4146
4247 services = {
48+
49+ journalbeat = let lt6 = builtins.compareVersions
50+ elk.journalbeat.version "6" < 0; in {
51+ enable = true;
52+ package = elk.journalbeat;
53+ extraConfig = mkOptionDefault (''
54+ logging:
55+ to_syslog: true
56+ level: warning
57+ metrics.enabled: false
58+ output.elasticsearch:
59+ hosts: [ "127.0.0.1:9200" ]
60+ ${optionalString lt6 "template.enabled: false"}
61+ '' + optionalString (!lt6) ''
62+ journalbeat.inputs:
63+ - paths: []
64+ seek: cursor
65+ '');
66+ };
67+
4368 logstash = {
4469 enable = true;
4570 package = elk.logstash;
@@ -107,14 +132,19 @@ let
107132 testScript = ''
108133 startAll;
109134
135+ # Wait until elasticsearch is listening for connections.
110136 $one->waitForUnit("elasticsearch.service");
137+ $one->waitForOpenPort(9200);
111138
112139 # Continue as long as the status is not "red". The status is probably
113140 # "yellow" instead of "green" because we are using a single elasticsearch
114141 # node which elasticsearch considers risky.
115142 #
116− # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green".
117− $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red");
143+ # TODO: extend this test with multiple elasticsearch nodes
144+ # and see if the status turns "green".
145+ $one->waitUntilSucceeds(
146+ "curl --silent --show-error '${esUrl}/_cluster/health' " .
147+ "| jq .status | grep -v red");
118148
119149 # Perform some simple logstash tests.
120150 $one->waitForUnit("logstash.service");
@@ -123,16 +153,28 @@ let
123153
124154 # See if kibana is healthy.
125155 $one->waitForUnit("kibana.service");
126− $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green");
156+ $one->waitUntilSucceeds(
157+ "curl --silent --show-error 'http://localhost:5601/api/status' " .
158+ "| jq .status.overall.state | grep green");
127159
128160 # See if logstash messages arive in elasticsearch.
129− $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
130− $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
161+ $one->waitUntilSucceeds("${totalHits "flowers"} | grep -v 0");
162+ $one->waitUntilSucceeds("${totalHits "dragons"} | grep 0");
163+
164+ # Test if a message logged to the journal
165+ # is ingested by elasticsearch via journalbeat.
166+ $one->waitForUnit("journalbeat.service");
167+ $one->execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat");
168+ $one->waitUntilSucceeds(
169+ "${totalHits "Supercalifragilisticexpialidocious"} | grep -v 0");
170+
131171 '' + optionalString (!elasticsearchGe7) ''
132172 # Test elasticsearch-curator.
133173 $one->systemctl("stop logstash");
134174 $one->systemctl("start elasticsearch-curator");
135− $one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1");
175+ $one->waitUntilSucceeds(
176+ "! curl --silent --show-error '${esUrl}/_cat/indices' " .
177+ "| grep logstash | grep -q ^$1");
136178 '';
137179 };
138180in mapAttrs mkElkTest {
@@ -140,6 +182,7 @@ in mapAttrs mkElkTest {
140182 elasticsearch = pkgs.elasticsearch5;
141183 logstash = pkgs.logstash5;
142184 kibana = pkgs.kibana5;
185+ journalbeat = pkgs.journalbeat5;
143186 };
144187 "ELK-6" =
145188 if enableUnfree
@@ -147,11 +190,13 @@ in mapAttrs mkElkTest {
147190 elasticsearch = pkgs.elasticsearch6;
148191 logstash = pkgs.logstash6;
149192 kibana = pkgs.kibana6;
193+ journalbeat = pkgs.journalbeat6;
150194 }
151195 else {
152196 elasticsearch = pkgs.elasticsearch6-oss;
153197 logstash = pkgs.logstash6-oss;
154198 kibana = pkgs.kibana6-oss;
199+ journalbeat = pkgs.journalbeat6;
155200 };
156201 "ELK-7" =
157202 if enableUnfree
@@ -159,10 +204,12 @@ in mapAttrs mkElkTest {
159204 elasticsearch = pkgs.elasticsearch7;
160205 logstash = pkgs.logstash7;
161206 kibana = pkgs.kibana7;
207+ journalbeat = pkgs.journalbeat7;
162208 }
163209 else {
164210 elasticsearch = pkgs.elasticsearch7-oss;
165211 logstash = pkgs.logstash7-oss;
166212 kibana = pkgs.kibana7-oss;
213+ journalbeat = pkgs.journalbeat7;
167214 };
168215}
pkgs/misc/logging/beats/6.x.nix+3
@@ -45,5 +45,8 @@ in {
4545 journal entries from Linuxes with systemd.
4646 '';
4747 buildInputs = [ systemd.dev ];
48+ postFixup = let libPath = stdenv.lib.makeLibraryPath [ systemd.lib ]; in ''
49+ patchelf --set-rpath ${libPath} "$bin/bin/journalbeat"
50+ '';
4851 };
4952}
pkgs/misc/logging/beats/7.x.nix+3
@@ -45,5 +45,8 @@ in {
4545 journal entries from Linuxes with systemd.
4646 '';
4747 buildInputs = [ systemd.dev ];
48+ postFixup = let libPath = stdenv.lib.makeLibraryPath [ systemd.lib ]; in ''
49+ patchelf --set-rpath ${libPath} "$bin/bin/journalbeat"
50+ '';
4851 };
4952}
pkgs/tools/system/journalbeat/default.nix+10−17
@@ -1,24 +1,9 @@
1−{ lib, pkgs, buildGoPackage, fetchFromGitHub, makeWrapper }:
2−
3−let
4−
5− libPath = lib.makeLibraryPath [ pkgs.systemd.lib ];
6−
7−in buildGoPackage rec {
1+{ lib, systemd, buildGoPackage, fetchFromGitHub, makeWrapper }:
82
3+buildGoPackage rec {
94 name = "journalbeat-${version}";
105 version = "5.6.8";
116
12− goPackagePath = "github.com/mheese/journalbeat";
13−
14− buildInputs = [ makeWrapper pkgs.systemd ];
15−
16− postInstall = ''
17− wrapProgram $bin/bin/journalbeat \
18− --argv0 journalbeat \
19− --prefix LD_LIBRARY_PATH : ${libPath}
20− '';
21−
227 src = fetchFromGitHub {
238 owner = "mheese";
249 repo = "journalbeat";
@@ -26,6 +11,14 @@ in buildGoPackage rec {
2611 sha256 = "1vgpwnwqjc93nvdpcd52748bwl3r371jb55l17bsgdzrmlcyfm8a";
2712 };
2813
14+ goPackagePath = "github.com/mheese/journalbeat";
15+
16+ buildInputs = [ systemd.dev ];
17+
18+ postFixup = let libPath = lib.makeLibraryPath [ systemd.lib ]; in ''
19+ patchelf --set-rpath ${libPath} "$bin/bin/journalbeat"
20+ '';
21+
2922 meta = with lib; {
3023 homepage = https://github.com/mheese/journalbeat;
3124 description = "Journalbeat is a log shipper from systemd/journald to Logstash/Elasticsearch";
pkgs/top-level/all-packages.nix+3−2
@@ -976,6 +976,8 @@ in
976976 metricbeat5
977977 packetbeat5;
978978
979+ journalbeat5 = callPackage ../tools/system/journalbeat { };
980+
979981 inherit (callPackages ../misc/logging/beats/6.x.nix {
980982 # XXX: this is failing with Go 1.12. Error is related to cgo, an
981983 # update to this package might fix it.
@@ -1002,6 +1004,7 @@ in
10021004 heartbeat = heartbeat6;
10031005 metricbeat = metricbeat6;
10041006 packetbeat = packetbeat6;
1007+ journalbeat = journalbeat6;
10051008
10061009 bfr = callPackage ../tools/misc/bfr { };
10071010
@@ -3787,8 +3790,6 @@ in
37873790
37883791 joplin-desktop = callPackage ../applications/misc/joplin-desktop { };
37893792
3790− journalbeat = callPackage ../tools/system/journalbeat { };
3791−
37923793 journaldriver = callPackage ../tools/misc/journaldriver { };
37933794
37943795 jp = callPackage ../development/tools/jp { };