@@ -12,6 +12,11 @@ with pkgs.lib;
1212let
1313 esUrl = "http://localhost:9200";
1414
15+ totalHits = message :
16+ "curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' " +
17+ ''-d '{\"query\" : { \"match\" : { \"message\" : \"${message}\"}}}' '' +
18+ "| jq .hits.total";
19+
1520 mkElkTest = name : elk :
1621 let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0;
1722 in makeTest {
@@ -21,7 +26,7 @@ let
2126 };
2227 nodes = {
2328 one =
24− { pkgs, ... }: {
29+ { pkgs, lib, ... }: {
2530 # Not giving the machine at least 2060MB results in elasticsearch failing with the following error:
2631 #
2732 # OpenJDK 64-Bit Server VM warning:
@@ -40,6 +45,26 @@ let
4045 environment.systemPackages = [ pkgs.jq ];
4146
4247 services = {
48+
49+ journalbeat = let lt6 = builtins.compareVersions
50+ elk.journalbeat.version "6" < 0; in {
51+ enable = true;
52+ package = elk.journalbeat;
53+ extraConfig = mkOptionDefault (''
54+ logging:
55+ to_syslog: true
56+ level: warning
57+ metrics.enabled: false
58+ output.elasticsearch:
59+ hosts: [ "127.0.0.1:9200" ]
60+ ${optionalString lt6 "template.enabled: false"}
61+ '' + optionalString (!lt6) ''
62+ journalbeat.inputs:
63+ - paths: []
64+ seek: cursor
65+ '');
66+ };
67+
4368 logstash = {
4469 enable = true;
4570 package = elk.logstash;
@@ -107,14 +132,19 @@ let
107132 testScript = ''
108133 startAll;
109134
135+ # Wait until elasticsearch is listening for connections.
110136 $one->waitForUnit("elasticsearch.service");
137+ $one->waitForOpenPort(9200);
111138
112139 # Continue as long as the status is not "red". The status is probably
113140 # "yellow" instead of "green" because we are using a single elasticsearch
114141 # node which elasticsearch considers risky.
115142 #
116− # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green".
117− $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red");
143+ # TODO: extend this test with multiple elasticsearch nodes
144+ # and see if the status turns "green".
145+ $one->waitUntilSucceeds(
146+ "curl --silent --show-error '${esUrl}/_cluster/health' " .
147+ "| jq .status | grep -v red");
118148
119149 # Perform some simple logstash tests.
120150 $one->waitForUnit("logstash.service");
@@ -123,16 +153,28 @@ let
123153
124154 # See if kibana is healthy.
125155 $one->waitForUnit("kibana.service");
126− $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green");
156+ $one->waitUntilSucceeds(
157+ "curl --silent --show-error 'http://localhost:5601/api/status' " .
158+ "| jq .status.overall.state | grep green");
127159
128160 # See if logstash messages arive in elasticsearch.
129− $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
130− $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
161+ $one->waitUntilSucceeds("${totalHits "flowers"} | grep -v 0");
162+ $one->waitUntilSucceeds("${totalHits "dragons"} | grep 0");
163+
164+ # Test if a message logged to the journal
165+ # is ingested by elasticsearch via journalbeat.
166+ $one->waitForUnit("journalbeat.service");
167+ $one->execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat");
168+ $one->waitUntilSucceeds(
169+ "${totalHits "Supercalifragilisticexpialidocious"} | grep -v 0");
170+
131171 '' + optionalString (!elasticsearchGe7) ''
132172 # Test elasticsearch-curator.
133173 $one->systemctl("stop logstash");
134174 $one->systemctl("start elasticsearch-curator");
135− $one->waitUntilSucceeds("! curl --silent --show-error '${esUrl}/_cat/indices' | grep logstash | grep -q ^$1");
175+ $one->waitUntilSucceeds(
176+ "! curl --silent --show-error '${esUrl}/_cat/indices' " .
177+ "| grep logstash | grep -q ^$1");
136178 '';
137179 };
138180in mapAttrs mkElkTest {
@@ -140,6 +182,7 @@ in mapAttrs mkElkTest {
140182 elasticsearch = pkgs.elasticsearch5;
141183 logstash = pkgs.logstash5;
142184 kibana = pkgs.kibana5;
185+ journalbeat = pkgs.journalbeat5;
143186 };
144187 "ELK-6" =
145188 if enableUnfree
@@ -147,11 +190,13 @@ in mapAttrs mkElkTest {
147190 elasticsearch = pkgs.elasticsearch6;
148191 logstash = pkgs.logstash6;
149192 kibana = pkgs.kibana6;
193+ journalbeat = pkgs.journalbeat6;
150194 }
151195 else {
152196 elasticsearch = pkgs.elasticsearch6-oss;
153197 logstash = pkgs.logstash6-oss;
154198 kibana = pkgs.kibana6-oss;
199+ journalbeat = pkgs.journalbeat6;
155200 };
156201 "ELK-7" =
157202 if enableUnfree
@@ -159,10 +204,12 @@ in mapAttrs mkElkTest {
159204 elasticsearch = pkgs.elasticsearch7;
160205 logstash = pkgs.logstash7;
161206 kibana = pkgs.kibana7;
207+ journalbeat = pkgs.journalbeat7;
162208 }
163209 else {
164210 elasticsearch = pkgs.elasticsearch7-oss;
165211 logstash = pkgs.logstash7-oss;
166212 kibana = pkgs.kibana7-oss;
213+ journalbeat = pkgs.journalbeat7;
167214 };
168215}