From f80797077a521ec3716d1cbeed67eed9519551dd Mon Sep 17 00:00:00 2001
From: talyz <kim.lindberger@gmail.com>
Date: Thu, 16 Dec 2021 16:32:59 +0100
Subject: [PATCH 1/8] Revert "elk7: 7.11.1 -> 7.16.1, 6.8.3 -> 6.8.21 + add
 filebeat module and tests (#150879)"

This reverts commit ebaa2268539a1d7fcf81773afb0bea671a6ec340 which was
a squash of multiple commits that shouldn't have been squashed.
---
 .../from_md/release-notes/rl-2205.section.xml |  10 +-
 .../manual/release-notes/rl-2205.section.md   |   2 -
 nixos/modules/module-list.nix                 |   1 -
 nixos/modules/services/logging/filebeat.nix   | 253 ------------------
 .../modules/services/logging/journalbeat.nix  |   3 +-
 nixos/tests/elk.nix                           |  90 ++-----
 nixos/tests/parsedmarc/default.nix            |  27 +-
 pkgs/development/tools/misc/kibana/6.x.nix    |   8 +-
 pkgs/development/tools/misc/kibana/7.x.nix    |  10 +-
 pkgs/misc/logging/beats/6.x.nix               |   2 +-
 pkgs/misc/logging/beats/7.x.nix               |  24 +-
 pkgs/servers/search/elasticsearch/6.x.nix     |   4 +-
 pkgs/servers/search/elasticsearch/7.x.nix     |   6 +-
 pkgs/servers/search/elasticsearch/plugins.nix |  44 +--
 pkgs/tools/misc/logstash/6.x.nix              |   4 +-
 pkgs/tools/misc/logstash/7.x.nix              |  12 +-
 pkgs/top-level/aliases.nix                    |   1 -
 pkgs/top-level/all-packages.nix               |   7 +-
 18 files changed, 94 insertions(+), 414 deletions(-)
 delete mode 100644 nixos/modules/services/logging/filebeat.nix

diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
index 2dd27649c52c..d02f951a03ca 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
@@ -19,7 +19,7 @@
   </section>
   <section xml:id="sec-release-22.05-new-services">
     <title>New Services</title>
-    <itemizedlist>
+    <itemizedlist spacing="compact">
       <listitem>
         <para>
           <link xlink:href="https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw">aesmd</link>,
@@ -28,14 +28,6 @@
           <link linkend="opt-services.aesmd.enable">services.aesmd</link>.
         </para>
       </listitem>
-      <listitem>
-        <para>
-          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html">filebeat</link>,
-          a lightweight shipper for forwarding and centralizing log
-          data. Available as
-          <link linkend="opt-services.filebeat.enable">services.filebeat</link>.
-        </para>
-      </listitem>
     </itemizedlist>
   </section>
   <section xml:id="sec-release-22.05-incompatibilities">
diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md
index 595785e732a4..11e5462b3316 100644
--- a/nixos/doc/manual/release-notes/rl-2205.section.md
+++ b/nixos/doc/manual/release-notes/rl-2205.section.md
@@ -10,8 +10,6 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - [aesmd](https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw), the Intel SGX Architectural Enclave Service Manager. Available as [services.aesmd](#opt-services.aesmd.enable).
 
-- [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html), a lightweight shipper for forwarding and centralizing log data. Available as [services.filebeat](#opt-services.filebeat.enable).
-
 ## Backward Incompatibilities {#sec-release-22.05-incompatibilities}
 
 - `pkgs.ghc` now refers to `pkgs.targetPackages.haskellPackages.ghc`.
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index cb2dd530de15..1f826220a0f3 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -448,7 +448,6 @@
   ./services/hardware/xow.nix
   ./services/logging/SystemdJournal2Gelf.nix
   ./services/logging/awstats.nix
-  ./services/logging/filebeat.nix
   ./services/logging/fluentd.nix
   ./services/logging/graylog.nix
   ./services/logging/heartbeat.nix
diff --git a/nixos/modules/services/logging/filebeat.nix b/nixos/modules/services/logging/filebeat.nix
deleted file mode 100644
index 223a993c505b..000000000000
--- a/nixos/modules/services/logging/filebeat.nix
+++ /dev/null
@@ -1,253 +0,0 @@
-{ config, lib, utils, pkgs, ... }:
-
-let
-  inherit (lib)
-    attrValues
-    literalExpression
-    mkEnableOption
-    mkIf
-    mkOption
-    types;
-
-  cfg = config.services.filebeat;
-
-  json = pkgs.formats.json {};
-in
-{
-  options = {
-
-    services.filebeat = {
-
-      enable = mkEnableOption "filebeat";
-
-      package = mkOption {
-        type = types.package;
-        default = pkgs.filebeat;
-        defaultText = literalExpression "pkgs.filebeat";
-        example = literalExpression "pkgs.filebeat7";
-        description = ''
-          The filebeat package to use.
-        '';
-      };
-
-      inputs = mkOption {
-        description = ''
-          Inputs specify how Filebeat locates and processes input data.
-
-          This is like <literal>services.filebeat.settings.filebeat.inputs</literal>,
-          but structured as an attribute set. This has the benefit
-          that multiple NixOS modules can contribute settings to a
-          single filebeat input.
-
-          An input type can be specified multiple times by choosing a
-          different <literal>&lt;name></literal> for each, but setting
-          <xref linkend="opt-services.filebeat.inputs._name_.type"/>
-          to the same value.
-
-          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
-        '';
-        default = {};
-        type = types.attrsOf (types.submodule ({ name, ... }: {
-          freeformType = json.type;
-          options = {
-            type = mkOption {
-              type = types.str;
-              default = name;
-              description = ''
-                The input type.
-
-                Look for the value after <literal>type:</literal> on
-                the individual input pages linked from
-                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
-              '';
-            };
-          };
-        }));
-        example = literalExpression ''
-          {
-            journald.id = "everything";  # Only for filebeat7
-            log = {
-              enabled = true;
-              paths = [
-                "/var/log/*.log"
-              ];
-            };
-          };
-        '';
-      };
-
-      modules = mkOption {
-        description = ''
-          Filebeat modules provide a quick way to get started
-          processing common log formats. They contain default
-          configurations, Elasticsearch ingest pipeline definitions,
-          and Kibana dashboards to help you implement and deploy a log
-          monitoring solution.
-
-          This is like <literal>services.filebeat.settings.filebeat.modules</literal>,
-          but structured as an attribute set. This has the benefit
-          that multiple NixOS modules can contribute settings to a
-          single filebeat module.
-
-          A module can be specified multiple times by choosing a
-          different <literal>&lt;name></literal> for each, but setting
-          <xref linkend="opt-services.filebeat.modules._name_.module"/>
-          to the same value.
-
-          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
-        '';
-        default = {};
-        type = types.attrsOf (types.submodule ({ name, ... }: {
-          freeformType = json.type;
-          options = {
-            module = mkOption {
-              type = types.str;
-              default = name;
-              description = ''
-                The name of the module.
-
-                Look for the value after <literal>module:</literal> on
-                the individual input pages linked from
-                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
-              '';
-            };
-          };
-        }));
-        example = literalExpression ''
-          {
-            nginx = {
-              access = {
-                enabled = true;
-                var.paths = [ "/path/to/log/nginx/access.log*" ];
-              };
-              error = {
-                enabled = true;
-                var.paths = [ "/path/to/log/nginx/error.log*" ];
-              };
-            };
-          };
-        '';
-      };
-
-      settings = mkOption {
-        type = types.submodule {
-          freeformType = json.type;
-
-          options = {
-
-            output.elasticsearch.hosts = mkOption {
-              type = with types; listOf str;
-              default = [ "127.0.0.1:9200" ];
-              example = [ "myEShost:9200" ];
-              description = ''
-                The list of Elasticsearch nodes to connect to.
-
-                The events are distributed to these nodes in round
-                robin order. If one node becomes unreachable, the
-                event is automatically sent to another node. Each
-                Elasticsearch node can be defined as a URL or
-                IP:PORT. For example:
-                <literal>http://192.15.3.2</literal>,
-                <literal>https://es.found.io:9230</literal> or
-                <literal>192.24.3.2:9300</literal>. If no port is
-                specified, <literal>9200</literal> is used.
-              '';
-            };
-
-            filebeat = {
-              inputs = mkOption {
-                type = types.listOf json.type;
-                default = [];
-                internal = true;
-                description = ''
-                  Inputs specify how Filebeat locates and processes
-                  input data. Use <xref
-                  linkend="opt-services.filebeat.inputs"/> instead.
-
-                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
-                '';
-              };
-              modules = mkOption {
-                type = types.listOf json.type;
-                default = [];
-                internal = true;
-                description = ''
-                  Filebeat modules provide a quick way to get started
-                  processing common log formats. They contain default
-                  configurations, Elasticsearch ingest pipeline
-                  definitions, and Kibana dashboards to help you
-                  implement and deploy a log monitoring solution.
-
-                  Use <xref linkend="opt-services.filebeat.modules"/> instead.
-
-                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
-                '';
-              };
-            };
-          };
-        };
-        default = {};
-        example = literalExpression ''
-          {
-            settings = {
-              output.elasticsearch = {
-                hosts = [ "myEShost:9200" ];
-                username = "filebeat_internal";
-                password = { _secret = "/var/keys/elasticsearch_password"; };
-              };
-              logging.level = "info";
-            };
-          };
-        '';
-
-        description = ''
-          Configuration for filebeat. See
-          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html"/>
-          for supported values.
-
-          Options containing secret data should be set to an attribute
-          set containing the attribute <literal>_secret</literal> - a
-          string pointing to a file containing the value the option
-          should be set to. See the example to get a better picture of
-          this: in the resulting
-          <filename>filebeat.yml</filename> file, the
-          <literal>output.elasticsearch.password</literal>
-          key will be set to the contents of the
-          <filename>/var/keys/elasticsearch_password</filename> file.
-        '';
-      };
-    };
-  };
-
-  config = mkIf cfg.enable {
-
-    services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs;
-    services.filebeat.settings.filebeat.modules = attrValues cfg.modules;
-
-    systemd.services.filebeat = {
-      description = "Filebeat log shipper";
-      wantedBy = [ "multi-user.target" ];
-      wants = [ "elasticsearch.service" ];
-      after = [ "elasticsearch.service" ];
-      serviceConfig = {
-        ExecStartPre = pkgs.writeShellScript "filebeat-exec-pre" ''
-          set -euo pipefail
-
-          umask u=rwx,g=,o=
-
-          ${utils.genJqSecretsReplacementSnippet
-              cfg.settings
-              "/var/lib/filebeat/filebeat.yml"
-           }
-        '';
-        ExecStart = ''
-          ${cfg.package}/bin/filebeat -e \
-            -c "/var/lib/filebeat/filebeat.yml" \
-            --path.data "/var/lib/filebeat"
-        '';
-        Restart = "always";
-        StateDirectory = "filebeat";
-      };
-    };
-  };
-}
diff --git a/nixos/modules/services/logging/journalbeat.nix b/nixos/modules/services/logging/journalbeat.nix
index 48b02142cb4c..2d98598c1bee 100644
--- a/nixos/modules/services/logging/journalbeat.nix
+++ b/nixos/modules/services/logging/journalbeat.nix
@@ -28,6 +28,7 @@ in
         type = types.package;
         default = pkgs.journalbeat;
         defaultText = literalExpression "pkgs.journalbeat";
+        example = literalExpression "pkgs.journalbeat7";
         description = ''
           The journalbeat package to use
         '';
@@ -88,8 +89,6 @@ in
     systemd.services.journalbeat = {
       description = "Journalbeat log shipper";
       wantedBy = [ "multi-user.target" ];
-      wants = [ "elasticsearch.service" ];
-      after = [ "elasticsearch.service" ];
       preStart = ''
         mkdir -p ${cfg.stateDir}/data
         mkdir -p ${cfg.stateDir}/logs
diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix
index f42be00f23b8..ae746d7e1f03 100644
--- a/nixos/tests/elk.nix
+++ b/nixos/tests/elk.nix
@@ -40,8 +40,9 @@ let
 
             services = {
 
-              journalbeat = {
-                enable = elk ? journalbeat;
+              journalbeat = let lt6 = builtins.compareVersions
+                                        elk.journalbeat.version "6" < 0; in {
+                enable = true;
                 package = elk.journalbeat;
                 extraConfig = pkgs.lib.mkOptionDefault (''
                   logging:
@@ -50,29 +51,14 @@ let
                     metrics.enabled: false
                   output.elasticsearch:
                     hosts: [ "127.0.0.1:9200" ]
+                    ${pkgs.lib.optionalString lt6 "template.enabled: false"}
+                '' + pkgs.lib.optionalString (!lt6) ''
                   journalbeat.inputs:
                   - paths: []
                     seek: cursor
                 '');
               };
 
-              filebeat = {
-                enable = elk ? filebeat;
-                package = elk.filebeat;
-                inputs.journald.id = "everything";
-
-                inputs.log = {
-                  enabled = true;
-                  paths = [
-                    "/var/lib/filebeat/test"
-                  ];
-                };
-
-                settings = {
-                  logging.level = "info";
-                };
-              };
-
               metricbeat = {
                 enable = true;
                 package = elk.metricbeat;
@@ -156,43 +142,27 @@ let
       };
 
     passthru.elkPackages = elk;
-    testScript =
-      let
-        valueObject = lib.optionalString (lib.versionAtLeast elk.elasticsearch.version "7") ".value";
-      in ''
+    testScript = ''
       import json
 
 
-      def expect_hits(message):
+      def total_hits(message):
           dictionary = {"query": {"match": {"message": message}}}
           return (
-              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
+              "curl --silent --show-error '${esUrl}/_search' "
               + "-H 'Content-Type: application/json' "
               + "-d '{}' ".format(json.dumps(dictionary))
-              + " | tee /dev/console"
-              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
-          )
-
-
-      def expect_no_hits(message):
-          dictionary = {"query": {"match": {"message": message}}}
-          return (
-              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
-              + "-H 'Content-Type: application/json' "
-              + "-d '{}' ".format(json.dumps(dictionary))
-              + " | tee /dev/console"
-              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} == 0 end'"
+              + "| jq .hits.total"
           )
 
 
       def has_metricbeat():
           dictionary = {"query": {"match": {"event.dataset": {"query": "system.cpu"}}}}
           return (
-              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
+              "curl --silent --show-error '${esUrl}/_search' "
               + "-H 'Content-Type: application/json' "
               + "-d '{}' ".format(json.dumps(dictionary))
-              + " | tee /dev/console"
-              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
+              + "| jq '.hits.total > 0'"
           )
 
 
@@ -208,8 +178,7 @@ let
       # TODO: extend this test with multiple elasticsearch nodes
       #       and see if the status turns "green".
       one.wait_until_succeeds(
-          "curl --silent --show-error --fail-with-body '${esUrl}/_cluster/health'"
-          + " | jq -es 'if . == [] then null else .[] | .status != \"red\" end'"
+          "curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"
       )
 
       with subtest("Perform some simple logstash tests"):
@@ -220,50 +189,33 @@ let
       with subtest("Kibana is healthy"):
           one.wait_for_unit("kibana.service")
           one.wait_until_succeeds(
-              "curl --silent --show-error --fail-with-body 'http://localhost:5601/api/status'"
-              + " | jq -es 'if . == [] then null else .[] | .status.overall.state == \"green\" end'"
+              "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"
           )
 
       with subtest("Metricbeat is running"):
           one.wait_for_unit("metricbeat.service")
 
       with subtest("Metricbeat metrics arrive in elasticsearch"):
-          one.wait_until_succeeds(has_metricbeat())
+          one.wait_until_succeeds(has_metricbeat() + " | tee /dev/console | grep 'true'")
 
       with subtest("Logstash messages arive in elasticsearch"):
-          one.wait_until_succeeds(expect_hits("flowers"))
-          one.wait_until_succeeds(expect_no_hits("dragons"))
+          one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
+          one.wait_until_succeeds(total_hits("dragons") + " | grep 0")
 
-    '' + lib.optionalString (elk ? journalbeat) ''
       with subtest(
           "A message logged to the journal is ingested by elasticsearch via journalbeat"
       ):
           one.wait_for_unit("journalbeat.service")
           one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat")
           one.wait_until_succeeds(
-              expect_hits("Supercalifragilisticexpialidocious")
+              total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
           )
-    '' + lib.optionalString (elk ? filebeat) ''
-      with subtest(
-          "A message logged to the journal is ingested by elasticsearch via filebeat"
-      ):
-          one.wait_for_unit("filebeat.service")
-          one.execute("echo 'Superdupercalifragilisticexpialidocious' | systemd-cat")
-          one.wait_until_succeeds(
-              expect_hits("Superdupercalifragilisticexpialidocious")
-          )
-          one.execute(
-              "echo 'SuperdupercalifragilisticexpialidociousIndeed' >> /var/lib/filebeat/test"
-          )
-          one.wait_until_succeeds(
-              expect_hits("SuperdupercalifragilisticexpialidociousIndeed")
-          )
-    '' + ''
+
       with subtest("Elasticsearch-curator works"):
           one.systemctl("stop logstash")
           one.systemctl("start elasticsearch-curator")
           one.wait_until_succeeds(
-              '! curl --silent --show-error --fail-with-body "${esUrl}/_cat/indices" | grep logstash | grep ^'
+              '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep ^'
           )
     '';
   }) { inherit pkgs system; };
@@ -283,7 +235,7 @@ in {
   #   elasticsearch = pkgs.elasticsearch7-oss;
   #   logstash      = pkgs.logstash7-oss;
   #   kibana        = pkgs.kibana7-oss;
-  #   filebeat      = pkgs.filebeat7;
+  #   journalbeat   = pkgs.journalbeat7;
   #   metricbeat    = pkgs.metricbeat7;
   # };
   unfree = lib.dontRecurseIntoAttrs {
@@ -298,7 +250,7 @@ in {
       elasticsearch = pkgs.elasticsearch7;
       logstash      = pkgs.logstash7;
       kibana        = pkgs.kibana7;
-      filebeat      = pkgs.filebeat7;
+      journalbeat   = pkgs.journalbeat7;
       metricbeat    = pkgs.metricbeat7;
     };
   };
diff --git a/nixos/tests/parsedmarc/default.nix b/nixos/tests/parsedmarc/default.nix
index 50b977723e9c..d838d3b6a39c 100644
--- a/nixos/tests/parsedmarc/default.nix
+++ b/nixos/tests/parsedmarc/default.nix
@@ -4,7 +4,6 @@
 { pkgs, ... }@args:
 let
   inherit (import ../../lib/testing-python.nix args) makeTest;
-  inherit (pkgs) lib;
 
   dmarcTestReport = builtins.fetchurl {
     name = "dmarc-test-report";
@@ -55,7 +54,7 @@ in
   localMail = makeTest
     {
       name = "parsedmarc-local-mail";
-      meta = with lib.maintainers; {
+      meta = with pkgs.lib.maintainers; {
         maintainers = [ talyz ];
       };
 
@@ -84,7 +83,7 @@ in
             };
           };
 
-          services.elasticsearch.package = pkgs.elasticsearch-oss;
+          services.elasticsearch.package = pkgs.elasticsearch7-oss;
 
           environment.systemPackages = [
             (sendEmail "dmarc@localhost")
@@ -95,7 +94,6 @@ in
       testScript = { nodes }:
         let
           esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
-          valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
         in ''
           parsedmarc.start()
           parsedmarc.wait_for_unit("postfix.service")
@@ -106,15 +104,11 @@ in
           )
 
           parsedmarc.fail(
-              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
-              + " | tee /dev/console"
-              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
+              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
           )
           parsedmarc.succeed("send-email")
           parsedmarc.wait_until_succeeds(
-              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
-              + " | tee /dev/console"
-              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
+              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
           )
         '';
     };
@@ -127,7 +121,7 @@ in
     in
       makeTest {
         name = "parsedmarc-external-mail";
-        meta = with lib.maintainers; {
+        meta = with pkgs.lib.maintainers; {
           maintainers = [ talyz ];
         };
 
@@ -159,7 +153,7 @@ in
                 };
               };
 
-              services.elasticsearch.package = pkgs.elasticsearch-oss;
+              services.elasticsearch.package = pkgs.elasticsearch7-oss;
 
               environment.systemPackages = [
                 pkgs.jq
@@ -207,7 +201,6 @@ in
         testScript = { nodes }:
           let
             esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
-            valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
           in ''
             mail.start()
             mail.wait_for_unit("postfix.service")
@@ -220,15 +213,11 @@ in
             )
 
             parsedmarc.fail(
-                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
-                + " | tee /dev/console"
-                + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
+                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
             )
             mail.succeed("send-email")
             parsedmarc.wait_until_succeeds(
-                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
-                + " | tee /dev/console"
-                + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
+                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
             )
           '';
       };
diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix
index cd81975ee445..c728f7375436 100644
--- a/pkgs/development/tools/misc/kibana/6.x.nix
+++ b/pkgs/development/tools/misc/kibana/6.x.nix
@@ -18,12 +18,12 @@ let
   shas =
     if enableUnfree
     then {
-      x86_64-linux  = "1a501lavxhckb3l93sbrbqyshicwkk6p89frry4x8p037xcfpy0x";
-      x86_64-darwin = "0zm45af30shhcg3mdhcma6rms1hyrx62rm5jzwnz9kxv4d30skbw";
+      x86_64-linux  = "1xwklhqxk5rmdrgy2simwvijzq29kyq5w2w3hy53xh2i1zlnyvq3";
+      x86_64-darwin = "1qpdn28mrpggd55khzqqld6r89l0hb870rigxcw2i8p2yx3jv106";
     }
     else {
-      x86_64-linux  = "0wfdipf21apyily7mvlqgyc7m5jpr96zgrryzwa854z3xb2vw8zg";
-      x86_64-darwin = "1nklfx4yz6hsxlljvnvwjy7pncv9mzngl84710xad5jlyras3sdj";
+      x86_64-linux  = "1wpnwal2rq5v2bsp5qil9j6dplif7ql5394sy4ia5ghp2fzifxmf";
+      x86_64-darwin = "12z8i0wbw10c097glbpdy350p0h3957433f51qfx2p0ghgkzkhzv";
     };
 
 in stdenv.mkDerivation rec {
diff --git a/pkgs/development/tools/misc/kibana/7.x.nix b/pkgs/development/tools/misc/kibana/7.x.nix
index 455f95a69eb0..7a2e0d1d3647 100644
--- a/pkgs/development/tools/misc/kibana/7.x.nix
+++ b/pkgs/development/tools/misc/kibana/7.x.nix
@@ -4,23 +4,23 @@
 , stdenv
 , makeWrapper
 , fetchurl
-, nodejs-16_x
+, nodejs-14_x
 , coreutils
 , which
 }:
 
 with lib;
 let
-  nodejs = nodejs-16_x;
+  nodejs = nodejs-14_x;
   inherit (builtins) elemAt;
   info = splitString "-" stdenv.hostPlatform.system;
   arch = elemAt info 0;
   plat = elemAt info 1;
   shas =
     {
-      x86_64-linux  = "0jivwsrq31n0qfznrsjfsn65sg3wpbd990afn2wzjnj4drq7plz6";
-      x86_64-darwin = "02483aqzrccq1x6rwznmcazijdd46yxj9vnbihnvp2xyp3w9as45";
-      aarch64-linux = "0iw155gkkl1hshc80lfj95rssg039ig21wz1l3srmmf2x4f934s9";
+      x86_64-linux  = "19p9s4sir982bb1zcldrbphhwfs9i11p0q28vgc421iqg10kjlf1";
+      x86_64-darwin = "0qq557ngwwakifidyrccga4cadj9k9pzhjwy4msmbcgf5pb86qyc";
+      aarch64-linux = "183cp1h8d3n7xfcpcys4hf36palczxa409afyp62kzyzckngy0j8";
     };
 
 in stdenv.mkDerivation rec {
diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix
index f5e31924791b..1808197498b5 100644
--- a/pkgs/misc/logging/beats/6.x.nix
+++ b/pkgs/misc/logging/beats/6.x.nix
@@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec {
         owner = "elastic";
         repo = "beats";
         rev = "v${version}";
-        sha256 = "1vnw9clsc10cfpjf6vxvc6m507b2q17sgsl079iwqbp4v0286il7";
+        sha256 = "0jkiz5dfdi9zsji04ipcmcj7pml9294v455y7s2c22k24gyzbaw8";
       };
 
       goPackagePath = "github.com/elastic/beats";
diff --git a/pkgs/misc/logging/beats/7.x.nix b/pkgs/misc/logging/beats/7.x.nix
index 9f5e550d9720..b8b82ed4b308 100644
--- a/pkgs/misc/logging/beats/7.x.nix
+++ b/pkgs/misc/logging/beats/7.x.nix
@@ -8,10 +8,10 @@ let beat = package: extraArgs: buildGoModule (rec {
     owner = "elastic";
     repo = "beats";
     rev = "v${version}";
-    sha256 = "sha256-9Jl5Xo1iKdOY9ZE5JXKSL4ee+NdsN3KCY2dDYuxlzPI=";
+    sha256 = "0gjyzprgj9nskvlkm2bf125b7qn3608llz4kh1fyzsvrw6zb7sm8";
   };
 
-  vendorSha256 = "sha256-tyxyM7RsTHTVVxc9gagPsSvFRaWGTmobKzyv9RODXBk=";
+  vendorSha256 = "04cwf96fh60ld3ndjzzssgirc9ssb53yq71j6ksx36m3y1x7fq9c";
 
   subPackages = [ package ];
 
@@ -24,14 +24,7 @@ let beat = package: extraArgs: buildGoModule (rec {
 } // extraArgs);
 in
 rec {
-  filebeat7 = beat "filebeat" {
-    meta.description = "Lightweight shipper for logfiles";
-    buildInputs = [ systemd ];
-    tags = [ "withjournald" ];
-    postFixup = ''
-      patchelf --set-rpath ${lib.makeLibraryPath [ (lib.getLib systemd) ]} "$out/bin/filebeat"
-    '';
-  };
+  filebeat7 = beat "filebeat" { meta.description = "Lightweight shipper for logfiles"; };
   heartbeat7 = beat "heartbeat" { meta.description = "Lightweight shipper for uptime monitoring"; };
   metricbeat7 = beat "metricbeat" {
     meta.description = "Lightweight shipper for metrics";
@@ -54,4 +47,15 @@ rec {
       PostgreSQL, Redis or Thrift and correlate the messages into transactions.
     '';
   };
+  journalbeat7 = beat "journalbeat" {
+    meta.description = ''
+      Journalbeat is an open source data collector to read and forward
+      journal entries from Linuxes with systemd.
+    '';
+    buildInputs = [ systemd.dev ];
+    postFixup = let libPath = lib.makeLibraryPath [ (lib.getLib systemd) ]; in
+      ''
+        patchelf --set-rpath ${libPath} "$out/bin/journalbeat"
+      '';
+  };
 }
diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix
index 56f0779c833a..4b92592f65df 100644
--- a/pkgs/servers/search/elasticsearch/6.x.nix
+++ b/pkgs/servers/search/elasticsearch/6.x.nix
@@ -19,8 +19,8 @@ stdenv.mkDerivation (rec {
     url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}.tar.gz";
     sha256 =
       if enableUnfree
-      then "1hkcgqsrnnx3zjpgar4424mxfaxrx0zbrp7n7n0dlbhphshwnkmd"
-      else "1pglg60aigy31xmpfchnxcc04nd18zwc3av4m0kyp00yk5mnlyqm";
+      then "09dy3iyzk460vra6na6vk7d3mzpbv4cl0pl7kjmybxy947j7hh42"
+      else "0s04xz3j4psyhawvy503sp2nl5s0gswmpd9wfvwnavgcrr23wk39";
   };
 
   patches = [ ./es-home-6.x.patch ];
diff --git a/pkgs/servers/search/elasticsearch/7.x.nix b/pkgs/servers/search/elasticsearch/7.x.nix
index 592cc947a42e..c254b733837b 100644
--- a/pkgs/servers/search/elasticsearch/7.x.nix
+++ b/pkgs/servers/search/elasticsearch/7.x.nix
@@ -18,9 +18,9 @@ let
   plat = elemAt info 1;
   shas =
     {
-      x86_64-linux  = "1s16l95wc589cr69pfbgmkn9rkvxn6sd6jlbiqpm6p6iyxiaxd6c";
-      x86_64-darwin = "05h7pvq4pb816wgcymnfklp3w6sv54x6138v2infw5219dnk8pfs";
-      aarch64-linux = "0q4xnjzhlx1b2lkikca88qh9glfxaifsm419k2bxxlrfrx31zlkq";
+      x86_64-linux  = "1ld7656b37l67vi4pyv0il865b168niqnbd4hzbvdnwrm35prp10";
+      x86_64-darwin = "11b180y11xw5q01l7aw6lyn15lp9ks8xmakjg1j7gp3z6c90hpn3";
+      aarch64-linux = "0s4ph79x17f90jk31wjwk259dk9dmhnmnkxdcn77m191wvf6m3wy";
     };
 in
 stdenv.mkDerivation rec {
diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix
index 03bb24d9a390..f71d7b9cc76c 100644
--- a/pkgs/servers/search/elasticsearch/plugins.nix
+++ b/pkgs/servers/search/elasticsearch/plugins.nix
@@ -38,8 +38,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.16.1" then "1sz858m9963xqr5kzjlwnq7k0a146rn60v6xijyfbp8y3brg618p"
-        else if version == "6.8.21" then "06b1pavyggzfp4wwdql0q9nm3r7i9px9cagp4yh4nhxhnk4w5fiq"
+        if version == "7.11.1" then "0mi6fmnjbqypa4n1w34dvlmyq793pz4wf1r5srcs7i84kkiddysy"
+        else if version == "6.8.3" then "0vbaqyj0lfy3ijl1c9h92b0nh605h5mjs57bk2zhycdvbw5sx2lv"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -55,8 +55,8 @@ in
     src = fetchurl {
       url = "https://github.com/vhyza/elasticsearch-${pluginName}/releases/download/v${version}/elasticsearch-${pluginName}-${version}-plugin.zip";
       sha256 =
-        if version == "7.16.1" then "0yjy9yhw77lmalivxnmv2rq8fk93ddxszkk73lgmpffladx2ikir"
-        else if version == "6.8.21" then "0m80cn7vkcvk95v4pdmi6vk5ww7p01k0hj2iqb9g870vs6x2qjzv"
+        if version == "7.11.1" then "0r2k2ndgqiqh27lch8dbay1m09f00h5kjcan87chcvyf623l40a3"
+        else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -72,8 +72,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.16.1" then "1w5ndgffqzj5ijglmykifrk1jsgh7qwn8m7sbpiv0r7n3aayhz1x"
-        else if version == "6.8.21" then "07w8s4a5gvr9lzjzf629y8rx3kvs6zd1vl07ksw1paghp42yb354"
+        if version == "7.11.1" then "10ln81zyf04qi9wv10mck8iz0xwfvwp4ni0hl1gkgvh44lf1n855"
+        else if version == "6.8.3" then "0ggdhf7w50bxsffmcznrjy14b578fps0f8arg3v54qvj94v9jc37"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -89,8 +89,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.16.1" then "16mv7b9nl96bcygabvjqidxp2sjk340va19mrmliblpq3mxa2sii"
-        else if version == "6.8.21" then "1kdpbrasxwr3dn21zjrklp1s389rwa51fairygdwl8px9liwwfa5"
+        if version == "7.11.1" then "09grfvqjmm2rznc48z84awh54afh81qa16amfqw3amsb8dr6czm6"
+        else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -106,8 +106,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.16.1" then "0bf8f8cybsp6s2ai3j04yay9kbhsafpgxivxjvzn2iy9qgc84ls4"
-        else if version == "6.8.21" then "0v31yyhjcdlqnjw1f9kihh7z3c6d31whc57hqqd1dn579n4s9rlz"
+        if version == "7.11.1" then "0imkf3w2fmspb78vkf9k6kqx1crm4f82qgnbk1qa7gbsa2j47hbs"
+        else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -123,8 +123,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.16.1" then "0sfa0ql3hh8jmha230dyhr51bvsvwmazyycf36ngpmxsysm8ccml"
-        else if version == "6.8.21" then "0sfh1az30q4f34zxig2fz8wn9gk53fmmxyg5pbi1svn9761p5awq"
+        if version == "7.11.1" then "0ahyb1plgwvq22id2kcx9g076ybb3kvybwakgcvsdjjdyi4cwgjs"
+        else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -140,8 +140,8 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.16.1" then "1b95hjr4qhiavm7r7k19bwk5c64r00f1g5s0ydnb6gzym9hdb5s1"
-        else if version == "6.8.21" then "00lwj00rfdk6850gk1n86chiz2w6afpqn7jn588jdbwv41qh5mrv"
+        if version == "7.11.1" then "0i98b905k1zwm3y9pfhr40v2fm5qdsp3icygibhxf7drffygk4l7"
+        else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -157,24 +157,24 @@ in
     pluginName = "search-guard";
     version =
       # https://docs.search-guard.com/latest/search-guard-versions
-      if esVersion == "7.16.1" then "${esVersion}-52.5.0"
-      else if esVersion == "6.8.21" then "${esVersion}-25.6"
+      if esVersion == "7.11.1" then "${esVersion}-50.0.0"
+      else if esVersion == "6.8.3" then "${esVersion}-25.5"
       else throw "unsupported version ${esVersion} for plugin ${pluginName}";
     src =
-      if esVersion == "7.16.1" then
+      if esVersion == "7.11.1" then
         fetchurl {
           url = "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip";
-          sha256 = "1m3nj35qyrkkh3mhmn66nippavima8h8qpaxddalhjsvf70lhnjb";
+          sha256 = "1lippygiy0xcxxlakylhvj3bj2i681k6jcfjsprkfk7hlaqsqxkm";
         }
-      else if esVersion == "6.8.21" then
+      else if esVersion == "6.8.3" then
         fetchurl {
-          url = "https://maven.search-guard.com/search-guard-release/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip";
-          sha256 = "19nj513wigwd0mzq747zax4fzvv5vi24f7j0636rydd9iv9cyhg2";
+          url = "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip";
+          sha256 = "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb";
         }
       else throw "unsupported version ${version} for plugin ${pluginName}";
     meta = with lib; {
       homepage = "https://search-guard.com";
-      description = "Elasticsearch plugin that offers encryption, authentication, and authorisation.";
+      description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. ";
       license = licenses.asl20;
     };
   };
diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix
index 0b3e17818dcd..b35e5a4aea43 100644
--- a/pkgs/tools/misc/logstash/6.x.nix
+++ b/pkgs/tools/misc/logstash/6.x.nix
@@ -17,8 +17,8 @@ let this = stdenv.mkDerivation rec {
     url = "https://artifacts.elastic.co/downloads/logstash/${pname}-${version}.tar.gz";
     sha256 =
       if enableUnfree
-      then "0hij1byw5b3xmk3vshr9p7gxwbjrywr7ylps05ydc2dmnz8q2a79"
-      else "1fa236pvhj7spys54nqi3k64rwzf6zi6gaccmqg4p4sh92jzsybv";
+      then "00pwi7clgdflzzg15bh3y30gzikvvy7p5fl88fww7xhhy47q8053"
+      else "0spxgqsyh72n0l0xh6rljp0lbqz46xmr02sqz25ybycr4qkxdhgk";
   };
 
   dontBuild         = true;
diff --git a/pkgs/tools/misc/logstash/7.x.nix b/pkgs/tools/misc/logstash/7.x.nix
index 636c380817ce..1e69fbc976d9 100644
--- a/pkgs/tools/misc/logstash/7.x.nix
+++ b/pkgs/tools/misc/logstash/7.x.nix
@@ -17,14 +17,14 @@ let
   shas =
     if enableUnfree
     then {
-      x86_64-linux  = "1vm53alq9q1qy2jcsjg9z339xrkac5r9qqpdafp53ny4zsv1n7vj";
-      x86_64-darwin = "0hhjyl04h3gd66rdk22272rj419br4v2i59lyrmaj6hmnsqbv968";
-      aarch64-linux = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p";
+      x86_64-linux  = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p";
+      x86_64-darwin = "0dqm66c89w1nvmbwqzphlqmf7avrycgv1nwd5b0k1z168fj0c3zm";
+      aarch64-linux = "11hjhyb48mjagmvqyxb780n57kr619h6p4adl2vs1zm97g9gslx8";
     }
     else {
-      x86_64-linux  = "1f3659vcgczm7v03q3fvsmp1ndp6wm3i7r2b2vbl4xq7hf9v7azk";
-      x86_64-darwin = "10zw9qc0lc0x9in0nkxc1aiazhyd69l8sya2ni46ivyyjwf0sqsn";
-      aarch64-linux = "1czhgmky2zf3mqykn5ww4257yfhd36mi4x6dq569ymly83pivf8v";
+      x86_64-linux  = "14b1649avjcalcsi0ffkgznq6d93qdk6m3j0i73mwfqka5d3dvy3";
+      x86_64-darwin = "0ypgdfklr5rxvsnc3czh231pa1z2h70366j1c6q5g64b3xnxpphs";
+      aarch64-linux = "01ainayr8fwwfix7dmxfhhmb23ji65dn4lbjwnj2w0pl0ym9h9w2";
     };
   this = stdenv.mkDerivation rec {
     version = elk7Version;
diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix
index 672fdf21b025..222c6ac0474f 100644
--- a/pkgs/top-level/aliases.nix
+++ b/pkgs/top-level/aliases.nix
@@ -384,7 +384,6 @@ mapAliases ({
   jbuilder = dune_1; # added 2018-09-09
   jikes = throw "jikes was deprecated on 2019-10-07: abandoned by upstream";
   joseki = apache-jena-fuseki; # added 2016-02-28
-  journalbeat7 = throw "journalbeat has been removed upstream. Use filebeat with the journald input instead.";
   jvmci8 = throw "graalvm8 and its tools were deprecated in favor of graalvm8-ce"; # added 2021-10-15
   json_glib = json-glib; # added 2018-02-25
   kafkacat = kcat; # added 2021-10-07
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 9c117ca4d712..8d98f53a3318 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -2140,7 +2140,8 @@ with pkgs;
     filebeat7
     heartbeat7
     metricbeat7
-    packetbeat7;
+    packetbeat7
+    journalbeat7;
 
   filebeat = filebeat6;
   heartbeat = heartbeat6;
@@ -4999,8 +5000,8 @@ with pkgs;
 
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
   # When updating make sure to update all plugins or they will break!
-  elk6Version = "6.8.21";
-  elk7Version = "7.16.1";
+  elk6Version = "6.8.3";
+  elk7Version = "7.11.1";
 
   elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix {
     util-linux = util-linuxMinimal;

From 6c5a5337970dc8aa9dd96c1b2b13cdc0250a66b4 Mon Sep 17 00:00:00 2001
From: talyz <kim.lindberger@gmail.com>
Date: Wed, 15 Dec 2021 19:14:12 +0100
Subject: [PATCH 2/8] elk7: 7.11.1 -> 7.16.1

---
 .../modules/services/logging/journalbeat.nix  |  1 -
 nixos/tests/elk.nix                           | 12 ++++------
 pkgs/development/tools/misc/kibana/7.x.nix    | 10 ++++----
 pkgs/misc/logging/beats/7.x.nix               | 24 ++++++++-----------
 pkgs/servers/search/elasticsearch/7.x.nix     |  6 ++---
 pkgs/servers/search/elasticsearch/plugins.nix | 20 ++++++++--------
 pkgs/tools/misc/logstash/7.x.nix              | 12 +++++-----
 pkgs/top-level/aliases.nix                    |  1 +
 pkgs/top-level/all-packages.nix               |  5 ++--
 9 files changed, 41 insertions(+), 50 deletions(-)

diff --git a/nixos/modules/services/logging/journalbeat.nix b/nixos/modules/services/logging/journalbeat.nix
index 2d98598c1bee..b34fb308f586 100644
--- a/nixos/modules/services/logging/journalbeat.nix
+++ b/nixos/modules/services/logging/journalbeat.nix
@@ -28,7 +28,6 @@ in
         type = types.package;
         default = pkgs.journalbeat;
         defaultText = literalExpression "pkgs.journalbeat";
-        example = literalExpression "pkgs.journalbeat7";
         description = ''
           The journalbeat package to use
         '';
diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix
index ae746d7e1f03..1dae625987f5 100644
--- a/nixos/tests/elk.nix
+++ b/nixos/tests/elk.nix
@@ -40,9 +40,8 @@ let
 
             services = {
 
-              journalbeat = let lt6 = builtins.compareVersions
-                                        elk.journalbeat.version "6" < 0; in {
-                enable = true;
+              journalbeat = {
+                enable = elk ? journalbeat;
                 package = elk.journalbeat;
                 extraConfig = pkgs.lib.mkOptionDefault (''
                   logging:
@@ -51,8 +50,6 @@ let
                     metrics.enabled: false
                   output.elasticsearch:
                     hosts: [ "127.0.0.1:9200" ]
-                    ${pkgs.lib.optionalString lt6 "template.enabled: false"}
-                '' + pkgs.lib.optionalString (!lt6) ''
                   journalbeat.inputs:
                   - paths: []
                     seek: cursor
@@ -202,6 +199,7 @@ let
           one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
           one.wait_until_succeeds(total_hits("dragons") + " | grep 0")
 
+    '' + lib.optionalString (elk ? journalbeat) ''
       with subtest(
           "A message logged to the journal is ingested by elasticsearch via journalbeat"
       ):
@@ -210,7 +208,7 @@ let
           one.wait_until_succeeds(
               total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
           )
-
+    '' + ''
       with subtest("Elasticsearch-curator works"):
           one.systemctl("stop logstash")
           one.systemctl("start elasticsearch-curator")
@@ -235,7 +233,6 @@ in {
   #   elasticsearch = pkgs.elasticsearch7-oss;
   #   logstash      = pkgs.logstash7-oss;
   #   kibana        = pkgs.kibana7-oss;
-  #   journalbeat   = pkgs.journalbeat7;
   #   metricbeat    = pkgs.metricbeat7;
   # };
   unfree = lib.dontRecurseIntoAttrs {
@@ -250,7 +247,6 @@ in {
       elasticsearch = pkgs.elasticsearch7;
       logstash      = pkgs.logstash7;
       kibana        = pkgs.kibana7;
-      journalbeat   = pkgs.journalbeat7;
       metricbeat    = pkgs.metricbeat7;
     };
   };
diff --git a/pkgs/development/tools/misc/kibana/7.x.nix b/pkgs/development/tools/misc/kibana/7.x.nix
index 7a2e0d1d3647..455f95a69eb0 100644
--- a/pkgs/development/tools/misc/kibana/7.x.nix
+++ b/pkgs/development/tools/misc/kibana/7.x.nix
@@ -4,23 +4,23 @@
 , stdenv
 , makeWrapper
 , fetchurl
-, nodejs-14_x
+, nodejs-16_x
 , coreutils
 , which
 }:
 
 with lib;
 let
-  nodejs = nodejs-14_x;
+  nodejs = nodejs-16_x;
   inherit (builtins) elemAt;
   info = splitString "-" stdenv.hostPlatform.system;
   arch = elemAt info 0;
   plat = elemAt info 1;
   shas =
     {
-      x86_64-linux  = "19p9s4sir982bb1zcldrbphhwfs9i11p0q28vgc421iqg10kjlf1";
-      x86_64-darwin = "0qq557ngwwakifidyrccga4cadj9k9pzhjwy4msmbcgf5pb86qyc";
-      aarch64-linux = "183cp1h8d3n7xfcpcys4hf36palczxa409afyp62kzyzckngy0j8";
+      x86_64-linux  = "0jivwsrq31n0qfznrsjfsn65sg3wpbd990afn2wzjnj4drq7plz6";
+      x86_64-darwin = "02483aqzrccq1x6rwznmcazijdd46yxj9vnbihnvp2xyp3w9as45";
+      aarch64-linux = "0iw155gkkl1hshc80lfj95rssg039ig21wz1l3srmmf2x4f934s9";
     };
 
 in stdenv.mkDerivation rec {
diff --git a/pkgs/misc/logging/beats/7.x.nix b/pkgs/misc/logging/beats/7.x.nix
index b8b82ed4b308..9f5e550d9720 100644
--- a/pkgs/misc/logging/beats/7.x.nix
+++ b/pkgs/misc/logging/beats/7.x.nix
@@ -8,10 +8,10 @@ let beat = package: extraArgs: buildGoModule (rec {
     owner = "elastic";
     repo = "beats";
     rev = "v${version}";
-    sha256 = "0gjyzprgj9nskvlkm2bf125b7qn3608llz4kh1fyzsvrw6zb7sm8";
+    sha256 = "sha256-9Jl5Xo1iKdOY9ZE5JXKSL4ee+NdsN3KCY2dDYuxlzPI=";
   };
 
-  vendorSha256 = "04cwf96fh60ld3ndjzzssgirc9ssb53yq71j6ksx36m3y1x7fq9c";
+  vendorSha256 = "sha256-tyxyM7RsTHTVVxc9gagPsSvFRaWGTmobKzyv9RODXBk=";
 
   subPackages = [ package ];
 
@@ -24,7 +24,14 @@ let beat = package: extraArgs: buildGoModule (rec {
 } // extraArgs);
 in
 rec {
-  filebeat7 = beat "filebeat" { meta.description = "Lightweight shipper for logfiles"; };
+  filebeat7 = beat "filebeat" {
+    meta.description = "Lightweight shipper for logfiles";
+    buildInputs = [ systemd ];
+    tags = [ "withjournald" ];
+    postFixup = ''
+      patchelf --set-rpath ${lib.makeLibraryPath [ (lib.getLib systemd) ]} "$out/bin/filebeat"
+    '';
+  };
   heartbeat7 = beat "heartbeat" { meta.description = "Lightweight shipper for uptime monitoring"; };
   metricbeat7 = beat "metricbeat" {
     meta.description = "Lightweight shipper for metrics";
@@ -47,15 +54,4 @@ rec {
       PostgreSQL, Redis or Thrift and correlate the messages into transactions.
     '';
   };
-  journalbeat7 = beat "journalbeat" {
-    meta.description = ''
-      Journalbeat is an open source data collector to read and forward
-      journal entries from Linuxes with systemd.
-    '';
-    buildInputs = [ systemd.dev ];
-    postFixup = let libPath = lib.makeLibraryPath [ (lib.getLib systemd) ]; in
-      ''
-        patchelf --set-rpath ${libPath} "$out/bin/journalbeat"
-      '';
-  };
 }
diff --git a/pkgs/servers/search/elasticsearch/7.x.nix b/pkgs/servers/search/elasticsearch/7.x.nix
index c254b733837b..592cc947a42e 100644
--- a/pkgs/servers/search/elasticsearch/7.x.nix
+++ b/pkgs/servers/search/elasticsearch/7.x.nix
@@ -18,9 +18,9 @@ let
   plat = elemAt info 1;
   shas =
     {
-      x86_64-linux  = "1ld7656b37l67vi4pyv0il865b168niqnbd4hzbvdnwrm35prp10";
-      x86_64-darwin = "11b180y11xw5q01l7aw6lyn15lp9ks8xmakjg1j7gp3z6c90hpn3";
-      aarch64-linux = "0s4ph79x17f90jk31wjwk259dk9dmhnmnkxdcn77m191wvf6m3wy";
+      x86_64-linux  = "1s16l95wc589cr69pfbgmkn9rkvxn6sd6jlbiqpm6p6iyxiaxd6c";
+      x86_64-darwin = "05h7pvq4pb816wgcymnfklp3w6sv54x6138v2infw5219dnk8pfs";
+      aarch64-linux = "0q4xnjzhlx1b2lkikca88qh9glfxaifsm419k2bxxlrfrx31zlkq";
     };
 in
 stdenv.mkDerivation rec {
diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix
index f71d7b9cc76c..baeeec5c67a7 100644
--- a/pkgs/servers/search/elasticsearch/plugins.nix
+++ b/pkgs/servers/search/elasticsearch/plugins.nix
@@ -38,7 +38,7 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "0mi6fmnjbqypa4n1w34dvlmyq793pz4wf1r5srcs7i84kkiddysy"
+        if version == "7.16.1" then "1sz858m9963xqr5kzjlwnq7k0a146rn60v6xijyfbp8y3brg618p"
         else if version == "6.8.3" then "0vbaqyj0lfy3ijl1c9h92b0nh605h5mjs57bk2zhycdvbw5sx2lv"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
@@ -55,7 +55,7 @@ in
     src = fetchurl {
       url = "https://github.com/vhyza/elasticsearch-${pluginName}/releases/download/v${version}/elasticsearch-${pluginName}-${version}-plugin.zip";
       sha256 =
-        if version == "7.11.1" then "0r2k2ndgqiqh27lch8dbay1m09f00h5kjcan87chcvyf623l40a3"
+        if version == "7.16.1" then "0yjy9yhw77lmalivxnmv2rq8fk93ddxszkk73lgmpffladx2ikir"
         else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
@@ -72,7 +72,7 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "10ln81zyf04qi9wv10mck8iz0xwfvwp4ni0hl1gkgvh44lf1n855"
+        if version == "7.16.1" then "1w5ndgffqzj5ijglmykifrk1jsgh7qwn8m7sbpiv0r7n3aayhz1x"
         else if version == "6.8.3" then "0ggdhf7w50bxsffmcznrjy14b578fps0f8arg3v54qvj94v9jc37"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
@@ -89,7 +89,7 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "09grfvqjmm2rznc48z84awh54afh81qa16amfqw3amsb8dr6czm6"
+        if version == "7.16.1" then "16mv7b9nl96bcygabvjqidxp2sjk340va19mrmliblpq3mxa2sii"
         else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
@@ -106,7 +106,7 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
-        if version == "7.11.1" then "0imkf3w2fmspb78vkf9k6kqx1crm4f82qgnbk1qa7gbsa2j47hbs"
+        if version == "7.16.1" then "0bf8f8cybsp6s2ai3j04yay9kbhsafpgxivxjvzn2iy9qgc84ls4"
         else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
@@ -123,7 +123,7 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.11.1" then "0ahyb1plgwvq22id2kcx9g076ybb3kvybwakgcvsdjjdyi4cwgjs"
+        if version == "7.16.1" then "0sfa0ql3hh8jmha230dyhr51bvsvwmazyycf36ngpmxsysm8ccml"
         else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
@@ -140,7 +140,7 @@ in
     src = fetchurl {
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
-        if version == "7.11.1" then "0i98b905k1zwm3y9pfhr40v2fm5qdsp3icygibhxf7drffygk4l7"
+        if version == "7.16.1" then "1b95hjr4qhiavm7r7k19bwk5c64r00f1g5s0ydnb6gzym9hdb5s1"
         else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
@@ -157,14 +157,14 @@ in
     pluginName = "search-guard";
     version =
       # https://docs.search-guard.com/latest/search-guard-versions
-      if esVersion == "7.11.1" then "${esVersion}-50.0.0"
+      if esVersion == "7.16.1" then "${esVersion}-52.5.0"
       else if esVersion == "6.8.3" then "${esVersion}-25.5"
       else throw "unsupported version ${esVersion} for plugin ${pluginName}";
     src =
-      if esVersion == "7.11.1" then
+      if esVersion == "7.16.1" then
         fetchurl {
           url = "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip";
-          sha256 = "1lippygiy0xcxxlakylhvj3bj2i681k6jcfjsprkfk7hlaqsqxkm";
+          sha256 = "1m3nj35qyrkkh3mhmn66nippavima8h8qpaxddalhjsvf70lhnjb";
         }
       else if esVersion == "6.8.3" then
         fetchurl {
diff --git a/pkgs/tools/misc/logstash/7.x.nix b/pkgs/tools/misc/logstash/7.x.nix
index 1e69fbc976d9..636c380817ce 100644
--- a/pkgs/tools/misc/logstash/7.x.nix
+++ b/pkgs/tools/misc/logstash/7.x.nix
@@ -17,14 +17,14 @@ let
   shas =
     if enableUnfree
     then {
-      x86_64-linux  = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p";
-      x86_64-darwin = "0dqm66c89w1nvmbwqzphlqmf7avrycgv1nwd5b0k1z168fj0c3zm";
-      aarch64-linux = "11hjhyb48mjagmvqyxb780n57kr619h6p4adl2vs1zm97g9gslx8";
+      x86_64-linux  = "1vm53alq9q1qy2jcsjg9z339xrkac5r9qqpdafp53ny4zsv1n7vj";
+      x86_64-darwin = "0hhjyl04h3gd66rdk22272rj419br4v2i59lyrmaj6hmnsqbv968";
+      aarch64-linux = "0yjaki7gjffrz86hvqgn1gzhd9dc9llcj50g2x1sgpyn88zk0z0p";
     }
     else {
-      x86_64-linux  = "14b1649avjcalcsi0ffkgznq6d93qdk6m3j0i73mwfqka5d3dvy3";
-      x86_64-darwin = "0ypgdfklr5rxvsnc3czh231pa1z2h70366j1c6q5g64b3xnxpphs";
-      aarch64-linux = "01ainayr8fwwfix7dmxfhhmb23ji65dn4lbjwnj2w0pl0ym9h9w2";
+      x86_64-linux  = "1f3659vcgczm7v03q3fvsmp1ndp6wm3i7r2b2vbl4xq7hf9v7azk";
+      x86_64-darwin = "10zw9qc0lc0x9in0nkxc1aiazhyd69l8sya2ni46ivyyjwf0sqsn";
+      aarch64-linux = "1czhgmky2zf3mqykn5ww4257yfhd36mi4x6dq569ymly83pivf8v";
     };
   this = stdenv.mkDerivation rec {
     version = elk7Version;
diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix
index 222c6ac0474f..672fdf21b025 100644
--- a/pkgs/top-level/aliases.nix
+++ b/pkgs/top-level/aliases.nix
@@ -384,6 +384,7 @@ mapAliases ({
   jbuilder = dune_1; # added 2018-09-09
   jikes = throw "jikes was deprecated on 2019-10-07: abandoned by upstream";
   joseki = apache-jena-fuseki; # added 2016-02-28
+  journalbeat7 = throw "journalbeat has been removed upstream. Use filebeat with the journald input instead.";
   jvmci8 = throw "graalvm8 and its tools were deprecated in favor of graalvm8-ce"; # added 2021-10-15
   json_glib = json-glib; # added 2018-02-25
   kafkacat = kcat; # added 2021-10-07
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 8d98f53a3318..a4ff85f6f848 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -2140,8 +2140,7 @@ with pkgs;
     filebeat7
     heartbeat7
     metricbeat7
-    packetbeat7
-    journalbeat7;
+    packetbeat7;
 
   filebeat = filebeat6;
   heartbeat = heartbeat6;
@@ -5001,7 +5000,7 @@ with pkgs;
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
   # When updating make sure to update all plugins or they will break!
   elk6Version = "6.8.3";
-  elk7Version = "7.11.1";
+  elk7Version = "7.16.1";
 
   elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix {
     util-linux = util-linuxMinimal;

From 9647a429eddd1fc70dd2b9a29c0b79233b3bf5aa Mon Sep 17 00:00:00 2001
From: talyz <kim.lindberger@gmail.com>
Date: Wed, 15 Dec 2021 19:22:48 +0100
Subject: [PATCH 3/8] nixosTests.elk: Improve reliability and compatibility
 with ELK 7.x

- Use comparisons in jq instead of grepping
- Match for `.hits.total.value` if version >= 7, otherwise it always
  passes
- Make curl fail if requests fails
---
 nixos/tests/elk.nix | 42 ++++++++++++++++++++++++++++++------------
 1 file changed, 30 insertions(+), 12 deletions(-)

diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix
index 1dae625987f5..8db49ecfb18a 100644
--- a/nixos/tests/elk.nix
+++ b/nixos/tests/elk.nix
@@ -139,27 +139,43 @@ let
       };
 
     passthru.elkPackages = elk;
-    testScript = ''
+    testScript =
+      let
+        valueObject = lib.optionalString (lib.versionAtLeast elk.elasticsearch.version "7") ".value";
+      in ''
       import json
 
 
-      def total_hits(message):
+      def expect_hits(message):
           dictionary = {"query": {"match": {"message": message}}}
           return (
-              "curl --silent --show-error '${esUrl}/_search' "
+              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
               + "-H 'Content-Type: application/json' "
               + "-d '{}' ".format(json.dumps(dictionary))
-              + "| jq .hits.total"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
+          )
+
+
+      def expect_no_hits(message):
+          dictionary = {"query": {"match": {"message": message}}}
+          return (
+              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
+              + "-H 'Content-Type: application/json' "
+              + "-d '{}' ".format(json.dumps(dictionary))
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} == 0 end'"
           )
 
 
       def has_metricbeat():
           dictionary = {"query": {"match": {"event.dataset": {"query": "system.cpu"}}}}
           return (
-              "curl --silent --show-error '${esUrl}/_search' "
+              "curl --silent --show-error --fail-with-body '${esUrl}/_search' "
               + "-H 'Content-Type: application/json' "
               + "-d '{}' ".format(json.dumps(dictionary))
-              + "| jq '.hits.total > 0'"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
 
 
@@ -175,7 +191,8 @@ let
       # TODO: extend this test with multiple elasticsearch nodes
       #       and see if the status turns "green".
       one.wait_until_succeeds(
-          "curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"
+          "curl --silent --show-error --fail-with-body '${esUrl}/_cluster/health'"
+          + " | jq -es 'if . == [] then null else .[] | .status != \"red\" end'"
       )
 
       with subtest("Perform some simple logstash tests"):
@@ -186,18 +203,19 @@ let
       with subtest("Kibana is healthy"):
           one.wait_for_unit("kibana.service")
           one.wait_until_succeeds(
-              "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"
+              "curl --silent --show-error --fail-with-body 'http://localhost:5601/api/status'"
+              + " | jq -es 'if . == [] then null else .[] | .status.overall.state == \"green\" end'"
           )
 
       with subtest("Metricbeat is running"):
           one.wait_for_unit("metricbeat.service")
 
       with subtest("Metricbeat metrics arrive in elasticsearch"):
-          one.wait_until_succeeds(has_metricbeat() + " | tee /dev/console | grep 'true'")
+          one.wait_until_succeeds(has_metricbeat())
 
       with subtest("Logstash messages arive in elasticsearch"):
-          one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0")
-          one.wait_until_succeeds(total_hits("dragons") + " | grep 0")
+          one.wait_until_succeeds(expect_hits("flowers"))
+          one.wait_until_succeeds(expect_no_hits("dragons"))
 
     '' + lib.optionalString (elk ? journalbeat) ''
       with subtest(
@@ -206,7 +224,7 @@ let
           one.wait_for_unit("journalbeat.service")
           one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat")
           one.wait_until_succeeds(
-              total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0"
+              expect_hits("Supercalifragilisticexpialidocious")
           )
     '' + ''
       with subtest("Elasticsearch-curator works"):

From 6c9c2b47346c3584dce0607e8821b961666a5c85 Mon Sep 17 00:00:00 2001
From: talyz <kim.lindberger@gmail.com>
Date: Wed, 15 Dec 2021 19:38:35 +0100
Subject: [PATCH 4/8] nixos/filebeat: Add initial module and test

Filebeat is an open source file harvester, mostly used to fetch logs
files and feed them into logstash.

This module can be used instead of journalbeat if used with
`filebeat7` and configured with the `journald` input.
---
 nixos/modules/module-list.nix               |   1 +
 nixos/modules/services/logging/filebeat.nix | 253 ++++++++++++++++++++
 nixos/tests/elk.nix                         |  36 ++-
 3 files changed, 289 insertions(+), 1 deletion(-)
 create mode 100644 nixos/modules/services/logging/filebeat.nix

diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index 1f826220a0f3..cb2dd530de15 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -448,6 +448,7 @@
   ./services/hardware/xow.nix
   ./services/logging/SystemdJournal2Gelf.nix
   ./services/logging/awstats.nix
+  ./services/logging/filebeat.nix
   ./services/logging/fluentd.nix
   ./services/logging/graylog.nix
   ./services/logging/heartbeat.nix
diff --git a/nixos/modules/services/logging/filebeat.nix b/nixos/modules/services/logging/filebeat.nix
new file mode 100644
index 000000000000..223a993c505b
--- /dev/null
+++ b/nixos/modules/services/logging/filebeat.nix
@@ -0,0 +1,253 @@
+{ config, lib, utils, pkgs, ... }:
+
+let
+  inherit (lib)
+    attrValues
+    literalExpression
+    mkEnableOption
+    mkIf
+    mkOption
+    types;
+
+  cfg = config.services.filebeat;
+
+  json = pkgs.formats.json {};
+in
+{
+  options = {
+
+    services.filebeat = {
+
+      enable = mkEnableOption "filebeat";
+
+      package = mkOption {
+        type = types.package;
+        default = pkgs.filebeat;
+        defaultText = literalExpression "pkgs.filebeat";
+        example = literalExpression "pkgs.filebeat7";
+        description = ''
+          The filebeat package to use.
+        '';
+      };
+
+      inputs = mkOption {
+        description = ''
+          Inputs specify how Filebeat locates and processes input data.
+
+          This is like <literal>services.filebeat.settings.filebeat.inputs</literal>,
+          but structured as an attribute set. This has the benefit
+          that multiple NixOS modules can contribute settings to a
+          single filebeat input.
+
+          An input type can be specified multiple times by choosing a
+          different <literal>&lt;name></literal> for each, but setting
+          <xref linkend="opt-services.filebeat.inputs._name_.type"/>
+          to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = json.type;
+          options = {
+            type = mkOption {
+              type = types.str;
+              default = name;
+              description = ''
+                The input type.
+
+                Look for the value after <literal>type:</literal> on
+                the individual input pages linked from
+                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+              '';
+            };
+          };
+        }));
+        example = literalExpression ''
+          {
+            journald.id = "everything";  # Only for filebeat7
+            log = {
+              enabled = true;
+              paths = [
+                "/var/log/*.log"
+              ];
+            };
+          };
+        '';
+      };
+
+      modules = mkOption {
+        description = ''
+          Filebeat modules provide a quick way to get started
+          processing common log formats. They contain default
+          configurations, Elasticsearch ingest pipeline definitions,
+          and Kibana dashboards to help you implement and deploy a log
+          monitoring solution.
+
+          This is like <literal>services.filebeat.settings.filebeat.modules</literal>,
+          but structured as an attribute set. This has the benefit
+          that multiple NixOS modules can contribute settings to a
+          single filebeat module.
+
+          A module can be specified multiple times by choosing a
+          different <literal>&lt;name></literal> for each, but setting
+          <xref linkend="opt-services.filebeat.modules._name_.module"/>
+          to the same value.
+
+          See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+        '';
+        default = {};
+        type = types.attrsOf (types.submodule ({ name, ... }: {
+          freeformType = json.type;
+          options = {
+            module = mkOption {
+              type = types.str;
+              default = name;
+              description = ''
+                The name of the module.
+
+                Look for the value after <literal>module:</literal> on
+                the individual input pages linked from
+                <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+              '';
+            };
+          };
+        }));
+        example = literalExpression ''
+          {
+            nginx = {
+              access = {
+                enabled = true;
+                var.paths = [ "/path/to/log/nginx/access.log*" ];
+              };
+              error = {
+                enabled = true;
+                var.paths = [ "/path/to/log/nginx/error.log*" ];
+              };
+            };
+          };
+        '';
+      };
+
+      settings = mkOption {
+        type = types.submodule {
+          freeformType = json.type;
+
+          options = {
+
+            output.elasticsearch.hosts = mkOption {
+              type = with types; listOf str;
+              default = [ "127.0.0.1:9200" ];
+              example = [ "myEShost:9200" ];
+              description = ''
+                The list of Elasticsearch nodes to connect to.
+
+                The events are distributed to these nodes in round
+                robin order. If one node becomes unreachable, the
+                event is automatically sent to another node. Each
+                Elasticsearch node can be defined as a URL or
+                IP:PORT. For example:
+                <literal>http://192.15.3.2</literal>,
+                <literal>https://es.found.io:9230</literal> or
+                <literal>192.24.3.2:9300</literal>. If no port is
+                specified, <literal>9200</literal> is used.
+              '';
+            };
+
+            filebeat = {
+              inputs = mkOption {
+                type = types.listOf json.type;
+                default = [];
+                internal = true;
+                description = ''
+                  Inputs specify how Filebeat locates and processes
+                  input data. Use <xref
+                  linkend="opt-services.filebeat.inputs"/> instead.
+
+                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/configuration-filebeat-options.html"/>.
+                '';
+              };
+              modules = mkOption {
+                type = types.listOf json.type;
+                default = [];
+                internal = true;
+                description = ''
+                  Filebeat modules provide a quick way to get started
+                  processing common log formats. They contain default
+                  configurations, Elasticsearch ingest pipeline
+                  definitions, and Kibana dashboards to help you
+                  implement and deploy a log monitoring solution.
+
+                  Use <xref linkend="opt-services.filebeat.modules"/> instead.
+
+                  See <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-modules.html"/>.
+                '';
+              };
+            };
+          };
+        };
+        default = {};
+        example = literalExpression ''
+          {
+            settings = {
+              output.elasticsearch = {
+                hosts = [ "myEShost:9200" ];
+                username = "filebeat_internal";
+                password = { _secret = "/var/keys/elasticsearch_password"; };
+              };
+              logging.level = "info";
+            };
+          };
+        '';
+
+        description = ''
+          Configuration for filebeat. See
+          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-reference-yml.html"/>
+          for supported values.
+
+          Options containing secret data should be set to an attribute
+          set containing the attribute <literal>_secret</literal> - a
+          string pointing to a file containing the value the option
+          should be set to. See the example to get a better picture of
+          this: in the resulting
+          <filename>filebeat.yml</filename> file, the
+          <literal>output.elasticsearch.password</literal>
+          key will be set to the contents of the
+          <filename>/var/keys/elasticsearch_password</filename> file.
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    services.filebeat.settings.filebeat.inputs = attrValues cfg.inputs;
+    services.filebeat.settings.filebeat.modules = attrValues cfg.modules;
+
+    systemd.services.filebeat = {
+      description = "Filebeat log shipper";
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "elasticsearch.service" ];
+      after = [ "elasticsearch.service" ];
+      serviceConfig = {
+        ExecStartPre = pkgs.writeShellScript "filebeat-exec-pre" ''
+          set -euo pipefail
+
+          umask u=rwx,g=,o=
+
+          ${utils.genJqSecretsReplacementSnippet
+              cfg.settings
+              "/var/lib/filebeat/filebeat.yml"
+           }
+        '';
+        ExecStart = ''
+          ${cfg.package}/bin/filebeat -e \
+            -c "/var/lib/filebeat/filebeat.yml" \
+            --path.data "/var/lib/filebeat"
+        '';
+        Restart = "always";
+        StateDirectory = "filebeat";
+      };
+    };
+  };
+}
diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix
index 8db49ecfb18a..f42be00f23b8 100644
--- a/nixos/tests/elk.nix
+++ b/nixos/tests/elk.nix
@@ -56,6 +56,23 @@ let
                 '');
               };
 
+              filebeat = {
+                enable = elk ? filebeat;
+                package = elk.filebeat;
+                inputs.journald.id = "everything";
+
+                inputs.log = {
+                  enabled = true;
+                  paths = [
+                    "/var/lib/filebeat/test"
+                  ];
+                };
+
+                settings = {
+                  logging.level = "info";
+                };
+              };
+
               metricbeat = {
                 enable = true;
                 package = elk.metricbeat;
@@ -226,12 +243,27 @@ let
           one.wait_until_succeeds(
               expect_hits("Supercalifragilisticexpialidocious")
           )
+    '' + lib.optionalString (elk ? filebeat) ''
+      with subtest(
+          "A message logged to the journal is ingested by elasticsearch via filebeat"
+      ):
+          one.wait_for_unit("filebeat.service")
+          one.execute("echo 'Superdupercalifragilisticexpialidocious' | systemd-cat")
+          one.wait_until_succeeds(
+              expect_hits("Superdupercalifragilisticexpialidocious")
+          )
+          one.execute(
+              "echo 'SuperdupercalifragilisticexpialidociousIndeed' >> /var/lib/filebeat/test"
+          )
+          one.wait_until_succeeds(
+              expect_hits("SuperdupercalifragilisticexpialidociousIndeed")
+          )
     '' + ''
       with subtest("Elasticsearch-curator works"):
           one.systemctl("stop logstash")
           one.systemctl("start elasticsearch-curator")
           one.wait_until_succeeds(
-              '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep ^'
+              '! curl --silent --show-error --fail-with-body "${esUrl}/_cat/indices" | grep logstash | grep ^'
           )
     '';
   }) { inherit pkgs system; };
@@ -251,6 +283,7 @@ in {
   #   elasticsearch = pkgs.elasticsearch7-oss;
   #   logstash      = pkgs.logstash7-oss;
   #   kibana        = pkgs.kibana7-oss;
+  #   filebeat      = pkgs.filebeat7;
   #   metricbeat    = pkgs.metricbeat7;
   # };
   unfree = lib.dontRecurseIntoAttrs {
@@ -265,6 +298,7 @@ in {
       elasticsearch = pkgs.elasticsearch7;
       logstash      = pkgs.logstash7;
       kibana        = pkgs.kibana7;
+      filebeat      = pkgs.filebeat7;
       metricbeat    = pkgs.metricbeat7;
     };
   };

From b38f44c8b7e0090c91199130e96ff79b485bda2d Mon Sep 17 00:00:00 2001
From: talyz <kim.lindberger@gmail.com>
Date: Wed, 15 Dec 2021 19:57:30 +0100
Subject: [PATCH 5/8] python3Packages.parsedmarc.tests: Fix breakage

- Don't use the deprecated elasticsearch7-oss package
- Improve jq query robustness and add tracing
---
 nixos/tests/parsedmarc/default.nix | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/nixos/tests/parsedmarc/default.nix b/nixos/tests/parsedmarc/default.nix
index d838d3b6a39c..50b977723e9c 100644
--- a/nixos/tests/parsedmarc/default.nix
+++ b/nixos/tests/parsedmarc/default.nix
@@ -4,6 +4,7 @@
 { pkgs, ... }@args:
 let
   inherit (import ../../lib/testing-python.nix args) makeTest;
+  inherit (pkgs) lib;
 
   dmarcTestReport = builtins.fetchurl {
     name = "dmarc-test-report";
@@ -54,7 +55,7 @@ in
   localMail = makeTest
     {
       name = "parsedmarc-local-mail";
-      meta = with pkgs.lib.maintainers; {
+      meta = with lib.maintainers; {
         maintainers = [ talyz ];
       };
 
@@ -83,7 +84,7 @@ in
             };
           };
 
-          services.elasticsearch.package = pkgs.elasticsearch7-oss;
+          services.elasticsearch.package = pkgs.elasticsearch-oss;
 
           environment.systemPackages = [
             (sendEmail "dmarc@localhost")
@@ -94,6 +95,7 @@ in
       testScript = { nodes }:
         let
           esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
+          valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
         in ''
           parsedmarc.start()
           parsedmarc.wait_for_unit("postfix.service")
@@ -104,11 +106,15 @@ in
           )
 
           parsedmarc.fail(
-              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
           parsedmarc.succeed("send-email")
           parsedmarc.wait_until_succeeds(
-              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+              "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+              + " | tee /dev/console"
+              + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
           )
         '';
     };
@@ -121,7 +127,7 @@ in
     in
       makeTest {
         name = "parsedmarc-external-mail";
-        meta = with pkgs.lib.maintainers; {
+        meta = with lib.maintainers; {
           maintainers = [ talyz ];
         };
 
@@ -153,7 +159,7 @@ in
                 };
               };
 
-              services.elasticsearch.package = pkgs.elasticsearch7-oss;
+              services.elasticsearch.package = pkgs.elasticsearch-oss;
 
               environment.systemPackages = [
                 pkgs.jq
@@ -201,6 +207,7 @@ in
         testScript = { nodes }:
           let
             esPort = toString nodes.parsedmarc.config.services.elasticsearch.port;
+            valueObject = lib.optionalString (lib.versionAtLeast nodes.parsedmarc.config.services.elasticsearch.package.version "7") ".value";
           in ''
             mail.start()
             mail.wait_for_unit("postfix.service")
@@ -213,11 +220,15 @@ in
             )
 
             parsedmarc.fail(
-                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+                + " | tee /dev/console"
+                + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
             )
             mail.succeed("send-email")
             parsedmarc.wait_until_succeeds(
-                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940 | jq -e 'if .hits.total.value > 0 then true else null end'"
+                "curl -sS -f http://localhost:${esPort}/_search?q=report_id:2940"
+                + " | tee /dev/console"
+                + " | jq -es 'if . == [] then null else .[] | .hits.total${valueObject} > 0 end'"
             )
           '';
       };

From 0a7d7d2fcebfdecf3cbb971a7149664182308c47 Mon Sep 17 00:00:00 2001
From: talyz <kim.lindberger@gmail.com>
Date: Wed, 15 Dec 2021 20:19:40 +0100
Subject: [PATCH 6/8] rl-2205: Note the addition of the filebeat service

---
 .../manual/from_md/release-notes/rl-2205.section.xml   | 10 +++++++++-
 nixos/doc/manual/release-notes/rl-2205.section.md      |  2 ++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
index d02f951a03ca..2dd27649c52c 100644
--- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
+++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml
@@ -19,7 +19,7 @@
   </section>
   <section xml:id="sec-release-22.05-new-services">
     <title>New Services</title>
-    <itemizedlist spacing="compact">
+    <itemizedlist>
       <listitem>
         <para>
           <link xlink:href="https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw">aesmd</link>,
@@ -28,6 +28,14 @@
           <link linkend="opt-services.aesmd.enable">services.aesmd</link>.
         </para>
       </listitem>
+      <listitem>
+        <para>
+          <link xlink:href="https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html">filebeat</link>,
+          a lightweight shipper for forwarding and centralizing log
+          data. Available as
+          <link linkend="opt-services.filebeat.enable">services.filebeat</link>.
+        </para>
+      </listitem>
     </itemizedlist>
   </section>
   <section xml:id="sec-release-22.05-incompatibilities">
diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md
index 11e5462b3316..595785e732a4 100644
--- a/nixos/doc/manual/release-notes/rl-2205.section.md
+++ b/nixos/doc/manual/release-notes/rl-2205.section.md
@@ -10,6 +10,8 @@ In addition to numerous new and upgraded packages, this release has the followin
 
 - [aesmd](https://github.com/intel/linux-sgx#install-the-intelr-sgx-psw), the Intel SGX Architectural Enclave Service Manager. Available as [services.aesmd](#opt-services.aesmd.enable).
 
+- [filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-overview.html), a lightweight shipper for forwarding and centralizing log data. Available as [services.filebeat](#opt-services.filebeat.enable).
+
 ## Backward Incompatibilities {#sec-release-22.05-incompatibilities}
 
 - `pkgs.ghc` now refers to `pkgs.targetPackages.haskellPackages.ghc`.

From b617526c2b807acebbc95f59affe2d54e66fa94e Mon Sep 17 00:00:00 2001
From: talyz <kim.lindberger@gmail.com>
Date: Thu, 16 Dec 2021 14:58:23 +0100
Subject: [PATCH 7/8] elk6: 6.8.3 -> 6.8.21

The latest version includes a fix for CVE-2021-44228.
---
 pkgs/development/tools/misc/kibana/6.x.nix    |  8 +++----
 pkgs/misc/logging/beats/6.x.nix               |  2 +-
 pkgs/servers/search/elasticsearch/6.x.nix     |  4 ++--
 pkgs/servers/search/elasticsearch/plugins.nix | 24 +++++++++----------
 pkgs/tools/misc/logstash/6.x.nix              |  4 ++--
 pkgs/top-level/all-packages.nix               |  2 +-
 6 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix
index c728f7375436..cd81975ee445 100644
--- a/pkgs/development/tools/misc/kibana/6.x.nix
+++ b/pkgs/development/tools/misc/kibana/6.x.nix
@@ -18,12 +18,12 @@ let
   shas =
     if enableUnfree
     then {
-      x86_64-linux  = "1xwklhqxk5rmdrgy2simwvijzq29kyq5w2w3hy53xh2i1zlnyvq3";
-      x86_64-darwin = "1qpdn28mrpggd55khzqqld6r89l0hb870rigxcw2i8p2yx3jv106";
+      x86_64-linux  = "1a501lavxhckb3l93sbrbqyshicwkk6p89frry4x8p037xcfpy0x";
+      x86_64-darwin = "0zm45af30shhcg3mdhcma6rms1hyrx62rm5jzwnz9kxv4d30skbw";
     }
     else {
-      x86_64-linux  = "1wpnwal2rq5v2bsp5qil9j6dplif7ql5394sy4ia5ghp2fzifxmf";
-      x86_64-darwin = "12z8i0wbw10c097glbpdy350p0h3957433f51qfx2p0ghgkzkhzv";
+      x86_64-linux  = "0wfdipf21apyily7mvlqgyc7m5jpr96zgrryzwa854z3xb2vw8zg";
+      x86_64-darwin = "1nklfx4yz6hsxlljvnvwjy7pncv9mzngl84710xad5jlyras3sdj";
     };
 
 in stdenv.mkDerivation rec {
diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix
index 1808197498b5..f5e31924791b 100644
--- a/pkgs/misc/logging/beats/6.x.nix
+++ b/pkgs/misc/logging/beats/6.x.nix
@@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec {
         owner = "elastic";
         repo = "beats";
         rev = "v${version}";
-        sha256 = "0jkiz5dfdi9zsji04ipcmcj7pml9294v455y7s2c22k24gyzbaw8";
+        sha256 = "1vnw9clsc10cfpjf6vxvc6m507b2q17sgsl079iwqbp4v0286il7";
       };
 
       goPackagePath = "github.com/elastic/beats";
diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix
index 4b92592f65df..56f0779c833a 100644
--- a/pkgs/servers/search/elasticsearch/6.x.nix
+++ b/pkgs/servers/search/elasticsearch/6.x.nix
@@ -19,8 +19,8 @@ stdenv.mkDerivation (rec {
     url = "https://artifacts.elastic.co/downloads/elasticsearch/${pname}-${version}.tar.gz";
     sha256 =
       if enableUnfree
-      then "09dy3iyzk460vra6na6vk7d3mzpbv4cl0pl7kjmybxy947j7hh42"
-      else "0s04xz3j4psyhawvy503sp2nl5s0gswmpd9wfvwnavgcrr23wk39";
+      then "1hkcgqsrnnx3zjpgar4424mxfaxrx0zbrp7n7n0dlbhphshwnkmd"
+      else "1pglg60aigy31xmpfchnxcc04nd18zwc3av4m0kyp00yk5mnlyqm";
   };
 
   patches = [ ./es-home-6.x.patch ];
diff --git a/pkgs/servers/search/elasticsearch/plugins.nix b/pkgs/servers/search/elasticsearch/plugins.nix
index baeeec5c67a7..03bb24d9a390 100644
--- a/pkgs/servers/search/elasticsearch/plugins.nix
+++ b/pkgs/servers/search/elasticsearch/plugins.nix
@@ -39,7 +39,7 @@ in
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
         if version == "7.16.1" then "1sz858m9963xqr5kzjlwnq7k0a146rn60v6xijyfbp8y3brg618p"
-        else if version == "6.8.3" then "0vbaqyj0lfy3ijl1c9h92b0nh605h5mjs57bk2zhycdvbw5sx2lv"
+        else if version == "6.8.21" then "06b1pavyggzfp4wwdql0q9nm3r7i9px9cagp4yh4nhxhnk4w5fiq"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -56,7 +56,7 @@ in
       url = "https://github.com/vhyza/elasticsearch-${pluginName}/releases/download/v${version}/elasticsearch-${pluginName}-${version}-plugin.zip";
       sha256 =
         if version == "7.16.1" then "0yjy9yhw77lmalivxnmv2rq8fk93ddxszkk73lgmpffladx2ikir"
-        else if version == "6.8.3" then "12bshvp01pp2lgwd0cn9l58axg8gdimsh4g9wfllxi1bdpv4cy53"
+        else if version == "6.8.21" then "0m80cn7vkcvk95v4pdmi6vk5ww7p01k0hj2iqb9g870vs6x2qjzv"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -73,7 +73,7 @@ in
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
         if version == "7.16.1" then "1w5ndgffqzj5ijglmykifrk1jsgh7qwn8m7sbpiv0r7n3aayhz1x"
-        else if version == "6.8.3" then "0ggdhf7w50bxsffmcznrjy14b578fps0f8arg3v54qvj94v9jc37"
+        else if version == "6.8.21" then "07w8s4a5gvr9lzjzf629y8rx3kvs6zd1vl07ksw1paghp42yb354"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -90,7 +90,7 @@ in
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
         if version == "7.16.1" then "16mv7b9nl96bcygabvjqidxp2sjk340va19mrmliblpq3mxa2sii"
-        else if version == "6.8.3" then "0pmffz761dqjpvmkl7i7xsyw1iyyspqpddxp89rjsznfc9pak5im"
+        else if version == "6.8.21" then "1kdpbrasxwr3dn21zjrklp1s389rwa51fairygdwl8px9liwwfa5"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -107,7 +107,7 @@ in
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${version}.zip";
       sha256 =
         if version == "7.16.1" then "0bf8f8cybsp6s2ai3j04yay9kbhsafpgxivxjvzn2iy9qgc84ls4"
-        else if version == "6.8.3" then "0kfr4i2rcwinjn31xrc2piicasjanaqcgnbif9xc7lnak2nnzmll"
+        else if version == "6.8.21" then "0v31yyhjcdlqnjw1f9kihh7z3c6d31whc57hqqd1dn579n4s9rlz"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -124,7 +124,7 @@ in
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
         if version == "7.16.1" then "0sfa0ql3hh8jmha230dyhr51bvsvwmazyycf36ngpmxsysm8ccml"
-        else if version == "6.8.3" then "1mm6hj2m1db68n81rzsvlw6nisflr5ikzk5zv9nmk0z641n5vh1x"
+        else if version == "6.8.21" then "0sfh1az30q4f34zxig2fz8wn9gk53fmmxyg5pbi1svn9761p5awq"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -141,7 +141,7 @@ in
       url = "https://artifacts.elastic.co/downloads/elasticsearch-plugins/${pluginName}/${pluginName}-${esVersion}.zip";
       sha256 =
         if version == "7.16.1" then "1b95hjr4qhiavm7r7k19bwk5c64r00f1g5s0ydnb6gzym9hdb5s1"
-        else if version == "6.8.3" then "1s2klpvnhpkrk53p64zbga3b66czi7h1a13f58kfn2cn0zfavnbk"
+        else if version == "6.8.21" then "00lwj00rfdk6850gk1n86chiz2w6afpqn7jn588jdbwv41qh5mrv"
         else throw "unsupported version ${version} for plugin ${pluginName}";
     };
     meta = with lib; {
@@ -158,7 +158,7 @@ in
     version =
       # https://docs.search-guard.com/latest/search-guard-versions
       if esVersion == "7.16.1" then "${esVersion}-52.5.0"
-      else if esVersion == "6.8.3" then "${esVersion}-25.5"
+      else if esVersion == "6.8.21" then "${esVersion}-25.6"
       else throw "unsupported version ${esVersion} for plugin ${pluginName}";
     src =
       if esVersion == "7.16.1" then
@@ -166,15 +166,15 @@ in
           url = "https://maven.search-guard.com/search-guard-suite-release/com/floragunn/search-guard-suite-plugin/${version}/search-guard-suite-plugin-${version}.zip";
           sha256 = "1m3nj35qyrkkh3mhmn66nippavima8h8qpaxddalhjsvf70lhnjb";
         }
-      else if esVersion == "6.8.3" then
+      else if esVersion == "6.8.21" then
         fetchurl {
-          url = "mirror://maven/com/floragunn/${pluginName}-${majorVersion}/${version}/${pluginName}-${majorVersion}-${version}.zip";
-          sha256 = "0a7ys9qinc0fjyka03cx9rv0pm7wnvslk234zv5vrphkrj52s1cb";
+          url = "https://maven.search-guard.com/search-guard-release/com/floragunn/search-guard-6/${version}/search-guard-6-${version}.zip";
+          sha256 = "19nj513wigwd0mzq747zax4fzvv5vi24f7j0636rydd9iv9cyhg2";
         }
       else throw "unsupported version ${version} for plugin ${pluginName}";
     meta = with lib; {
       homepage = "https://search-guard.com";
-      description = "Elasticsearch plugin that offers encryption, authentication, and authorisation. ";
+      description = "Elasticsearch plugin that offers encryption, authentication, and authorisation.";
       license = licenses.asl20;
     };
   };
diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix
index b35e5a4aea43..0b3e17818dcd 100644
--- a/pkgs/tools/misc/logstash/6.x.nix
+++ b/pkgs/tools/misc/logstash/6.x.nix
@@ -17,8 +17,8 @@ let this = stdenv.mkDerivation rec {
     url = "https://artifacts.elastic.co/downloads/logstash/${pname}-${version}.tar.gz";
     sha256 =
       if enableUnfree
-      then "00pwi7clgdflzzg15bh3y30gzikvvy7p5fl88fww7xhhy47q8053"
-      else "0spxgqsyh72n0l0xh6rljp0lbqz46xmr02sqz25ybycr4qkxdhgk";
+      then "0hij1byw5b3xmk3vshr9p7gxwbjrywr7ylps05ydc2dmnz8q2a79"
+      else "1fa236pvhj7spys54nqi3k64rwzf6zi6gaccmqg4p4sh92jzsybv";
   };
 
   dontBuild         = true;
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index a4ff85f6f848..9c117ca4d712 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -4999,7 +4999,7 @@ with pkgs;
 
   # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
   # When updating make sure to update all plugins or they will break!
-  elk6Version = "6.8.3";
+  elk6Version = "6.8.21";
   elk7Version = "7.16.1";
 
   elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix {

From 813afad3048a31fb3ad2a54a25f1b2679eb4ba59 Mon Sep 17 00:00:00 2001
From: talyz <kim.lindberger@gmail.com>
Date: Thu, 16 Dec 2021 15:02:39 +0100
Subject: [PATCH 8/8] nixos/journalbeat: Add a loose dependency on
 elasticsearch

Avoid unnecssary back-off when elasticsearch is running on the same
host.
---
 nixos/modules/services/logging/journalbeat.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/nixos/modules/services/logging/journalbeat.nix b/nixos/modules/services/logging/journalbeat.nix
index b34fb308f586..48b02142cb4c 100644
--- a/nixos/modules/services/logging/journalbeat.nix
+++ b/nixos/modules/services/logging/journalbeat.nix
@@ -88,6 +88,8 @@ in
     systemd.services.journalbeat = {
       description = "Journalbeat log shipper";
       wantedBy = [ "multi-user.target" ];
+      wants = [ "elasticsearch.service" ];
+      after = [ "elasticsearch.service" ];
       preStart = ''
         mkdir -p ${cfg.stateDir}/data
         mkdir -p ${cfg.stateDir}/logs