diff --git a/nixos/modules/services/search/elasticsearch.nix b/nixos/modules/services/search/elasticsearch.nix index c51dd5d94655..adef500b7b5c 100644 --- a/nixos/modules/services/search/elasticsearch.nix +++ b/nixos/modules/services/search/elasticsearch.nix @@ -6,6 +6,7 @@ let cfg = config.services.elasticsearch; es5 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "5" >= 0; + es6 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "6" >= 0; esConfig = '' network.host: ${cfg.listenAddress} @@ -92,8 +93,6 @@ in { node.name: "elasticsearch" node.master: true node.data: false - index.number_of_shards: 5 - index.number_of_replicas: 1 ''; }; @@ -165,7 +164,10 @@ in { path = [ pkgs.inetutils ]; environment = { ES_HOME = cfg.dataDir; - ES_JAVA_OPTS = toString ([ "-Des.path.conf=${configDir}" ] ++ cfg.extraJavaOptions); + ES_JAVA_OPTS = toString ( optional (!es6) [ "-Des.path.conf=${configDir}" ] + ++ cfg.extraJavaOptions); + } // optionalAttrs es6 { + ES_PATH_CONF = configDir; }; serviceConfig = { ExecStart = "${cfg.package}/bin/elasticsearch ${toString cfg.extraCmdLineOptions}"; diff --git a/nixos/release.nix b/nixos/release.nix index cf3fe6abd48c..33916243e97b 100644 --- a/nixos/release.nix +++ b/nixos/release.nix @@ -260,7 +260,7 @@ in rec { tests.etcd = hydraJob (import tests/etcd.nix { system = "x86_64-linux"; }); tests.ec2-nixops = hydraJob (import tests/ec2.nix { system = "x86_64-linux"; }).boot-ec2-nixops; tests.ec2-config = hydraJob (import tests/ec2.nix { system = "x86_64-linux"; }).boot-ec2-config; - tests.elk = hydraJob (import tests/elk.nix { system = "x86_64-linux"; }); + tests.elk = callSubTests tests/elk.nix { system = "x86_64-linux"; }; tests.env = callTest tests/env.nix {}; tests.ferm = callTest tests/ferm.nix {}; tests.firefox = callTest tests/firefox.nix {}; diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index 65ff1cac070b..ed656b3628b9 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -1,95 +1,107 @@ -# Test the ELK stack: Elasticsearch, Logstash and Kibana. - -import ./make-test.nix ({ pkgs, ...} : +{ system ? builtins.currentSystem }: +with import ../lib/testing.nix { inherit system; }; +with pkgs.lib; let esUrl = "http://localhost:9200"; -in { - name = "ELK"; - meta = with pkgs.stdenv.lib.maintainers; { - maintainers = [ eelco chaoflow offline basvandijk ]; - }; - nodes = { - one = - { config, pkgs, ... }: { - # Not giving the machine at least 2060MB results in elasticsearch failing with the following error: - # - # OpenJDK 64-Bit Server VM warning: - # INFO: os::commit_memory(0x0000000085330000, 2060255232, 0) - # failed; error='Cannot allocate memory' (errno=12) - # - # There is insufficient memory for the Java Runtime Environment to continue. - # Native memory allocation (mmap) failed to map 2060255232 bytes for committing reserved memory. - # - # When setting this to 2500 I got "Kernel panic - not syncing: Out of - # memory: compulsory panic_on_oom is enabled" so lets give it even a - # bit more room: - virtualisation.memorySize = 3000; + mkElkTest = name : elk : makeTest { + inherit name; + meta = with pkgs.stdenv.lib.maintainers; { + maintainers = [ eelco chaoflow offline basvandijk ]; + }; + nodes = { + one = + { config, pkgs, ... }: { + # Not giving the machine at least 2060MB results in elasticsearch failing with the following error: + # + # OpenJDK 64-Bit Server VM warning: + # INFO: os::commit_memory(0x0000000085330000, 2060255232, 0) + # failed; error='Cannot allocate memory' (errno=12) + # + # There is insufficient memory for the Java Runtime Environment to continue. + # Native memory allocation (mmap) failed to map 2060255232 bytes for committing reserved memory. + # + # When setting this to 2500 I got "Kernel panic - not syncing: Out of + # memory: compulsory panic_on_oom is enabled" so lets give it even a + # bit more room: + virtualisation.memorySize = 3000; - # For querying JSON objects returned from elasticsearch and kibana. - environment.systemPackages = [ pkgs.jq ]; + # For querying JSON objects returned from elasticsearch and kibana. + environment.systemPackages = [ pkgs.jq ]; - services = { - logstash = { - enable = true; - package = pkgs.logstash5; - inputConfig = '' - exec { command => "echo -n flowers" interval => 1 type => "test" } - exec { command => "echo -n dragons" interval => 1 type => "test" } - ''; - filterConfig = '' - if [message] =~ /dragons/ { - drop {} - } - ''; - outputConfig = '' - file { - path => "/tmp/logstash.out" - codec => line { format => "%{message}" } - } - elasticsearch { - hosts => [ "${esUrl}" ] - } - ''; - }; + services = { + logstash = { + enable = true; + package = elk.logstash; + inputConfig = '' + exec { command => "echo -n flowers" interval => 1 type => "test" } + exec { command => "echo -n dragons" interval => 1 type => "test" } + ''; + filterConfig = '' + if [message] =~ /dragons/ { + drop {} + } + ''; + outputConfig = '' + file { + path => "/tmp/logstash.out" + codec => line { format => "%{message}" } + } + elasticsearch { + hosts => [ "${esUrl}" ] + } + ''; + }; - elasticsearch = { - enable = true; - package = pkgs.elasticsearch5; - }; + elasticsearch = { + enable = true; + package = elk.elasticsearch; + }; - kibana = { - enable = true; - package = pkgs.kibana5; - elasticsearch.url = esUrl; + kibana = { + enable = true; + package = elk.kibana; + elasticsearch.url = esUrl; + }; }; }; - }; - }; + }; - testScript = '' - startAll; + testScript = '' + startAll; - $one->waitForUnit("elasticsearch.service"); + $one->waitForUnit("elasticsearch.service"); - # Continue as long as the status is not "red". The status is probably - # "yellow" instead of "green" because we are using a single elasticsearch - # node which elasticsearch considers risky. - # - # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green". - $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"); + # Continue as long as the status is not "red". The status is probably + # "yellow" instead of "green" because we are using a single elasticsearch + # node which elasticsearch considers risky. + # + # TODO: extend this test with multiple elasticsearch nodes and see if the status turns "green". + $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red"); - # Perform some simple logstash tests. - $one->waitForUnit("logstash.service"); - $one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers"); - $one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons"); + # Perform some simple logstash tests. + $one->waitForUnit("logstash.service"); + $one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers"); + $one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons"); - # See if kibana is healthy. - $one->waitForUnit("kibana.service"); - $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"); + # See if kibana is healthy. + $one->waitForUnit("kibana.service"); + $one->waitUntilSucceeds("curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green"); - # See if logstash messages arive in elasticsearch. - $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0"); - $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0"); - ''; -}) + # See if logstash messages arive in elasticsearch. + $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0"); + $one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0"); + ''; + }; +in mapAttrs mkElkTest { + "ELK-5" = { + elasticsearch = pkgs.elasticsearch5; + logstash = pkgs.logstash5; + kibana = pkgs.kibana5; + }; + "ELK-6" = { + elasticsearch = pkgs.elasticsearch6; + logstash = pkgs.logstash6; + kibana = pkgs.kibana6; + }; +} diff --git a/pkgs/development/tools/misc/kibana/5.x.nix b/pkgs/development/tools/misc/kibana/5.x.nix index 735f85f74569..428f24cd4e92 100644 --- a/pkgs/development/tools/misc/kibana/5.x.nix +++ b/pkgs/development/tools/misc/kibana/5.x.nix @@ -11,9 +11,9 @@ let elasticArch = archOverrides."${arch}" or arch; plat = elemAt info 1; shas = { - "x86_64-linux" = "02dhhp16pmkrpi2dfrca9qzz1q7jrxhaw6l3cfflgxx77hz0hlnw"; - "i686-linux" = "1h1zr342dq7nngvzpf9pn9mvwsi7aksa3qjyqpcc4yvbmmyrlk0m"; - "x86_64-darwin" = "0van8cnir6s520crc20bf2clbkf822c3ylpk7iiq7da8hwvsypp9"; + "x86_64-linux" = "09bck05dfq4j1csyghlpw86nzn28kpx8ikli3v1s4si2hbxb1ifr"; + "i686-linux" = "0ql1611wg7i9vwqr4wmz04606hjj7w224ak34svfsn6qxyrh2dbb"; + "x86_64-darwin" = "1x24rqkkc9slm7jbyy41q5c2rbn17h85m0k6h3ijiafky6cv0cz2"; }; in stdenv.mkDerivation rec { name = "kibana-${version}"; diff --git a/pkgs/development/tools/misc/kibana/6.x.nix b/pkgs/development/tools/misc/kibana/6.x.nix new file mode 100644 index 000000000000..bfa1d92eb52a --- /dev/null +++ b/pkgs/development/tools/misc/kibana/6.x.nix @@ -0,0 +1,40 @@ +{ stdenv, makeWrapper, fetchurl, elk6Version, nodejs, coreutils, which }: + +with stdenv.lib; +let + inherit (builtins) elemAt; + info = splitString "-" stdenv.system; + arch = elemAt info 0; + plat = elemAt info 1; + shas = { + "x86_64-linux" = "0847flk4sfimcdx9wqkaglk7bvbnz1iyindz10z0d1fvbldivp46"; + "x86_64-darwin" = "03f7l91r6nczzzlqxsxkpzzwafpy45fx4lss4g6kg022rwisdma7"; + }; +in stdenv.mkDerivation rec { + name = "kibana-${version}"; + version = elk6Version; + + src = fetchurl { + url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz"; + sha256 = shas."${stdenv.system}" or (throw "Unknown architecture"); + }; + + buildInputs = [ makeWrapper ]; + + installPhase = '' + mkdir -p $out/libexec/kibana $out/bin + mv * $out/libexec/kibana/ + rm -r $out/libexec/kibana/node + makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \ + --prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}" + sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana + ''; + + meta = { + description = "Visualize logs and time-stamped data"; + homepage = http://www.elasticsearch.org/overview/kibana; + license = licenses.asl20; + maintainers = with maintainers; [ offline rickynils basvandijk ]; + platforms = with platforms; unix; + }; +} diff --git a/pkgs/misc/logging/beats/default.nix b/pkgs/misc/logging/beats/5.x.nix similarity index 95% rename from pkgs/misc/logging/beats/default.nix rename to pkgs/misc/logging/beats/5.x.nix index e3333fa5caac..ed2a2eadb65a 100644 --- a/pkgs/misc/logging/beats/default.nix +++ b/pkgs/misc/logging/beats/5.x.nix @@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec { owner = "elastic"; repo = "beats"; rev = "v${version}"; - sha256 = "1lbdi4c0y4bfkmim9q98ravknv4yw0dl3z57c3w5aqhi2sx0w23h"; + sha256 = "0pp4in66byggcfmvf8yx0m1vra98cs77m7mbr45sdla4hinvaqar"; }; goPackagePath = "github.com/elastic/beats"; diff --git a/pkgs/misc/logging/beats/6.x.nix b/pkgs/misc/logging/beats/6.x.nix new file mode 100644 index 000000000000..48c2fb6973fe --- /dev/null +++ b/pkgs/misc/logging/beats/6.x.nix @@ -0,0 +1,42 @@ +{ stdenv, fetchFromGitHub, elk6Version, buildGoPackage, libpcap }: + +let beat = package : extraArgs : buildGoPackage (rec { + name = "${package}-${version}"; + version = elk6Version; + + src = fetchFromGitHub { + owner = "elastic"; + repo = "beats"; + rev = "v${version}"; + sha256 = "1vifxa0v6ha29ijvgnrkx02syckhydg6vjxjqbm8y8zysvnh1869"; + }; + + goPackagePath = "github.com/elastic/beats"; + + subPackages = [ package ]; + + meta = with stdenv.lib; { + homepage = https://www.elastic.co/products/beats; + license = licenses.asl20; + maintainers = with maintainers; [ fadenb basvandijk ]; + platforms = platforms.linux; + }; + } // extraArgs); +in { + filebeat = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";}; + heartbeat = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";}; + metricbeat = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";}; + packetbeat = beat "packetbeat" { + buildInputs = [ libpcap ]; + meta.description = "Network packet analyzer that ships data to Elasticsearch"; + meta.longDescription = '' + Packetbeat is an open source network packet analyzer that ships the + data to Elasticsearch. + + Think of it like a distributed real-time Wireshark with a lot more + analytics features. The Packetbeat shippers sniff the traffic between + your application processes, parse on the fly protocols like HTTP, MySQL, + PostgreSQL, Redis or Thrift and correlate the messages into transactions. + ''; + }; +} diff --git a/pkgs/servers/search/elasticsearch/5.x.nix b/pkgs/servers/search/elasticsearch/5.x.nix index fb988b546308..e6bbc787251d 100644 --- a/pkgs/servers/search/elasticsearch/5.x.nix +++ b/pkgs/servers/search/elasticsearch/5.x.nix @@ -8,7 +8,7 @@ stdenv.mkDerivation rec { src = fetchurl { url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz"; - sha256 = "0pvi6akicg0i3bz3lbc6k9rznxw7d25flg9wbs2dyxv8i2rrqvq0"; + sha256 = "1cks75227mxyri2r0hykc7jlvk6c4f4vaxh14mgmfmw4krwvrzxs"; }; patches = [ ./es-home-5.x.patch ./es-classpath-5.x.patch ]; diff --git a/pkgs/servers/search/elasticsearch/6.x.nix b/pkgs/servers/search/elasticsearch/6.x.nix new file mode 100644 index 000000000000..23d60ca0bef3 --- /dev/null +++ b/pkgs/servers/search/elasticsearch/6.x.nix @@ -0,0 +1,45 @@ +{ stdenv, fetchurl, elk6Version, makeWrapper, jre_headless, utillinux, getopt }: + +with stdenv.lib; + +stdenv.mkDerivation rec { + version = elk6Version; + name = "elasticsearch-${version}"; + + src = fetchurl { + url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz"; + sha256 = "1dkl7crha5g8h9c1zs1ahcmv221cpipzkvk574g99gdi586ckb8c"; + }; + + patches = [ ./es-home-6.x.patch ]; + + postPatch = '' + sed -i "s|ES_CLASSPATH=\"\$ES_HOME/lib/\*\"|ES_CLASSPATH=\"$out/lib/*\"|" ./bin/elasticsearch-env + ''; + + buildInputs = [ makeWrapper jre_headless ] ++ + (if (!stdenv.isDarwin) then [utillinux] else [getopt]); + + installPhase = '' + mkdir -p $out + cp -R bin config lib modules plugins $out + + chmod -x $out/bin/*.* + + wrapProgram $out/bin/elasticsearch \ + ${if (!stdenv.isDarwin) + then ''--prefix PATH : "${utillinux}/bin/"'' + else ''--prefix PATH : "${getopt}/bin"''} \ + --set JAVA_HOME "${jre_headless}" \ + --set ES_JVM_OPTIONS "$out/config/jvm.options" + + wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}" + ''; + + meta = { + description = "Open Source, Distributed, RESTful Search Engine"; + license = licenses.asl20; + platforms = platforms.unix; + maintainers = with maintainers; [ apeschar basvandijk ]; + }; +} diff --git a/pkgs/servers/search/elasticsearch/es-home-6.x.patch b/pkgs/servers/search/elasticsearch/es-home-6.x.patch new file mode 100644 index 000000000000..aba8d396a69c --- /dev/null +++ b/pkgs/servers/search/elasticsearch/es-home-6.x.patch @@ -0,0 +1,26 @@ +diff -Naur a/bin/elasticsearch-env b/bin/elasticsearch-env +--- a/bin/elasticsearch-env 2017-12-12 13:31:51.000000000 +0100 ++++ b/bin/elasticsearch-env 2017-12-18 19:51:12.282809695 +0100 +@@ -19,18 +19,10 @@ + fi + done + +-# determine Elasticsearch home; to do this, we strip from the path until we find +-# bin, and then strip bin (there is an assumption here that there is no nested +-# directory under bin also named bin) +-ES_HOME=`dirname "$SCRIPT"` +- +-# now make ES_HOME absolute +-ES_HOME=`cd "$ES_HOME"; pwd` +- +-while [ "`basename "$ES_HOME"`" != "bin" ]; do +- ES_HOME=`dirname "$ES_HOME"` +-done +-ES_HOME=`dirname "$ES_HOME"` ++if [ -z "$ES_HOME" ]; then ++ echo "You must set the ES_HOME var" >&2 ++ exit 1 ++fi + + # now set the classpath + ES_CLASSPATH="$ES_HOME/lib/*" diff --git a/pkgs/tools/misc/logstash/5.x.nix b/pkgs/tools/misc/logstash/5.x.nix index 4ecc0be783eb..da165446d397 100644 --- a/pkgs/tools/misc/logstash/5.x.nix +++ b/pkgs/tools/misc/logstash/5.x.nix @@ -6,7 +6,7 @@ stdenv.mkDerivation rec { src = fetchurl { url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz"; - sha256 = "11qg8i0svsccr1wd0yj0ivfzpza2hd68221g38v88shvj0bb737f"; + sha256 = "18k2bhyzpxc2pad64wz0rpy43xp0nv843igjflav53jsglifh1yk"; }; dontBuild = true; diff --git a/pkgs/tools/misc/logstash/6.x.nix b/pkgs/tools/misc/logstash/6.x.nix new file mode 100644 index 000000000000..b59fb9012efa --- /dev/null +++ b/pkgs/tools/misc/logstash/6.x.nix @@ -0,0 +1,39 @@ +{ stdenv, fetchurl, elk6Version, makeWrapper, jre }: + +stdenv.mkDerivation rec { + version = elk6Version; + name = "logstash-${version}"; + + src = fetchurl { + url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz"; + sha256 = "07apb0135rlbraqw3pmwf13jjhzgflr6qik0b0qxp8im0hwx082p"; + }; + + dontBuild = true; + dontPatchELF = true; + dontStrip = true; + dontPatchShebangs = true; + + buildInputs = [ + makeWrapper jre + ]; + + installPhase = '' + mkdir -p $out + cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out + + wrapProgram $out/bin/logstash \ + --set JAVA_HOME "${jre}" + + wrapProgram $out/bin/logstash-plugin \ + --set JAVA_HOME "${jre}" + ''; + + meta = with stdenv.lib; { + description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems"; + homepage = https://www.elastic.co/products/logstash; + license = licenses.asl20; + platforms = platforms.unix; + maintainers = with maintainers; [ wjlroe offline basvandijk ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c2d5bb5d22e1..0b90f81a1972 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -746,12 +746,23 @@ with pkgs; bchunk = callPackage ../tools/cd-dvd/bchunk { }; - inherit (callPackages ../misc/logging/beats { }) + inherit (callPackages ../misc/logging/beats/5.x.nix { }) filebeat heartbeat metricbeat packetbeat; + inherit (let beats6 = callPackages ../misc/logging/beats/6.x.nix { }; in { + filebeat6 = beats6.filebeat; + heartbeat6 = beats6.heartbeat; + metricbeat6 = beats6.metricbeat; + packetbeat6 = beats6.packetbeat; + }) + filebeat6 + heartbeat6 + metricbeat6 + packetbeat6; + bfr = callPackage ../tools/misc/bfr { }; bibtool = callPackage ../tools/misc/bibtool { }; @@ -1963,11 +1974,13 @@ with pkgs; evemu = callPackage ../tools/system/evemu { }; # The latest version used by elasticsearch, logstash, kibana and the the beats from elastic. - elk5Version = "5.6.1"; + elk5Version = "5.6.5"; + elk6Version = "6.1.1"; elasticsearch = callPackage ../servers/search/elasticsearch { }; elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { }; elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { }; + elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { }; elasticsearchPlugins = recurseIntoAttrs ( callPackage ../servers/search/elasticsearch/plugins.nix { } @@ -2975,6 +2988,7 @@ with pkgs; kibana = callPackage ../development/tools/misc/kibana { }; kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { }; + kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { }; kismet = callPackage ../applications/networking/sniffers/kismet { }; @@ -3041,6 +3055,7 @@ with pkgs; logstash = callPackage ../tools/misc/logstash { }; logstash5 = callPackage ../tools/misc/logstash/5.x.nix { }; + logstash6 = callPackage ../tools/misc/logstash/6.x.nix { }; logstash-contrib = callPackage ../tools/misc/logstash/contrib.nix { };