ilyakooo0/nixpkgs
1
0
Fork 0
mirror of https://github.com/ilyakooo0/nixpkgs.git synced 2024-11-19 02:44:17 +03:00
Code Issues Projects Releases Wiki Activity

Merge pull request #32822 from LumiGuide/elk6

Browse Source 
ELK: 5.6.1 -> 5.6.5 & add ELK 6.1.0
This commit is contained in:
Sarah Brofeldt 2018-01-14 10:40:50 +01:00 committed by GitHub
commit ee4e6ebbfa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 313 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
nixos/modules/services/search/elasticsearch.nix
View File

@ -6,6 +6,7 @@ let
cfg = config.services.elasticsearch;
es5 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "5" >= 0;
es6 = builtins.compareVersions (builtins.parseDrvName cfg.package.name).version "6" >= 0;
esConfig = ''
network.host: ${cfg.listenAddress}
@ -92,8 +93,6 @@ in {
node.name: "elasticsearch"
node.master: true
node.data: false
index.number_of_shards: 5
index.number_of_replicas: 1
'';
};
@ -165,7 +164,10 @@ in {
path = [ pkgs.inetutils ];
environment = {
ES_HOME = cfg.dataDir;
ES_JAVA_OPTS = toString ([ "-Des.path.conf=${configDir}" ] ++ cfg.extraJavaOptions);
ES_JAVA_OPTS = toString ( optional (!es6) [ "-Des.path.conf=${configDir}" ]
++ cfg.extraJavaOptions);
} // optionalAttrs es6 {
ES_PATH_CONF = configDir;
};
serviceConfig = {
ExecStart = "${cfg.package}/bin/elasticsearch ${toString cfg.extraCmdLineOptions}";

2
nixos/release.nix
View File

@ -260,7 +260,7 @@ in rec {
tests.etcd = hydraJob (import tests/etcd.nix { system = "x86_64-linux"; });
tests.ec2-nixops = hydraJob (import tests/ec2.nix { system = "x86_64-linux"; }).boot-ec2-nixops;
tests.ec2-config = hydraJob (import tests/ec2.nix { system = "x86_64-linux"; }).boot-ec2-config;
tests.elk = hydraJob (import tests/elk.nix { system = "x86_64-linux"; });
tests.elk = callSubTests tests/elk.nix { system = "x86_64-linux"; };
tests.env = callTest tests/env.nix {};
tests.ferm = callTest tests/ferm.nix {};
tests.firefox = callTest tests/firefox.nix {};

32
nixos/tests/elk.nix
View File

@ -1,14 +1,14 @@
# Test the ELK stack: Elasticsearch, Logstash and Kibana.
import ./make-test.nix ({ pkgs, ...} :
{ system ? builtins.currentSystem }:
with import ../lib/testing.nix { inherit system; };
with pkgs.lib;
let
esUrl = "http://localhost:9200";
in {
name = "ELK";
mkElkTest = name : elk : makeTest {
inherit name;
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ eelco chaoflow offline basvandijk ];
};
nodes = {
one =
{ config, pkgs, ... }: {
@ -32,7 +32,7 @@ in {
services = {
logstash = {
enable = true;
package = pkgs.logstash5;
package = elk.logstash;
inputConfig = ''
exec { command => "echo -n flowers" interval => 1 type => "test" }
exec { command => "echo -n dragons" interval => 1 type => "test" }
@ -55,12 +55,12 @@ in {
elasticsearch = {
enable = true;
package = pkgs.elasticsearch5;
package = elk.elasticsearch;
};
kibana = {
enable = true;
package = pkgs.kibana5;
package = elk.kibana;
elasticsearch.url = esUrl;
};
};
@ -92,4 +92,16 @@ in {
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"flowers\"}}}' | jq .hits.total | grep -v 0");
$one->waitUntilSucceeds("curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' -d '{\"query\" : { \"match\" : { \"message\" : \"dragons\"}}}' | jq .hits.total | grep 0");
'';
})
};
in mapAttrs mkElkTest {
"ELK-5" = {
elasticsearch = pkgs.elasticsearch5;
logstash = pkgs.logstash5;
kibana = pkgs.kibana5;
};
"ELK-6" = {
elasticsearch = pkgs.elasticsearch6;
logstash = pkgs.logstash6;
kibana = pkgs.kibana6;
};
}

6
pkgs/development/tools/misc/kibana/5.x.nix
View File

@ -11,9 +11,9 @@ let
elasticArch = archOverrides."${arch}" or arch;
plat = elemAt info 1;
shas = {
"x86_64-linux" = "02dhhp16pmkrpi2dfrca9qzz1q7jrxhaw6l3cfflgxx77hz0hlnw";
"i686-linux" = "1h1zr342dq7nngvzpf9pn9mvwsi7aksa3qjyqpcc4yvbmmyrlk0m";
"x86_64-darwin" = "0van8cnir6s520crc20bf2clbkf822c3ylpk7iiq7da8hwvsypp9";
"x86_64-linux" = "09bck05dfq4j1csyghlpw86nzn28kpx8ikli3v1s4si2hbxb1ifr";
"i686-linux" = "0ql1611wg7i9vwqr4wmz04606hjj7w224ak34svfsn6qxyrh2dbb";
"x86_64-darwin" = "1x24rqkkc9slm7jbyy41q5c2rbn17h85m0k6h3ijiafky6cv0cz2";
};
in stdenv.mkDerivation rec {
name = "kibana-${version}";

40
pkgs/development/tools/misc/kibana/6.x.nix Normal file
View File

@ -0,0 +1,40 @@
{ stdenv, makeWrapper, fetchurl, elk6Version, nodejs, coreutils, which }:
with stdenv.lib;
let
inherit (builtins) elemAt;
info = splitString "-" stdenv.system;
arch = elemAt info 0;
plat = elemAt info 1;
shas = {
"x86_64-linux" = "0847flk4sfimcdx9wqkaglk7bvbnz1iyindz10z0d1fvbldivp46";
"x86_64-darwin" = "03f7l91r6nczzzlqxsxkpzzwafpy45fx4lss4g6kg022rwisdma7";
};
in stdenv.mkDerivation rec {
name = "kibana-${version}";
version = elk6Version;
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/kibana/${name}-${plat}-${arch}.tar.gz";
sha256 = shas."${stdenv.system}" or (throw "Unknown architecture");
};
buildInputs = [ makeWrapper ];
installPhase = ''
mkdir -p $out/libexec/kibana $out/bin
mv * $out/libexec/kibana/
rm -r $out/libexec/kibana/node
makeWrapper $out/libexec/kibana/bin/kibana $out/bin/kibana \
--prefix PATH : "${stdenv.lib.makeBinPath [ nodejs coreutils which ]}"
sed -i 's@NODE=.*@NODE=${nodejs}/bin/node@' $out/libexec/kibana/bin/kibana
'';
meta = {
description = "Visualize logs and time-stamped data";
homepage = http://www.elasticsearch.org/overview/kibana;
license = licenses.asl20;
maintainers = with maintainers; [ offline rickynils basvandijk ];
platforms = with platforms; unix;
};
}

2
pkgs/misc/logging/beats/default.nix → pkgs/misc/logging/beats/5.x.nix
View File

@ -8,7 +8,7 @@ let beat = package : extraArgs : buildGoPackage (rec {
owner = "elastic";
repo = "beats";
rev = "v${version}";
sha256 = "1lbdi4c0y4bfkmim9q98ravknv4yw0dl3z57c3w5aqhi2sx0w23h";
sha256 = "0pp4in66byggcfmvf8yx0m1vra98cs77m7mbr45sdla4hinvaqar";
};
goPackagePath = "github.com/elastic/beats";

42
pkgs/misc/logging/beats/6.x.nix Normal file
View File

@ -0,0 +1,42 @@
{ stdenv, fetchFromGitHub, elk6Version, buildGoPackage, libpcap }:
let beat = package : extraArgs : buildGoPackage (rec {
name = "${package}-${version}";
version = elk6Version;
src = fetchFromGitHub {
owner = "elastic";
repo = "beats";
rev = "v${version}";
sha256 = "1vifxa0v6ha29ijvgnrkx02syckhydg6vjxjqbm8y8zysvnh1869";
};
goPackagePath = "github.com/elastic/beats";
subPackages = [ package ];
meta = with stdenv.lib; {
homepage = https://www.elastic.co/products/beats;
license = licenses.asl20;
maintainers = with maintainers; [ fadenb basvandijk ];
platforms = platforms.linux;
};
} // extraArgs);
in {
filebeat = beat "filebeat" {meta.description = "Lightweight shipper for logfiles";};
heartbeat = beat "heartbeat" {meta.description = "Lightweight shipper for uptime monitoring";};
metricbeat = beat "metricbeat" {meta.description = "Lightweight shipper for metrics";};
packetbeat = beat "packetbeat" {
buildInputs = [ libpcap ];
meta.description = "Network packet analyzer that ships data to Elasticsearch";
meta.longDescription = ''
Packetbeat is an open source network packet analyzer that ships the
data to Elasticsearch.
Think of it like a distributed real-time Wireshark with a lot more
analytics features. The Packetbeat shippers sniff the traffic between
your application processes, parse on the fly protocols like HTTP, MySQL,
PostgreSQL, Redis or Thrift and correlate the messages into transactions.
'';
};
}

2
pkgs/servers/search/elasticsearch/5.x.nix
View File

@ -8,7 +8,7 @@ stdenv.mkDerivation rec {
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz";
sha256 = "0pvi6akicg0i3bz3lbc6k9rznxw7d25flg9wbs2dyxv8i2rrqvq0";
sha256 = "1cks75227mxyri2r0hykc7jlvk6c4f4vaxh14mgmfmw4krwvrzxs";
};
patches = [ ./es-home-5.x.patch ./es-classpath-5.x.patch ];

45
pkgs/servers/search/elasticsearch/6.x.nix Normal file
View File

@ -0,0 +1,45 @@
{ stdenv, fetchurl, elk6Version, makeWrapper, jre_headless, utillinux, getopt }:
with stdenv.lib;
stdenv.mkDerivation rec {
version = elk6Version;
name = "elasticsearch-${version}";
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/elasticsearch/${name}.tar.gz";
sha256 = "1dkl7crha5g8h9c1zs1ahcmv221cpipzkvk574g99gdi586ckb8c";
};
patches = [ ./es-home-6.x.patch ];
postPatch = ''
sed -i "s|ES_CLASSPATH=\"\$ES_HOME/lib/\*\"|ES_CLASSPATH=\"$out/lib/*\"|" ./bin/elasticsearch-env
'';
buildInputs = [ makeWrapper jre_headless ] ++
(if (!stdenv.isDarwin) then [utillinux] else [getopt]);
installPhase = ''
mkdir -p $out
cp -R bin config lib modules plugins $out
chmod -x $out/bin/*.*
wrapProgram $out/bin/elasticsearch \
${if (!stdenv.isDarwin)
then ''--prefix PATH : "${utillinux}/bin/"''
else ''--prefix PATH : "${getopt}/bin"''} \
--set JAVA_HOME "${jre_headless}" \
--set ES_JVM_OPTIONS "$out/config/jvm.options"
wrapProgram $out/bin/elasticsearch-plugin --set JAVA_HOME "${jre_headless}"
'';
meta = {
description = "Open Source, Distributed, RESTful Search Engine";
license = licenses.asl20;
platforms = platforms.unix;
maintainers = with maintainers; [ apeschar basvandijk ];
};
}

26
pkgs/servers/search/elasticsearch/es-home-6.x.patch Normal file
View File

@ -0,0 +1,26 @@
diff -Naur a/bin/elasticsearch-env b/bin/elasticsearch-env
--- a/bin/elasticsearch-env 2017-12-12 13:31:51.000000000 +0100
+++ b/bin/elasticsearch-env 2017-12-18 19:51:12.282809695 +0100
@@ -19,18 +19,10 @@
fi
done
-# determine Elasticsearch home; to do this, we strip from the path until we find
-# bin, and then strip bin (there is an assumption here that there is no nested
-# directory under bin also named bin)
-ES_HOME=`dirname "$SCRIPT"`
-
-# now make ES_HOME absolute
-ES_HOME=`cd "$ES_HOME"; pwd`
-
-while [ "`basename "$ES_HOME"`" != "bin" ]; do
- ES_HOME=`dirname "$ES_HOME"`
-done
-ES_HOME=`dirname "$ES_HOME"`
+if [ -z "$ES_HOME" ]; then
+ echo "You must set the ES_HOME var" >&2
+ exit 1
+fi
# now set the classpath
ES_CLASSPATH="$ES_HOME/lib/*"

2
pkgs/tools/misc/logstash/5.x.nix
View File

@ -6,7 +6,7 @@ stdenv.mkDerivation rec {
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
sha256 = "11qg8i0svsccr1wd0yj0ivfzpza2hd68221g38v88shvj0bb737f";
sha256 = "18k2bhyzpxc2pad64wz0rpy43xp0nv843igjflav53jsglifh1yk";
};
dontBuild = true;

39
pkgs/tools/misc/logstash/6.x.nix Normal file
View File

@ -0,0 +1,39 @@
{ stdenv, fetchurl, elk6Version, makeWrapper, jre }:
stdenv.mkDerivation rec {
version = elk6Version;
name = "logstash-${version}";
src = fetchurl {
url = "https://artifacts.elastic.co/downloads/logstash/${name}.tar.gz";
sha256 = "07apb0135rlbraqw3pmwf13jjhzgflr6qik0b0qxp8im0hwx082p";
};
dontBuild = true;
dontPatchELF = true;
dontStrip = true;
dontPatchShebangs = true;
buildInputs = [
makeWrapper jre
];
installPhase = ''
mkdir -p $out
cp -r {Gemfile*,modules,vendor,lib,bin,config,data,logstash-core,logstash-core-plugin-api} $out
wrapProgram $out/bin/logstash \
--set JAVA_HOME "${jre}"
wrapProgram $out/bin/logstash-plugin \
--set JAVA_HOME "${jre}"
'';
meta = with stdenv.lib; {
description = "Logstash is a data pipeline that helps you process logs and other event data from a variety of systems";
homepage = https://www.elastic.co/products/logstash;
license = licenses.asl20;
platforms = platforms.unix;
maintainers = with maintainers; [ wjlroe offline basvandijk ];
};
}

19
pkgs/top-level/all-packages.nix
View File

@ -746,12 +746,23 @@ with pkgs;
bchunk = callPackage ../tools/cd-dvd/bchunk { };
inherit (callPackages ../misc/logging/beats { })
inherit (callPackages ../misc/logging/beats/5.x.nix { })
filebeat
heartbeat
metricbeat
packetbeat;
inherit (let beats6 = callPackages ../misc/logging/beats/6.x.nix { }; in {
filebeat6 = beats6.filebeat;
heartbeat6 = beats6.heartbeat;
metricbeat6 = beats6.metricbeat;
packetbeat6 = beats6.packetbeat;
})
filebeat6
heartbeat6
metricbeat6
packetbeat6;
bfr = callPackage ../tools/misc/bfr { };
bibtool = callPackage ../tools/misc/bibtool { };
@ -1963,11 +1974,13 @@ with pkgs;
evemu = callPackage ../tools/system/evemu { };
# The latest version used by elasticsearch, logstash, kibana and the the beats from elastic.
elk5Version = "5.6.1";
elk5Version = "5.6.5";
elk6Version = "6.1.1";
elasticsearch = callPackage ../servers/search/elasticsearch { };
elasticsearch2 = callPackage ../servers/search/elasticsearch/2.x.nix { };
elasticsearch5 = callPackage ../servers/search/elasticsearch/5.x.nix { };
elasticsearch6 = callPackage ../servers/search/elasticsearch/6.x.nix { };
elasticsearchPlugins = recurseIntoAttrs (
callPackage ../servers/search/elasticsearch/plugins.nix { }
@ -2975,6 +2988,7 @@ with pkgs;
kibana = callPackage ../development/tools/misc/kibana { };
kibana5 = callPackage ../development/tools/misc/kibana/5.x.nix { };
kibana6 = callPackage ../development/tools/misc/kibana/6.x.nix { };
kismet = callPackage ../applications/networking/sniffers/kismet { };
@ -3041,6 +3055,7 @@ with pkgs;
logstash = callPackage ../tools/misc/logstash { };
logstash5 = callPackage ../tools/misc/logstash/5.x.nix { };
logstash6 = callPackage ../tools/misc/logstash/6.x.nix { };
logstash-contrib = callPackage ../tools/misc/logstash/contrib.nix { };