diff --git a/doc/prometheus.md b/doc/prometheus.md new file mode 100644 index 0000000..efffdbc --- /dev/null +++ b/doc/prometheus.md @@ -0,0 +1,41 @@ +# Prometheus + +[Prometheus] is a systems and service monitoring system. It collects metrics from configured targets at given intervals, evaluates rule expressions, displays the results, and can trigger alerts when specified conditions are observed. + +[Prometheus]: https://github.com/prometheus/prometheus + +## Getting Started + +```nix +# In `perSystem.process-compose.` +{ + services.prometheus."pro1".enable = true; +} +``` + +{#tips} +## Tips & Tricks + +{#scrape-configs} +### Adding Scrape Configs + +`scrape_configs` controls what resources Prometheus monitors. + +Since Prometheus also exposes data about itself as an HTTP endpoint it can scrape and monitor its own health. In the [default example configuration](https://github.com/prometheus/prometheus/blob/3f686cad8bee405229b2532584ef181ce9f6a8b3/documentation/examples/prometheus.yml) there is a single job, called prometheus. We can add it to `scrape_configs` using the following config: + +```nix +{ + services.prometheus."pro1" = { + enable = true; + # scrape prometheus + extraConfig = { + scrape_configs = [{ + job_name = "prometheus"; + static_configs = [{ + targets = [ "localhost:9090" ]; + }]; + }]; + }; + }; +} +``` diff --git a/doc/services.md b/doc/services.md index 184930a..9b988f6 100644 --- a/doc/services.md +++ b/doc/services.md @@ -17,6 +17,7 @@ short-title: Services - [ ] Redis Cluster - [ ] Zookeeper - [x] [[grafana]]# +- [X] [[prometheus]]# - [ ] ... [gh]: https://github.com/juspay/services-flake diff --git a/nix/default.nix b/nix/default.nix index d393bc7..5e65d1a 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -14,5 +14,6 @@ in ./redis.nix ./zookeeper.nix ./grafana.nix + ./prometheus.nix ]; } diff --git a/nix/prometheus.nix b/nix/prometheus.nix new file mode 100644 index 0000000..4a904ab --- /dev/null +++ b/nix/prometheus.nix @@ -0,0 +1,109 @@ +{ pkgs, lib, name, config, ... }: +let + inherit (lib) types; + yamlFormat = pkgs.formats.yaml { }; +in +{ + options = { + enable = lib.mkEnableOption name; + + package = lib.mkPackageOption pkgs "prometheus" { }; + + port = lib.mkOption { + type = types.port; + default = 9090; + description = "Port to listen on"; + }; + + listenAddress = lib.mkOption { + type = types.str; + default = "0.0.0.0"; + description = lib.mdDoc "Address to listen on for the web interface, API, and telemetry"; + }; + + dataDir = lib.mkOption { + type = types.str; + default = "./data/${name}"; + description = "The prometheus data directory"; + }; + + extraFlags = lib.mkOption { + type = types.listOf types.str; + default = [ ]; + description = "Extra commandline options when launching Prometheus"; + }; + + defaultExtraConfig = lib.mkOption { + type = yamlFormat.type; + internal = true; + readOnly = true; + default = { + global = { + scrape_interval = "15s"; + evaluation_interval = "15s"; + }; + }; + }; + + extraConfig = lib.mkOption { + type = yamlFormat.type; + default = { }; + description = "Additional config for prometheus"; + example = '' + # scrape prometheus itself + scrape_configs = [{ + job_name = "prometheus"; + static_configs = [{ + targets = [ "localhost:9090" ]; + }]; + }]; + ''; + }; + + outputs.settings = lib.mkOption { + type = types.deferredModule; + internal = true; + readOnly = true; + default = { + processes = { + "${name}" = + let + prometheusConfig = yamlFormat.generate "prometheus.yaml" ( + lib.recursiveUpdate config.defaultExtraConfig config.extraConfig + ); + execFlags = builtins.concatStringsSep " \\\n" ([ + "--config.file=${prometheusConfig}" + "--storage.tsdb.path=${config.dataDir}" + "--web.listen-address=${config.listenAddress}:${builtins.toString config.port}" + ] ++ config.extraFlags); + + startScript = pkgs.writeShellApplication { + name = "start-prometheus"; + runtimeInputs = [ config.package ]; + text = "prometheus ${execFlags}"; + }; + in + { + command = "${startScript}/bin/start-prometheus"; + readiness_probe = { + http_get = { + host = config.listenAddress; + port = config.port; + path = "/-/ready"; + }; + initial_delay_seconds = 2; + period_seconds = 10; + timeout_seconds = 4; + success_threshold = 1; + failure_threshold = 5; + }; + namespace = name; + + # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy + availability.restart = "on_failure"; + }; + }; + }; + }; + }; +} diff --git a/nix/prometheus_test.nix b/nix/prometheus_test.nix new file mode 100644 index 0000000..bb98e4f --- /dev/null +++ b/nix/prometheus_test.nix @@ -0,0 +1,30 @@ +{ pkgs, config, ... }: { + services.prometheus."pro1" = { + enable = true; + # scrape prometheus + extraConfig = { + scrape_configs = [{ + job_name = "prometheus"; + static_configs = [{ + targets = [ "localhost:9090" ]; + }]; + }]; + }; + }; + + settings.processes.test = + let + cfg = config.services.prometheus."pro1"; + in + { + command = pkgs.writeShellApplication { + runtimeInputs = [ cfg.package pkgs.curl pkgs.gnugrep ]; + text = '' + curl -sS ${cfg.listenAddress}:${builtins.toString cfg.port}/-/healthy + curl -s -o /dev/null -w "%{http_code}" ${cfg.listenAddress}:${builtins.toString cfg.port}/metrics + ''; + name = "prometheus-test"; + }; + depends_on."pro1".condition = "process_healthy"; + }; +} diff --git a/test/flake.nix b/test/flake.nix index 64191f8..f40b510 100644 --- a/test/flake.nix +++ b/test/flake.nix @@ -47,6 +47,7 @@ "${inputs.services-flake}/nix/redis-cluster_test.nix" "${inputs.services-flake}/nix/zookeeper_test.nix" "${inputs.services-flake}/nix/grafana_test.nix" + "${inputs.services-flake}/nix/prometheus_test.nix" ]); }; };