add prometheus service (#114)

* prometheus: init service

* prometheus: add doc

* prometheus: update doc

doc/prometheus.md

@@ -0,0 +1,41 @@
+# Prometheus
+
+[Prometheus] is a systems and service monitoring system. It collects metrics from configured targets at given intervals, evaluates rule expressions, displays the results, and can trigger alerts when specified conditions are observed.
+
+[Prometheus]: https://github.com/prometheus/prometheus
+
+## Getting Started
+
+```nix
+# In `perSystem.process-compose.<name>`
+{
+  services.prometheus."pro1".enable = true;
+}
+```
+
+{#tips}
+## Tips & Tricks
+
+{#scrape-configs}
+### Adding Scrape Configs
+
+`scrape_configs` controls what resources Prometheus monitors.
+
+Since Prometheus also exposes data about itself as an HTTP endpoint it can scrape and monitor its own health. In the [default example configuration](https://github.com/prometheus/prometheus/blob/3f686cad8bee405229b2532584ef181ce9f6a8b3/documentation/examples/prometheus.yml) there is a single job, called prometheus. We can add it to `scrape_configs` using the following config:
+
+```nix
+{
+  services.prometheus."pro1" = {
+    enable = true;
+    # scrape prometheus
+    extraConfig = {
+      scrape_configs = [{
+        job_name = "prometheus";
+        static_configs = [{
+          targets = [ "localhost:9090" ];
+        }];
+      }];
+    };
+  };
+}
+```

doc/services.md

@@ -17,6 +17,7 @@ short-title: Services
 - [ ] Redis Cluster
 - [ ] Zookeeper
 - [x] [[grafana]]#
+- [X] [[prometheus]]#
 - [ ] ...
 
 [gh]: https://github.com/juspay/services-flake

nix/default.nix

@@ -14,5 +14,6 @@ in
     ./redis.nix
     ./zookeeper.nix
     ./grafana.nix
+    ./prometheus.nix
   ];
 }

nix/prometheus.nix

@@ -0,0 +1,109 @@
+{ pkgs, lib, name, config, ... }:
+let
+  inherit (lib) types;
+  yamlFormat = pkgs.formats.yaml { };
+in
+{
+  options = {
+    enable = lib.mkEnableOption name;
+
+    package = lib.mkPackageOption pkgs "prometheus" { };
+
+    port = lib.mkOption {
+      type = types.port;
+      default = 9090;
+      description = "Port to listen on";
+    };
+
+    listenAddress = lib.mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      description = lib.mdDoc "Address to listen on for the web interface, API, and telemetry";
+    };
+
+    dataDir = lib.mkOption {
+      type = types.str;
+      default = "./data/${name}";
+      description = "The prometheus data directory";
+    };
+
+    extraFlags = lib.mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      description = "Extra commandline options when launching Prometheus";
+    };
+
+    defaultExtraConfig = lib.mkOption {
+      type = yamlFormat.type;
+      internal = true;
+      readOnly = true;
+      default = {
+        global = {
+          scrape_interval = "15s";
+          evaluation_interval = "15s";
+        };
+      };
+    };
+
+    extraConfig = lib.mkOption {
+      type = yamlFormat.type;
+      default = { };
+      description = "Additional config for prometheus";
+      example = ''
+        # scrape prometheus itself
+        scrape_configs = [{
+          job_name = "prometheus";
+          static_configs = [{
+            targets = [ "localhost:9090" ];
+          }];
+        }];
+      '';
+    };
+
+    outputs.settings = lib.mkOption {
+      type = types.deferredModule;
+      internal = true;
+      readOnly = true;
+      default = {
+        processes = {
+          "${name}" =
+            let
+              prometheusConfig = yamlFormat.generate "prometheus.yaml" (
+                lib.recursiveUpdate config.defaultExtraConfig config.extraConfig
+              );
+              execFlags = builtins.concatStringsSep " \\\n" ([
+                "--config.file=${prometheusConfig}"
+                "--storage.tsdb.path=${config.dataDir}"
+                "--web.listen-address=${config.listenAddress}:${builtins.toString config.port}"
+              ] ++ config.extraFlags);
+
+              startScript = pkgs.writeShellApplication {
+                name = "start-prometheus";
+                runtimeInputs = [ config.package ];
+                text = "prometheus ${execFlags}";
+              };
+            in
+            {
+              command = "${startScript}/bin/start-prometheus";
+              readiness_probe = {
+                http_get = {
+                  host = config.listenAddress;
+                  port = config.port;
+                  path = "/-/ready";
+                };
+                initial_delay_seconds = 2;
+                period_seconds = 10;
+                timeout_seconds = 4;
+                success_threshold = 1;
+                failure_threshold = 5;
+              };
+              namespace = name;
+
+              # https://github.com/F1bonacc1/process-compose#-auto-restart-if-not-healthy
+              availability.restart = "on_failure";
+            };
+        };
+      };
+    };
+  };
+}

nix/prometheus_test.nix

@@ -0,0 +1,30 @@
+{ pkgs, config, ... }: {
+  services.prometheus."pro1" = {
+    enable = true;
+    # scrape prometheus
+    extraConfig = {
+      scrape_configs = [{
+        job_name = "prometheus";
+        static_configs = [{
+          targets = [ "localhost:9090" ];
+        }];
+      }];
+    };
+  };
+
+  settings.processes.test =
+    let
+      cfg = config.services.prometheus."pro1";
+    in
+    {
+      command = pkgs.writeShellApplication {
+        runtimeInputs = [ cfg.package pkgs.curl pkgs.gnugrep ];
+        text = ''
+          curl -sS ${cfg.listenAddress}:${builtins.toString cfg.port}/-/healthy
+          curl -s -o /dev/null -w "%{http_code}" ${cfg.listenAddress}:${builtins.toString cfg.port}/metrics
+        '';
+        name = "prometheus-test";
+      };
+      depends_on."pro1".condition = "process_healthy";
+    };
+}

test/flake.nix

@@ -47,6 +47,7 @@
             "${inputs.services-flake}/nix/redis-cluster_test.nix"
             "${inputs.services-flake}/nix/zookeeper_test.nix"
             "${inputs.services-flake}/nix/grafana_test.nix"
+            "${inputs.services-flake}/nix/prometheus_test.nix"
           ]);
       };
     };