nixpkgs/nixos/tests/prometheus/alertmanager.nix

import ../make-test-python.nix ({ lib, pkgs, ... }:

{
  name = "prometheus-alertmanager";

  nodes = {
    prometheus = { config, pkgs, ... }: {
      environment.systemPackages = [ pkgs.jq ];

      networking.firewall.allowedTCPPorts = [ config.services.prometheus.port ];

      services.prometheus = {
        enable = true;
        globalConfig.scrape_interval = "2s";

        alertmanagers = [
          {
            scheme = "http";
            static_configs = [
              {
                targets = [
                  "alertmanager:${toString config.services.prometheus.alertmanager.port}"
                ];
              }
            ];
          }
        ];

        rules = [
          ''
            groups:
              - name: test
                rules:
                  - alert: InstanceDown
                    expr: up == 0
                    for: 5s
                    labels:
                      severity: page
                    annotations:
                      summary: "Instance {{ $labels.instance }} down"
          ''
        ];

        scrapeConfigs = [
          {
            job_name = "alertmanager";
            static_configs = [
              {
                targets = [
                  "alertmanager:${toString config.services.prometheus.alertmanager.port}"
                ];
              }
            ];
          }
          {
            job_name = "node";
            static_configs = [
              {
                targets = [
                  "node:${toString config.services.prometheus.exporters.node.port}"
                ];
              }
            ];
          }
        ];
      };
    };

    alertmanager = { config, pkgs, ... }: {
      services.prometheus.alertmanager = {
        enable = true;
        openFirewall = true;

        configuration = {
          global = {
            resolve_timeout = "1m";
          };

          route = {
            # Root route node
            receiver = "test";
            group_by = ["..."];
            continue = false;
            group_wait = "1s";
            group_interval = "15s";
            repeat_interval = "24h";
          };

          receivers = [
            {
              name = "test";
              webhook_configs = [
                {
                  url = "http://logger:6725";
                  send_resolved = true;
                  max_alerts = 0;
                }
              ];
            }
          ];
        };
      };
    };

    logger = { config, pkgs, ... }: {
      networking.firewall.allowedTCPPorts = [ 6725 ];

      services.prometheus.alertmanagerWebhookLogger.enable = true;
    };
  };

  testScript = ''
    alertmanager.wait_for_unit("alertmanager")
    alertmanager.wait_for_open_port(9093)
    alertmanager.wait_until_succeeds("curl -s http://127.0.0.1:9093/-/ready")
    #alertmanager.wait_until_succeeds("journalctl -o cat -u alertmanager.service | grep 'version=${pkgs.prometheus-alertmanager.version}'")

    logger.wait_for_unit("alertmanager-webhook-logger")
    logger.wait_for_open_port(6725)

    prometheus.wait_for_unit("prometheus")
    prometheus.wait_for_open_port(9090)

    prometheus.wait_until_succeeds(
      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"alertmanager\"\}==1)' | "
      + "jq '.data.result[0].value[1]' | grep '\"1\"'"
    )

    prometheus.wait_until_succeeds(
      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=sum(alertmanager_build_info)%20by%20(version)' | "
      + "jq '.data.result[0].metric.version' | grep '\"${pkgs.prometheus-alertmanager.version}\"'"
    )

    prometheus.wait_until_succeeds(
      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=count(up\{job=\"node\"\}!=1)' | "
      + "jq '.data.result[0].value[1]' | grep '\"1\"'"
    )

    prometheus.wait_until_succeeds(
      "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=alertmanager_notifications_total\{integration=\"webhook\"\}' | "
      + "jq '.data.result[0].value[1]' | grep -v '\"0\"'"
    )

    logger.wait_until_succeeds(
      "journalctl -o cat -u alertmanager-webhook-logger.service | grep '\"alertname\":\"InstanceDown\"'"
    )

    logger.log(logger.succeed("systemd-analyze security alertmanager-webhook-logger.service | grep -v '✓'"))

    alertmanager.log(alertmanager.succeed("systemd-analyze security alertmanager.service | grep -v '✓'"))
  '';
})