Configured local prometheus and pushgateway in docker-compose (#21538)

ref
https://linear.app/ghost/issue/ENG-1746/enable-ghost-to-push-metrics-to-a-pushgateway

- Added prometheus job to scrape the pushgateway
- Updated grafana dashboard to use the metrics from the pushgateway
- Added some logging to prometheus client to log errors when pushing
metrics to pushgateway

.github/scripts/grafana/dashboards/main-dashboard.json

@@ -312,21 +312,21 @@
         "steppedLine": false,
         "targets": [
           {
-            "expr": "irate(ghost_process_cpu_user_seconds_total{job=~\"$job\", instance=~\"$instance\"}[1m]) * 100",
+            "expr": "irate(ghost_process_cpu_user_seconds_total{job=~\"$job\"}[1m]) * 100",
             "interval": "",
-            "legendFormat": "User CPU - {{instance}}",
+            "legendFormat": "User CPU - {{job}}",
             "refId": "A"
           },
           {
-            "expr": "irate(ghost_process_cpu_system_seconds_total{job=~\"$job\", instance=~\"$instance\"}[1m]) * 100",
+            "expr": "irate(ghost_process_cpu_system_seconds_total{job=~\"$job\"}[1m]) * 100",
             "interval": "",
-            "legendFormat": "System CPU - {{instance}}",
+            "legendFormat": "System CPU - {{job}}",
             "refId": "B"
           },
           {
-            "expr": "irate(ghost_process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[1m]) * 100",
+            "expr": "irate(ghost_process_cpu_seconds_total{job=~\"$job\"}[1m]) * 100",
             "interval": "",
-            "legendFormat": "Total CPU - {{instance}}",
+            "legendFormat": "Total CPU - {{job}}",
             "refId": "C"
           }
         ],

.github/scripts/prometheus/prometheus.yml

@@ -1,11 +1,6 @@
 global:
   scrape_interval:     15s # By default, scrape targets every 15 seconds.
 
-  # Attach these labels to any time series or alerts when communicating with
-  # external systems (federation, remote storage, Alertmanager).
-  external_labels:
-    monitor: 'codelab-monitor'
-
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.
 scrape_configs:
@@ -18,14 +13,14 @@ scrape_configs:
     static_configs:
       - targets: ['localhost:9090']
 
-  - job_name: 'ghost'
+  - job_name: 'pushgateway'
 
     scrape_interval: 1s
 
     static_configs:
-      - targets: ['host.docker.internal:9416']
+      - targets: ['pushgateway:9091']
 
-    metrics_path: '/metrics'
+    honor_labels: true
 
 remote_write:
   - url: http://grafana:3000/api/prom/push

ghost/prometheus-metrics/package.json

@@ -32,6 +32,7 @@
     "typescript": "5.6.2"
   },
   "dependencies": {
+    "@tryghost/logging": "2.4.19",
     "express": "4.21.1",
     "prom-client": "15.1.3",
     "stoppable": "1.1.0"

ghost/prometheus-metrics/src/PrometheusClient.ts

@@ -1,5 +1,6 @@
 import {Request, Response} from 'express';
 import client from 'prom-client';
+import logging from '@tryghost/logging';
 
 type PrometheusClientConfig = {
     register?: client.Registry;
@@ -23,13 +24,11 @@ export class PrometheusClient {
         this.config = prometheusConfig;
         this.client = client;
         this.prefix = 'ghost_';
-        this.register = this.config.register || client.register;
     }
 
     public client;
     private config: PrometheusClientConfig;
     private prefix;
-    public register: client.Registry; // public for testing
     public gateway: client.Pushgateway<client.RegistryContentType> | undefined; // public for testing
     private pushInterval: ReturnType<typeof setInterval> | undefined;
 
@@ -54,7 +53,12 @@ export class PrometheusClient {
     async pushMetrics() {
         if (this.config.pushgateway?.enabled && this.gateway) {
             const jobName = this.config.pushgateway?.jobName || 'ghost';
-            await this.gateway.pushAdd({jobName});
+            try {
+                await this.gateway.pushAdd({jobName});
+                logging.debug('Metrics pushed to pushgateway - jobName: ', jobName);
+            } catch (error) {
+                logging.error('Error pushing metrics to pushgateway - jobName: ', jobName);
+            }
         }
     }
 
@@ -73,7 +77,7 @@ export class PrometheusClient {
      * Only called once on init
      */
     collectDefaultMetrics() {
-        this.client.collectDefaultMetrics({prefix: this.prefix, register: this.register});
+        this.client.collectDefaultMetrics({prefix: this.prefix});
     }
 
     /**
@@ -98,13 +102,13 @@ export class PrometheusClient {
      * Returns the metrics from the registry
      */
     async getMetrics() {
-        return this.register.metrics();
+        return this.client.register.metrics();
     }
 
     /**
      * Returns the content type for the metrics
      */
     getContentType() {
-        return this.register.contentType;
+        return this.client.register.contentType;
     }
 }

ghost/prometheus-metrics/src/libraries.d.ts

@@ -1 +1,2 @@
-declare module '@tryghost/debug';
+declare module '@tryghost/debug';
+declare module '@tryghost/logging';

ghost/prometheus-metrics/test/prometheus-client.test.ts

@@ -12,7 +12,7 @@ describe('Prometheus Client', function () {
     afterEach(function () {
         if (instance) {
             instance.stop();
-            instance.register.clear();
+            instance.client.register.clear();
         }
     });
 

yarn.lock

@@ -7653,7 +7653,7 @@
     lodash "^4.17.21"
     luxon "^1.26.0"
 
-"@tryghost/logging@2.4.10", "@tryghost/logging@2.4.18", "@tryghost/logging@^2.4.7":
+"@tryghost/logging@2.4.10", "@tryghost/logging@2.4.18", "@tryghost/logging@2.4.19", "@tryghost/logging@^2.4.7":
   version "2.4.18"
   resolved "https://registry.yarnpkg.com/@tryghost/logging/-/logging-2.4.18.tgz#5d7ddb2d0a66dc6834a6048ebbf48418420445d5"
   integrity sha512-mMJkdCFDXa0ohS0FlDTvOrJQd7VamBIqjljGYvNECdVli7BMjdUYgZyWr8bEJ/d7scsq8OE2bVVBJWLxvPxLAg==