From 85408d10b75485ccde1e9bf852f127e8defe4a79 Mon Sep 17 00:00:00 2001 From: Chris Raible Date: Thu, 7 Nov 2024 23:01:34 -0800 Subject: [PATCH] Added connection pool metrics to prometheus client (#21576) ref https://linear.app/ghost/issue/ENG-1592/start-monitoring-connection-pool-utilization-in-ghost - This commit adds prometheus metrics to the connection pool so we can start to track connection pool utilization, number of pending acquires, and also adds some basic SQL query summary metrics like queries per minute and query duration percentiles. - The connection pool has now been theorized to be a main constraint of Ghost for some time, but it's been challenging to get actual visibility into the state of the connection pool. With this change, we should be able to directly observe, monitor and alert on the connection pool. - Updated grafana version to fix a bug in the query editor that was fixed in 8.3, even though this is a couple versions ahead of production --- .github/scripts/docker-compose.yml | 3 +- .github/scripts/grafana/dashboard.yml | 2 +- .../grafana/dashboards/main-dashboard.json | 3842 ++++++++++------- ghost/core/core/server/data/db/connection.js | 4 + ghost/prometheus-metrics/package.json | 3 +- .../src/PrometheusClient.ts | 111 +- .../test/prometheus-client.test.ts | 250 +- yarn.lock | 2 +- 8 files changed, 2579 insertions(+), 1638 deletions(-) diff --git a/.github/scripts/docker-compose.yml b/.github/scripts/docker-compose.yml index e6986b2a55..1f61ef3958 100644 --- a/.github/scripts/docker-compose.yml +++ b/.github/scripts/docker-compose.yml @@ -37,13 +37,14 @@ services: - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml grafana: profiles: [monitoring] - image: grafana/grafana:8.2.3 + image: grafana/grafana:8.3.0 container_name: ghost-grafana ports: - "3000:3000" restart: always environment: - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin volumes: - ./grafana/datasources:/etc/grafana/provisioning/datasources - ./grafana/dashboard.yml:/etc/grafana/provisioning/dashboards/main.yaml diff --git a/.github/scripts/grafana/dashboard.yml b/.github/scripts/grafana/dashboard.yml index d50b265869..bb0b691a76 100644 --- a/.github/scripts/grafana/dashboard.yml +++ b/.github/scripts/grafana/dashboard.yml @@ -9,7 +9,7 @@ providers: type: file disableDeletion: false updateIntervalSeconds: 10 - allowUiUpdates: false + allowUiUpdates: true options: path: /var/lib/grafana/dashboards foldersFromFilesStructure: true \ No newline at end of file diff --git a/.github/scripts/grafana/dashboards/main-dashboard.json b/.github/scripts/grafana/dashboards/main-dashboard.json index a1347c8d1c..0215fb1db9 100644 --- a/.github/scripts/grafana/dashboards/main-dashboard.json +++ b/.github/scripts/grafana/dashboards/main-dashboard.json @@ -1,1634 +1,2244 @@ { - "__inputs": [], - "__requires": [ + "annotations": { + "list": [ { - "id": "grafana", - "name": "Grafana", - "type": "grafana", - "version": "7.4.3" - }, - { - "id": "graph", - "name": "Graph", - "type": "panel", - "version": "" - }, - { - "id": "prometheus", - "name": "Prometheus", - "type": "datasource", - "version": "1.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], "type": "dashboard" }, - { - "datasource": "$datasource", - "enable": true, - "expr": "ghost_process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"} * 1000", - "hide": false, - "iconColor": "#B877D9", - "name": "Ghost Start", - "showIn": 0, - "textFormat": "{{instance}}", - "titleFormat": "Ghost Start", - "useValueForTime": true - } - ] - }, - "description": "An overview of the Ghost process metrics.", - "editable": true, - "gnetId": 14058, - "graphTooltip": 0, - "id": 1, - "iteration": 1608497517213, - "links": [], - "panels": [ - { - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 1, - "title": "Overview", - "type": "row" + "type": "dashboard" }, { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], "datasource": "$datasource", - "description": "The version of Node.js.", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 0, - "y": 1 - }, - "id": 2, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "pluginVersion": "6.6.2", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false, - "ymax": null, - "ymin": null - }, - "tableColumn": "version", - "targets": [ - { - "expr": "ghost_nodejs_version_info", - "format": "table", - "instant": true, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Node.js Version", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "first" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "$datasource", - "description": "The number of times Node.js restarted.", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 2, - "w": 3, - "x": 3, - "y": 1 - }, - "id": 3, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "pluginVersion": "6.6.2", - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false, - "ymax": null, - "ymin": null - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(changes(ghost_process_start_time_seconds{job=~\"$job\", instance=~\"$instance\", hostname=~\"$hostname\"}[$__range]))", - "instant": false, - "legendFormat": "Node.js", - "refId": "A" - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Node.js Restarts", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [], - "valueName": "current" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 4, - "panels": [], - "title": "Process", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "CPU usage.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 4 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(ghost_process_cpu_user_seconds_total{job=~\"$job\"}[1m]) * 100", - "interval": "", - "legendFormat": "User CPU - {{job}}", - "refId": "A" - }, - { - "expr": "rate(ghost_process_cpu_system_seconds_total{job=~\"$job\"}[1m]) * 100", - "interval": "", - "legendFormat": "System CPU - {{job}}", - "refId": "B" - }, - { - "expr": "rate(ghost_process_cpu_seconds_total{job=~\"$job\"}[1m]) * 100", - "interval": "", - "legendFormat": "Total CPU - {{job}}", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "CPU time spent.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 4 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(ghost_process_cpu_seconds_total{job=~\"$job\", instance=~\"$instance\"}[$interval])", - "interval": "", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU Time Spent", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Memory usage.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 16, - "y": 4 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": true, - "min": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "Process Memory - {{instance}}", - "refId": "A" - }, - { - "expr": "ghost_nodejs_heap_size_total_bytes{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "Heap Total - {{instance}}", - "refId": "B" - }, - { - "expr": "ghost_nodejs_heap_size_used_bytes{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "Heap Used - {{instance}}", - "refId": "C" - }, - { - "expr": "ghost_nodejs_external_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "External Memory - {{instance}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Number of active handle and active requests.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 0, - "y": 13 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ghost_nodejs_active_handles_total{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "Active Handler - {{instance}}", - "refId": "A" - }, - { - "expr": "ghost_nodejs_active_requests_total{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "Active Request - {{instance}}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Active Handlers and Requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Latency of the event loop.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 9, - "w": 8, - "x": 8, - "y": 13 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ghost_nodejs_eventloop_lag_seconds{job=~\"$job\", instance=~\"$instance\"}", - "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Last - {{instance}}", - "refId": "A" - }, - { - "expr": "ghost_nodejs_eventloop_lag_p99_seconds{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "P99 - {{instance}}", - "refId": "B" - }, - { - "expr": "ghost_nodejs_eventloop_lag_p50_seconds{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "P50 - {{instance}}", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Event Loop Latency", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 22 - }, - "id": 10, - "panels": [], - "title": "Garbage Collector", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Rate of garbage collection duration.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 23 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(ghost_nodejs_gc_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}[$interval])", - "interval": "", - "legendFormat": "{{kind}} - {{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GC Duration Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "s", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Duration of garbage collection.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 23 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ghost_nodejs_gc_duration_seconds_sum{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{kind}} - {{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GC Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "s", - "label": "", - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Usage of heap memory.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 23 - }, - "hiddenSeries": false, - "id": 13, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ghost_nodejs_heap_size_total_bytes{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "Total - {{instance}}", - "refId": "A" - }, - { - "expr": "ghost_nodejs_heap_size_used_bytes{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "Used - {{instance}}", - "refId": "B" - }, - { - "expr": "process_resident_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "Resident - {{instance}}", - "refId": "C" - }, - { - "expr": "ghost_nodejs_external_memory_bytes{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "External - {{instance}}", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Heap Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Rate of garbage collection.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 31 - }, - "hiddenSeries": false, - "id": 14, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(ghost_nodejs_gc_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}[$interval])", - "interval": "", - "legendFormat": "{{instance}} - {{kind}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GC Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Count of garbage collection.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 31 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "ghost_nodejs_gc_duration_seconds_count{job=~\"$job\", instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{kind}} - {{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GC Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": "", - "logBase": 2, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "description": "Usage of heap space.", - "fieldConfig": { - "defaults": { - "custom": {}, - "links": [] - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 31 - }, - "hiddenSeries": false, - "id": 16, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "sort": "current", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.3.0-beta2", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "ghost_nodejs_heap_space_size_used_bytes{instance=~\"$instance\"}", - "interval": "", - "legendFormat": "{{space}} - {{instance}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Heap Space Used", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } + "enable": true, + "expr": "ghost_process_start_time_seconds{job=~\"$job\", instance=~\"$instance\"} * 1000", + "hide": false, + "iconColor": "#B877D9", + "name": "Ghost Start", + "showIn": 0, + "textFormat": "{{instance}}", + "titleFormat": "Ghost Start", + "useValueForTime": true } - ], - "refresh": false, - "schemaVersion": 26, - "style": "dark", - "tags": [ - "node.js", - "nodejs" - ], - "templating": { - "list": [ - { - "current": { - "selected": false, - "text": "prometheus", - "value": "prometheus" + ] + }, + "description": "An overview of the Ghost process metrics.", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 14058, + "graphTooltip": 0, + "id": 1, + "iteration": 1731033697061, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "The version of Node.js.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "error": null, - "hide": 0, - "includeAll": false, - "label": "Datasource", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "queryValue": "", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": null, - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } ] }, - "datasource": "$datasource", - "definition": "label_values(ghost_nodejs_eventloop_lag_seconds, job)", - "error": null, - "hide": 0, - "includeAll": true, - "label": "Job", - "multi": true, - "name": "job", - "options": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 2, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "first" + ], + "fields": "/^version$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "ghost_nodejs_version_info{}", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Node.js Version", + "type": "stat" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "The number of times Node.js restarted.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 3, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "sum(changes(ghost_process_start_time_seconds{job=~\"$job\"}[$__range]))", + "instant": false, + "interval": "", + "legendFormat": "Node.js", + "refId": "A" + } + ], + "title": "Node.js Restarts", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 4, + "panels": [], + "title": "Process", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "CPU usage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 5, + "interval": "1", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "expr": "rate(ghost_process_cpu_user_seconds_total{job=~\"$job\"}[1m]) * 100", + "interval": "", + "legendFormat": "User CPU - {{job}}", + "refId": "A" + }, + { + "expr": "rate(ghost_process_cpu_system_seconds_total{job=~\"$job\"}[1m]) * 100", + "interval": "", + "legendFormat": "System CPU - {{job}}", + "refId": "B" + }, + { + "expr": "rate(ghost_process_cpu_seconds_total{job=~\"$job\"}[1m]) * 100", + "interval": "", + "legendFormat": "Total CPU - {{job}}", + "refId": "C" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "CPU time spent.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 6, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "rate(ghost_process_cpu_seconds_total{job=~\"$job\"}[$interval])", + "interval": "", + "legendFormat": "Total - {{job}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "rate(ghost_process_cpu_system_seconds_total{job=~\"$job\"}[$interval])", + "hide": false, + "interval": "", + "legendFormat": "System - {{job}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "rate(ghost_process_cpu_user_seconds_total{job=~\"$job\"}[$interval])", + "hide": false, + "interval": "", + "legendFormat": "User - {{job}}", + "refId": "C" + } + ], + "title": "CPU Time Spent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Memory usage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 7, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "process_resident_memory_bytes{job=~\"$job\"}", + "interval": "", + "legendFormat": "Process Memory - {{job}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "ghost_nodejs_heap_size_total_bytes{job=~\"$job\"}", + "interval": "", + "legendFormat": "Heap Total - {{job}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "ghost_nodejs_heap_size_used_bytes{job=~\"$job\"}", + "interval": "", + "legendFormat": "Heap Used - {{job}}", + "refId": "C" + }, + { + "exemplar": true, + "expr": "ghost_nodejs_external_memory_bytes{job=~\"$job\"}", + "interval": "", + "legendFormat": "External Memory - {{job}}", + "refId": "D" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Number of active handle and active requests.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 13 + }, + "id": 8, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "ghost_nodejs_active_handles_total{job=~\"$job\"}", + "interval": "", + "legendFormat": "Active Handler - {{job}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "ghost_nodejs_active_requests_total{job=~\"$job\"}", + "interval": "", + "legendFormat": "Active Request - {{job}}", + "refId": "B" + } + ], + "title": "Active Handlers and Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Latency of the event loop.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 13 + }, + "id": 9, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "ghost_nodejs_eventloop_lag_seconds{job=~\"$job\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Last - {{job}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "ghost_nodejs_eventloop_lag_p99_seconds{job=~\"$job\"}", + "interval": "", + "legendFormat": "P99 - {{job}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "ghost_nodejs_eventloop_lag_p50_seconds{job=~\"$job\"}", + "interval": "", + "legendFormat": "P50 - {{job}}", + "refId": "C" + } + ], + "title": "Event Loop Latency", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 18, + "panels": [], + "title": "Database: Connection Pool", + "type": "row" + }, + { + "description": "The number of active connections maintained with the database. These connections can be in use or idle.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 20, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_connection_pool_max{job=\"$job\"}", + "interval": "", + "legendFormat": "Max - {{job}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_connection_pool_active{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "Active - {{job}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_connection_pool_min{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "Min - {{job}}", + "refId": "C" + } + ], + "title": "Active Connections", + "type": "timeseries" + }, + { + "description": "The number of connections in the pool in active use (i.e. to run a query).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 23, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_connection_pool_max{job=\"$job\"}", + "interval": "", + "legendFormat": "Max - {{job}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_connection_pool_used{job=\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "Used - {{job}}", + "refId": "B" + } + ], + "title": "Used Connections", + "type": "timeseries" + }, + { + "description": "The percent of active or used connections out of the pool max.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 23 + }, + "id": 25, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_connection_pool_used{job=~\"$job\"} / ghost_db_connection_pool_max{job=~\"$job\"}", + "interval": "", + "legendFormat": "Used - {{job}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_connection_pool_active{job=~\"$job\"} / ghost_db_connection_pool_max{job=~\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "Active - {{job}}", + "refId": "B" + } + ], + "title": "Utilization", + "type": "timeseries" + }, + { + "description": "The number of queries that are currently waiting for a free connection.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 24, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_connection_pool_pending_acquires{job=~\"$job\"}", + "interval": "", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Pending Acquires", + "type": "timeseries" + }, + { + "description": "The number of connections waiting to be established with the database.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 22, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_connection_pool_pending_creates{job=~\"$job\"}", + "interval": "", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Pending Creates", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 27, + "panels": [], + "title": "Database: Queries", + "type": "row" + }, + { + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dtdurationms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 40 + }, + "id": 31, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_query_duration_milliseconds{quantile=\"0.5\", job=~\"$job\"}", + "interval": "", + "legendFormat": "P50 - {{job}}", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_query_duration_milliseconds{quantile=\"0.9\", job=~\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "P90 - {{job}}", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "ghost_db_query_duration_milliseconds{quantile=\"0.99\", job=~\"$job\"}", + "hide": false, + "interval": "", + "legendFormat": "P99 - {{job}}", + "refId": "C" + } + ], + "title": "Query Duration", + "type": "timeseries" + }, + { + "description": "The number of queries executed in the trailing 60s.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "QPM" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 40 + }, + "id": 29, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rate(ghost_db_query_count{job=~\"$job\"}[1m])", + "interval": "", + "legendFormat": "{{job}}", + "refId": "A" + } + ], + "title": "Query Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 10, + "panels": [], + "title": "Garbage Collector", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Rate of garbage collection duration.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 49 + }, + "id": 11, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "rate(ghost_nodejs_gc_duration_seconds_sum{job=~\"$job\"}[$interval])", + "interval": "", + "legendFormat": "{{kind}} - {{job}}", + "refId": "A" + } + ], + "title": "GC Duration Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Duration of garbage collection.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 49 + }, + "id": 12, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "ghost_nodejs_gc_duration_seconds_sum{job=~\"$job\"}", + "interval": "", + "legendFormat": "{{kind}} - {{job}}", + "refId": "A" + } + ], + "title": "GC Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Usage of heap memory.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 49 + }, + "id": 13, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "ghost_nodejs_heap_size_total_bytes{job=~\"$job\"}", + "interval": "", + "legendFormat": "Total - {{job}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "ghost_nodejs_heap_size_used_bytes{job=~\"$job\"}", + "interval": "", + "legendFormat": "Used - {{job}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "process_resident_memory_bytes{job=~\"$job\"}", + "interval": "", + "legendFormat": "Resident - {{job}}", + "refId": "C" + }, + { + "exemplar": true, + "expr": "ghost_nodejs_external_memory_bytes{job=~\"$job\"}", + "interval": "", + "legendFormat": "External - {{job}}", + "refId": "D" + } + ], + "title": "Heap Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Rate of garbage collection.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 57 + }, + "id": 14, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "rate(ghost_nodejs_gc_duration_seconds_count{job=~\"$job\"}[$interval])", + "interval": "", + "legendFormat": "{{kind}} - {{job}}", + "refId": "A" + } + ], + "title": "GC Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Count of garbage collection.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 57 + }, + "id": 15, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "ghost_nodejs_gc_duration_seconds_count{job=~\"$job\"}", + "interval": "", + "legendFormat": "{{kind}} - {{job}}", + "refId": "A" + } + ], + "title": "GC Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "Usage of heap space.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 57 + }, + "id": 16, + "interval": "1s", + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.0", + "targets": [ + { + "exemplar": true, + "expr": "ghost_nodejs_heap_space_size_used_bytes{job=~\"$job\"}", + "interval": "", + "legendFormat": "{{space}} - {{job}}", + "refId": "A" + } + ], + "title": "Heap Space Used", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 33, + "style": "dark", + "tags": [ + "node.js", + "nodejs" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": [ + "ghost-chris-local" + ], + "value": [ + "ghost-chris-local" + ] + }, + "datasource": { + "uid": "$datasource" + }, + "definition": "label_values(ghost_nodejs_eventloop_lag_seconds, job)", + "hide": 0, + "includeAll": true, + "label": "Job", + "multi": true, + "name": "job", + "options": [], + "query": { "query": "label_values(ghost_nodejs_eventloop_lag_seconds, job)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "refId": "Prometheus-job-Variable-Query" }, - { - "allValue": null, - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "$datasource", - "definition": "label_values(ghost_nodejs_eventloop_lag_seconds, instance)", - "error": null, - "hide": 0, - "includeAll": true, - "label": "Instance", - "multi": true, - "name": "instance", - "options": [], + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "uid": "$datasource" + }, + "definition": "label_values(ghost_nodejs_eventloop_lag_seconds, instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": true, + "name": "instance", + "options": [], + "query": { "query": "label_values(ghost_nodejs_eventloop_lag_seconds, instance)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false + "refId": "Prometheus-instance-Variable-Query" }, - { - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "1m", + "value": "1m" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, "text": "1m", "value": "1m" }, - "error": null, - "hide": 0, - "label": "Interval", - "name": "interval", - "options": [ - { - "selected": true, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "6h", - "value": "6h" - }, - { - "selected": false, - "text": "12h", - "value": "12h" - }, - { - "selected": false, - "text": "1d", - "value": "1d" - }, - { - "selected": false, - "text": "7d", - "value": "7d" - }, - { - "selected": false, - "text": "14d", - "value": "14d" - }, - { - "selected": false, - "text": "30d", - "value": "30d" - } - ], - "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", - "refresh": 2, - "skipUrlSync": false, - "type": "interval" - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "", - "title": "Ghost Dashboard", - "uid": "yX2d7k1Gk", - "version": 1 - } \ No newline at end of file + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Ghost Dashboard", + "uid": "yX2d7k1Gk", + "version": 12, + "weekStart": "" +} \ No newline at end of file diff --git a/ghost/core/core/server/data/db/connection.js b/ghost/core/core/server/data/db/connection.js index 4cb555de50..056d0bda8c 100644 --- a/ghost/core/core/server/data/db/connection.js +++ b/ghost/core/core/server/data/db/connection.js @@ -68,6 +68,10 @@ if (!knexInstance && config.get('database') && config.get('database').client) { const instrumentation = new ConnectionPoolInstrumentation({knex: knexInstance, logging, metrics, config}); instrumentation.instrument(); } + if (config.get('prometheus:enabled')) { + const prometheusClient = require('../../../shared/prometheus-client'); + prometheusClient.instrumentKnex(knexInstance); + } } module.exports = knexInstance; diff --git a/ghost/prometheus-metrics/package.json b/ghost/prometheus-metrics/package.json index ac054443a2..46dae48d3d 100644 --- a/ghost/prometheus-metrics/package.json +++ b/ghost/prometheus-metrics/package.json @@ -25,14 +25,15 @@ "@types/express": "4.17.21", "@types/stoppable": "1.1.0", "c8": "10.1.2", + "knex": "2.4.2", "mocha": "10.7.3", + "nock": "13.5.5", "sinon": "19.0.2", "supertest": "7.0.0", "ts-node": "10.9.2", "typescript": "5.6.2" }, "dependencies": { - "@tryghost/errors": "1.3.6", "@tryghost/logging": "2.4.19", "express": "4.21.1", "prom-client": "15.1.3", diff --git a/ghost/prometheus-metrics/src/PrometheusClient.ts b/ghost/prometheus-metrics/src/PrometheusClient.ts index df760aee47..734b137e0e 100644 --- a/ghost/prometheus-metrics/src/PrometheusClient.ts +++ b/ghost/prometheus-metrics/src/PrometheusClient.ts @@ -1,7 +1,7 @@ import {Request, Response} from 'express'; import client from 'prom-client'; +import type {Knex} from 'knex'; import logging from '@tryghost/logging'; -import errors from '@tryghost/errors'; type PrometheusClientConfig = { register?: client.Registry; @@ -21,17 +21,22 @@ export class PrometheusClient { * Creates a new PrometheusClient instance * @param prometheusConfig - The configuration for the PrometheusClient */ - constructor(prometheusConfig: PrometheusClientConfig = {}) { + constructor(prometheusConfig: PrometheusClientConfig = {}, logger: any = logging) { this.config = prometheusConfig; this.client = client; this.prefix = 'ghost_'; + this.logger = logger; } public client; + public gateway: client.Pushgateway | undefined; // public for testing + public customMetrics: Map = new Map(); + public queries: Map = new Map(); + private config: PrometheusClientConfig; private prefix; - public gateway: client.Pushgateway | undefined; // public for testing private pushInterval: ReturnType | undefined; + private logger: any; /** * Initializes the prometheus client, setting up the pushgateway if enabled @@ -42,6 +47,7 @@ export class PrometheusClient { const gatewayUrl = this.config.pushgateway.url || 'http://localhost:9091'; const interval = this.config.pushgateway.interval || 5000; this.gateway = new client.Pushgateway(gatewayUrl); + this.pushMetrics(); this.pushInterval = setInterval(() => { this.pushMetrics(); }, interval); @@ -56,15 +62,15 @@ export class PrometheusClient { const jobName = this.config.pushgateway?.jobName || 'ghost'; try { await this.gateway.pushAdd({jobName}); - logging.debug('Metrics pushed to pushgateway - jobName: ', jobName); + this.logger.debug('Metrics pushed to pushgateway - jobName: ', jobName); } catch (err) { let error; if (typeof err === 'object' && err !== null && 'code' in err) { - error = new errors.InternalServerError({message: 'Error pushing metrics to pushgateway: ' + err.code, code: err.code as string}); + error = 'Error pushing metrics to pushgateway: ' + err.code as string; } else { - error = new errors.InternalServerError({message: 'Error pushing metrics to pushgateway: Unknown error'}); + error = 'Error pushing metrics to pushgateway: Unknown error'; } - logging.error(error); + this.logger.error(error); } } } @@ -118,4 +124,95 @@ export class PrometheusClient { getContentType() { return this.client.register.contentType; } + + // Utility functions for creating custom metrics + + /** + * Instruments the knex connection pool and queries + * @param knexInstance - The knex instance + */ + instrumentKnex(knexInstance: Knex) { + // Create some gauges for tracking the connection pool + this.customMetrics.set(`${this.prefix}db_connection_pool_max`, new this.client.Gauge({ + name: `${this.prefix}db_connection_pool_max`, + help: 'The maximum number of connections allowed in the pool', + collect() { + this.set(knexInstance.client.pool.max); + } + })); + + this.customMetrics.set(`${this.prefix}db_connection_pool_min`, new this.client.Gauge({ + name: `${this.prefix}db_connection_pool_min`, + help: 'The minimum number of connections allowed in the pool', + collect() { + this.set(knexInstance.client.pool.min); + } + })); + + this.customMetrics.set(`${this.prefix}db_connection_pool_active`, new this.client.Gauge({ + name: `${this.prefix}db_connection_pool_active`, + help: 'The number of active connections to the database, which can be in use or idle', + collect() { + this.set(knexInstance.client.pool.numUsed() + knexInstance.client.pool.numFree()); + } + })); + + this.customMetrics.set(`${this.prefix}db_connection_pool_used`, new this.client.Gauge({ + name: `${this.prefix}db_connection_pool_used`, + help: 'The number of connections currently in use by the database', + collect() { + this.set(knexInstance.client.pool.numUsed()); + } + })); + + this.customMetrics.set(`${this.prefix}db_connection_pool_idle`, new this.client.Gauge({ + name: `${this.prefix}db_connection_pool_idle`, + help: 'The number of active connections currently idle in pool', + collect() { + this.set(knexInstance.client.pool.numFree()); + } + })); + + this.customMetrics.set(`${this.prefix}db_connection_pool_pending_acquires`, new this.client.Gauge({ + name: `${this.prefix}db_connection_pool_pending_acquires`, + help: 'The number of connections currently waiting to be acquired from the pool', + collect() { + this.set(knexInstance.client.pool.numPendingAcquires()); + } + })); + + this.customMetrics.set(`${this.prefix}db_connection_pool_pending_creates`, new this.client.Gauge({ + name: `${this.prefix}db_connection_pool_pending_creates`, + help: 'The number of connections currently waiting to be created', + collect() { + this.set(knexInstance.client.pool.numPendingCreates()); + } + })); + + this.customMetrics.set(`${this.prefix}db_query_count`, new this.client.Counter({ + name: `${this.prefix}db_query_count`, + help: 'The number of queries executed' + })); + + this.customMetrics.set(`${this.prefix}db_query_duration_milliseconds`, new this.client.Summary({ + name: `${this.prefix}db_query_duration_milliseconds`, + help: 'The duration of queries in milliseconds', + percentiles: [0.5, 0.9, 0.99] + })); + + knexInstance.on('query', (query) => { + // Increment the query counter + (this.customMetrics.get(`${this.prefix}db_query_count`) as client.Counter).inc(); + // Add the query to the map + this.queries.set(query.__knexQueryUid, new Date()); + }); + + knexInstance.on('query-response', (err, query) => { + const start = this.queries.get(query.__knexQueryUid); + if (start) { + const duration = new Date().getTime() - start.getTime(); + (this.customMetrics.get(`${this.prefix}db_query_duration_milliseconds`) as client.Summary).observe(duration); + } + }); + } } diff --git a/ghost/prometheus-metrics/test/prometheus-client.test.ts b/ghost/prometheus-metrics/test/prometheus-client.test.ts index 65c10abaf0..e3f2aa7382 100644 --- a/ghost/prometheus-metrics/test/prometheus-client.test.ts +++ b/ghost/prometheus-metrics/test/prometheus-client.test.ts @@ -2,11 +2,22 @@ import assert from 'assert/strict'; import {PrometheusClient} from '../src'; import {Request, Response} from 'express'; import * as sinon from 'sinon'; +import type {Knex} from 'knex'; +import nock from 'nock'; +import {EventEmitter} from 'events'; +import type {EventEmitter as EventEmitterType} from 'events'; +import type {Gauge, Counter, Summary, Pushgateway, RegistryContentType} from 'prom-client'; describe('Prometheus Client', function () { let instance: PrometheusClient; + let logger: any; + beforeEach(function () { sinon.restore(); + logger = { + info: sinon.stub(), + error: sinon.stub() + }; }); afterEach(function () { @@ -14,6 +25,7 @@ describe('Prometheus Client', function () { instance.stop(); instance.client.register.clear(); } + nock.cleanAll(); }); describe('constructor', function () { @@ -31,19 +43,21 @@ describe('Prometheus Client', function () { assert.ok(collectDefaultMetricsSpy.called); }); - it('should create the pushgateway client if the pushgateway is enabled', function () { - instance = new PrometheusClient({pushgateway: {enabled: true}}); + it('should create the pushgateway client if the pushgateway is enabled', async function () { + const clock = sinon.useFakeTimers(); + nock('http://localhost:9091') + .persist() + .post('/metrics/job/ghost') + .reply(200); + + instance = new PrometheusClient({pushgateway: {enabled: true, interval: 20}}); + const pushMetricsStub = sinon.stub(instance, 'pushMetrics').resolves(); instance.init(); assert.ok(instance.gateway); - }); - - it('should push metrics to the pushgateway if it is enabled', async function () { - const clock = sinon.useFakeTimers(); - instance = new PrometheusClient({pushgateway: {enabled: true}}); - const pushMetricsSpy = sinon.spy(instance, 'pushMetrics'); - instance.init(); - clock.tick(10000); - assert.ok(pushMetricsSpy.called); + assert.ok(pushMetricsStub.called, 'pushMetrics should be called immediately'); + clock.tick(30); + assert.ok(pushMetricsStub.calledTwice, 'pushMetrics should be called again after the interval'); + clock.restore(); }); }); @@ -56,6 +70,43 @@ describe('Prometheus Client', function () { }); }); + describe('pushMetrics', function () { + it('should push metrics to the pushgateway', async function () { + const scope = nock('http://localhost:9091') + .persist() + .post('/metrics/job/ghost') + .reply(200); + instance = new PrometheusClient({pushgateway: {enabled: true}}); + instance.init(); + await instance.pushMetrics(); + scope.done(); + }); + + it('should log an error with error code if pushing metrics to the gateway fails', async function () { + instance = new PrometheusClient({pushgateway: {enabled: true}}, logger); + instance.init(); + instance.gateway = { + pushAdd: sinon.stub().rejects({code: 'ECONNRESET'}) + } as unknown as Pushgateway; + await instance.pushMetrics(); + assert.ok(logger.error.called); + const [[error]] = logger.error.args; + assert.match(error, /ECONNRESET/); + }); + + it('should log a generic error if the error is unknown', async function () { + instance = new PrometheusClient({pushgateway: {enabled: true}}, logger); + instance.init(); + instance.gateway = { + pushAdd: sinon.stub().rejects() + } as unknown as Pushgateway; + await instance.pushMetrics(); + assert.ok(logger.error.called); + const [[error]] = logger.error.args; + assert.match(error, /Unknown error/); + }); + }); + describe('handleMetricsRequest', function () { it('should return the metrics', async function () { const setStub = sinon.stub(); @@ -111,4 +162,181 @@ describe('Prometheus Client', function () { assert.match(metrics, /^# HELP/); }); }); + + describe('instrumentKnex', function () { + let knexMock: Knex; + let eventEmitter: EventEmitterType; + + function simulateQuery(queryUid: string, duration: number) { + const clock = sinon.useFakeTimers(); + eventEmitter.emit('query', {__knexQueryUid: queryUid, sql: 'SELECT 1'}); + clock.tick(duration); + eventEmitter.emit('query-response', null, {__knexQueryUid: queryUid, sql: 'SELECT 1'}); + clock.restore(); + } + + function simulateQueries(durations: number[]) { + durations.forEach((duration, index) => { + simulateQuery(`${index}`, duration); + }); + } + + beforeEach(function () { + eventEmitter = new EventEmitter(); + knexMock = { + on: sinon.stub().callsFake((event, callback) => { + eventEmitter.on(event, callback); + }), + client: { + pool: { + max: 10, + min: 1, + numUsed: sinon.stub().returns(0), + numFree: sinon.stub().returns(0), + numPendingAcquires: sinon.stub().returns(0), + numPendingCreates: sinon.stub().returns(0) + } + } + } as unknown as Knex; + }); + + afterEach(function () { + sinon.restore(); + }); + + it('should create all the custom metrics for the connection pool and queries', function () { + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + const metrics = Array.from(instance.customMetrics.keys()); + assert.deepEqual(metrics, [ + 'ghost_db_connection_pool_max', + 'ghost_db_connection_pool_min', + 'ghost_db_connection_pool_active', + 'ghost_db_connection_pool_used', + 'ghost_db_connection_pool_idle', + 'ghost_db_connection_pool_pending_acquires', + 'ghost_db_connection_pool_pending_creates', + 'ghost_db_query_count', + 'ghost_db_query_duration_milliseconds' + ]); + }); + + it('should collect the connection pool max metric', async function () { + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + const connectionPoolMaxGauge = instance.customMetrics.get('ghost_db_connection_pool_max') as Gauge; + const result = await connectionPoolMaxGauge.get(); + assert.equal(result.values[0].value, 10); + }); + + it('should collect the connection pool min metric', async function () { + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + const connectionPoolMinGauge = instance.customMetrics.get('ghost_db_connection_pool_min') as Gauge; + const result = await connectionPoolMinGauge.get(); + assert.equal(result.values[0].value, 1); + }); + + it('should collect the connection pool active metric', async function () { + knexMock.client.pool.numUsed = sinon.stub().returns(3); + knexMock.client.pool.numFree = sinon.stub().returns(7); + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + const connectionPoolActiveGauge = instance.customMetrics.get('ghost_db_connection_pool_active') as Gauge; + const result = await connectionPoolActiveGauge.get(); + assert.equal(result.values[0].value, 10); + }); + + it('should collect the connection pool used metric', async function () { + knexMock.client.pool.numUsed = sinon.stub().returns(3); + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + const connectionPoolUsedGauge = instance.customMetrics.get('ghost_db_connection_pool_used') as Gauge; + const result = await connectionPoolUsedGauge.get(); + assert.equal(result.values[0].value, 3); + }); + + it('should collect the connection pool idle metric', async function () { + knexMock.client.pool.numFree = sinon.stub().returns(7); + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + const connectionPoolIdleGauge = instance.customMetrics.get('ghost_db_connection_pool_idle') as Gauge; + const result = await connectionPoolIdleGauge.get(); + assert.equal(result.values[0].value, 7); + }); + + it('should collect the connection pool pending acquires metric', async function () { + knexMock.client.pool.numPendingAcquires = sinon.stub().returns(3); + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + const connectionPoolPendingAcquiresGauge = instance.customMetrics.get('ghost_db_connection_pool_pending_acquires') as Gauge; + const result = await connectionPoolPendingAcquiresGauge.get(); + assert.equal(result.values[0].value, 3); + }); + + it('should collect the connection pool pending creates metric', async function () { + knexMock.client.pool.numPendingCreates = sinon.stub().returns(3); + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + const connectionPoolPendingCreatesGauge = instance.customMetrics.get('ghost_db_connection_pool_pending_creates') as Gauge; + const result = await connectionPoolPendingCreatesGauge.get(); + assert.equal(result.values[0].value, 3); + }); + + it('should collect the db query count metric', async function () { + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + const dbQueryCountGauge = instance.customMetrics.get('ghost_db_query_count') as Counter; + const result = await dbQueryCountGauge.get(); + assert.equal(result.values[0].value, 0); + }); + + it('should increment the db query count metric when a query is executed', async function () { + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + eventEmitter.emit('query', {__knexQueryUid: '1', sql: 'SELECT 1'}); + const dbQueryCountGauge = instance.customMetrics.get('ghost_db_query_count') as Counter; + const result = await dbQueryCountGauge.get(); + assert.equal(result.values[0].value, 1); + assert.equal(instance.queries.size, 1); + assert.ok(instance.queries.has('1')); + }); + + it('should collect the db query duration metric when a query is executed', async function () { + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + eventEmitter.emit('query', {__knexQueryUid: '1', sql: 'SELECT 1'}); + const dbQueryDurationSummary = instance.customMetrics.get('ghost_db_query_duration_milliseconds') as Summary; + const result = await dbQueryDurationSummary.get(); + assert.equal(result.values[0].value, 0); + }); + + it('should accurately calculate the query duration of a query', async function () { + instance = new PrometheusClient(); + instance.init(); + instance.instrumentKnex(knexMock); + const durations = [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000]; + simulateQueries(durations); + const dbQueryDurationSummary = instance.customMetrics.get('ghost_db_query_duration_milliseconds') as Summary; + const result = await dbQueryDurationSummary.get(); + assert.deepEqual(result.values, [ + {labels: {quantile: 0.5}, value: 550}, + {labels: {quantile: 0.9}, value: 950}, + {labels: {quantile: 0.99}, value: 1000}, + {metricName: 'ghost_db_query_duration_milliseconds_sum', labels: {}, value: 5500}, + {metricName: 'ghost_db_query_duration_milliseconds_count', labels: {}, value: 10} + ]); + }); + }); }); diff --git a/yarn.lock b/yarn.lock index 497a0f0624..94e14f0c6b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7430,7 +7430,7 @@ focus-trap "^6.7.2" postcss-preset-env "^7.3.1" -"@tryghost/errors@1.3.1", "@tryghost/errors@1.3.5", "@tryghost/errors@1.3.6", "@tryghost/errors@^1.2.26", "@tryghost/errors@^1.2.3", "@tryghost/errors@^1.3.5", "@tryghost/errors@^1.3.6": +"@tryghost/errors@1.3.1", "@tryghost/errors@1.3.5", "@tryghost/errors@^1.2.26", "@tryghost/errors@^1.2.3", "@tryghost/errors@^1.3.5", "@tryghost/errors@^1.3.6": version "1.3.5" resolved "https://registry.yarnpkg.com/@tryghost/errors/-/errors-1.3.5.tgz#f4ef8e5c41a8a37456f2285271124180685827ae" integrity sha512-iOkiHGnYFqSdFM9AVlgiL56Qcx6V9iQ3kbDKxyOAxrhMKq1OnOmOm7tr1CgGK1YDte9XYEZmR9hUZEg+ujn/jQ==