Added OpenTelemetry instrumentation to Ghost backend (#20144)

This commit adds OpenTelemetry instrumentation to Ghost's backend, which
allows us to view traces similar to what we see in Sentry Performance
locally.

OpenTelemetry is enabled if `NODE_ENV === 'development'` or if it is
explicitly enabled via config with `opentelemetry:enabled`.

It also adds a [Jaeger](https://www.jaegertracing.io/) container to
Ghost's docker-compose file for viewing the traces. There's no setup
required (beyond running `yarn docker:reset` to pickup the changes in
the docker-compose file the first time — but this will also reset your
DB so be careful). This will launch the Jaeger container, and you can
view the UI to see the traces at `http://localhost:16686/search`.

.github/scripts/docker-compose.yml

@@ -27,3 +27,13 @@ services:
     ports:
       - "6379:6379"
     restart: always
+  jaeger:
+      image: jaegertracing/all-in-one:1.56
+      container_name: ghost-jaeger
+      ports:
+        - "4318:4318"
+        - "16686:16686"
+        - "9411:9411"
+      restart: always
+      environment:
+        COLLECTOR_ZIPKIN_HOST_PORT: :9411

ghost/core/core/boot.js

@@ -509,6 +509,11 @@ async function bootGhost({backend = true, frontend = true, server = true} = {})
 
     try {
         // Step 1 - require more fundamental components
+        // OpenTelemetry should be configured as early as possible
+        debug('Begin: Load OpenTelemetry');
+        const opentelemetryInstrumentation = require('./shared/instrumentation');
+        opentelemetryInstrumentation.initOpenTelemetry({config});
+        debug('End: Load OpenTelemetry');
 
         // Sentry must be initialized early, but requires config
         debug('Begin: Load sentry');
@@ -531,8 +536,9 @@ async function bootGhost({backend = true, frontend = true, server = true} = {})
         debug('Begin: Get DB ready');
         await initDatabase({config});
         bootLogger.log('database ready');
+        const connection = require('./server/data/db/connection');
         sentry.initQueryTracing(
-            require('./server/data/db/connection')
+            connection
         );
         debug('End: Get DB ready');
 

ghost/core/core/server/data/db/index.js

@@ -1,8 +1,3 @@
-const logging = require('@tryghost/logging');
-const metrics = require('@tryghost/metrics');
-const config = require('../../../shared/config');
-const ConnectionPoolInstrumentation = require('./ConnectionPoolInstrumentation');
-
 let connection;
 
 Object.defineProperty(exports, 'knex', {
@@ -10,10 +5,6 @@ Object.defineProperty(exports, 'knex', {
     configurable: true,
     get: function get() {
         connection = connection || require('./connection');
-        if (config.get('telemetry:connectionPool')) {
-            const instrumentation = new ConnectionPoolInstrumentation({knex: connection, logging, metrics, config});
-            instrumentation.instrument();
-        }
         return connection;
     }
 });

ghost/core/core/shared/instrumentation.js

@@ -0,0 +1,31 @@
+const {NodeSDK} = require('@opentelemetry/sdk-node');
+const {OTLPTraceExporter} = require('@opentelemetry/exporter-trace-otlp-http');
+const {getNodeAutoInstrumentations} = require('@opentelemetry/auto-instrumentations-node');
+
+async function initOpenTelemetry({config}) {
+    // Always enable in development environment
+    // In production, only enable if explicitly enabled via config `opentelemetry:enabled`
+    const isDevelopment = process.env.NODE_ENV === 'development';
+    const isConfigured = config.get('opentelemetry:enabled');
+    const enabled = isDevelopment || isConfigured;
+    if (!enabled) {
+        return;
+    }
+    const collectorOptions = {
+        url: config.get('opentelemetry:exporter:endpoint') || 'http://localhost:4318/v1/traces',
+        headers: {},
+        concurrencyLimit: 10
+    };
+    const sdk = new NodeSDK({
+        serviceName: 'ghost',
+        traceExporter: new OTLPTraceExporter(collectorOptions),
+        instrumentations: [
+            getNodeAutoInstrumentations()
+        ]
+    });
+    sdk.start();
+}
+
+module.exports = {
+    initOpenTelemetry
+};

ghost/core/package.json

@@ -59,6 +59,15 @@
   "dependencies": {
     "@extractus/oembed-extractor": "3.2.1",
     "@sentry/node": "7.117.0",
+    "@opentelemetry/api": "^1.8.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.46.0",
+    "@opentelemetry/exporter-prometheus": "^0.51.1",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.51.0",
+    "@opentelemetry/instrumentation-knex": "^0.36.1",
+    "@opentelemetry/instrumentation-runtime-node": "^0.4.0",
+    "@opentelemetry/sdk-metrics": "^1.24.0",
+    "@opentelemetry/sdk-node": "^0.51.0",
+    "@opentelemetry/sdk-trace-node": "^1.24.0",
     "@tryghost/adapter-base-cache": "0.1.12",
     "@tryghost/adapter-cache-redis": "0.0.0",
     "@tryghost/adapter-manager": "0.0.0",