mirror of
https://github.com/hasura/graphql-engine.git
synced 2024-10-03 21:37:40 +03:00
docs: update latest changes of OpenTelemetry, best practices and deprecate Cloud integration
## Description [Rendered](https://pr-10880---docusaurus-5kcgrtdh3a-wl.a.run.app/docs/latest/observability/opentelemetry/graphql-engine/) https://hasurahq.atlassian.net/browse/INFV2-20 - Update Grafana dashboards for the latest metrics changes. - Improve documentation for OpenTelemetry and best practices. - Remove the `beta` label to move OpenTelemetry to GA - Add deprecation notes for Hasura Cloud observability integrations PR-URL: https://github.com/hasura/graphql-engine-mono/pull/10880 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Sean Park-Ross <94021366+seanparkross@users.noreply.github.com> GitOrigin-RevId: fcfaefef3a08747fb4f0c582bc91c8b61142ab13
This commit is contained in:
Toan Nguyen
hasura-bot
parent
e43c6b5421
commit
1199b669cb
@ -1,5 +1,3 @@
|
||||
version: "3.6"
|
||||
|
||||
volumes:
|
||||
prometheus_data: {}
|
||||
grafana_data: {}
|
||||
@ -177,7 +175,7 @@ services:
|
||||
- 6831:6831/udp
|
||||
- 6832:6832/udp
|
||||
- 5778:5778
|
||||
- 4002:16686
|
||||
- ${JAEGER_UI_PORT}:16686
|
||||
- 14250:14250
|
||||
- 14268:14268
|
||||
- 14269:14269
|
||||
@ -185,7 +183,6 @@ services:
|
||||
- 4318:4318 # OTLP HTTP
|
||||
- 9411:9411
|
||||
environment:
|
||||
COLLECTOR_ZIPKIN_HOST_PORT: ":9411"
|
||||
COLLECTOR_OTLP_ENABLED: "true"
|
||||
|
||||
loki:
|
||||
@ -212,6 +209,7 @@ services:
|
||||
- "55679" # zpages extension
|
||||
environment:
|
||||
DD_API_KEY: ${DD_API_KEY}
|
||||
NEW_RELIC_API_KEY: ${NEW_RELIC_API_KEY}
|
||||
HONEYCOMB_API_KEY: ${HONEYCOMB_API_KEY}
|
||||
HONEYCOMB_DATASET: ${HONEYCOMB_DATASET}
|
||||
# GOOGLE_APPLICATION_CREDENTIALS: /etc/otel/gcp-sa.json
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# HGE
|
||||
HGE_IMAGE=hasura/graphql-engine:v2.35.0.cli-migrations-v3
|
||||
HGE_IMAGE=hasura/graphql-engine:v2.40.0.cli-migrations-v3
|
||||
HGE_PORT=8080
|
||||
HGE_EE_LICENSE_KEY=
|
||||
HGE_ADMIN_SECRET=myadminsecretkey
|
||||
@ -25,39 +25,36 @@ PG_PASSWORD=postgrespassword
|
||||
PG_REPLICATION_PASSWORD=repl_password
|
||||
|
||||
# Prometheus
|
||||
PROM_IMAGE=prom/prometheus:v2.44.0
|
||||
PROM_IMAGE=prom/prometheus:v2.52.0
|
||||
PROM_PORT=9090
|
||||
|
||||
# Node-exporter
|
||||
NODE_EXPORTER_IMAGE=prom/node-exporter:v1.5.0
|
||||
NODE_EXPORTER_IMAGE=prom/node-exporter:v1.8.0
|
||||
NODE_EXPORTER_PORT=9100
|
||||
|
||||
# Alert Manager
|
||||
ALERT_MANAGER_IMAGE=prom/alertmanager:v0.25.0
|
||||
ALERT_MANAGER_IMAGE=prom/alertmanager:v0.27.0
|
||||
ALERT_MANAGER_PORT=9093
|
||||
|
||||
# Blackbox Exporter
|
||||
BLACKBOX_IMAGE=prom/blackbox-exporter:v0.24.0
|
||||
BLACKBOX_IMAGE=prom/blackbox-exporter:v0.25.0
|
||||
BLACKBOX_PORT=9115
|
||||
|
||||
# Grafana
|
||||
GRAFANA_IMAGE=grafana/grafana:9.5.2
|
||||
GRAFANA_IMAGE=grafana/grafana:10.4.2
|
||||
GRAFANA_PORT=3000
|
||||
GRAFANA_ADMIN_PASSWORD=HGEMonitoring!
|
||||
|
||||
# Jaeger
|
||||
JAEGER_IMAGE=jaegertracing/all-in-one:1.45
|
||||
JAEGER_IMAGE=jaegertracing/all-in-one:1.57
|
||||
JAEGER_UI_PORT=4002
|
||||
|
||||
# Loki
|
||||
LOKI_IMAGE=grafana/loki:2.8.2
|
||||
LOKI_IMAGE=grafana/loki:2.9.8
|
||||
LOKI_PORT=3100
|
||||
|
||||
# Promtail
|
||||
PROMTAIL_IMAGE=grafana/promtail:2.8.2
|
||||
|
||||
# OpenTelemetry Collector
|
||||
OTEL_COLLECTOR_IMAGE=otel/opentelemetry-collector-contrib:0.84.0
|
||||
OTEL_COLLECTOR_IMAGE=otel/opentelemetry-collector-contrib:0.100.0
|
||||
|
||||
# DataDog
|
||||
DD_API_KEY=
|
||||
@ -65,3 +62,6 @@ DD_API_KEY=
|
||||
# HoneyComb
|
||||
HONEYCOMB_API_KEY=
|
||||
HONEYCOMB_DATASET=
|
||||
|
||||
# New Relic
|
||||
NEW_RELIC_API_KEY=
|
||||
@ -1,50 +1,50 @@
|
||||
{
|
||||
"__elements": {},
|
||||
"__inputs": [
|
||||
{
|
||||
"description": "",
|
||||
"label": "Prometheus",
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "Prometheus",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "prometheus",
|
||||
"pluginName": "Prometheus",
|
||||
"type": "datasource"
|
||||
"pluginName": "Prometheus"
|
||||
}
|
||||
],
|
||||
"__elements": {},
|
||||
"__requires": [
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "gauge",
|
||||
"name": "Gauge",
|
||||
"type": "panel",
|
||||
"version": ""
|
||||
},
|
||||
{
|
||||
"type": "grafana",
|
||||
"id": "grafana",
|
||||
"name": "Grafana",
|
||||
"type": "grafana",
|
||||
"version": "9.5.2"
|
||||
"version": "10.4.2"
|
||||
},
|
||||
{
|
||||
"type": "datasource",
|
||||
"id": "prometheus",
|
||||
"name": "Prometheus",
|
||||
"type": "datasource",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "stat",
|
||||
"name": "Stat",
|
||||
"type": "panel",
|
||||
"version": ""
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "table",
|
||||
"name": "Table",
|
||||
"type": "panel",
|
||||
"version": ""
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "timeseries",
|
||||
"name": "Time series",
|
||||
"type": "panel",
|
||||
"version": ""
|
||||
}
|
||||
],
|
||||
@ -130,13 +130,17 @@
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "9.5.2",
|
||||
"pluginVersion": "10.4.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -208,13 +212,17 @@
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "9.5.2",
|
||||
"pluginVersion": "10.4.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -265,16 +273,21 @@
|
||||
"id": 31,
|
||||
"interval": "0s",
|
||||
"options": {
|
||||
"minVizHeight": 75,
|
||||
"minVizWidth": 75,
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true
|
||||
"showThresholdMarkers": true,
|
||||
"sizing": "auto"
|
||||
},
|
||||
"pluginVersion": "9.5.2",
|
||||
"pluginVersion": "10.4.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -330,13 +343,17 @@
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "9.5.2",
|
||||
"pluginVersion": "10.4.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -409,7 +426,20 @@
|
||||
},
|
||||
"unit": "none"
|
||||
},
|
||||
"overrides": []
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Service"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 82
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
@ -422,14 +452,17 @@
|
||||
"cellHeight": "sm",
|
||||
"footer": {
|
||||
"countRows": false,
|
||||
"enablePagination": true,
|
||||
"fields": "",
|
||||
"reducer": ["sum"],
|
||||
"reducer": [
|
||||
"sum"
|
||||
],
|
||||
"show": false
|
||||
},
|
||||
"showHeader": true,
|
||||
"sortBy": []
|
||||
},
|
||||
"pluginVersion": "9.5.2",
|
||||
"pluginVersion": "10.4.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -438,7 +471,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": false,
|
||||
"expr": "topk(5, sum by (source_name,trigger_name) (increase(hasura_event_processed_total{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__range]))) > 0",
|
||||
"expr": "topk(10, sum by (job,source_name,trigger_name) (increase(hasura_event_processed_total{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__range]))) > 0",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
@ -453,7 +486,12 @@
|
||||
"id": "filterFieldsByName",
|
||||
"options": {
|
||||
"include": {
|
||||
"names": ["Value", "trigger_name", "source_name"]
|
||||
"names": [
|
||||
"source_name",
|
||||
"trigger_name",
|
||||
"Value",
|
||||
"job"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
@ -473,12 +511,16 @@
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {},
|
||||
"includeByName": {},
|
||||
"indexByName": {
|
||||
"Value": 2,
|
||||
"Value": 3,
|
||||
"job": 2,
|
||||
"source_name": 1,
|
||||
"trigger_name": 0
|
||||
},
|
||||
"renameByName": {
|
||||
"Value": "Count",
|
||||
"job": "Service",
|
||||
"operation_name": "Operation Name",
|
||||
"parameterized_query_hash": "Query Hash",
|
||||
"source_name": "Source",
|
||||
@ -520,7 +562,32 @@
|
||||
},
|
||||
"unit": "percent"
|
||||
},
|
||||
"overrides": []
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Service"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 100
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Source"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 129
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
@ -533,14 +600,17 @@
|
||||
"cellHeight": "sm",
|
||||
"footer": {
|
||||
"countRows": false,
|
||||
"enablePagination": true,
|
||||
"fields": "",
|
||||
"reducer": ["sum"],
|
||||
"reducer": [
|
||||
"sum"
|
||||
],
|
||||
"show": false
|
||||
},
|
||||
"showHeader": true,
|
||||
"sortBy": []
|
||||
},
|
||||
"pluginVersion": "9.5.2",
|
||||
"pluginVersion": "10.4.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -549,7 +619,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": false,
|
||||
"expr": "topk(5, sum by (source_name,trigger_name) (increase(hasura_event_processed_total{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\",status=\"failed\"}[$__range])) \n/ sum by (source_name,trigger_name) (increase(hasura_event_processed_total{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__range]))\n) > 0",
|
||||
"expr": "topk(10, sum by (job,source_name,trigger_name) (increase(hasura_event_processed_total{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\",status=\"failed\"}[$__range])) \n/ sum by (job,source_name,trigger_name) (increase(hasura_event_processed_total{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__range]))\n) > 0",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
@ -564,7 +634,12 @@
|
||||
"id": "filterFieldsByName",
|
||||
"options": {
|
||||
"include": {
|
||||
"names": ["Value", "trigger_name", "source_name"]
|
||||
"names": [
|
||||
"source_name",
|
||||
"trigger_name",
|
||||
"Value",
|
||||
"job"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
@ -584,12 +659,16 @@
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {},
|
||||
"includeByName": {},
|
||||
"indexByName": {
|
||||
"Value": 2,
|
||||
"Value": 3,
|
||||
"job": 2,
|
||||
"source_name": 1,
|
||||
"trigger_name": 0
|
||||
},
|
||||
"renameByName": {
|
||||
"Value": "",
|
||||
"job": "Service",
|
||||
"operation_name": "Operation Name",
|
||||
"parameterized_query_hash": "Query Hash",
|
||||
"source_name": "Source",
|
||||
@ -635,7 +714,32 @@
|
||||
},
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
"overrides": [
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Service"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 105
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"matcher": {
|
||||
"id": "byName",
|
||||
"options": "Source"
|
||||
},
|
||||
"properties": [
|
||||
{
|
||||
"id": "custom.width",
|
||||
"value": 100
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"gridPos": {
|
||||
"h": 8,
|
||||
@ -648,14 +752,17 @@
|
||||
"cellHeight": "sm",
|
||||
"footer": {
|
||||
"countRows": false,
|
||||
"enablePagination": true,
|
||||
"fields": "",
|
||||
"reducer": ["sum"],
|
||||
"reducer": [
|
||||
"sum"
|
||||
],
|
||||
"show": false
|
||||
},
|
||||
"showHeader": true,
|
||||
"sortBy": []
|
||||
},
|
||||
"pluginVersion": "9.5.2",
|
||||
"pluginVersion": "10.4.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -664,7 +771,7 @@
|
||||
},
|
||||
"editorMode": "code",
|
||||
"exemplar": false,
|
||||
"expr": "topk(5, histogram_quantile(0.95, sum(rate(hasura_event_processing_time_seconds_bucket{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__range])) by (source_name, trigger_name, le)))",
|
||||
"expr": "topk(5, histogram_quantile(0.95, sum(rate(hasura_event_processing_time_seconds_bucket{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__range])) by (job,source_name, trigger_name, le)))",
|
||||
"format": "table",
|
||||
"instant": true,
|
||||
"interval": "",
|
||||
@ -679,7 +786,12 @@
|
||||
"id": "filterFieldsByName",
|
||||
"options": {
|
||||
"include": {
|
||||
"names": ["Value", "trigger_name", "source_name"]
|
||||
"names": [
|
||||
"source_name",
|
||||
"trigger_name",
|
||||
"Value",
|
||||
"job"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
@ -699,12 +811,15 @@
|
||||
"id": "organize",
|
||||
"options": {
|
||||
"excludeByName": {},
|
||||
"includeByName": {},
|
||||
"indexByName": {
|
||||
"Value": 2,
|
||||
"Value": 3,
|
||||
"job": 2,
|
||||
"source_name": 1,
|
||||
"trigger_name": 0
|
||||
},
|
||||
"renameByName": {
|
||||
"job": "Service",
|
||||
"operation_name": "Operation Name",
|
||||
"parameterized_query_hash": "Query Hash",
|
||||
"source_name": "Source",
|
||||
@ -728,6 +843,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -741,6 +857,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
@ -798,8 +915,8 @@
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (source_name,trigger_name) (rate(hasura_event_processed_total{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{trigger_name}} ({{source_name}})",
|
||||
"expr": "sum by (job,source_name,trigger_name) (rate(hasura_event_processed_total{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{job}} - {{trigger_name}} ({{source_name}})",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
@ -819,6 +936,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -832,6 +950,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
@ -888,8 +1007,8 @@
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (source_name,trigger_name) (rate(hasura_event_processed_total{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\",status=\"failed\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{trigger_name}} ({{source_name}})",
|
||||
"expr": "sum by (job,source_name,trigger_name) (rate(hasura_event_processed_total{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\",status=\"failed\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{job}} - {{trigger_name}} ({{source_name}})",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
@ -909,6 +1028,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -922,6 +1042,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
@ -943,7 +1064,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
@ -982,8 +1104,8 @@
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(hasura_event_processing_time_seconds_bucket{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__rate_interval])) by (source_name, trigger_name, le))",
|
||||
"legendFormat": "{{trigger_name}} ({{source_name}})",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(hasura_event_processing_time_seconds_bucket{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__rate_interval])) by (job,source_name, trigger_name, le))",
|
||||
"legendFormat": "{{job}} - {{trigger_name}} ({{source_name}})",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
@ -1003,6 +1125,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -1016,6 +1139,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
@ -1037,7 +1161,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
@ -1076,8 +1201,8 @@
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(hasura_event_webhook_processing_time_seconds_bucket{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__rate_interval])) by (source_name, trigger_name, le))",
|
||||
"legendFormat": "{{trigger_name}} (source_name)",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(hasura_event_webhook_processing_time_seconds_bucket{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__rate_interval])) by (job, source_name, trigger_name, le))",
|
||||
"legendFormat": "{{job}} - {{trigger_name}} (source_name)",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
@ -1097,6 +1222,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -1110,6 +1236,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
@ -1131,7 +1258,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
@ -1190,6 +1318,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -1203,6 +1332,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
@ -1224,7 +1354,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
@ -1262,9 +1393,9 @@
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(hasura_event_queue_time_seconds_bucket{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__rate_interval])) by (source_name,trigger_name,le))",
|
||||
"expr": "histogram_quantile(0.95, sum(rate(hasura_event_queue_time_seconds_bucket{job=~\"$job\",instance=~\"$instance\",source_name=~\"$event_source\",trigger_name=~\"$trigger_name\"}[$__rate_interval])) by (job,source_name,trigger_name,le))",
|
||||
"interval": "",
|
||||
"legendFormat": "{{trigger_name}} ({{source_name}})",
|
||||
"legendFormat": "{{job}} - {{trigger_name}} ({{source_name}})",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
@ -1284,6 +1415,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -1297,6 +1429,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
@ -1318,7 +1451,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1373,6 +1507,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -1386,6 +1521,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
@ -1407,7 +1543,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
@ -1476,6 +1613,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -1489,6 +1627,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
@ -1510,7 +1649,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
@ -1565,6 +1705,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -1578,6 +1719,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
@ -1599,7 +1741,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
@ -1654,6 +1797,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -1667,6 +1811,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
@ -1688,7 +1833,8 @@
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green"
|
||||
"color": "green",
|
||||
"value": null
|
||||
}
|
||||
]
|
||||
},
|
||||
@ -1735,16 +1881,17 @@
|
||||
],
|
||||
"refresh": "10s",
|
||||
"revision": 1,
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": ["hasura"],
|
||||
"schemaVersion": 39,
|
||||
"tags": [
|
||||
"hasura"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "Prometheus",
|
||||
"value": "Prometheus"
|
||||
"value": "PBFA97CFB590B2093"
|
||||
},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
@ -1864,6 +2011,6 @@
|
||||
"timezone": "",
|
||||
"title": "Hasura Events",
|
||||
"uid": "-nBhdJEVz",
|
||||
"version": 1,
|
||||
"version": 2,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,38 +1,38 @@
|
||||
{
|
||||
"__elements": {},
|
||||
"__inputs": [
|
||||
{
|
||||
"description": "",
|
||||
"label": "Prometheus",
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "Prometheus",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "prometheus",
|
||||
"pluginName": "Prometheus",
|
||||
"type": "datasource"
|
||||
"pluginName": "Prometheus"
|
||||
}
|
||||
],
|
||||
"__elements": {},
|
||||
"__requires": [
|
||||
{
|
||||
"type": "grafana",
|
||||
"id": "grafana",
|
||||
"name": "Grafana",
|
||||
"type": "grafana",
|
||||
"version": "9.5.2"
|
||||
"version": "10.4.2"
|
||||
},
|
||||
{
|
||||
"type": "datasource",
|
||||
"id": "prometheus",
|
||||
"name": "Prometheus",
|
||||
"type": "datasource",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "stat",
|
||||
"name": "Stat",
|
||||
"type": "panel",
|
||||
"version": ""
|
||||
},
|
||||
{
|
||||
"type": "panel",
|
||||
"id": "timeseries",
|
||||
"name": "Time series",
|
||||
"type": "panel",
|
||||
"version": ""
|
||||
}
|
||||
],
|
||||
@ -110,7 +110,6 @@
|
||||
"y": 0
|
||||
},
|
||||
"id": 6,
|
||||
"links": [],
|
||||
"maxDataPoints": 100,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
@ -118,13 +117,17 @@
|
||||
"justifyMode": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "9.5.2",
|
||||
"pluginVersion": "10.4.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -199,7 +202,6 @@
|
||||
"y": 0
|
||||
},
|
||||
"id": 7,
|
||||
"links": [],
|
||||
"maxDataPoints": 100,
|
||||
"options": {
|
||||
"colorMode": "background",
|
||||
@ -207,13 +209,17 @@
|
||||
"justifyMode": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "9.5.2",
|
||||
"pluginVersion": "10.4.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -221,8 +227,8 @@
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "max by (source_name) (hasura_source_health{job=~\"$job\"})",
|
||||
"legendFormat": "__auto",
|
||||
"expr": "max by (job, source_name) (hasura_source_health{job=~\"$job\"})",
|
||||
"legendFormat": "{{source_name}} ({{job}})",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
}
|
||||
@ -262,7 +268,6 @@
|
||||
"y": 0
|
||||
},
|
||||
"id": 9,
|
||||
"links": [],
|
||||
"maxDataPoints": 100,
|
||||
"options": {
|
||||
"colorMode": "none",
|
||||
@ -270,13 +275,17 @@
|
||||
"justifyMode": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": ["lastNotNull"],
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"textMode": "auto"
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
},
|
||||
"pluginVersion": "9.5.2",
|
||||
"pluginVersion": "10.4.2",
|
||||
"targets": [
|
||||
{
|
||||
"datasource": {
|
||||
@ -306,6 +315,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -319,6 +329,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "linear",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 4,
|
||||
@ -360,10 +371,14 @@
|
||||
"y": 6
|
||||
},
|
||||
"id": 4,
|
||||
"links": [],
|
||||
"options": {
|
||||
"legend": {
|
||||
"calcs": ["mean", "lastNotNull", "max", "min"],
|
||||
"calcs": [
|
||||
"mean",
|
||||
"lastNotNull",
|
||||
"max",
|
||||
"min"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "bottom",
|
||||
"showLegend": true
|
||||
@ -404,6 +419,7 @@
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
@ -417,6 +433,7 @@
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineStyle": {
|
||||
"fill": "solid"
|
||||
@ -480,7 +497,7 @@
|
||||
"uid": "${DS_PROMETHEUS}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "sum by (role,conn_info,source_name) (hasura_postgres_connections{job=~\"$job\",instance=~\"$instance\"})",
|
||||
"expr": "sum by (job,role,conn_info,source_name) (hasura_postgres_connections{job=~\"$job\",instance=~\"$instance\"})",
|
||||
"legendFormat": "__auto",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@ -491,16 +508,17 @@
|
||||
}
|
||||
],
|
||||
"refresh": "1m",
|
||||
"schemaVersion": 38,
|
||||
"style": "dark",
|
||||
"tags": ["hasura"],
|
||||
"schemaVersion": 39,
|
||||
"tags": [
|
||||
"hasura"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": false,
|
||||
"text": "Prometheus",
|
||||
"value": "Prometheus"
|
||||
"value": "PBFA97CFB590B2093"
|
||||
},
|
||||
"description": "Prometheus source",
|
||||
"hide": 0,
|
||||
@ -584,11 +602,21 @@
|
||||
"2h",
|
||||
"1d"
|
||||
],
|
||||
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"]
|
||||
"time_options": [
|
||||
"5m",
|
||||
"15m",
|
||||
"1h",
|
||||
"6h",
|
||||
"12h",
|
||||
"24h",
|
||||
"2d",
|
||||
"7d",
|
||||
"30d"
|
||||
]
|
||||
},
|
||||
"timezone": "browser",
|
||||
"title": "Hasura Health",
|
||||
"uid": "f477a6f2-95ce-4bf8-9483-7af6879f48d2",
|
||||
"version": 1,
|
||||
"version": 2,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
@ -58,7 +58,7 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"description": "Hasura dashboards for HTTP GraphQL requests",
|
||||
"description": "Hasura GraphQL Engine dashboards for monitoring OpenTelemetry exporter",
|
||||
"editable": true,
|
||||
"fiscalYearStartMonth": 0,
|
||||
"graphTooltip": 0,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -9,7 +9,7 @@ exporters:
|
||||
endpoint: "0.0.0.0:8889"
|
||||
|
||||
# Data sources: traces, metrics, logs
|
||||
logging:
|
||||
debug:
|
||||
verbosity: detailed
|
||||
|
||||
loki:
|
||||
@ -19,17 +19,23 @@ exporters:
|
||||
# api:
|
||||
# key: ${DD_API_KEY}
|
||||
|
||||
## deprecated: can push traces directly to http://jaeger:4318/v1/traces
|
||||
## use this pipeline if you want exporting traces to multi-channels
|
||||
##
|
||||
jaeger:
|
||||
endpoint: jaeger:14250
|
||||
## New Relic exporter
|
||||
otlp/newrelic:
|
||||
endpoint: otlp.nr-data.net:4317
|
||||
headers:
|
||||
api-key: ${NEW_RELIC_API_KEY}
|
||||
|
||||
## Export traces to Jaeger via OTLP protocol.
|
||||
## You can also export traces directly to jaeger to graphql-engine.
|
||||
## However, the collector supports
|
||||
otlp/jaeger:
|
||||
endpoint: jaeger:4317
|
||||
tls:
|
||||
insecure: true
|
||||
|
||||
## Honeycomb exporter
|
||||
## Note: logs are sent to the service.name in trace (hasura), not to x-honeycomb-dataset
|
||||
# otlp:
|
||||
# otlp/honeycomb:
|
||||
# endpoint: api.honeycomb.io:443
|
||||
# headers:
|
||||
# x-honeycomb-team: ${HONEYCOMB_API_KEY}
|
||||
@ -40,7 +46,7 @@ exporters:
|
||||
## - roles/monitoring.metricWriter
|
||||
## - roles/cloudtrace.agent
|
||||
## - roles/logging.logWriter
|
||||
## download and rename to gcp-sa.json so the otel-collector can mount the service account
|
||||
## download and rename to gcp-sa.json so the otel-collector can mount the service account
|
||||
##
|
||||
# googlecloud:
|
||||
# log:
|
||||
@ -82,19 +88,44 @@ processors:
|
||||
limit_percentage: 65
|
||||
spike_limit_percentage: 20
|
||||
|
||||
## New Relic can parse body in JSON string only.
|
||||
## You must enable this transformer to convert the body to string
|
||||
##
|
||||
# transform/new_relic_logs:
|
||||
# log_statements:
|
||||
# - context: log
|
||||
# statements:
|
||||
# - set(body, Concat([body, ""], ""))
|
||||
|
||||
filter:
|
||||
error_mode: ignore
|
||||
traces:
|
||||
span:
|
||||
# You can filter unused traces to save cost, for example:
|
||||
# - 'IsMatch(name, "Event trigger")'
|
||||
# - 'IsMatch(name, "Scheduled trigger")'
|
||||
# - 'IsMatch(name, "websocket")'
|
||||
- 'IsMatch(name, "/v1/version")'
|
||||
- 'IsMatch(name, "/v1/entitlement")'
|
||||
- 'IsMatch(name, "/v1alpha1/config")'
|
||||
|
||||
logs:
|
||||
log_record:
|
||||
# You can filter unused logs to save cost, or configure HASURA_GRAPHQL_ENABLED_LOG_TYPES in GraphQL Engine to disable unused log types
|
||||
- 'attributes["type"] == "query-log" and IsMatch(body["query"]["operationName"], "UnknownQuery")'
|
||||
- 'attributes["type"] == "http-log" and IsMatch(body["operation"]["query"]["operationName"], "UnknownQuery")'
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
## enable traces to DataDogs if require
|
||||
##
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [jaeger]
|
||||
processors: [filter, memory_limiter, batch]
|
||||
exporters: [otlp/jaeger, debug]
|
||||
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [logging, loki]
|
||||
processors: [filter, memory_limiter, batch]
|
||||
exporters: [loki]
|
||||
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
|
||||
@ -23,6 +23,13 @@ import ProductBadge from '@site/src/components/ProductBadge';
|
||||
|
||||
## Introduction
|
||||
|
||||
:::caution Deprecation
|
||||
|
||||
Hasura Cloud observability integrations are deprecated in favour of using
|
||||
[OpenTelemetry exporter](/observability/opentelemetry/index.mdx) to get advanced traces, metrics and logs.
|
||||
|
||||
:::
|
||||
|
||||
You can export metrics, operation logs and traces of your Hasura Cloud project to
|
||||
[Azure Monitor](https://azure.microsoft.com/en-in/services/monitor/). This can be configured on the integrations tab on
|
||||
the project's setting page.
|
||||
@ -34,13 +41,6 @@ above.
|
||||
|
||||
:::
|
||||
|
||||
:::tip More metrics on OpenTelemetry exporter
|
||||
|
||||
Try the new [OpenTelemetry exporter](/observability/opentelemetry/index.mdx) to get advanced metrics and traces to
|
||||
connect with Azure Monitor. This integration will export the metrics as detailed [here](#view-metrics).
|
||||
|
||||
:::
|
||||
|
||||
## Pre-requisites
|
||||
|
||||
- Create a
|
||||
|
||||
@ -23,6 +23,13 @@ import ProductBadge from '@site/src/components/ProductBadge';
|
||||
|
||||
## Introduction
|
||||
|
||||
:::caution Deprecation
|
||||
|
||||
Hasura Cloud observability integrations are deprecated in favour of using
|
||||
[OpenTelemetry exporter](/observability/opentelemetry/index.mdx) to get advanced traces, metrics and logs.
|
||||
|
||||
:::
|
||||
|
||||
You can export metrics, operation logs and traces of your Hasura Cloud project to your organization's Datadog dashboard.
|
||||
This can be configured on the integrations tab on the project's setting page.
|
||||
|
||||
@ -32,13 +39,6 @@ For Hasura Cloud projects, the Datadog Integration is only available on the `Sta
|
||||
|
||||
:::
|
||||
|
||||
:::tip More metrics on OpenTelemetry exporter
|
||||
|
||||
Try the new [OpenTelemetry exporter](/observability/opentelemetry/index.mdx) to get advanced metrics and traces to
|
||||
connect with Datadog. This integration will export the metrics as detailed [here](#view-metrics).
|
||||
|
||||
:::
|
||||
|
||||
## Configure Datadog integration
|
||||
|
||||
Navigate to the integrations tab on project settings page to find Datadog integration.
|
||||
|
||||
@ -25,6 +25,13 @@ import ProductBadge from '@site/src/components/ProductBadge';
|
||||
|
||||
## Introduction
|
||||
|
||||
:::caution Deprecation
|
||||
|
||||
Hasura Cloud observability integrations are deprecated in favour of using
|
||||
[OpenTelemetry exporter](/observability/opentelemetry/index.mdx) to get advanced traces, metrics and logs.
|
||||
|
||||
:::
|
||||
|
||||
To be able to effectively monitor, diagnose and troubleshoot your application stack in production, Hasura Cloud will
|
||||
export metrics, logs and traces to observability tools / APM vendors.
|
||||
|
||||
|
||||
@ -24,6 +24,13 @@ import ProductBadge from '@site/src/components/ProductBadge';
|
||||
|
||||
## Introduction
|
||||
|
||||
:::caution Deprecation
|
||||
|
||||
Hasura Cloud observability integrations are deprecated in favour of using
|
||||
[OpenTelemetry exporter](/observability/opentelemetry/index.mdx) to get advanced traces, metrics and logs.
|
||||
|
||||
:::
|
||||
|
||||
You can export metrics and operation logs of your Hasura Cloud project to [New Relic](https://newrelic.com/). This can
|
||||
be configured on the integrations tab on the project's setting page.
|
||||
|
||||
@ -33,13 +40,6 @@ For Hasura Cloud projects, the New Relic Integration is only available on the `S
|
||||
|
||||
:::
|
||||
|
||||
:::tip More metrics on OpenTelemetry exporter
|
||||
|
||||
Try the new [OpenTelemetry exporter](/observability/opentelemetry/index.mdx) to get advanced metrics and traces to
|
||||
connect with New Relic. This integration will export the metrics as detailed [here](#view-metrics).
|
||||
|
||||
:::
|
||||
|
||||
## Configure New Relic integration
|
||||
|
||||
Navigate to the integrations tab on project settings page to find New Relic integration.
|
||||
|
||||
@ -24,6 +24,13 @@ import ProductBadge from '@site/src/components/ProductBadge';
|
||||
|
||||
## Introduction
|
||||
|
||||
:::caution Deprecation
|
||||
|
||||
Hasura Cloud observability integrations are deprecated in favour of using
|
||||
[OpenTelemetry exporter](/observability/opentelemetry/index.mdx) to get advanced traces, metrics and logs.
|
||||
|
||||
:::
|
||||
|
||||
You can export metrics of your Hasura Cloud project to [Prometheus](https://prometheus.io/). This can be configured on
|
||||
the integrations tab on the project's setting page.
|
||||
|
||||
@ -34,14 +41,6 @@ above.
|
||||
|
||||
:::
|
||||
|
||||
:::tip More metrics on OpenTelemetry exporter
|
||||
|
||||
Try the new [OpenTelemetry exporter](/observability/opentelemetry/index.mdx) to get advanced metrics and traces to
|
||||
connect with Prometheus. This integration will export the metrics as detailed
|
||||
[here](/observability/cloud/prometheus.mdx#metrics-details).
|
||||
|
||||
:::
|
||||
|
||||
## Configure Prometheus integration
|
||||
|
||||
Navigate to the integrations tab on project settings page to find Prometheus integration.
|
||||
|
||||
111
docs/docs/observability/opentelemetry/best-practices.mdx
Normal file
111
docs/docs/observability/opentelemetry/best-practices.mdx
Normal file
@ -0,0 +1,111 @@
|
||||
---
|
||||
sidebar_label: Best Practices
|
||||
description: Best practices for OpenTelemetry of Hasura GraphQL Engine
|
||||
title: 'OpenTelemetry Best Practices'
|
||||
keywords:
|
||||
- hasura
|
||||
- docs
|
||||
- enterprise
|
||||
- opentelemetry
|
||||
- open telemetry
|
||||
- traces
|
||||
- metrics
|
||||
- logs
|
||||
sidebar_position: 4
|
||||
---
|
||||
|
||||
import ProductBadge from '@site/src/components/ProductBadge';
|
||||
|
||||
# OpenTelemetry Best Practices
|
||||
|
||||
<div className="badge-container">
|
||||
<ProductBadge free pro ee self />
|
||||
</div>
|
||||
|
||||
## Identify different applications
|
||||
|
||||
GraphQL Engine uses the `hasura` service name by default. If you have many different GraphQL Engine applications you
|
||||
should configure different `service.name` attributes for each application to identify and filter metrics more easily.
|
||||
|
||||
## Use OpenTelemetry Collector
|
||||
|
||||
Most of observability services require the
|
||||
[OpenTelemetry Collector](https://github.com/open-telemetry/opentelemetry-collector) as a proxy to push data. Using
|
||||
OpenTelemetry Collector also helps:
|
||||
|
||||
- Route and export data to multiple services.
|
||||
- Optimize the cost with sampling and filter processors.
|
||||
- Transform and calculate aggregate metrics.
|
||||
|
||||
## Cost optimization
|
||||
|
||||
### Sampling
|
||||
|
||||
The OpenTelemetry Collector supports many strategies to sample spans and log records. See OpenTelemetry Collector Contrib
|
||||
docs for more context.
|
||||
|
||||
- [Probabilistic Sampling Processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/probabilisticsamplerprocessor).
|
||||
- [Tail Sampling Processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/tailsamplingprocessor).
|
||||
|
||||
### Trace
|
||||
|
||||
You may not want to trace all operations, for example, you only want to trace GraphQL requests or individual GraphQL
|
||||
operations. Let's configure the
|
||||
[Filter Processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/filterprocessor).
|
||||
|
||||
```yaml
|
||||
processors:
|
||||
filter:
|
||||
error_mode: ignore
|
||||
traces:
|
||||
span:
|
||||
# - 'IsMatch(name, "Event trigger")'
|
||||
# - 'IsMatch(name, "Scheduled trigger")'
|
||||
# - 'IsMatch(name, "websocket")'
|
||||
- 'IsMatch(name, "/v1/version")'
|
||||
- 'IsMatch(name, "/v1/entitlement")'
|
||||
- 'IsMatch(name, "/v1alpha1/config")'
|
||||
# filter unused graphql operation
|
||||
- attributes["graphql.operation.name"] == "MyQuery"
|
||||
```
|
||||
|
||||
### Logs
|
||||
|
||||
Similar to traces, besides disabling log types via
|
||||
[environment variables](/observability/opentelemetry/graphql-engine.mdx#configurations), you may want to filter
|
||||
individual logs:
|
||||
|
||||
```yaml
|
||||
processors:
|
||||
filter:
|
||||
error_mode: ignore
|
||||
logs:
|
||||
log_record:
|
||||
- 'attributes["type"] == "query-log" and IsMatch(body["query"]["operationName"], "UnknownQuery")'
|
||||
- 'attributes["type"] == "http-log" and IsMatch(body["operation"]["query"]["operationName"], "UnknownQuery")'
|
||||
```
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Logging
|
||||
|
||||
OpenTelemetry logs are printed in an `unstructured` type with a `warn` level if runtime errors happen.
|
||||
|
||||
```json
|
||||
{
|
||||
"detail": "OTel exporter: Failed to deliver logs: Encountered retryable HTTP exception: ConnectionFailure ...",
|
||||
"level": "warn",
|
||||
"timestamp": "2024-06-19T06:39:51.704+0000",
|
||||
"type": "unstructured"
|
||||
}
|
||||
```
|
||||
|
||||
### Metrics
|
||||
|
||||
GraphQL Engine exports metrics to monitor the number of sent and dropped (failed) trace spans or log records. You can
|
||||
collect them via OpenTelemetry metrics or the native Prometheus exporter.
|
||||
|
||||
See the list of available metrics
|
||||
[here](/observability/enterprise-edition/prometheus/metrics.mdx#opentelemetry-otlp-export-metrics).
|
||||
|
||||
Download and install [the Grafana dashboard](https://grafana.com/grafana/dashboards/21363) if you are using Prometheus.
|
||||
@ -2,7 +2,6 @@
|
||||
sidebar_label: Data Connectors
|
||||
description: Traces via OpenTelemetry for Hasura GraphQL Engine Data Connectors in Enterprise Edition
|
||||
title: 'Data Connector Traces via OpenTelemetry'
|
||||
sidebar_class_name: beta-icon
|
||||
keywords:
|
||||
- hasura
|
||||
- docs
|
||||
@ -10,11 +9,11 @@ keywords:
|
||||
- opentelemetry
|
||||
- open telemetry
|
||||
- traces
|
||||
- metrics
|
||||
- logs
|
||||
sidebar_position: 3
|
||||
---
|
||||
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
import Thumbnail from '@site/src/components/Thumbnail';
|
||||
import ProductBadge from '@site/src/components/ProductBadge';
|
||||
|
||||
@ -22,7 +21,6 @@ import ProductBadge from '@site/src/components/ProductBadge';
|
||||
|
||||
<div className="badge-container">
|
||||
<ProductBadge free pro ee self />
|
||||
<div className="badge badge-gray heading-badge">Beta</div>
|
||||
</div>
|
||||
|
||||
## Introduction
|
||||
|
||||
@ -2,7 +2,6 @@
|
||||
sidebar_label: GraphQL Engine
|
||||
description: Traces via OpenTelemetry for Hasura GraphQL Engine Enterprise Edition
|
||||
title: 'GraphQL Engine Traces via OpenTelemetry'
|
||||
sidebar_class_name: beta-icon
|
||||
keywords:
|
||||
- hasura
|
||||
- docs
|
||||
@ -10,6 +9,8 @@ keywords:
|
||||
- opentelemetry
|
||||
- open telemetry
|
||||
- traces
|
||||
- metrics
|
||||
- logs
|
||||
sidebar_position: 2
|
||||
---
|
||||
|
||||
@ -22,7 +23,6 @@ import ProductBadge from '@site/src/components/ProductBadge';
|
||||
|
||||
<div className="badge-container">
|
||||
<ProductBadge free pro ee self />
|
||||
<div className="badge badge-gray heading-badge">Beta</div>
|
||||
</div>
|
||||
|
||||
## Introduction
|
||||
@ -38,11 +38,6 @@ subscriptions with the [OpenTelemetry](https://opentelemetry.io/docs/concepts/si
|
||||
be exported directly from your Hasura instances to your observability tool that supports OpenTelemetry traces. This can
|
||||
be configured in the `Settings` section of the Hasura Console.
|
||||
|
||||
## Available Metrics
|
||||
|
||||
The available OpenTelemetry metrics are the same as those available via
|
||||
[Prometheus](/observability/enterprise-edition/prometheus/metrics.mdx).
|
||||
|
||||
## Configure the OpenTelemetry receiver
|
||||
|
||||
:::info Supported from
|
||||
@ -305,6 +300,160 @@ be found in the [OpenTelemetry Collector repository](https://github.com/open-tel
|
||||
|
||||
:::
|
||||
|
||||
### Available Traces
|
||||
|
||||
GraphQL Engine traces activities of:
|
||||
|
||||
- Metadata APIs (`/v1/metadata`)
|
||||
- Schema APIs (`/v2/query`)
|
||||
- GraphQL API (`/v1/graphql`)
|
||||
- Event triggers
|
||||
- Scheduled triggers
|
||||
|
||||
Multiple operations can be linked together with the same trace ID. For example, the following trace includes Scheduled
|
||||
Trigger, GraphQL, Event Trigger and Schema requests in the same flame graph.
|
||||
|
||||
<Thumbnail
|
||||
src="/img/observability/opentelemetry-trace-scheduled-trigger.png"
|
||||
alt="Flame graph for multiple operations in the same trace entry"
|
||||
width="1146px"
|
||||
/>
|
||||
|
||||
**Metadata API**
|
||||
|
||||
<Thumbnail
|
||||
src="/img/observability/opentelemetry-trace-metadata.png"
|
||||
alt="Flame graph for /v1/metadata trace"
|
||||
width="1146px"
|
||||
/>
|
||||
|
||||
**Schema API**
|
||||
|
||||
<Thumbnail
|
||||
src="/img/observability/opentelemetry-trace-query.png"
|
||||
alt="Flame graph for /v2/query trace"
|
||||
width="1146px"
|
||||
/>
|
||||
|
||||
**Subscription**
|
||||
|
||||
<Thumbnail
|
||||
src="/img/observability/opentelemetry-trace-subscription.png"
|
||||
alt="Flame graph for subscription"
|
||||
width="1146px"
|
||||
/>
|
||||
|
||||
## Consuming and visualizing metrics
|
||||
|
||||
:::info Compare with Native Prometheus metrics
|
||||
|
||||
OpenTelemetry and [Native Prometheus](/observability/enterprise-edition/prometheus/index.mdx) metrics are exchangeable.
|
||||
You should enable only one of these exporters to avoid receiving duplicated data.
|
||||
|
||||
:::
|
||||
|
||||
### OpenTelemetry Collector
|
||||
|
||||
The OpenTelemetry Collector is recommended, especially if your production infrastructure has many GraphQL Engine replicas.
|
||||
The replication instance ID is in the global resource attributes array. However, some observability services don't
|
||||
automatically convert resource attributes to data point labels. Aggregate values of metrics may not be accurate. You
|
||||
should add a processor to assign the instance ID.
|
||||
|
||||
```yaml
|
||||
processors:
|
||||
transform:
|
||||
error_mode: ignore
|
||||
metric_statements:
|
||||
- context: datapoint
|
||||
statements:
|
||||
- set(attributes["instance"], resource.attributes["host.name"])
|
||||
```
|
||||
|
||||
See the full configuration example
|
||||
[here](https://github.com/hasura/graphql-engine/blob/master/community/boilerplates/observability/enterprise/otel-collector/otel-collector-config.yaml).
|
||||
|
||||
### Available Metrics
|
||||
|
||||
The available OpenTelemetry metrics are the same as those available via
|
||||
[Prometheus](/observability/enterprise-edition/prometheus/metrics.mdx).
|
||||
|
||||
### Visualizing metrics via Prometheus and Grafana
|
||||
|
||||
OpenTelemetry and Native Prometheus metrics are exchangeable. [Grafana dashboards](https://grafana.com/orgs/hasurahq)
|
||||
are compatible with those metrics as well. However, because Prometheus renames the `job` label from exported metrics by
|
||||
default you need to enable `honor_labels` to allow overriding the `job` label from OpenTelemetry Collector.
|
||||
|
||||
```yaml
|
||||
scrape_configs:
|
||||
- job_name: 'otel-collector-hasura'
|
||||
scrape_interval: 30s
|
||||
honor_labels: true
|
||||
static_configs:
|
||||
- targets: ['otel-collector:8889']
|
||||
```
|
||||
|
||||
## Logs
|
||||
|
||||
### Log Structure
|
||||
|
||||
The overall structure of OpenTelemetry logs is similar to [standard GraphQL Engine logs](/deployment/logging.mdx).
|
||||
However, several minor changes are made to be compatible with the specification of
|
||||
[OpenTelemetry Logs Data Model](https://opentelemetry.io/docs/specs/otel/logs/data-model/).
|
||||
|
||||
| Standard | OpenTelemetry |
|
||||
| -------- | ----------------- |
|
||||
| `detail` | `body` |
|
||||
| `level` | `severity` |
|
||||
| `type` | `attributes.type` |
|
||||
|
||||
<Thumbnail
|
||||
src="/img/observability/opentelemetry-logs-structure-comparison.png"
|
||||
alt="OpenTelemetry Logging structure comparison"
|
||||
width="1146px"
|
||||
/>
|
||||
|
||||
### Configurations
|
||||
|
||||
All logs which are printed to the output stream are exported to OpenTelemetry receivers. Therefore, besides native
|
||||
OpenTelemetry configurations, these configurations take effect to OpenTelemetry logs as well:
|
||||
|
||||
- [HASURA_GRAPHQL_LOG_LEVEL](/deployment/graphql-engine-flags/reference.mdx#log-level)
|
||||
- [HASURA_GRAPHQL_ENABLED_LOG_TYPES](/deployment/graphql-engine-flags/reference.mdx#enabled-log-types)
|
||||
|
||||
### New Relic
|
||||
|
||||
New Relic can parse the body in JSON string format only. If you export logs directly to public OpenTelemetry endpoints
|
||||
of New Relic the body content will be removed. The workaround is using the OpenTelemetry collector with a transform
|
||||
processor to convert the log body to string.
|
||||
|
||||
```yaml
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
http:
|
||||
|
||||
exporters:
|
||||
otlp/newrelic:
|
||||
endpoint: otlp.nr-data.net:4317
|
||||
headers:
|
||||
api-key: ${NEW_RELIC_API_KEY}
|
||||
|
||||
processors:
|
||||
transform/new_relic_logs:
|
||||
log_statements:
|
||||
- context: log
|
||||
statements:
|
||||
- set(body, Concat([body, ""], ""))
|
||||
|
||||
service:
|
||||
pipelines:
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [transform/new_relic_logs]
|
||||
exporters: [otlp/newrelic]
|
||||
```
|
||||
|
||||
## Connect Logs and Traces
|
||||
|
||||
Trace and Span ID are included in the root of the log body. GraphQL Engine follows
|
||||
@ -377,3 +526,8 @@ matched with that attribute.
|
||||
alt="Honeycomb traces and logs correlation"
|
||||
width="1000px"
|
||||
/>
|
||||
|
||||
## Examples
|
||||
|
||||
Check out the
|
||||
[enterprise observability boilerplate](https://github.com/hasura/graphql-engine/tree/master/community/boilerplates/observability/enterprise).
|
||||
|
||||
@ -29,3 +29,4 @@ OpenTelemetry's capabilities to meet your observability needs.
|
||||
|
||||
- [Export OTEL information for Hasura GraphQL Engine](/observability/opentelemetry/graphql-engine.mdx)
|
||||
- [Export OTEL information for data connectors](/observability/opentelemetry/data-connectors.mdx)
|
||||
- [OpenTelemetry best practices](/observability/opentelemetry/best-practices.mdx)
|
||||
|
||||
BIN
docs/static/img/observability/opentelemetry-logs-structure-comparison.png
vendored
Normal file
BIN
docs/static/img/observability/opentelemetry-logs-structure-comparison.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 162 KiB |
BIN
docs/static/img/observability/opentelemetry-trace-metadata.png
vendored
Normal file
BIN
docs/static/img/observability/opentelemetry-trace-metadata.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 220 KiB |
BIN
docs/static/img/observability/opentelemetry-trace-query.png
vendored
Normal file
BIN
docs/static/img/observability/opentelemetry-trace-query.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 217 KiB |
BIN
docs/static/img/observability/opentelemetry-trace-scheduled-trigger.png
vendored
Normal file
BIN
docs/static/img/observability/opentelemetry-trace-scheduled-trigger.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 200 KiB |
BIN
docs/static/img/observability/opentelemetry-trace-subscription.png
vendored
Normal file
BIN
docs/static/img/observability/opentelemetry-trace-subscription.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 219 KiB |
Loading…
Reference in New Issue
Block a user