mirror of
https://github.com/StanGirard/quivr.git
synced 2024-11-30 21:28:56 +03:00
87138635aa
# Description Please include a summary of the changes and the related issue. Please also include relevant motivation and context. ## Checklist before requesting a review Please delete options that are not relevant. - [ ] My code follows the style guidelines of this project - [ ] I have performed a self-review of my code - [ ] I have commented hard-to-understand areas - [ ] I have ideally added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged ## Screenshots (if appropriate):
233 lines
7.1 KiB
YAML
233 lines
7.1 KiB
YAML
api:
|
|
enabled: true
|
|
address: 0.0.0.0:9001
|
|
|
|
sources:
|
|
docker_host:
|
|
type: docker_logs
|
|
exclude_containers:
|
|
- supabase-vector
|
|
|
|
transforms:
|
|
project_logs:
|
|
type: remap
|
|
inputs:
|
|
- docker_host
|
|
source: |-
|
|
.project = "default"
|
|
.event_message = del(.message)
|
|
.appname = del(.container_name)
|
|
del(.container_created_at)
|
|
del(.container_id)
|
|
del(.source_type)
|
|
del(.stream)
|
|
del(.label)
|
|
del(.image)
|
|
del(.host)
|
|
del(.stream)
|
|
router:
|
|
type: route
|
|
inputs:
|
|
- project_logs
|
|
route:
|
|
kong: '.appname == "supabase-kong"'
|
|
auth: '.appname == "supabase-auth"'
|
|
rest: '.appname == "supabase-rest"'
|
|
realtime: '.appname == "supabase-realtime"'
|
|
storage: '.appname == "supabase-storage"'
|
|
functions: '.appname == "supabase-functions"'
|
|
db: '.appname == "supabase-db"'
|
|
# Ignores non nginx errors since they are related with kong booting up
|
|
kong_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.kong
|
|
source: |-
|
|
req, err = parse_nginx_log(.event_message, "combined")
|
|
if err == null {
|
|
.timestamp = req.timestamp
|
|
.metadata.request.headers.referer = req.referer
|
|
.metadata.request.headers.user_agent = req.agent
|
|
.metadata.request.headers.cf_connecting_ip = req.client
|
|
.metadata.request.method = req.method
|
|
.metadata.request.path = req.path
|
|
.metadata.request.protocol = req.protocol
|
|
.metadata.response.status_code = req.status
|
|
}
|
|
if err != null {
|
|
abort
|
|
}
|
|
# Ignores non nginx errors since they are related with kong booting up
|
|
kong_err:
|
|
type: remap
|
|
inputs:
|
|
- router.kong
|
|
source: |-
|
|
.metadata.request.method = "GET"
|
|
.metadata.response.status_code = 200
|
|
parsed, err = parse_nginx_log(.event_message, "error")
|
|
if err == null {
|
|
.timestamp = parsed.timestamp
|
|
.severity = parsed.severity
|
|
.metadata.request.host = parsed.host
|
|
.metadata.request.headers.cf_connecting_ip = parsed.client
|
|
url, err = split(parsed.request, " ")
|
|
if err == null {
|
|
.metadata.request.method = url[0]
|
|
.metadata.request.path = url[1]
|
|
.metadata.request.protocol = url[2]
|
|
}
|
|
}
|
|
if err != null {
|
|
abort
|
|
}
|
|
# Gotrue logs are structured json strings which frontend parses directly. But we keep metadata for consistency.
|
|
auth_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.auth
|
|
source: |-
|
|
parsed, err = parse_json(.event_message)
|
|
if err == null {
|
|
.metadata.timestamp = parsed.time
|
|
.metadata = merge!(.metadata, parsed)
|
|
}
|
|
# PostgREST logs are structured so we separate timestamp from message using regex
|
|
rest_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.rest
|
|
source: |-
|
|
parsed, err = parse_regex(.event_message, r'^(?P<time>.*): (?P<msg>.*)$')
|
|
if err == null {
|
|
.event_message = parsed.msg
|
|
.timestamp = to_timestamp!(parsed.time)
|
|
.metadata.host = .project
|
|
}
|
|
# Realtime logs are structured so we parse the severity level using regex (ignore time because it has no date)
|
|
realtime_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.realtime
|
|
source: |-
|
|
.metadata.project = del(.project)
|
|
.metadata.external_id = .metadata.project
|
|
parsed, err = parse_regex(.event_message, r'^(?P<time>\d+:\d+:\d+\.\d+) \[(?P<level>\w+)\] (?P<msg>.*)$')
|
|
if err == null {
|
|
.event_message = parsed.msg
|
|
.metadata.level = parsed.level
|
|
}
|
|
# Storage logs may contain json objects so we parse them for completeness
|
|
storage_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.storage
|
|
source: |-
|
|
.metadata.project = del(.project)
|
|
.metadata.tenantId = .metadata.project
|
|
parsed, err = parse_json(.event_message)
|
|
if err == null {
|
|
.event_message = parsed.msg
|
|
.metadata.level = parsed.level
|
|
.metadata.timestamp = parsed.time
|
|
.metadata.context[0].host = parsed.hostname
|
|
.metadata.context[0].pid = parsed.pid
|
|
}
|
|
# Postgres logs some messages to stderr which we map to warning severity level
|
|
db_logs:
|
|
type: remap
|
|
inputs:
|
|
- router.db
|
|
source: |-
|
|
.metadata.host = "db-default"
|
|
.metadata.parsed.timestamp = .timestamp
|
|
|
|
parsed, err = parse_regex(.event_message, r'.*(?P<level>INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC?):.*', numeric_groups: true)
|
|
|
|
if err != null || parsed == null {
|
|
.metadata.parsed.error_severity = "info"
|
|
}
|
|
if parsed != null {
|
|
.metadata.parsed.error_severity = parsed.level
|
|
}
|
|
if .metadata.parsed.error_severity == "info" {
|
|
.metadata.parsed.error_severity = "log"
|
|
}
|
|
.metadata.parsed.error_severity = upcase!(.metadata.parsed.error_severity)
|
|
|
|
sinks:
|
|
logflare_auth:
|
|
type: 'http'
|
|
inputs:
|
|
- auth_logs
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://analytics:4000/api/logs?source_name=gotrue.logs.prod&api_key=your-super-secret-and-long-logflare-key'
|
|
logflare_realtime:
|
|
type: 'http'
|
|
inputs:
|
|
- realtime_logs
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://analytics:4000/api/logs?source_name=realtime.logs.prod&api_key=your-super-secret-and-long-logflare-key'
|
|
logflare_rest:
|
|
type: 'http'
|
|
inputs:
|
|
- rest_logs
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://analytics:4000/api/logs?source_name=postgREST.logs.prod&api_key=your-super-secret-and-long-logflare-key'
|
|
logflare_db:
|
|
type: 'http'
|
|
inputs:
|
|
- db_logs
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
# We must route the sink through kong because ingesting logs before logflare is fully initialised will
|
|
# lead to broken queries from studio. This works by the assumption that containers are started in the
|
|
# following order: vector > db > logflare > kong
|
|
uri: 'http://kong:8000/analytics/v1/api/logs?source_name=postgres.logs&api_key=your-super-secret-and-long-logflare-key'
|
|
logflare_functions:
|
|
type: 'http'
|
|
inputs:
|
|
- router.functions
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://analytics:4000/api/logs?source_name=deno-relay-logs&api_key=your-super-secret-and-long-logflare-key'
|
|
logflare_storage:
|
|
type: 'http'
|
|
inputs:
|
|
- storage_logs
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://analytics:4000/api/logs?source_name=storage.logs.prod.2&api_key=your-super-secret-and-long-logflare-key'
|
|
logflare_kong:
|
|
type: 'http'
|
|
inputs:
|
|
- kong_logs
|
|
- kong_err
|
|
encoding:
|
|
codec: 'json'
|
|
method: 'post'
|
|
request:
|
|
retry_max_duration_secs: 10
|
|
uri: 'http://analytics:4000/api/logs?source_name=cloudflare.logs.prod&api_key=your-super-secret-and-long-logflare-key'
|