@@ -161,9 +161,6 @@ backgroundPruneInterval = 30000
161-- Long-poll hold timeout before returning empty to the client (25 s).
161-- Long-poll hold timeout before returning empty to the client (25 s).
162pollTimeoutMs : Int<Ms>
162pollTimeoutMs : Int<Ms>
163pollTimeoutMs = 25000
163pollTimeoutMs = 25000
164−-- Concurrent active long-polls permitted per pubkey.
167-- Cadence at which this server refreshes its own presence to peers (5 min).
164-- Cadence at which this server refreshes its own presence to peers (5 min).
168presenceRefreshInterval : Int<Ms>
165presenceRefreshInterval : Int<Ms>
169presenceRefreshInterval = 300000
166presenceRefreshInterval = 300000
@@ -264,10 +261,6 @@ type ForwardFailure = {server: Text, failedAt: Timestamp, failures: Int, firstFa
264-- time at which the current window began; once `t - windowStart` exceeds the
261-- time at which the current window began; once `t - windowStart` exceeds the
265-- configured window length the entry is recycled (counter resets to 1).
262-- configured window length the entry is recycled (counter resets to 1).
266type RateEntry = {key: Text, windowStart: Timestamp, count: Int}
263type RateEntry = {key: Text, windowStart: Timestamp, count: Int}
267−-- A live long-poll slot held by a pubkey; used to cap concurrency per key.
268−-- `startedAt` lets `pruneExpired` reap slots whose holder vanished without
269−-- cleanly exiting `pollOnce` (network drop mid-poll).
270−type ActivePoll = {pubkey: PubkeyHex, startedAt: Timestamp}
272-- ============================================
265-- ============================================
273-- Persisted State (all flat lists, no sharding)
266-- Persisted State (all flat lists, no sharding)
@@ -292,8 +285,6 @@ type ActivePoll = {pubkey: PubkeyHex, startedAt: Timestamp}
292*forwards : [ForwardEntry]
285*forwards : [ForwardEntry]
293-- Per-peer failure counters; drives `retryBackoff` and the federation health view.
286-- Per-peer failure counters; drives `retryBackoff` and the federation health view.
294*forwardFailures : [ForwardFailure]
287*forwardFailures : [ForwardFailure]
295−-- Currently-blocking long-poll slots, used to cap concurrent /poll holds per pubkey.
296−*activePolls : [ActivePoll]
298-- Per-key fixed-window counters for /messages send-side rate limiting.
289-- Per-key fixed-window counters for /messages send-side rate limiting.
299*sendRates : [RateEntry]
290*sendRates : [RateEntry]
@@ -404,9 +395,9 @@ type KeyPresenceEntry = {server: Text, expiresAt: Timestamp}
404type RecipientCount = {pubkey: PubkeyHex, count: Int}
395type RecipientCount = {pubkey: PubkeyHex, count: Int}
406route StatsApi where
397route StatsApi where
407− GET /stats headers {authorization: Text} -> {ok: Bool, version: Text, capacity: {sessions: Int, sessionsMax: Int, sessionsPct: Int, messages: Int, totalMessageCount: Int, presence: Int, forwards: Int, challenges: Int, challengesMax: Int, forwardNonces: Int, forwardNoncesMax: Int, nonceEvictions: Int, activePolls: Int, recipientCounts: Int, messageBucketMax: Int, messageBucketMin: Int}, federation: {enabled: Bool, inFlight: Int, hungSlots: Int, avgLatencyMs: Int, maxLatencyMs: Int, callCount: Int, slowVerifies: Int, verifyFailures: Int, forwardFailures: Int, oldestForwardAge: Int, recentGossip: Int, blockedPeers: Int, forwardServers: Int, maxForwardQueueDepth: Int, topFailingPeers: [{server: Text, failures: Int}], gossipSuccess: Int, gossipFail: Int, latencyBuckets: {lt100: Int, lt500: Int, lt2000: Int, gte2000: Int}}, rates: {send: Int, challenge: Int, federation: Int, federationPair: Int, global: Int, ip: Int, lookup: Int, recv: Int, pair: Int, gossipEvent: Int, verifyIp: Int, uniqueKeyIp: Int, rejectedGlobal: Int, rejectedIp: Int, rejectedAuth: Int, rejectedPerKey: Int, rejectedFederation: Int}, timing: {pruneCount: Int, pruneRunning: Timestamp, heartbeatCount: Int, uptime: Int, lastPruneAt: Timestamp, lastHeartbeatAt: Timestamp, totalEntries: Int, activeRequests: Int, peakActiveRequests: Int, pollAvgMs: Int, pollMaxMs: Int, pollCount: Int, pollLatencyBuckets: {lt50: Int, lt200: Int, lt1000: Int, gte1000: Int}, sendAvgMs: Int, sendMaxMs: Int, sendCount: Int, sendLatencyBuckets: {lt50: Int, lt200: Int, lt1000: Int, gte1000: Int}, authAvgMs: Int, authMaxMs: Int, authCount: Int}} = Stats
398+ GET /stats headers {authorization: Text} -> {ok: Bool, version: Text, capacity: {sessions: Int, sessionsMax: Int, sessionsPct: Int, messages: Int, totalMessageCount: Int, presence: Int, forwards: Int, challenges: Int, challengesMax: Int, forwardNonces: Int, forwardNoncesMax: Int, nonceEvictions: Int, recipientCounts: Int, messageBucketMax: Int, messageBucketMin: Int}, federation: {enabled: Bool, inFlight: Int, hungSlots: Int, avgLatencyMs: Int, maxLatencyMs: Int, callCount: Int, slowVerifies: Int, verifyFailures: Int, forwardFailures: Int, oldestForwardAge: Int, recentGossip: Int, blockedPeers: Int, forwardServers: Int, maxForwardQueueDepth: Int, topFailingPeers: [{server: Text, failures: Int}], gossipSuccess: Int, gossipFail: Int, latencyBuckets: {lt100: Int, lt500: Int, lt2000: Int, gte2000: Int}}, rates: {send: Int, challenge: Int, federation: Int, federationPair: Int, global: Int, ip: Int, lookup: Int, recv: Int, pair: Int, gossipEvent: Int, verifyIp: Int, uniqueKeyIp: Int, rejectedGlobal: Int, rejectedIp: Int, rejectedAuth: Int, rejectedPerKey: Int, rejectedFederation: Int}, timing: {pruneCount: Int, pruneRunning: Timestamp, heartbeatCount: Int, uptime: Int, lastPruneAt: Timestamp, lastHeartbeatAt: Timestamp, totalEntries: Int, activeRequests: Int, peakActiveRequests: Int, pollAvgMs: Int, pollMaxMs: Int, pollCount: Int, pollLatencyBuckets: {lt50: Int, lt200: Int, lt1000: Int, gte1000: Int}, sendAvgMs: Int, sendMaxMs: Int, sendCount: Int, sendLatencyBuckets: {lt50: Int, lt200: Int, lt1000: Int, gte1000: Int}, authAvgMs: Int, authMaxMs: Int, authCount: Int}} = Stats
408 GET /stats/peers headers {authorization: Text} -> {peers: [PeerStatus], ok: Bool} = StatsPeers
399 GET /stats/peers headers {authorization: Text} -> {peers: [PeerStatus], ok: Bool} = StatsPeers
409− GET /stats/key/{pubkey: PubkeyHex} headers {authorization: Text} -> {messages: Int, presence: [KeyPresenceEntry], sessions: Int, forwards: Int, forwardDetails: [{server: Text, retries: Int}], activePolls: Int, ok: Bool} = StatsKey
400+ GET /stats/key/{pubkey: PubkeyHex} headers {authorization: Text} -> {messages: Int, presence: [KeyPresenceEntry], sessions: Int, forwards: Int, forwardDetails: [{server: Text, retries: Int}], ok: Bool} = StatsKey
410 POST {limit: Int} /stats/top-recipients headers {authorization: Text} -> {recipients: [RecipientCount], ok: Bool, total: Int} = TopRecipients
401 POST {limit: Int} /stats/top-recipients headers {authorization: Text} -> {recipients: [RecipientCount], ok: Bool, total: Int} = TopRecipients
411 GET /stats/forwards headers {authorization: Text} -> {servers: [{server: Text, count: Int, totalRetries: Int}], ok: Bool, total: Int} = StatsForwards
402 GET /stats/forwards headers {authorization: Text} -> {servers: [{server: Text, count: Int, totalRetries: Int}], ok: Bool, total: Int} = StatsForwards
412 GET /stats/rates headers {authorization: Text} -> {ok: Bool, scalePct: Int, globalRequest: Int, ipRate: Int, sendRate: Int, federationRate: Int, authGlobal: Int, recvRate: Int, challengeRate: Int, pairRate: Int, lookupRate: Int} = StatsRates
403 GET /stats/rates headers {authorization: Text} -> {ok: Bool, scalePct: Int, globalRequest: Int, ipRate: Int, sendRate: Int, federationRate: Int, authGlobal: Int, recvRate: Int, challengeRate: Int, pairRate: Int, lookupRate: Int} = StatsRates
@@ -813,7 +804,7 @@ knownPeerServersFromPresence = \presence -> collectPeers presence (\p -> p.serve
813-- One sweep over every state ref: drops expired rows in a single atomic block.
804-- One sweep over every state ref: drops expired rows in a single atomic block.
814-- Returns the locally-expired presence rows whose pubkey has no other live
805-- Returns the locally-expired presence rows whose pubkey has no other live
815-- local session, so the caller can broadcast offline gossip for them.
806-- local session, so the caller can broadcast offline gossip for them.
816−pruneExpired : Int<Ms> -> IO {rw *activePolls, rw *challenges, rw *forwardFailures, rw *messages, rw *presence, rw *sessions} [{expiresAt: Timestamp, pubkey: PubkeyHex, server: ServerName}]
807+pruneExpired : Int<Ms> -> IO {rw *challenges, rw *forwardFailures, rw *messages, rw *presence, rw *sessions} [{expiresAt: Timestamp, pubkey: PubkeyHex, server: ServerName}]
817pruneExpired = \t -> atomic do
808pruneExpired = \t -> atomic do
818 ch <- *challenges
809 ch <- *challenges
819 *challenges = filter ch (\x -> x.expiresAt > t)
810 *challenges = filter ch (\x -> x.expiresAt > t)
@@ -823,8 +814,6 @@ pruneExpired = \t -> atomic do
823 *messages = filter msgs (\m -> t - m.receivedAt < messageMaxAge)
814 *messages = filter msgs (\m -> t - m.receivedAt < messageMaxAge)
824 ff <- *forwardFailures
815 ff <- *forwardFailures
825 *forwardFailures = filter ff (\f -> t - f.failedAt < 86400000)
816 *forwardFailures = filter ff (\f -> t - f.failedAt < 86400000)
827− *activePolls = filter aps (\p -> t - p.startedAt < pollTimeoutMs * 3)
828 -- presence: remember locally-expired keys for offline gossip
817 -- presence: remember locally-expired keys for offline gossip
829 pres <- *presence
818 pres <- *presence
830 *presence = filter pres (\p -> p.expiresAt > t)
819 *presence = filter pres (\p -> p.expiresAt > t)
@@ -1059,12 +1048,11 @@ handleVerifyAuth = \pubkey challenge signature revokeOthers clientIp -> do
1059-- * Ack-and-drop everything <= the cursor (treating poll progress as ack).
1048-- * Ack-and-drop everything <= the cursor (treating poll progress as ack).
1060-- * Wait on `pollWakeup` (bumped by send/heartbeat) and return new events
1049-- * Wait on `pollWakeup` (bumped by send/heartbeat) and return new events
1061-- plus piggybacked sender-side expirations/deliveries.
1050-- plus piggybacked sender-side expirations/deliveries.
1062−-- Per-pubkey concurrency is capped via `activePolls` to bound long-poll holds.
1064-- POST /poll: top-level dispatch. Validates the global rate gate, detects a
1052-- POST /poll: top-level dispatch. Validates the global rate gate, detects a
1065-- generation mismatch (server restart) to force a cursor reset, then hands
1053-- generation mismatch (server restart) to force a cursor reset, then hands
1066-- off to `pollOnce` for the actual long-poll body.
1054-- off to `pollOnce` for the actual long-poll body.
1067−handlePoll : Int -> Int -> Text -> Text -> IO {r *pollWakeup, r *serverGeneration, r *sessions, rw *activePolls, rw *forwards, rw *globalRateCount, rw *globalRateWindowStart, rw *messages, clock} {authorized: Bool, cursor: Timestamp, cursorReset: Bool, cursorStale: Bool, events: [PollEvent], generation: Int, hasMore: Bool, heartbeat: Bool, loadPct: Int, rateLimited: Bool, retryAfterMs: Int}
1055+handlePoll : Int -> Int -> Text -> Text -> IO {r *pollWakeup, r *serverGeneration, r *sessions, rw *forwards, rw *globalRateCount, rw *globalRateWindowStart, rw *messages, clock} {authorized: Bool, cursor: Timestamp, cursorReset: Bool, cursorStale: Bool, events: [PollEvent], generation: Int, hasMore: Bool, heartbeat: Bool, loadPct: Int, rateLimited: Bool, retryAfterMs: Int}
1068handlePoll = \cursor generation authorization clientIp -> do
1056handlePoll = \cursor generation authorization clientIp -> do
1069 t <- now
1057 t <- now
1070 serverGen <- *serverGeneration
1058 serverGen <- *serverGeneration
@@ -1076,9 +1064,9 @@ handlePoll = \cursor generation authorization clientIp -> do
1076 else pollOnce authorization (if cursorWasReset then 0 else cursor) cursorWasReset clientIp serverGen t
1064 else pollOnce authorization (if cursorWasReset then 0 else cursor) cursorWasReset clientIp serverGen t
1078-- Body of a single poll. Treats the cursor advance as an implicit ack
1066-- Body of a single poll. Treats the cursor advance as an implicit ack
1079−-- (`ackPolledMessages`), acquires a per-pubkey poll slot, runs `doPoll` to
1067+-- (`ackPolledMessages`), then runs `doPoll` to collect events (or block
1080−-- collect events (or block until wakeup), then assembles the response.
1068+-- until wakeup) and assembles the response.
1081−pollOnce : Text -> Int -> Bool -> Text -> Int -> Int<Ms> -> IO {r *pollWakeup, r *sessions, rw *activePolls, rw *forwards, rw *messages, clock} {authorized: Bool, cursor: Timestamp, cursorReset: Bool, cursorStale: Bool, events: [PollEvent], generation: Int, hasMore: Bool, heartbeat: Bool, loadPct: Int, rateLimited: Bool, retryAfterMs: Int}
1069+pollOnce : Text -> Int -> Bool -> Text -> Int -> Int<Ms> -> IO {r *pollWakeup, r *sessions, rw *forwards, rw *messages, clock} {authorized: Bool, cursor: Timestamp, cursorReset: Bool, cursorStale: Bool, events: [PollEvent], generation: Int, hasMore: Bool, heartbeat: Bool, loadPct: Int, rateLimited: Bool, retryAfterMs: Int}
1082pollOnce = \authorization cursor cursorWasReset clientIp serverGen t -> do
1070pollOnce = \authorization cursor cursorWasReset clientIp serverGen t -> do
1083 setup <- atomic withSessionAuth authorization t clientIp (yield (Nothing {})) (\pubkey -> do
1071 setup <- atomic withSessionAuth authorization t clientIp (yield (Nothing {})) (\pubkey -> do
1084 let clampedCursor = clamp 0 t cursor
1072 let clampedCursor = clamp 0 t cursor
@@ -1088,31 +1076,20 @@ pollOnce = \authorization cursor cursorWasReset clientIp serverGen t -> do
1088 case setup of
1076 case setup of
1089 Nothing {} -> yield (emptyPollResp 0 (False {}) (False {}) serverGen 0 cursorWasReset)
1077 Nothing {} -> yield (emptyPollResp 0 (False {}) (False {}) serverGen 0 cursorWasReset)
1090 Just {value: s} -> do
1078 Just {value: s} -> do
1091− acquired <- atomic (do
1092− aps <- *activePolls
1093− if countWhere aps (\p -> p.pubkey == s.pubkey) >= maxPollsPerKey
1094− then yield (False {})
1096− *activePolls = union aps [{s.pubkey, startedAt: t}]
1099− then yield (emptyPollResp cursor (True {}) (True {}) serverGen pollTimeoutMs cursorWasReset)
1101− pollResult <- doPoll s t
1079+ pollResult <- doPoll s t
1102− atomic (do aps <- *activePolls; *activePolls = filter aps (\p -> p.pubkey != s.pubkey))
1103− yield {
1080+ yield {
1104− pollResult.events,
1081+ pollResult.events,
1105− pollResult.cursor,
1082+ pollResult.cursor,
1106− authorized: True {},
1083+ authorized: True {},
1107− rateLimited: False {},
1084+ rateLimited: False {},
1108− generation: serverGen,
1085+ generation: serverGen,
1109− heartbeat: count pollResult.events == 0,
1086+ heartbeat: count pollResult.events == 0,
1110− retryAfterMs: 0,
1087+ retryAfterMs: 0,
1111− cursorReset: cursorWasReset,
1088+ cursorReset: cursorWasReset,
1112− pollResult.hasMore,
1089+ pollResult.hasMore,
1113− cursorStale: False {},
1090+ cursorStale: False {},
1114− loadPct: 0
1091+ loadPct: 0
1117-- One pass: collect events at `t`. If empty, retry-wait on `pollWakeup`
1094-- One pass: collect events at `t`. If empty, retry-wait on `pollWakeup`
1118-- (bumped by send/heartbeat) and re-collect once at the new wall time.
1095-- (bumped by send/heartbeat) and re-collect once at the new wall time.
@@ -1400,7 +1377,7 @@ handleFederationBatchForward = \forwards _ -> do
1400-- POST /maintenance/prune: trigger an out-of-band `pruneAndRetry` tick.
1377-- POST /maintenance/prune: trigger an out-of-band `pruneAndRetry` tick.
1401-- Useful when the operator wants to force expiry/forward retries without
1378-- Useful when the operator wants to force expiry/forward retries without
1402-- waiting for the next `backgroundPrune` interval.
1379-- waiting for the next `backgroundPrune` interval.
1403−handlePrune : Text -> IO {r *runtimeMaintenanceSecret, rw *activePolls, rw *challenges, rw *forwardFailures, r *forwards, rw *lastPresenceRefresh, rw *lastPrunedAt, rw *messages, rw *presence, rw *sessions, clock} {ok: Bool}
1380+handlePrune : Text -> IO {r *runtimeMaintenanceSecret, rw *challenges, rw *forwardFailures, r *forwards, rw *lastPresenceRefresh, rw *lastPrunedAt, rw *messages, rw *presence, rw *sessions, clock} {ok: Bool}
1404handlePrune = \authorization -> do
1381handlePrune = \authorization -> do
1405 t <- now
1382 t <- now
1406 authorized <- verifyMaintenanceAuth authorization
1383 authorized <- verifyMaintenanceAuth authorization
@@ -1585,7 +1562,7 @@ handleGetConfig = \authorization -> do
1585-- GET /stats: capacity gauges + minimal federation/timing snapshot. Most of
1562-- GET /stats: capacity gauges + minimal federation/timing snapshot. Most of
1586-- the historical metric fields are zero-filled (see `zeroStats`) — only the
1563-- the historical metric fields are zero-filled (see `zeroStats`) — only the
1587-- counts that come straight from the state refs are populated.
1564-- counts that come straight from the state refs are populated.
1588−handleStats : Text -> IO {r *activePolls, r *challenges, r *forwards, r *heartbeatCount, r *lastHeartbeatAt, r *lastPrunedAt, r *messages, r *presence, r *pruneCount, r *runtimeMaintenanceSecret, r *sessions, r *startedAt, clock} {capacity: {activePolls: Int, challenges: Int, challengesMax: Int, forwardNonces: Int, forwardNoncesMax: Int, forwards: Int, messageBucketMax: Int, messageBucketMin: Int, messages: Int, nonceEvictions: Int, presence: Int, recipientCounts: Int, sessions: Int, sessionsMax: Int, sessionsPct: Int, totalMessageCount: Int}, federation: {avgLatencyMs: Int, blockedPeers: Int, callCount: Int, enabled: Bool, forwardFailures: Int, forwardServers: Int, gossipFail: Int, gossipSuccess: Int, hungSlots: Int, inFlight: Int, latencyBuckets: {gte2000: Int, lt100: Int, lt2000: Int, lt500: Int}, maxForwardQueueDepth: Int, maxLatencyMs: Int, oldestForwardAge: Int, recentGossip: Int, slowVerifies: Int, topFailingPeers: [{failures: Int, server: Text}], verifyFailures: Int}, ok: Bool, rates: {challenge: Int, federation: Int, federationPair: Int, global: Int, gossipEvent: Int, ip: Int, lookup: Int, pair: Int, recv: Int, rejectedAuth: Int, rejectedFederation: Int, rejectedGlobal: Int, rejectedIp: Int, rejectedPerKey: Int, send: Int, uniqueKeyIp: Int, verifyIp: Int}, timing: {activeRequests: Int, authAvgMs: Int, authCount: Int, authMaxMs: Int, heartbeatCount: Int, lastHeartbeatAt: Timestamp, lastPruneAt: Timestamp, peakActiveRequests: Int, pollAvgMs: Int, pollCount: Int, pollLatencyBuckets: {gte1000: Int, lt1000: Int, lt200: Int, lt50: Int}, pollMaxMs: Int, pruneCount: Int, pruneRunning: Timestamp, sendAvgMs: Int, sendCount: Int, sendLatencyBuckets: {gte1000: Int, lt1000: Int, lt200: Int, lt50: Int}, sendMaxMs: Int, totalEntries: Int, uptime: Int}, version: Text}
1565+handleStats : Text -> IO {r *challenges, r *forwards, r *heartbeatCount, r *lastHeartbeatAt, r *lastPrunedAt, r *messages, r *presence, r *pruneCount, r *runtimeMaintenanceSecret, r *sessions, r *startedAt, clock} {capacity: {challenges: Int, challengesMax: Int, forwardNonces: Int, forwardNoncesMax: Int, forwards: Int, messageBucketMax: Int, messageBucketMin: Int, messages: Int, nonceEvictions: Int, presence: Int, recipientCounts: Int, sessions: Int, sessionsMax: Int, sessionsPct: Int, totalMessageCount: Int}, federation: {avgLatencyMs: Int, blockedPeers: Int, callCount: Int, enabled: Bool, forwardFailures: Int, forwardServers: Int, gossipFail: Int, gossipSuccess: Int, hungSlots: Int, inFlight: Int, latencyBuckets: {gte2000: Int, lt100: Int, lt2000: Int, lt500: Int}, maxForwardQueueDepth: Int, maxLatencyMs: Int, oldestForwardAge: Int, recentGossip: Int, slowVerifies: Int, topFailingPeers: [{failures: Int, server: Text}], verifyFailures: Int}, ok: Bool, rates: {challenge: Int, federation: Int, federationPair: Int, global: Int, gossipEvent: Int, ip: Int, lookup: Int, pair: Int, recv: Int, rejectedAuth: Int, rejectedFederation: Int, rejectedGlobal: Int, rejectedIp: Int, rejectedPerKey: Int, send: Int, uniqueKeyIp: Int, verifyIp: Int}, timing: {activeRequests: Int, authAvgMs: Int, authCount: Int, authMaxMs: Int, heartbeatCount: Int, lastHeartbeatAt: Timestamp, lastPruneAt: Timestamp, peakActiveRequests: Int, pollAvgMs: Int, pollCount: Int, pollLatencyBuckets: {gte1000: Int, lt1000: Int, lt200: Int, lt50: Int}, pollMaxMs: Int, pruneCount: Int, pruneRunning: Timestamp, sendAvgMs: Int, sendCount: Int, sendLatencyBuckets: {gte1000: Int, lt1000: Int, lt200: Int, lt50: Int}, sendMaxMs: Int, totalEntries: Int, uptime: Int}, version: Text}
1589handleStats = \authorization -> withMaintAuth authorization (yield zeroStats) (do
1566handleStats = \authorization -> withMaintAuth authorization (yield zeroStats) (do
1590 t <- now
1567 t <- now
1591 sa <- *startedAt
1568 sa <- *startedAt
@@ -1593,7 +1570,6 @@ handleStats = \authorization -> withMaintAuth authorization (yield zeroStats) (d
1593 msgs <- *messages
1570 msgs <- *messages
1594 pres <- *presence
1571 pres <- *presence
1595 fwds <- *forwards
1572 fwds <- *forwards
1596− aps <- *activePolls
1597 chs <- *challenges
1573 chs <- *challenges
1598 pc <- *pruneCount
1574 pc <- *pruneCount
1599 hc <- *heartbeatCount
1575 hc <- *heartbeatCount
@@ -1619,7 +1595,6 @@ handleStats = \authorization -> withMaintAuth authorization (yield zeroStats) (d
1619 forwardNonces: 0,
1595 forwardNonces: 0,
1620 forwardNoncesMax: 0,
1596 forwardNoncesMax: 0,
1621 nonceEvictions: 0,
1597 nonceEvictions: 0,
1622− activePolls: count aps,
1623 recipientCounts: 0,
1598 recipientCounts: 0,
1624 messageBucketMax: 0,
1599 messageBucketMax: 0,
1625 messageBucketMin: 0
1600 messageBucketMin: 0
@@ -1690,7 +1665,7 @@ handleStats = \authorization -> withMaintAuth authorization (yield zeroStats) (d
1690-- All-zero /stats payload returned to unauthorized callers. The shape is kept
1665-- All-zero /stats payload returned to unauthorized callers. The shape is kept
1691-- in lockstep with the response in `handleStats` so older operator tooling
1666-- in lockstep with the response in `handleStats` so older operator tooling
1692-- never sees a missing field.
1667-- never sees a missing field.
1693−zeroStats : {capacity: {activePolls: Int, challenges: Int, challengesMax: Int, forwardNonces: Int, forwardNoncesMax: Int, forwards: Int, messageBucketMax: Int, messageBucketMin: Int, messages: Int, nonceEvictions: Int, presence: Int, recipientCounts: Int, sessions: Int, sessionsMax: Int, sessionsPct: Int, totalMessageCount: Int}, federation: {avgLatencyMs: Int, blockedPeers: Int, callCount: Int, enabled: Bool, forwardFailures: Int, forwardServers: Int, gossipFail: Int, gossipSuccess: Int, hungSlots: Int, inFlight: Int, latencyBuckets: {gte2000: Int, lt100: Int, lt2000: Int, lt500: Int}, maxForwardQueueDepth: Int, maxLatencyMs: Int, oldestForwardAge: Int, recentGossip: Int, slowVerifies: Int, topFailingPeers: [{failures: Int, server: Text}], verifyFailures: Int}, ok: Bool, rates: {challenge: Int, federation: Int, federationPair: Int, global: Int, gossipEvent: Int, ip: Int, lookup: Int, pair: Int, recv: Int, rejectedAuth: Int, rejectedFederation: Int, rejectedGlobal: Int, rejectedIp: Int, rejectedPerKey: Int, send: Int, uniqueKeyIp: Int, verifyIp: Int}, timing: {activeRequests: Int, authAvgMs: Int, authCount: Int, authMaxMs: Int, heartbeatCount: Int, lastHeartbeatAt: Timestamp, lastPruneAt: Timestamp, peakActiveRequests: Int, pollAvgMs: Int, pollCount: Int, pollLatencyBuckets: {gte1000: Int, lt1000: Int, lt200: Int, lt50: Int}, pollMaxMs: Int, pruneCount: Int, pruneRunning: Timestamp, sendAvgMs: Int, sendCount: Int, sendLatencyBuckets: {gte1000: Int, lt1000: Int, lt200: Int, lt50: Int}, sendMaxMs: Int, totalEntries: Int, uptime: Int}, version: Text}
1668+zeroStats : {capacity: {challenges: Int, challengesMax: Int, forwardNonces: Int, forwardNoncesMax: Int, forwards: Int, messageBucketMax: Int, messageBucketMin: Int, messages: Int, nonceEvictions: Int, presence: Int, recipientCounts: Int, sessions: Int, sessionsMax: Int, sessionsPct: Int, totalMessageCount: Int}, federation: {avgLatencyMs: Int, blockedPeers: Int, callCount: Int, enabled: Bool, forwardFailures: Int, forwardServers: Int, gossipFail: Int, gossipSuccess: Int, hungSlots: Int, inFlight: Int, latencyBuckets: {gte2000: Int, lt100: Int, lt2000: Int, lt500: Int}, maxForwardQueueDepth: Int, maxLatencyMs: Int, oldestForwardAge: Int, recentGossip: Int, slowVerifies: Int, topFailingPeers: [{failures: Int, server: Text}], verifyFailures: Int}, ok: Bool, rates: {challenge: Int, federation: Int, federationPair: Int, global: Int, gossipEvent: Int, ip: Int, lookup: Int, pair: Int, recv: Int, rejectedAuth: Int, rejectedFederation: Int, rejectedGlobal: Int, rejectedIp: Int, rejectedPerKey: Int, send: Int, uniqueKeyIp: Int, verifyIp: Int}, timing: {activeRequests: Int, authAvgMs: Int, authCount: Int, authMaxMs: Int, heartbeatCount: Int, lastHeartbeatAt: Timestamp, lastPruneAt: Timestamp, peakActiveRequests: Int, pollAvgMs: Int, pollCount: Int, pollLatencyBuckets: {gte1000: Int, lt1000: Int, lt200: Int, lt50: Int}, pollMaxMs: Int, pruneCount: Int, pruneRunning: Timestamp, sendAvgMs: Int, sendCount: Int, sendLatencyBuckets: {gte1000: Int, lt1000: Int, lt200: Int, lt50: Int}, sendMaxMs: Int, totalEntries: Int, uptime: Int}, version: Text}
1694zeroStats = {
1669zeroStats = {
1695 ok: False {},
1670 ok: False {},
1696 version: serverBuildVersion,
1671 version: serverBuildVersion,
@@ -1707,7 +1682,6 @@ zeroStats = {
1707 forwardNonces: 0,
1682 forwardNonces: 0,
1708 forwardNoncesMax: 0,
1683 forwardNoncesMax: 0,
1709 nonceEvictions: 0,
1684 nonceEvictions: 0,
1711 recipientCounts: 0,
1685 recipientCounts: 0,
1712 messageBucketMax: 0,
1686 messageBucketMax: 0,
1713 messageBucketMin: 0
1687 messageBucketMin: 0
@@ -1800,15 +1774,14 @@ handleStatsPeers = \authorization -> withMaintAuth authorization (yield {peers:
1802-- GET /stats/key/{pubkey}: per-key footprint — message count, presence rows,
1776-- GET /stats/key/{pubkey}: per-key footprint — message count, presence rows,
1803−-- session count, queued forwards (with details), and active long-poll count.
1777+-- session count, and queued forwards (with details).
1804−handleStatsKey : PubkeyHex -> Text -> IO {r *activePolls, r *forwards, r *messages, r *presence, r *runtimeMaintenanceSecret, r *sessions, clock} {activePolls: Int, forwardDetails: [{retries: Int, server: ServerName}], forwards: Int, messages: Int, ok: Bool, presence: [{expiresAt: Int<Ms>, server: ServerName}], sessions: Int}
1778+handleStatsKey : PubkeyHex -> Text -> IO {r *forwards, r *messages, r *presence, r *runtimeMaintenanceSecret, r *sessions, clock} {forwardDetails: [{retries: Int, server: ServerName}], forwards: Int, messages: Int, ok: Bool, presence: [{expiresAt: Int<Ms>, server: ServerName}], sessions: Int}
1805handleStatsKey = \pubkey authorization -> withMaintAuth authorization (yield {
1779handleStatsKey = \pubkey authorization -> withMaintAuth authorization (yield {
1806 messages: 0,
1780 messages: 0,
1807 presence: [],
1781 presence: [],
1808 sessions: 0,
1782 sessions: 0,
1809 forwards: 0,
1783 forwards: 0,
1810 forwardDetails: [],
1784 forwardDetails: [],
1812 ok: False {}
1785 ok: False {}
1814 t <- now
1787 t <- now
@@ -1816,7 +1789,6 @@ handleStatsKey = \pubkey authorization -> withMaintAuth authorization (yield {
1816 pres <- *presence
1789 pres <- *presence
1817 ses <- *sessions
1790 ses <- *sessions
1818 fwds <- *forwards
1791 fwds <- *forwards
1819− aps <- *activePolls
1820 let myFwds = filter fwds (\x -> x.toKey == pubkey)
1792 let myFwds = filter fwds (\x -> x.toKey == pubkey)
1822 messages: countWhere msgs (\x -> x.toKey == pubkey),
1794 messages: countWhere msgs (\x -> x.toKey == pubkey),
@@ -1827,7 +1799,6 @@ handleStatsKey = \pubkey authorization -> withMaintAuth authorization (yield {
1827 sessions: countWhere ses (\x -> x.pubkey == pubkey),
1799 sessions: countWhere ses (\x -> x.pubkey == pubkey),
1828 forwards: count myFwds,
1800 forwards: count myFwds,
1829 forwardDetails: pluck myFwds (\x -> {server: x.toServer, x.retries}),
1801 forwardDetails: pluck myFwds (\x -> {server: x.toServer, x.retries}),
1830− activePolls: countWhere aps (\x -> x.pubkey == pubkey),
1831 ok: True {}
1802 ok: True {}
@@ -1886,14 +1857,13 @@ handleStatsRates = \authorization -> do
1886-- GET /metrics: Prometheus-style text exposition of the most useful gauges.
1857-- GET /metrics: Prometheus-style text exposition of the most useful gauges.
1887-- The `authorization` header is accepted but not validated here — operators
1858-- The `authorization` header is accepted but not validated here — operators
1888-- relying on auth must front this endpoint with a reverse proxy that enforces it.
1859-- relying on auth must front this endpoint with a reverse proxy that enforces it.
1889−handleMetrics : a -> IO {r *activePolls, r *forwards, r *messages, r *presence, r *sessions, r *startedAt, clock} {metrics: Text, ok: Bool}
1860+handleMetrics : a -> IO {r *forwards, r *messages, r *presence, r *sessions, r *startedAt, clock} {metrics: Text, ok: Bool}
1890handleMetrics = \authorization -> do
1861handleMetrics = \authorization -> do
1891 t <- now
1862 t <- now
1892 msgs <- *messages
1863 msgs <- *messages
1893 ses <- *sessions
1864 ses <- *sessions
1894 pres <- *presence
1865 pres <- *presence
1895 fwds <- *forwards
1866 fwds <- *forwards
1896− aps <- *activePolls
1897 sa <- *startedAt
1867 sa <- *startedAt
1899 metrics: fold (\acc r -> acc ++ r.k ++ " " ++ show r.v ++ "\n") "" [
1869 metrics: fold (\acc r -> acc ++ r.k ++ " " ++ show r.v ++ "\n") "" [
@@ -1901,7 +1871,6 @@ handleMetrics = \authorization -> do
1901 {k: "skrepka_sessions_total", v: count ses},
1871 {k: "skrepka_sessions_total", v: count ses},
1902 {k: "skrepka_presence_total", v: count pres},
1872 {k: "skrepka_presence_total", v: count pres},
1903 {k: "skrepka_forwards_total", v: count fwds},
1873 {k: "skrepka_forwards_total", v: count fwds},
1904− {k: "skrepka_active_polls", v: count aps},
1905 {k: "skrepka_uptime_seconds", v: (t - sa) / 1000}
1874 {k: "skrepka_uptime_seconds", v: (t - sa) / 1000}
1907 ok: True {}
1876 ok: True {}
@@ -1958,7 +1927,7 @@ api = serve Api where
1959-- Endless tick: every `backgroundPruneInterval` run `pruneAndRetry` and bump
1928-- Endless tick: every `backgroundPruneInterval` run `pruneAndRetry` and bump
1960-- `pruneCount` for /stats and /health.
1929-- `pruneCount` for /stats and /health.
1961−backgroundPrune : IO {rw *activePolls, rw *challenges, rw *forwardFailures, r *forwards, rw *lastPresenceRefresh, rw *lastPrunedAt, rw *messages, rw *presence, rw *pruneCount, rw *sessions, clock} a
1930+backgroundPrune : IO {rw *challenges, rw *forwardFailures, r *forwards, rw *lastPresenceRefresh, rw *lastPrunedAt, rw *messages, rw *presence, rw *pruneCount, rw *sessions, clock} a
1962backgroundPrune = do
1931backgroundPrune = do
1963 sleep backgroundPruneInterval
1932 sleep backgroundPruneInterval
1964 t <- now
1933 t <- now