44b945b · skrepka · git.iko.soy

git.iko.soy/skrepka

Implement 15 server improvements: federation cancel/forward batching, snapshot v8 with senderExpirations, runtime secret rotation, stats enhancements, rate limiting, cache/IP-binding fixes, structured logging

44b945biko6 months ago

server.knot+175−69

compare with master

unifiedsplit

@@ -165,7 +165,7 @@ federationLatencyWindowMs : Int<Ms>

165federationLatencyWindowMs = 300000 -- 5 minutes; reset latency metrics after this interval

166

167-- Snapshot format version (increment on schema changes to prevent corrupt restores)

168−snapshotVersion = 7 -- bumped for pendingAckCleanups field in snapshot

168+snapshotVersion = 8 -- bumped for senderExpirations field in snapshot

169

170-- Server build version: included in /health and /stats responses for deployment verification.

171-- Update this string when deploying a new build so operators can confirm which version is running.

@@ -543,6 +543,8 @@ type SenderExpiration = {senderKey: Text, messageId: MessageIdHex, toKey: Pubkey

543-- Round-robin presence refresh: tracks which bucket group (0-3) to scan next.

544-- Each refresh cycle scans 4 buckets (0-3, 4-7, 8-11, 12-15), cycling through

545-- all 16 over 4 refresh intervals. Spreads gossip traffic across prune cycles.

546+-- NOTE: group only advances when shouldRefresh is true (enough time passed since

547+-- lastPresenceRefresh), so skipped/delayed prune cycles don't cause bucket gaps.

546*presenceRefreshGroup : Int

548*presenceRefreshGroup : Int

547

549

548*challengeRates : [ChallengeRateEntry]

550*challengeRates : [ChallengeRateEntry]

@@ -766,6 +768,11 @@ type SenderExpiration = {senderKey: Text, messageId: MessageIdHex, toKey: Pubkey

766-- Populated during prune for locally-originated messages (senderKey != "").

768-- Populated during prune for locally-originated messages (senderKey != "").

767*senderExpirations : [SenderExpiration]

769*senderExpirations : [SenderExpiration]

768

770

771+-- Runtime maintenance secret: when non-empty, used instead of compile-time maintenanceSecret.

772+-- Set via /maintenance/rotate-secret to enable zero-downtime secret rotation.

773+-- Empty = use compile-time maintenanceSecret (default).

774+*runtimeMaintenanceSecret : Text

775+

769-- Runtime-configurable rate limit overrides (0 = use compile-time default).

776-- Runtime-configurable rate limit overrides (0 = use compile-time default).

770-- Adjusted via /maintenance/rate-limits POST endpoint without recompile+restart.

777-- Adjusted via /maintenance/rate-limits POST endpoint without recompile+restart.

771*rateOverrideGlobalRequest : Int

778*rateOverrideGlobalRequest : Int

@@ -941,12 +948,12 @@ route MaintenanceApi where

941 -- Messages are excluded (ephemeral by design; clients handle re-delivery).

948 -- Messages are excluded (ephemeral by design; clients handle re-delivery).

942 -- Includes presence, acked/delivered IDs (dedup state), and peer failure records.

949 -- Includes presence, acked/delivered IDs (dedup state), and peer failure records.

943 GET /maintenance/snapshot headers {authorization: Text}

950 GET /maintenance/snapshot headers {authorization: Text}

944− -> {ok: Bool, version: Int, checksum: Int, presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure], forwards: [ForwardEntry], forwardNonces: [ForwardNonce], blockedPeers: [Text], allowedPeers: [Text], pendingAckCleanups: [{messageId: MessageIdHex, toKey: PubkeyHex, cleanupAfter: Timestamp}]}

951+ -> {ok: Bool, version: Int, checksum: Int, presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure], forwards: [ForwardEntry], forwardNonces: [ForwardNonce], blockedPeers: [Text], allowedPeers: [Text], pendingAckCleanups: [{messageId: MessageIdHex, toKey: PubkeyHex, cleanupAfter: Timestamp}], senderExpirations: [SenderExpiration]}

945 = Snapshot

952 = Snapshot

946 -- Restore: import a previously-exported snapshot. Server must be in drain mode

953 -- Restore: import a previously-exported snapshot. Server must be in drain mode

947 -- to prevent conflicts with live state. Run /maintenance/undrain after restore.

954 -- to prevent conflicts with live state. Run /maintenance/undrain after restore.

948 -- checksum: integrity check (0 = skip verification, for pre-v6 snapshots)

955 -- checksum: integrity check (0 = skip verification, for pre-v6 snapshots)

949− POST {version: Int, checksum: Int, presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure], forwards: [ForwardEntry], forwardNonces: [ForwardNonce], blockedPeers: [Text], allowedPeers: [Text], pendingAckCleanups: [{messageId: MessageIdHex, toKey: PubkeyHex, cleanupAfter: Timestamp}]} /maintenance/restore headers {authorization: Text}

956+ POST {version: Int, checksum: Int, presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure], forwards: [ForwardEntry], forwardNonces: [ForwardNonce], blockedPeers: [Text], allowedPeers: [Text], pendingAckCleanups: [{messageId: MessageIdHex, toKey: PubkeyHex, cleanupAfter: Timestamp}], senderExpirations: [SenderExpiration]} /maintenance/restore headers {authorization: Text}

950 -> {ok: Bool, error: Text, presenceCount: Int, ackedIdsCount: Int, deliveredIdsCount: Int}

957 -> {ok: Bool, error: Text, presenceCount: Int, ackedIdsCount: Int, deliveredIdsCount: Int}

951 = Restore

958 = Restore

952 -- Block a federation peer (rejects all inbound/outbound federation with this server)

959 -- Block a federation peer (rejects all inbound/outbound federation with this server)

@@ -1016,6 +1023,12 @@ route MaintenanceApi where

1016 POST {} /maintenance/reconcile headers {authorization: Text}

1023 POST {} /maintenance/reconcile headers {authorization: Text}

1017 -> {ok: Bool, messageDrift: Int, presenceDrift: Int, sessionDrift: Int}

1024 -> {ok: Bool, messageDrift: Int, presenceDrift: Int, sessionDrift: Int}

1018 = Reconcile

1025 = Reconcile

1026+ -- Rotate maintenance secret at runtime (zero-downtime rotation).

1027+ -- Authenticated with the current secret; sets a new runtime secret that overrides

1028+ -- the compile-time default. Pass empty newSecret to revert to compile-time secret.

1029+ POST {newSecret: Text} /maintenance/rotate-secret headers {authorization: Text}

1030+ -> {ok: Bool, error: Text}

1031+ = RotateSecret

1019

1032

1020route HealthApi where

1033route HealthApi where

1021 GET /health headers {authorization: Text} -> {ok: Bool, version: Text, pruneHealthy: Bool, heartbeatHealthy: Bool, capacityHealthy: Bool, federationHealthy: Bool, snapshotHealthy: Bool, watchdogHealthy: Bool, draining: Bool} = Health

1034 GET /health headers {authorization: Text} -> {ok: Bool, version: Text, pruneHealthy: Bool, heartbeatHealthy: Bool, capacityHealthy: Bool, federationHealthy: Bool, snapshotHealthy: Bool, watchdogHealthy: Bool, draining: Bool} = Health

@@ -1028,7 +1041,7 @@ route HealthApi where

1028-- BREAKING CHANGE from flat format — existing scrapers must update field paths.

1041-- BREAKING CHANGE from flat format — existing scrapers must update field paths.

1029route StatsApi where

1042route StatsApi where

1030 GET /stats headers {authorization: Text}

1043 GET /stats headers {authorization: Text}

1031− -> {ok: Bool, version: Text, capacity: {sessions: Int, sessionsMax: Int, sessionsPct: Int, messages: Int, messagesMax: Int, messagesPct: Int, totalMessageCount: Int, presence: Int, presenceMax: Int, presencePct: Int, forwards: Int, forwardsMax: Int, forwardsPct: Int, challenges: Int, challengesMax: Int, ackedIds: Int, ackedIdsMax: Int, ackOverflow: Int, ackOverflowMax: Int, deliveredIds: Int, deliveredIdsMax: Int, forwardNonces: Int, forwardNoncesMax: Int, activePolls: Int, activePollsMax: Int, recipientCounts: Int}, federation: {enabled: Bool, inFlight: Int, hungSlots: Int, avgLatencyMs: Int, maxLatencyMs: Int, callCount: Int, slowVerifies: Int, verifyFailures: Int, forwardFailures: Int, oldestForwardAge: Int, oldestAckAge: Int, recentGossip: Int, blockedPeers: Int, drainCompletedAt: Timestamp, forwardServers: Int, maxForwardQueueDepth: Int}, rates: {send: Int, challenge: Int, federation: Int, federationPair: Int, global: Int, ip: Int, lookup: Int, recv: Int, pair: Int, gossipEvent: Int, verifyIp: Int, uniqueKeyIp: Int, pendingAckCleanups: Int, senderExpirations: Int, rejectedGlobal: Int, rejectedIp: Int, rejectedAuth: Int, rejectedPerKey: Int, rejectedFederation: Int}, timing: {pruneCount: Int, pruneRunning: Timestamp, heartbeatCount: Int, uptime: Int, lastPruneAt: Timestamp, lastHeartbeatAt: Timestamp, draining: Bool, drainPhase: Int}}

1044+ -> {ok: Bool, version: Text, capacity: {sessions: Int, sessionsMax: Int, sessionsPct: Int, messages: Int, messagesMax: Int, messagesPct: Int, totalMessageCount: Int, presence: Int, presenceMax: Int, presencePct: Int, forwards: Int, forwardsMax: Int, forwardsPct: Int, challenges: Int, challengesMax: Int, ackedIds: Int, ackedIdsMax: Int, ackOverflow: Int, ackOverflowMax: Int, deliveredIds: Int, deliveredIdsMax: Int, forwardNonces: Int, forwardNoncesMax: Int, activePolls: Int, activePollsMax: Int, recipientCounts: Int}, federation: {enabled: Bool, inFlight: Int, hungSlots: Int, avgLatencyMs: Int, maxLatencyMs: Int, callCount: Int, slowVerifies: Int, verifyFailures: Int, forwardFailures: Int, oldestForwardAge: Int, oldestAckAge: Int, recentGossip: Int, blockedPeers: Int, drainCompletedAt: Timestamp, forwardServers: Int, maxForwardQueueDepth: Int}, rates: {send: Int, challenge: Int, federation: Int, federationPair: Int, global: Int, ip: Int, lookup: Int, recv: Int, pair: Int, gossipEvent: Int, verifyIp: Int, uniqueKeyIp: Int, pendingAckCleanups: Int, senderExpirations: Int, rejectedGlobal: Int, rejectedIp: Int, rejectedAuth: Int, rejectedPerKey: Int, rejectedFederation: Int}, timing: {pruneCount: Int, pruneRunning: Timestamp, heartbeatCount: Int, uptime: Int, lastPruneAt: Timestamp, lastHeartbeatAt: Timestamp, draining: Bool, drainPhase: Int, prunesSinceSnapshot: Int, totalEntries: Int}}

1032 = Stats

1045 = Stats

1033 GET /stats/peers headers {authorization: Text}

1046 GET /stats/peers headers {authorization: Text}

1034 -> {peers: [PeerStatus], ok: Bool}

1047 -> {peers: [PeerStatus], ok: Bool}

@@ -1060,6 +1073,13 @@ logInfo = \msg -> println ("[INFO] " ++ msg)

1060logWarn = \msg -> println ("[WARN] " ++ msg)

1073logWarn = \msg -> println ("[WARN] " ++ msg)

1061logError = \msg -> println ("[ERROR] " ++ msg)

1074logError = \msg -> println ("[ERROR] " ++ msg)

1062

1075

1076+-- JSON structured log helper for machine-parseable events.

1077+-- Outputs: {"level":"<level>","event":"<event>","msg":"<message>"}

1078+-- Use for key operational events that monitoring pipelines need to parse

1079+-- (rate limit rejections, federation failures, capacity warnings).

1080+-- Plain-text logInfo/logWarn/logError remain for human-readable context.

1081+logEvent = \level event msg -> println ("{\"level\":\"" ++ level ++ "\",\"event\":\"" ++ event ++ "\",\"msg\":\"" ++ msg ++ "\"}")

1082+

1063any : [a] -> (a -> Bool) -> Bool

1083any : [a] -> (a -> Bool) -> Bool

1064any = \xs pred -> fold (\acc x -> if acc then acc else pred x) (False {}) xs

1084any = \xs pred -> fold (\acc x -> if acc then acc else pred x) (False {}) xs

1065none : [a] -> (a -> Bool) -> Bool

1085none : [a] -> (a -> Bool) -> Bool

@@ -1381,8 +1401,14 @@ verifyAuthWith = \check authorization ->

1381 Nothing {} -> False {}

1401 Nothing {} -> False {}

1382 Just {value: token} -> check token

1402 Just {value: token} -> check token

1383

1403

1404+-- Check maintenance auth: tries runtime secret first (if set), then compile-time secret.

1405+-- This allows zero-downtime rotation: set the new secret via /maintenance/rotate-secret,

1406+-- then update the compile-time config and restart at convenience.

1384−verifyMaintenanceAuth : Text -> Bool

1407+verifyMaintenanceAuth : Text -> IO Bool

1408+verifyMaintenanceAuth = \authorization -> do

1409+ runtimeSecret <- *runtimeMaintenanceSecret

1410+ let checkSecret = if runtimeSecret != "" then runtimeSecret else maintenanceSecret

1385−verifyMaintenanceAuth = verifyAuthWith (\token -> verifySecret token maintenanceSecret)

1411+ yield (verifyAuthWith (\token -> verifySecret token checkSecret) authorization)

1386

1412

1387-- Check if a server is on the blocklist

1413-- Check if a server is on the blocklist

1388isBlockedPeer : Text -> [Text] -> Bool

1414isBlockedPeer : Text -> [Text] -> Bool

@@ -2394,16 +2420,21 @@ computeForwardStatsInner = \b t acc ->

2394-- Remove forward entries matching a server and message IDs, scanning per-bucket.

2420-- Remove forward entries matching a server and message IDs, scanning per-bucket.

2395-- Only rewrites buckets that actually changed (avoids writeAllForwards repartitioning).

2421-- Only rewrites buckets that actually changed (avoids writeAllForwards repartitioning).

2396-- Returns all removed forward entries across all buckets.

2422-- Returns all removed forward entries across all buckets.

2423+-- Batches the totalForwardCount decrement into a single atomic write at the end

2424+-- to reduce STM contention (vs per-bucket decrements contending on the shared counter).

2397removeForwardsByServer = \server messageIds b acc ->

2425removeForwardsByServer = \server messageIds b acc ->

2398− if b > 15 then yield acc

2426+ if b > 15 then do

2427+ let totalRemoved = count acc

2428+ when (totalRemoved > 0) (atomic do

2429+ tfc <- *totalForwardCount

2430+ set *totalForwardCount = max 0 (tfc - totalRemoved))

2431+ yield acc

2399 else do

2432 else do

2400 removed <- atomic do

2433 removed <- atomic do

2401 bucket <- readFwdBucket b

2434 bucket <- readFwdBucket b

2402 let matched = do f <- bucket; where f.toServer == server && any messageIds (\mid -> f.messageId == mid); yield f

2435 let matched = do f <- bucket; where f.toServer == server && any messageIds (\mid -> f.messageId == mid); yield f

2403− when (count matched > 0) (do

2436+ when (count matched > 0) (

2404− writeFwdBucket b (do f <- bucket; where not (f.toServer == server && any messageIds (\mid -> f.messageId == mid)); yield f)

2437+ writeFwdBucket b (do f <- bucket; where not (f.toServer == server && any messageIds (\mid -> f.messageId == mid)); yield f))

2405− tfc <- *totalForwardCount

2406− set *totalForwardCount = max 0 (tfc - count matched))

2407 yield matched

2438 yield matched

2408 removeForwardsByServer server messageIds (b + 1) (if count removed > 0 then union acc removed else acc)

2439 removeForwardsByServer server messageIds (b + 1) (if count removed > 0 then union acc removed else acc)

2409

2440

@@ -3611,7 +3642,9 @@ retryPendingForwards = \pending -> do

3611 t <- now

3642 t <- now

3612 failures <- *forwardFailures

3643 failures <- *forwardFailures

3613 if isFederationCircuitOpen failures t

3644 if isFederationCircuitOpen failures t

3645+ then do

3614− then logWarn ("federation: circuit breaker open (" ++ show (count failures) ++ " peers failing), skipping forward retries")

3646+ logWarn ("federation: circuit breaker open (" ++ show (count failures) ++ " peers failing), skipping forward retries")

3647+ logEvent "warn" "circuit_breaker_open" (show (count failures) ++ " peers failing")

3615 else do

3648 else do

3616 inFlight <- *federationInFlight

3649 inFlight <- *federationInFlight

3617 blocked <- *blockedPeers

3650 blocked <- *blockedPeers

@@ -4088,6 +4121,10 @@ ackPolledMessages = \pubkey cursor t -> do

4088

4121

4089pollNewEvents = \pubkey clampedCursor t -> do

4122pollNewEvents = \pubkey clampedCursor t -> do

4090 bucket <- readMessageBucket (pubkeyBucket pubkey)

4123 bucket <- readMessageBucket (pubkeyBucket pubkey)

4124+ -- NOTE: sortBy is technically redundant since receivedAt is monotonically increasing

4125+ -- (via nextSeqTs) and union appends new messages at the end. Kept as a defensive

4126+ -- measure in case bucket ordering assumptions are ever violated (e.g., by a future

4127+ -- bulk-import feature or runtime changes to list representation).

4091 let limitedMsgs = takeRelation maxPollEvents (sortBy (\m -> m.receivedAt) (pollableMessages bucket pubkey clampedCursor t))

4128 let limitedMsgs = takeRelation maxPollEvents (sortBy (\m -> m.receivedAt) (pollableMessages bucket pubkey clampedCursor t))

4092 let evts = do

4129 let evts = do

4093 m <- limitedMsgs

4130 m <- limitedMsgs

@@ -4245,8 +4282,15 @@ forwardCandidates = \targets messages forwards t ->

4245 originServer: origin, retries: 0})) [] targets

4282 originServer: origin, retries: 0})) [] targets

4246

4283

4247

4284

4285+-- Challenge-IP binding rules:

4286+-- 1. Both sides have IPs: must match (standard proxy case)

4287+-- 2. Challenge created without IP, verify without IP: accept (no-proxy case)

4288+-- 3. Challenge created without IP, verify with IP: accept (proxy added mid-flow)

4289+-- 4. Challenge created with IP, verify without IP: REJECT (proxy removed or bypassed)

4290+-- Case 4 prevents an attacker who intercepts the challenge string from verifying

4291+-- without going through the same reverse proxy that issued the challenge.

4248challengeMatch = \challenges pubkey challenge ip t ->

4292challengeMatch = \challenges pubkey challenge ip t ->

4249− any challenges (\c -> c.pubkey == pubkey && safeCompare c.challenge challenge && c.expiresAt > t && (ip == "" || c.ip == "" || c.ip == ip))

4293+ any challenges (\c -> c.pubkey == pubkey && safeCompare c.challenge challenge && c.expiresAt > t && (if c.ip == "" then True {} else if ip == "" then False {} else c.ip == ip))

4250

4294

4251challengesExcluding = \challenges pubkey challenge ->

4295challengesExcluding = \challenges pubkey challenge ->

4252 do c <- challenges; where c.pubkey != pubkey || c.challenge != challenge; yield c

4296 do c <- challenges; where c.pubkey != pubkey || c.challenge != challenge; yield c

@@ -4966,7 +5010,9 @@ handleVerifyToken = \token scope clientIp respond -> do

4966 when match (set *pendingVerifications = do p <- pending; where not (safeCompare p.token token); yield p)

5010 when match (set *pendingVerifications = do p <- pending; where not (safeCompare p.token token); yield p)

4967 yield match

5011 yield match

4968 -- Omit scope from log to prevent scope enumeration by attackers probing the endpoint

5012 -- Omit scope from log to prevent scope enumeration by attackers probing the endpoint

5013+ when (not found) (do

4969− when (not found) (logWarn ("federation/verify: failed attempt from " ++ clientIp))

5014+ logWarn ("federation/verify: failed attempt from " ++ clientIp)

5015+ logEvent "warn" "verify_failed" clientIp)

4970 yield (respond {ok: found})

5016 yield (respond {ok: found})

4971

5017

4972-- POST /federation/gossip: Receive presence gossip from a peer server.

5018-- POST /federation/gossip: Receive presence gossip from a peer server.

@@ -5286,7 +5332,12 @@ handleFederationCancel = \messageId toKey fromServer requestTimestamp correlatio

5286 writeRcBucket b (adjustRecipientCount rc toKey (0 - 1))

5332 writeRcBucket b (adjustRecipientCount rc toKey (0 - 1))

5287 -- Also cancel pending forwards for this message

5333 -- Also cancel pending forwards for this message

5288 fwdBkt <- readFwdBucket b

5334 fwdBkt <- readFwdBucket b

5289− writeFwdBucket b (do f <- fwdBkt; where not (f.messageId == messageId && f.toKey == toKey); yield f)

5335+ let keptFwds = do f <- fwdBkt; where not (f.messageId == messageId && f.toKey == toKey); yield f

5336+ writeFwdBucket b keptFwds

5337+ let fwdRemoved = count fwdBkt - count keptFwds

5338+ when (fwdRemoved > 0) (do

5339+ tfc <- *totalForwardCount

5340+ set *totalForwardCount = max 0 (tfc - fwdRemoved))

5290 yield {cancelled: True {}}

5341 yield {cancelled: True {}}

5291 when result.cancelled

5342 when result.cancelled

5292 (logInfo ("federation/cancel: cancelled " ++ messageId ++ " for " ++ toKey ++ " (from " ++ fromServer ++ ")"))

5343 (logInfo ("federation/cancel: cancelled " ++ messageId ++ " for " ++ toKey ++ " (from " ++ fromServer ++ ")"))

@@ -5298,7 +5349,7 @@ handleFederationCancel = \messageId toKey fromServer requestTimestamp correlatio

5298-- phase: 0 or 2 = full drain (backward compat default), 1 = soft drain (block new auth only)

5349-- phase: 0 or 2 = full drain (backward compat default), 1 = soft drain (block new auth only)

5299handleDrain = \phase authorization respond -> do

5350handleDrain = \phase authorization respond -> do

5300 t <- now

5351 t <- now

5301− let authorized = verifyMaintenanceAuth authorization

5352+ authorized <- verifyMaintenanceAuth authorization

5302 if not authorized then yield (respond {ok: False {}, drained: 0, phase: 0})

5353 if not authorized then yield (respond {ok: False {}, drained: 0, phase: 0})

5303 else do

5354 else do

5304 -- Effective phase: 0 or >=2 means full drain (backward compat for clients sending empty body)

5355 -- Effective phase: 0 or >=2 means full drain (backward compat for clients sending empty body)

@@ -5349,8 +5400,9 @@ handleDrain = \phase authorization respond -> do

5349 bp <- *blockedPeers

5400 bp <- *blockedPeers

5350 ap <- *allowedPeers

5401 ap <- *allowedPeers

5351 pac <- *pendingAckCleanups

5402 pac <- *pendingAckCleanups

5403+ se <- *senderExpirations

5352 let drainCs = snapshotChecksum (count drainResult.localEntries) (count a) (count d) (count flushState.forwards) (count fn)

5404 let drainCs = snapshotChecksum (count drainResult.localEntries) (count a) (count d) (count flushState.forwards) (count fn)

5353− yield {version: snapshotVersion, checksum: drainCs, presence: drainResult.localEntries, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: flushState.forwards, forwardNonces: fn, blockedPeers: bp, allowedPeers: ap, pendingAckCleanups: pac}

5405+ yield {version: snapshotVersion, checksum: drainCs, presence: drainResult.localEntries, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: flushState.forwards, forwardNonces: fn, blockedPeers: bp, allowedPeers: ap, pendingAckCleanups: pac, senderExpirations: se}

5354 -- Sentinel-wrapped: sidecar processes should verify [SNAPSHOT_END] follows to detect truncation

5406 -- Sentinel-wrapped: sidecar processes should verify [SNAPSHOT_END] follows to detect truncation

5355 println ("[SNAPSHOT_DATA] " ++ toJson drainSnapshot)

5407 println ("[SNAPSHOT_DATA] " ++ toJson drainSnapshot)

5356 println "[SNAPSHOT_END]"

5408 println "[SNAPSHOT_END]"

@@ -5396,7 +5448,7 @@ handleDrain = \phase authorization respond -> do

5396-- POST /maintenance/undrain: Resume accepting requests after a drain.

5448-- POST /maintenance/undrain: Resume accepting requests after a drain.

5397-- Background loops (prune, heartbeat) automatically resume on next iteration.

5449-- Background loops (prune, heartbeat) automatically resume on next iteration.

5398handleUndrain = \authorization respond -> do

5450handleUndrain = \authorization respond -> do

5399− let authorized = verifyMaintenanceAuth authorization

5451+ authorized <- verifyMaintenanceAuth authorization

5400 if not authorized then yield (respond {ok: False {}})

5452 if not authorized then yield (respond {ok: False {}})

5401 else do

5453 else do

5402 wasDraining <- atomic do

5454 wasDraining <- atomic do

@@ -5404,6 +5456,9 @@ handleUndrain = \authorization respond -> do

5404 set *drainPhase = 0

5456 set *drainPhase = 0

5405 set *drainCompletedAt = 0

5457 set *drainCompletedAt = 0

5406 set *shutdownRequested = False {}

5458 set *shutdownRequested = False {}

5459+ -- Clear stale verification cache: entries cached during drain may reference

5460+ -- peers whose trust state changed during the drain/undrain cycle.

5461+ set *verificationCache = []

5407 yield (d > 0)

5462 yield (d > 0)

5408 when wasDraining (logInfo "undrain: server resumed, accepting new requests")

5463 when wasDraining (logInfo "undrain: server resumed, accepting new requests")

5409 when (not wasDraining) (logInfo "undrain: server was not draining")

5464 when (not wasDraining) (logInfo "undrain: server was not draining")

@@ -5414,7 +5469,7 @@ handleUndrain = \authorization respond -> do

5414-- Unlike drain, shutdown is not undoable (requires process restart).

5469-- Unlike drain, shutdown is not undoable (requires process restart).

5415handleShutdown = \authorization respond -> do

5470handleShutdown = \authorization respond -> do

5416 t <- now

5471 t <- now

5417− let authorized = verifyMaintenanceAuth authorization

5472+ authorized <- verifyMaintenanceAuth authorization

5418 if not authorized then yield (respond {ok: False {}, drained: 0})

5473 if not authorized then yield (respond {ok: False {}, drained: 0})

5419 else do

5474 else do

5420 dp <- *drainPhase

5475 dp <- *drainPhase

@@ -5433,7 +5488,8 @@ handleShutdown = \authorization respond -> do

5433handlePrune = \authorization respond -> do

5488handlePrune = \authorization respond -> do

5434 t <- now

5489 t <- now

5435 dp <- *drainPhase

5490 dp <- *drainPhase

5491+ authOk <- verifyMaintenanceAuth authorization

5436− let authorized = verifyMaintenanceAuth authorization && dp < 2

5492+ let authorized = authOk && dp < 2

5437 when authorized (fork (pruneAndRetry t))

5493 when authorized (fork (pruneAndRetry t))

5438 yield (respond {ok: authorized})

5494 yield (respond {ok: authorized})

5439

5495

@@ -5441,7 +5497,7 @@ handlePrune = \authorization respond -> do

5441-- Returns drift values for messages, presence, and sessions. Non-zero drift means

5497-- Returns drift values for messages, presence, and sessions. Non-zero drift means

5442-- counters were corrected. Use when /stats shows suspicious counter values.

5498-- counters were corrected. Use when /stats shows suspicious counter values.

5443handleReconcile = \authorization respond -> do

5499handleReconcile = \authorization respond -> do

5444− let authorized = verifyMaintenanceAuth authorization

5500+ authorized <- verifyMaintenanceAuth authorization

5445 if not authorized then yield (respond {ok: False {}, messageDrift: 0, presenceDrift: 0, sessionDrift: 0})

5501 if not authorized then yield (respond {ok: False {}, messageDrift: 0, presenceDrift: 0, sessionDrift: 0})

5446 else do

5502 else do

5447 presDrift <- atomic do

5503 presDrift <- atomic do

@@ -5536,8 +5592,9 @@ handleHealth = \authorization respond -> do

5536 yield {ok: cOk && dp == 0, version: serverBuildVersion, pruneHealthy: cP, heartbeatHealthy: cH, capacityHealthy: cC, federationHealthy: cF, snapshotHealthy: cS, watchdogHealthy: cW, draining: dp > 0}

5592 yield {ok: cOk && dp == 0, version: serverBuildVersion, pruneHealthy: cP, heartbeatHealthy: cH, capacityHealthy: cC, federationHealthy: cF, snapshotHealthy: cS, watchdogHealthy: cW, draining: dp > 0}

5537 yield (respond cached)

5593 yield (respond cached)

5538 else do

5594 else do

5539− let isAuthed = verifyMaintenanceAuth authorization

5595+ isAuthed <- verifyMaintenanceAuth authorization

5540 result <- computeAndCacheHealth t

5596 result <- computeAndCacheHealth t

5597+ let allOk = not result.isDraining && result.pruneOk && result.heartbeatOk && result.capacityOk && result.federationOk && result.snapshotOk && result.watchdogOk

5541 -- Unauthenticated: return real ok boolean so load balancers can route around unhealthy nodes,

5598 -- Unauthenticated: return real ok boolean so load balancers can route around unhealthy nodes,

5542 -- but hide subsystem breakdown to prevent targeted load-correlation attacks.

5599 -- but hide subsystem breakdown to prevent targeted load-correlation attacks.

5543 -- Draining status is always visible (load balancers need it for routing decisions).

5600 -- Draining status is always visible (load balancers need it for routing decisions).

@@ -5557,14 +5614,14 @@ handleHealth = \authorization respond -> do

5557 watchdogHealthy: True {},

5614 watchdogHealthy: True {},

5558 draining: result.isDraining})

5615 draining: result.isDraining})

5559

5616

5560−zeroStats = {ok: False {}, version: serverBuildVersion, capacity: {sessions: 0, sessionsMax: 0, sessionsPct: 0, messages: 0, messagesMax: 0, messagesPct: 0, totalMessageCount: 0, presence: 0, presenceMax: 0, presencePct: 0, forwards: 0, forwardsMax: 0, forwardsPct: 0, challenges: 0, challengesMax: 0, ackedIds: 0, ackedIdsMax: 0, ackOverflow: 0, ackOverflowMax: 0, deliveredIds: 0, deliveredIdsMax: 0, forwardNonces: 0, forwardNoncesMax: 0, activePolls: 0, activePollsMax: 0, recipientCounts: 0}, federation: {enabled: False {}, inFlight: 0, hungSlots: 0, avgLatencyMs: 0, maxLatencyMs: 0, callCount: 0, slowVerifies: 0, verifyFailures: 0, forwardFailures: 0, oldestForwardAge: 0, oldestAckAge: 0, recentGossip: 0, blockedPeers: 0, drainCompletedAt: 0, forwardServers: 0, maxForwardQueueDepth: 0}, rates: {send: 0, challenge: 0, federation: 0, federationPair: 0, global: 0, ip: 0, lookup: 0, recv: 0, pair: 0, gossipEvent: 0, verifyIp: 0, uniqueKeyIp: 0, pendingAckCleanups: 0, senderExpirations: 0, rejectedGlobal: 0, rejectedIp: 0, rejectedAuth: 0, rejectedPerKey: 0, rejectedFederation: 0}, timing: {pruneCount: 0, pruneRunning: 0, heartbeatCount: 0, uptime: 0, lastPruneAt: 0, lastHeartbeatAt: 0, draining: False {}, drainPhase: 0}}

5617+zeroStats = {ok: False {}, version: serverBuildVersion, capacity: {sessions: 0, sessionsMax: 0, sessionsPct: 0, messages: 0, messagesMax: 0, messagesPct: 0, totalMessageCount: 0, presence: 0, presenceMax: 0, presencePct: 0, forwards: 0, forwardsMax: 0, forwardsPct: 0, challenges: 0, challengesMax: 0, ackedIds: 0, ackedIdsMax: 0, ackOverflow: 0, ackOverflowMax: 0, deliveredIds: 0, deliveredIdsMax: 0, forwardNonces: 0, forwardNoncesMax: 0, activePolls: 0, activePollsMax: 0, recipientCounts: 0}, federation: {enabled: False {}, inFlight: 0, hungSlots: 0, avgLatencyMs: 0, maxLatencyMs: 0, callCount: 0, slowVerifies: 0, verifyFailures: 0, forwardFailures: 0, oldestForwardAge: 0, oldestAckAge: 0, recentGossip: 0, blockedPeers: 0, drainCompletedAt: 0, forwardServers: 0, maxForwardQueueDepth: 0}, rates: {send: 0, challenge: 0, federation: 0, federationPair: 0, global: 0, ip: 0, lookup: 0, recv: 0, pair: 0, gossipEvent: 0, verifyIp: 0, uniqueKeyIp: 0, pendingAckCleanups: 0, senderExpirations: 0, rejectedGlobal: 0, rejectedIp: 0, rejectedAuth: 0, rejectedPerKey: 0, rejectedFederation: 0}, timing: {pruneCount: 0, pruneRunning: 0, heartbeatCount: 0, uptime: 0, lastPruneAt: 0, lastHeartbeatAt: 0, draining: False {}, drainPhase: 0, prunesSinceSnapshot: 0, totalEntries: 0}}

5561

5618

5562handleStats = \authorization respond -> do

5619handleStats = \authorization respond -> do

5563 t <- now

5620 t <- now

5564 globalRateLimited <- checkGlobalRate t

5621 globalRateLimited <- checkGlobalRate t

5565 if globalRateLimited then yield (respond zeroStats)

5622 if globalRateLimited then yield (respond zeroStats)

5566 else do

5623 else do

5567− let authorized = verifyMaintenanceAuth authorization

5624+ authorized <- verifyMaintenanceAuth authorization

5568 if authorized

5625 if authorized

5569 then do

5626 then do

5570 -- Split into two atomic reads: core capacity state (consistent) and counters/timing

5627 -- Split into two atomic reads: core capacity state (consistent) and counters/timing

@@ -5612,6 +5669,7 @@ handleStats = \authorization respond -> do

5612 lp <- *lastPrunedAt

5669 lp <- *lastPrunedAt

5613 lh <- *lastHeartbeatAt

5670 lh <- *lastHeartbeatAt

5614 dr <- *drainPhase

5671 dr <- *drainPhase

5672+ pss <- *prunesSinceSnapshot

5615 flt <- *federationLatencyTotal

5673 flt <- *federationLatencyTotal

5616 flc <- *federationLatencyCount

5674 flc <- *federationLatencyCount

5617 flm <- *federationLatencyMax

5675 flm <- *federationLatencyMax

@@ -5623,7 +5681,7 @@ handleStats = \authorization respond -> do

5623 rla <- *rateLimitedAuth

5681 rla <- *rateLimitedAuth

5624 rlk <- *rateLimitedPerKey

5682 rlk <- *rateLimitedPerKey

5625 rlf <- *rateLimitedFederation

5683 rlf <- *rateLimitedFederation

5626− yield {sr: sr, cr: cr, fr: fr, fpr: fpr, grc: grc, ir: ir, lr: lr, rr: rr, prr: prr, pc: pc, pr: pr, hc: hc, sa: sa, lp: lp, lh: lh, dr: dr, flt: flt, flc: flc, flm: flm, svc: svc, vfc: vfc, dca: dca, rlg: rlg, rli: rli, rla: rla, rlk: rlk, rlf: rlf}

5684+ yield {sr: sr, cr: cr, fr: fr, fpr: fpr, grc: grc, ir: ir, lr: lr, rr: rr, prr: prr, pc: pc, pr: pr, hc: hc, sa: sa, lp: lp, lh: lh, dr: dr, pss: pss, flt: flt, flc: flc, flm: flm, svc: svc, vfc: vfc, dca: dca, rlg: rlg, rli: rli, rla: rla, rlk: rlk, rlf: rlf}

5627 -- Compute derived metrics

5685 -- Compute derived metrics

5628 let sessions = core.sc

5686 let sessions = core.sc

5629 let messages = core.tmc

5687 let messages = core.tmc

@@ -5646,7 +5704,7 @@ handleStats = \authorization respond -> do

5646 capacity: {sessions: sessions, sessionsMax: maxSessions, sessionsPct: sessPct, messages: messages, messagesMax: maxMessages, messagesPct: msgPct, totalMessageCount: messages, presence: presence, presenceMax: maxPresenceEntries, presencePct: presPct, forwards: fwds, forwardsMax: maxForwardsEntries, forwardsPct: fwdPct, challenges: count core.c, challengesMax: maxChallenges, ackedIds: core.aCount, ackedIdsMax: maxAckedIds, ackOverflow: count core.ao, ackOverflowMax: maxAckOverflow, deliveredIds: core.dCount, deliveredIdsMax: maxDeliveredIds, forwardNonces: core.fnCount, forwardNoncesMax: maxForwardNonces, activePolls: count core.ap, activePollsMax: maxActivePolls, recipientCounts: count core.rc},

5704 capacity: {sessions: sessions, sessionsMax: maxSessions, sessionsPct: sessPct, messages: messages, messagesMax: maxMessages, messagesPct: msgPct, totalMessageCount: messages, presence: presence, presenceMax: maxPresenceEntries, presencePct: presPct, forwards: fwds, forwardsMax: maxForwardsEntries, forwardsPct: fwdPct, challenges: count core.c, challengesMax: maxChallenges, ackedIds: core.aCount, ackedIdsMax: maxAckedIds, ackOverflow: count core.ao, ackOverflowMax: maxAckOverflow, deliveredIds: core.dCount, deliveredIdsMax: maxDeliveredIds, forwardNonces: core.fnCount, forwardNoncesMax: maxForwardNonces, activePolls: count core.ap, activePollsMax: maxActivePolls, recipientCounts: count core.rc},

5647 federation: {enabled: federationEnabled, inFlight: count core.fi, hungSlots: hungFedSlots, avgLatencyMs: fedAvg, maxLatencyMs: counters.flm, callCount: counters.flc, slowVerifies: counters.svc, verifyFailures: counters.vfc, forwardFailures: count core.ff, oldestForwardAge: oldestFwdAge, oldestAckAge: oldestAckAge, recentGossip: count core.rg, blockedPeers: count core.bp, drainCompletedAt: counters.dca, forwardServers: fwdServerCount, maxForwardQueueDepth: maxFwdDepth},

5705 federation: {enabled: federationEnabled, inFlight: count core.fi, hungSlots: hungFedSlots, avgLatencyMs: fedAvg, maxLatencyMs: counters.flm, callCount: counters.flc, slowVerifies: counters.svc, verifyFailures: counters.vfc, forwardFailures: count core.ff, oldestForwardAge: oldestFwdAge, oldestAckAge: oldestAckAge, recentGossip: count core.rg, blockedPeers: count core.bp, drainCompletedAt: counters.dca, forwardServers: fwdServerCount, maxForwardQueueDepth: maxFwdDepth},

5648 rates: {send: count counters.sr, challenge: count counters.cr, federation: count counters.fr, federationPair: count counters.fpr, global: counters.grc, ip: count counters.ir, lookup: count counters.lr, recv: count counters.rr, pair: count counters.prr, gossipEvent: count core.ger, verifyIp: count core.vir, uniqueKeyIp: count core.ukir, pendingAckCleanups: count core.pac, senderExpirations: count core.se, rejectedGlobal: counters.rlg, rejectedIp: counters.rli, rejectedAuth: counters.rla, rejectedPerKey: counters.rlk, rejectedFederation: counters.rlf},

5706 rates: {send: count counters.sr, challenge: count counters.cr, federation: count counters.fr, federationPair: count counters.fpr, global: counters.grc, ip: count counters.ir, lookup: count counters.lr, recv: count counters.rr, pair: count counters.prr, gossipEvent: count core.ger, verifyIp: count core.vir, uniqueKeyIp: count core.ukir, pendingAckCleanups: count core.pac, senderExpirations: count core.se, rejectedGlobal: counters.rlg, rejectedIp: counters.rli, rejectedAuth: counters.rla, rejectedPerKey: counters.rlk, rejectedFederation: counters.rlf},

5649− timing: {pruneCount: counters.pc, pruneRunning: counters.pr, heartbeatCount: counters.hc, uptime: t - counters.sa, lastPruneAt: counters.lp, lastHeartbeatAt: counters.lh, draining: counters.dr > 0, drainPhase: counters.dr}})

5707+ timing: {pruneCount: counters.pc, pruneRunning: counters.pr, heartbeatCount: counters.hc, uptime: t - counters.sa, lastPruneAt: counters.lp, lastHeartbeatAt: counters.lh, draining: counters.dr > 0, drainPhase: counters.dr, prunesSinceSnapshot: counters.pss, totalEntries: sessions + messages + presence + fwds + count core.c + core.aCount + count core.ao + core.dCount + core.fnCount + count core.ap + count core.fi}})

5650 else yield (respond zeroStats)

5708 else yield (respond zeroStats)

5651

5709

5652handleStatsPeers = \authorization respond -> do

5710handleStatsPeers = \authorization respond -> do

@@ -5654,7 +5712,7 @@ handleStatsPeers = \authorization respond -> do

5654 globalRateLimited <- checkGlobalRate t

5712 globalRateLimited <- checkGlobalRate t

5655 if globalRateLimited then yield (respond {peers: [], ok: False {}})

5713 if globalRateLimited then yield (respond {peers: [], ok: False {}})

5656 else do

5714 else do

5657− let authorized = verifyMaintenanceAuth authorization

5715+ authorized <- verifyMaintenanceAuth authorization

5658 if authorized

5716 if authorized

5659 then do

5717 then do

5660 failures <- *forwardFailures

5718 failures <- *forwardFailures

@@ -5674,7 +5732,7 @@ handleStatsKey = \pubkey authorization respond -> do

5674 globalRateLimited <- checkGlobalRate t

5732 globalRateLimited <- checkGlobalRate t

5675 if globalRateLimited then yield (respond {messages: 0, presence: [], sessions: 0, forwards: 0, activePolls: 0, ok: False {}})

5733 if globalRateLimited then yield (respond {messages: 0, presence: [], sessions: 0, forwards: 0, activePolls: 0, ok: False {}})

5676 else do

5734 else do

5677− let authorized = verifyMaintenanceAuth authorization

5735+ authorized <- verifyMaintenanceAuth authorization

5678 if authorized

5736 if authorized

5679 then do

5737 then do

5680 result <- atomic do

5738 result <- atomic do

@@ -5703,7 +5761,7 @@ handleTopRecipients = \limit authorization respond -> do

5703 globalRateLimited <- checkGlobalRate t

5761 globalRateLimited <- checkGlobalRate t

5704 if globalRateLimited then yield (respond {recipients: [], ok: False {}, total: 0})

5762 if globalRateLimited then yield (respond {recipients: [], ok: False {}, total: 0})

5705 else do

5763 else do

5706− let authorized = verifyMaintenanceAuth authorization

5764+ authorized <- verifyMaintenanceAuth authorization

5707 if authorized

5765 if authorized

5708 then do

5766 then do

5709 let effectiveLimit = if limit <= 0 then 20 else clamp 1 100 limit

5767 let effectiveLimit = if limit <= 0 then 20 else clamp 1 100 limit

@@ -5719,20 +5777,26 @@ handleTopRecipients = \limit authorization respond -> do

5719-- Returns false until the first prune cycle completes (server fully initialized).

5777-- Returns false until the first prune cycle completes (server fully initialized).

5720handleReady = \respond -> do

5778handleReady = \respond -> do

5721 t <- now

5779 t <- now

5780+ -- Rate-limit /ready to prevent DoS (shares health rate limiter since both are

5781+ -- unauthenticated probes). Without this, /ready can be hit unlimited times

5782+ -- for cheap reconnaissance (detecting restarts) or resource exhaustion.

5783+ healthLimited <- checkHealthRate t

5784+ if healthLimited then yield (respond {ready: True {}})

5785+ else do

5722− sa <- *startedAt

5786+ sa <- *startedAt

5723− lastPrune <- *lastPrunedAt

5787+ lastPrune <- *lastPrunedAt

5724− -- Ready once the first prune cycle has run (indicates all background loops are alive)

5788+ -- Ready once the first prune cycle has run (indicates all background loops are alive)

5725− let isReady = lastPrune > 0 && lastPrune >= sa

5789+ let isReady = lastPrune > 0 && lastPrune >= sa

5726− yield (respond {ready: isReady})

5790+ yield (respond {ready: isReady})

5727

5791

5728−emptySnapshot = {ok: False {}, version: 0, checksum: 0, presence: [], ackedIds: [], ackOverflow: [], deliveredIds: [], forwardFailures: [], forwards: [], forwardNonces: [], blockedPeers: [], allowedPeers: [], pendingAckCleanups: []}

5792+emptySnapshot = {ok: False {}, version: 0, checksum: 0, presence: [], ackedIds: [], ackOverflow: [], deliveredIds: [], forwardFailures: [], forwards: [], forwardNonces: [], blockedPeers: [], allowedPeers: [], pendingAckCleanups: [], senderExpirations: []}

5729

5793

5730-- GET /maintenance/snapshot: Export federation metadata for persistence.

5794-- GET /maintenance/snapshot: Export federation metadata for persistence.

5731-- Reads all dedup state (ackedIds, deliveredIds), presence, peer failure records,

5795-- Reads all dedup state (ackedIds, deliveredIds), presence, peer failure records,

5732-- and pending forwards in a single atomic snapshot. Forward retry counters are

5796-- and pending forwards in a single atomic snapshot. Forward retry counters are

5733-- preserved so they survive restarts (prevents re-trying permanently-failing forwards).

5797-- preserved so they survive restarts (prevents re-trying permanently-failing forwards).

5734handleSnapshot = \authorization respond -> do

5798handleSnapshot = \authorization respond -> do

5735− let authorized = verifyMaintenanceAuth authorization

5799+ authorized <- verifyMaintenanceAuth authorization

5736 if not authorized then yield (respond emptySnapshot)

5800 if not authorized then yield (respond emptySnapshot)

5737 else do

5801 else do

5738 t <- now

5802 t <- now

@@ -5747,27 +5811,28 @@ handleSnapshot = \authorization respond -> do

5747 bp <- *blockedPeers

5811 bp <- *blockedPeers

5748 ap <- *allowedPeers

5812 ap <- *allowedPeers

5749 pac <- *pendingAckCleanups

5813 pac <- *pendingAckCleanups

5814+ se <- *senderExpirations

5750− yield {presence: p, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: f, forwardNonces: fn, blockedPeers: bp, allowedPeers: ap, pendingAckCleanups: pac}

5815+ yield {presence: p, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: f, forwardNonces: fn, blockedPeers: bp, allowedPeers: ap, pendingAckCleanups: pac, senderExpirations: se}

5751 atomic do

5816 atomic do

5752 set *lastSnapshotExportAt = t

5817 set *lastSnapshotExportAt = t

5753 set *prunesSinceSnapshot = 0

5818 set *prunesSinceSnapshot = 0

5754 let cs = snapshotChecksum (count result.presence) (count result.ackedIds) (count result.deliveredIds) (count result.forwards) (count result.forwardNonces)

5819 let cs = snapshotChecksum (count result.presence) (count result.ackedIds) (count result.deliveredIds) (count result.forwards) (count result.forwardNonces)

5755− logInfo ("snapshot: exported " ++ show (count result.presence) ++ " presence, " ++ show (count result.ackedIds) ++ " ackedIds, " ++ show (count result.deliveredIds) ++ " deliveredIds, " ++ show (count result.forwards) ++ " forwards, " ++ show (count result.forwardNonces) ++ " forwardNonces, " ++ show (count result.blockedPeers) ++ " blockedPeers, " ++ show (count result.allowedPeers) ++ " allowedPeers, " ++ show (count result.pendingAckCleanups) ++ " pendingAckCleanups, checksum=" ++ show cs)

5820+ logInfo ("snapshot: exported " ++ show (count result.presence) ++ " presence, " ++ show (count result.ackedIds) ++ " ackedIds, " ++ show (count result.deliveredIds) ++ " deliveredIds, " ++ show (count result.forwards) ++ " forwards, " ++ show (count result.forwardNonces) ++ " forwardNonces, " ++ show (count result.blockedPeers) ++ " blockedPeers, " ++ show (count result.allowedPeers) ++ " allowedPeers, " ++ show (count result.pendingAckCleanups) ++ " pendingAckCleanups, " ++ show (count result.senderExpirations) ++ " senderExpirations, checksum=" ++ show cs)

5756− yield (respond {ok: True {}, version: snapshotVersion, checksum: cs, presence: result.presence, ackedIds: result.ackedIds, ackOverflow: result.ackOverflow, deliveredIds: result.deliveredIds, forwardFailures: result.forwardFailures, forwards: result.forwards, forwardNonces: result.forwardNonces, blockedPeers: result.blockedPeers, allowedPeers: result.allowedPeers, pendingAckCleanups: result.pendingAckCleanups})

5821+ yield (respond {ok: True {}, version: snapshotVersion, checksum: cs, presence: result.presence, ackedIds: result.ackedIds, ackOverflow: result.ackOverflow, deliveredIds: result.deliveredIds, forwardFailures: result.forwardFailures, forwards: result.forwards, forwardNonces: result.forwardNonces, blockedPeers: result.blockedPeers, allowedPeers: result.allowedPeers, pendingAckCleanups: result.pendingAckCleanups, senderExpirations: result.senderExpirations})

5757

5822

5758-- POST /maintenance/restore: Import a previously-exported snapshot.

5823-- POST /maintenance/restore: Import a previously-exported snapshot.

5759-- Requires drain mode to prevent conflicts with live state. Merges imported data

5824-- Requires drain mode to prevent conflicts with live state. Merges imported data

5760-- with existing state (dedup entries are unioned, presence is merged by freshness).

5825-- with existing state (dedup entries are unioned, presence is merged by freshness).

5761-- After restore, call /maintenance/undrain to resume, then /maintenance/prune to

5826-- After restore, call /maintenance/undrain to resume, then /maintenance/prune to

5762-- trigger federation forward creation from messages + restored presence.

5827-- trigger federation forward creation from messages + restored presence.

5763−handleRestore = \version checksumIn presenceData ackedIdsData ackOverflowData deliveredIdsData forwardFailuresData forwardsData forwardNoncesData blockedPeersData allowedPeersData pendingAckCleanupsData authorization respond -> do

5828+handleRestore = \version checksumIn presenceData ackedIdsData ackOverflowData deliveredIdsData forwardFailuresData forwardsData forwardNoncesData blockedPeersData allowedPeersData pendingAckCleanupsData senderExpirationsData authorization respond -> do

5764− let authorized = verifyMaintenanceAuth authorization

5829+ authorized <- verifyMaintenanceAuth authorization

5765 if not authorized then yield (respond {ok: False {}, error: "unauthorized", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})

5830 if not authorized then yield (respond {ok: False {}, error: "unauthorized", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})

5766 else do

5831 else do

5767 -- Version 0 = pre-versioning snapshot, version 1 = pre-forwardNonces, version 2 = pre-blockedPeers,

5832 -- Version 0 = pre-versioning snapshot, version 1 = pre-forwardNonces, version 2 = pre-blockedPeers,

5768 -- version 3 = pre-ForwardEntry changes, version 4 = pre-allowedPeers, version 5 = pre-checksum

5833 -- version 3 = pre-ForwardEntry changes, version 4 = pre-allowedPeers, version 5 = pre-checksum

5769 -- (accept all for backwards compatibility; missing fields default to empty list or 0)

5834 -- (accept all for backwards compatibility; missing fields default to empty list or 0)

5770− if version != 0 && version != 1 && version != 2 && version != 3 && version != 4 && version != 5 && version != 6 && version != snapshotVersion then do

5835+ if version != 0 && version != 1 && version != 2 && version != 3 && version != 4 && version != 5 && version != 6 && version != 7 && version != snapshotVersion then do

5771 logError ("restore: incompatible snapshot version " ++ show version ++ " (expected " ++ show snapshotVersion ++ ")")

5836 logError ("restore: incompatible snapshot version " ++ show version ++ " (expected " ++ show snapshotVersion ++ ")")

5772 yield (respond {ok: False {}, error: "incompatible_version", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})

5837 yield (respond {ok: False {}, error: "incompatible_version", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})

5773 else do

5838 else do

@@ -5853,6 +5918,11 @@ handleRestore = \version checksumIn presenceData ackedIdsData ackOverflowData de

5853 existingCleanups <- *pendingAckCleanups

5918 existingCleanups <- *pendingAckCleanups

5854 let mergedCleanups = union existingCleanups pendingAckCleanupsData

5919 let mergedCleanups = union existingCleanups pendingAckCleanupsData

5855 set *pendingAckCleanups = takeRelation maxPendingAckCleanups mergedCleanups

5920 set *pendingAckCleanups = takeRelation maxPendingAckCleanups mergedCleanups

5921+ -- Restore sender expiration notifications (union with existing, cap at maxSenderExpirations)

5922+ -- pre-v8 snapshots have empty senderExpirations (field defaults to [])

5923+ existingSenderExp <- *senderExpirations

5924+ let mergedSenderExp = union existingSenderExp senderExpirationsData

5925+ set *senderExpirations = takeRelation maxSenderExpirations mergedSenderExp

5856 yield {presenceCount: count cappedPresence, ackedIdsCount: count cappedAcked, deliveredIdsCount: count cappedDelivered}

5926 yield {presenceCount: count cappedPresence, ackedIdsCount: count cappedAcked, deliveredIdsCount: count cappedDelivered}

5857 logInfo ("restore: imported " ++ show result.presenceCount ++ " presence, " ++ show result.ackedIdsCount ++ " ackedIds, " ++ show result.deliveredIdsCount ++ " deliveredIds")

5927 logInfo ("restore: imported " ++ show result.presenceCount ++ " presence, " ++ show result.ackedIdsCount ++ " ackedIds, " ++ show result.deliveredIdsCount ++ " deliveredIds")

5858 yield (respond {ok: True {}, error: "", presenceCount: result.presenceCount, ackedIdsCount: result.ackedIdsCount, deliveredIdsCount: result.deliveredIdsCount})

5928 yield (respond {ok: True {}, error: "", presenceCount: result.presenceCount, ackedIdsCount: result.ackedIdsCount, deliveredIdsCount: result.deliveredIdsCount})

@@ -6149,7 +6219,7 @@ sendBatchInner = \senderKey messages t acc ->

6149-- POST /maintenance/block: Add a server to the federation blocklist.

6219-- POST /maintenance/block: Add a server to the federation blocklist.

6150-- Blocked peers are rejected from all inbound and outbound federation operations.

6220-- Blocked peers are rejected from all inbound and outbound federation operations.

6151handleBlockPeer = \server authorization respond -> do

6221handleBlockPeer = \server authorization respond -> do

6152− let authorized = verifyMaintenanceAuth authorization

6222+ authorized <- verifyMaintenanceAuth authorization

6153 if not authorized then yield (respond {ok: False {}, blocked: 0})

6223 if not authorized then yield (respond {ok: False {}, blocked: 0})

6154 else do

6224 else do

6155 result <- atomic do

6225 result <- atomic do

@@ -6169,7 +6239,7 @@ handleBlockPeer = \server authorization respond -> do

6169

6239

6170-- POST /maintenance/unblock: Remove a server from the federation blocklist.

6240-- POST /maintenance/unblock: Remove a server from the federation blocklist.

6171handleUnblockPeer = \server authorization respond -> do

6241handleUnblockPeer = \server authorization respond -> do

6172− let authorized = verifyMaintenanceAuth authorization

6242+ authorized <- verifyMaintenanceAuth authorization

6173 if not authorized then yield (respond {ok: False {}, unblocked: False {}})

6243 if not authorized then yield (respond {ok: False {}, unblocked: False {}})

6174 else do

6244 else do

6175 removed <- atomic do

6245 removed <- atomic do

@@ -6182,7 +6252,7 @@ handleUnblockPeer = \server authorization respond -> do

6182

6252

6183-- GET /maintenance/blocked: List currently blocked federation peers.

6253-- GET /maintenance/blocked: List currently blocked federation peers.

6184handleListBlocked = \authorization respond -> do

6254handleListBlocked = \authorization respond -> do

6185− let authorized = verifyMaintenanceAuth authorization

6255+ authorized <- verifyMaintenanceAuth authorization

6186 if not authorized then yield (respond {peers: [], ok: False {}})

6256 if not authorized then yield (respond {peers: [], ok: False {}})

6187 else do

6257 else do

6188 blocked <- *blockedPeers

6258 blocked <- *blockedPeers

@@ -6192,7 +6262,7 @@ handleListBlocked = \authorization respond -> do

6192-- When the allowlist is non-empty, only servers on the list can federate with us.

6262-- When the allowlist is non-empty, only servers on the list can federate with us.

6193-- Adding the first server transitions from open federation to allowlist-only mode.

6263-- Adding the first server transitions from open federation to allowlist-only mode.

6194handleAllowPeer = \server authorization respond -> do

6264handleAllowPeer = \server authorization respond -> do

6195− let authorized = verifyMaintenanceAuth authorization

6265+ authorized <- verifyMaintenanceAuth authorization

6196 if not authorized then yield (respond {ok: False {}, allowed: 0})

6266 if not authorized then yield (respond {ok: False {}, allowed: 0})

6197 else do

6267 else do

6198 result <- atomic do

6268 result <- atomic do

@@ -6212,7 +6282,7 @@ handleAllowPeer = \server authorization respond -> do

6212-- POST /maintenance/unallow: Remove a server from the federation allowlist.

6282-- POST /maintenance/unallow: Remove a server from the federation allowlist.

6213-- Removing the last server transitions back to open federation mode.

6283-- Removing the last server transitions back to open federation mode.

6214handleUnallowPeer = \server authorization respond -> do

6284handleUnallowPeer = \server authorization respond -> do

6215− let authorized = verifyMaintenanceAuth authorization

6285+ authorized <- verifyMaintenanceAuth authorization

6216 if not authorized then yield (respond {ok: False {}, removed: False {}})

6286 if not authorized then yield (respond {ok: False {}, removed: False {}})

6217 else do

6287 else do

6218 removed <- atomic do

6288 removed <- atomic do

@@ -6226,7 +6296,7 @@ handleUnallowPeer = \server authorization respond -> do

6226-- GET /maintenance/allowed: List currently allowed federation peers.

6296-- GET /maintenance/allowed: List currently allowed federation peers.

6227-- Empty list = open federation (all peers allowed).

6297-- Empty list = open federation (all peers allowed).

6228handleListAllowed = \authorization respond -> do

6298handleListAllowed = \authorization respond -> do

6229− let authorized = verifyMaintenanceAuth authorization

6299+ authorized <- verifyMaintenanceAuth authorization

6230 if not authorized then yield (respond {peers: [], ok: False {}})

6300 if not authorized then yield (respond {peers: [], ok: False {}})

6231 else do

6301 else do

6232 allowed <- *allowedPeers

6302 allowed <- *allowedPeers

@@ -6238,7 +6308,7 @@ handleListAllowed = \authorization respond -> do

6238-- then processes the empty gossip (no mutations, no re-broadcast since hops=1).

6308-- then processes the empty gossip (no mutations, no re-broadcast since hops=1).

6239-- This exercises the real bidirectional connectivity and auth flow.

6309-- This exercises the real bidirectional connectivity and auth flow.

6240handlePingPeer = \server authorization respond -> do

6310handlePingPeer = \server authorization respond -> do

6241− let authorized = verifyMaintenanceAuth authorization

6311+ authorized <- verifyMaintenanceAuth authorization

6242 if not authorized then yield (respond {ok: False {}, latencyMs: 0, error: "unauthorized"})

6312 if not authorized then yield (respond {ok: False {}, latencyMs: 0, error: "unauthorized"})

6243 else do

6313 else do

6244 startTime <- now

6314 startTime <- now

@@ -6260,7 +6330,7 @@ handlePingPeer = \server authorization respond -> do

6260-- Uses per-bucket scan (collectSessionsForKey) instead of readAllSessions to avoid

6330-- Uses per-bucket scan (collectSessionsForKey) instead of readAllSessions to avoid

6261-- materializing all 16 session buckets into one list just to filter by pubkey.

6331-- materializing all 16 session buckets into one list just to filter by pubkey.

6262handleAdminListSessions = \pubkey authorization respond -> do

6332handleAdminListSessions = \pubkey authorization respond -> do

6263− let authorized = verifyMaintenanceAuth authorization

6333+ authorized <- verifyMaintenanceAuth authorization

6264 if not authorized then yield (respond {sessions: [], count: 0, ok: False {}})

6334 if not authorized then yield (respond {sessions: [], count: 0, ok: False {}})

6265 else do

6335 else do

6266 keySessions <- collectSessionsForKey pubkey 0

6336 keySessions <- collectSessionsForKey pubkey 0

@@ -6273,7 +6343,7 @@ handleAdminListSessions = \pubkey authorization respond -> do

6273-- POST /maintenance/sessions/revoke-all: Revoke all sessions for a pubkey (abuse response).

6343-- POST /maintenance/sessions/revoke-all: Revoke all sessions for a pubkey (abuse response).

6274-- Also clears local presence and gossips offline if federation is enabled.

6344-- Also clears local presence and gossips offline if federation is enabled.

6275handleAdminRevokeSessions = \pubkey authorization respond -> do

6345handleAdminRevokeSessions = \pubkey authorization respond -> do

6276− let authorized = verifyMaintenanceAuth authorization

6346+ authorized <- verifyMaintenanceAuth authorization

6277 if not authorized then yield (respond {ok: False {}, revokedCount: 0})

6347 if not authorized then yield (respond {ok: False {}, revokedCount: 0})

6278 else do

6348 else do

6279 t <- now

6349 t <- now

@@ -6304,7 +6374,7 @@ handleAdminRevokeSessions = \pubkey authorization respond -> do

6304-- Federated messages (originServer != "") generate ack entries so origin servers

6374-- Federated messages (originServer != "") generate ack entries so origin servers

6305-- can reclaim their forward entries instead of retrying until prune.

6375-- can reclaim their forward entries instead of retrying until prune.

6306handleAdminPurgeMessages = \pubkey authorization respond -> do

6376handleAdminPurgeMessages = \pubkey authorization respond -> do

6307− let authorized = verifyMaintenanceAuth authorization

6377+ authorized <- verifyMaintenanceAuth authorization

6308 if not authorized then yield (respond {ok: False {}, purgedCount: 0})

6378 if not authorized then yield (respond {ok: False {}, purgedCount: 0})

6309 else do

6379 else do

6310 t <- now

6380 t <- now

@@ -6352,7 +6422,7 @@ handleAdminPurgeMessages = \pubkey authorization respond -> do

6352-- Scans all 16 session buckets, removes matching sessions, and gossips offline

6422-- Scans all 16 session buckets, removes matching sessions, and gossips offline

6353-- for any pubkey that lost its last session.

6423-- for any pubkey that lost its last session.

6354handleAdminRevokeIp = \ip authorization respond -> do

6424handleAdminRevokeIp = \ip authorization respond -> do

6355− let authorized = verifyMaintenanceAuth authorization

6425+ authorized <- verifyMaintenanceAuth authorization

6356 if not authorized then yield (respond {ok: False {}, revokedCount: 0, affectedKeys: 0})

6426 if not authorized then yield (respond {ok: False {}, revokedCount: 0, affectedKeys: 0})

6357 else if ip == "" then yield (respond {ok: False {}, revokedCount: 0, affectedKeys: 0})

6427 else if ip == "" then yield (respond {ok: False {}, revokedCount: 0, affectedKeys: 0})

6358 else do

6428 else do

@@ -6408,31 +6478,39 @@ checkCleanupForKeys = \keys t acc ->

6408-- Useful for detecting suspicious multi-key activity (e.g., one IP authenticating many keys)

6478-- Useful for detecting suspicious multi-key activity (e.g., one IP authenticating many keys)

6409-- without needing to know the pubkey in advance. Groups sessions by pubkey.

6479-- without needing to know the pubkey in advance. Groups sessions by pubkey.

6410handleAdminListAllSessions = \limit authorization respond -> do

6480handleAdminListAllSessions = \limit authorization respond -> do

6411− let authorized = verifyMaintenanceAuth authorization

6481+ authorized <- verifyMaintenanceAuth authorization

6412 if not authorized then yield (respond {sessions: [], ok: False {}, total: 0})

6482 if not authorized then yield (respond {sessions: [], ok: False {}, total: 0})

6413 else do

6483 else do

6414 let effectiveLimit = if limit <= 0 then 100 else clamp 1 500 limit

6484 let effectiveLimit = if limit <= 0 then 100 else clamp 1 500 limit

6415− allSessions <- readAllSessions

6416 t <- now

6485 t <- now

6417− let active = do s <- allSessions; where s.expiresAt > t && t - s.createdAt < maxSessionLifetime; yield s

6486+ -- Per-bucket scan: reads one session bucket at a time instead of materializing

6418− -- Group by pubkey: accumulate count, unique IPs, and latest expiry per pubkey.

6487+ -- all 16 into one list (avoids O(n) union across all sessions).

6419− -- O(n*k) where k = unique pubkeys; acceptable since readAllSessions already materialized all.

6488+ grouped <- groupSessionsByBucket 0 t []

6420− let grouped = fold (\acc s ->

6421− if any acc (\a -> a.pubkey == s.pubkey)

6422− then do a <- acc; yield (if a.pubkey == s.pubkey then {pubkey: a.pubkey, count: a.count + 1, ips: if any a.ips (\i -> i == s.ip) || s.ip == "" then a.ips else union a.ips [s.ip], latestExpiry: max a.latestExpiry s.expiresAt} else a)

6423− else union acc [{pubkey: s.pubkey, count: 1, ips: if s.ip == "" then [] else [s.ip], latestExpiry: s.expiresAt}])

6424− [] active

6425 -- Sort by count descending (most sessions first) for abuse investigation

6489 -- Sort by count descending (most sessions first) for abuse investigation

6426 let sorted = sortBy (\g -> 0 - g.count) grouped

6490 let sorted = sortBy (\g -> 0 - g.count) grouped

6427 let capped = takeRelation effectiveLimit sorted

6491 let capped = takeRelation effectiveLimit sorted

6428 yield (respond {sessions: capped, ok: True {}, total: count grouped})

6492 yield (respond {sessions: capped, ok: True {}, total: count grouped})

6429

6493

6494+-- Per-bucket session grouping: reads one bucket at a time, merging active sessions

6495+-- into the running grouped accumulator. Avoids readAllSessions O(n) materialization.

6496+groupSessionsByBucket = \b t acc ->

6497+ if b > 15 then yield acc

6498+ else do

6499+ bucket <- readSessionBucket b

6500+ let active = do s <- bucket; where s.expiresAt > t && t - s.createdAt < maxSessionLifetime; yield s

6501+ let merged = fold (\grouped s ->

6502+ if any grouped (\a -> a.pubkey == s.pubkey)

6503+ then do a <- grouped; yield (if a.pubkey == s.pubkey then {pubkey: a.pubkey, count: a.count + 1, ips: if any a.ips (\i -> i == s.ip) || s.ip == "" then a.ips else union a.ips [s.ip], latestExpiry: max a.latestExpiry s.expiresAt} else a)

6504+ else union grouped [{pubkey: s.pubkey, count: 1, ips: if s.ip == "" then [] else [s.ip], latestExpiry: s.expiresAt}])

6505+ acc active

6506+ groupSessionsByBucket (b + 1) t merged

6507+

6430-- POST /maintenance/nonces/flush: Clear all forward nonces (recovery from corruption).

6508-- POST /maintenance/nonces/flush: Clear all forward nonces (recovery from corruption).

6431-- Nonces provide extended replay protection beyond message/ack dedup lifetime.

6509-- Nonces provide extended replay protection beyond message/ack dedup lifetime.

6432-- Flushing temporarily reopens a replay window for messages older than 30 days but

6510-- Flushing temporarily reopens a replay window for messages older than 30 days but

6433-- younger than 60 days. Use only for recovery, not routine maintenance.

6511-- younger than 60 days. Use only for recovery, not routine maintenance.

6434handleFlushNonces = \authorization respond -> do

6512handleFlushNonces = \authorization respond -> do

6435− let authorized = verifyMaintenanceAuth authorization

6513+ authorized <- verifyMaintenanceAuth authorization

6436 if not authorized then yield (respond {ok: False {}, flushedCount: 0})

6514 if not authorized then yield (respond {ok: False {}, flushedCount: 0})

6437 else do

6515 else do

6438 flushedCount <- atomic do

6516 flushedCount <- atomic do

@@ -6442,9 +6520,27 @@ handleFlushNonces = \authorization respond -> do

6442 logInfo ("nonces/flush: flushed " ++ show flushedCount ++ " forward nonces")

6520 logInfo ("nonces/flush: flushed " ++ show flushedCount ++ " forward nonces")

6443 yield (respond {ok: True {}, flushedCount: flushedCount})

6521 yield (respond {ok: True {}, flushedCount: flushedCount})

6444

6522

6523+-- POST /maintenance/rotate-secret: Rotate the maintenance secret at runtime.

6524+-- Authenticated with the current secret. The new secret takes effect immediately

6525+-- for all subsequent requests. Pass empty newSecret to revert to the compile-time default.

6526+handleRotateSecret = \newSecret authorization respond -> do

6527+ authorized <- verifyMaintenanceAuth authorization

6528+ if not authorized then yield (respond {ok: False {}, error: "unauthorized"})

6529+ else if newSecret != "" && not (tooLong newSecret 31) then do

6530+ logWarn "rotate-secret: new secret is shorter than 32 characters — 32+ recommended for production"

6531+ atomic do set *runtimeMaintenanceSecret = newSecret

6532+ logInfo "rotate-secret: maintenance secret rotated (warning: short secret)"

6533+ yield (respond {ok: True {}, error: ""})

6534+ else do

6535+ atomic do set *runtimeMaintenanceSecret = newSecret

6536+ if newSecret == ""

6537+ then logInfo "rotate-secret: reverted to compile-time maintenance secret"

6538+ else logInfo "rotate-secret: maintenance secret rotated"

6539+ yield (respond {ok: True {}, error: ""})

6540+

6445-- GET /maintenance/rate-limits: Return current effective rate limits.

6541-- GET /maintenance/rate-limits: Return current effective rate limits.

6446handleGetRateLimits = \authorization respond -> do

6542handleGetRateLimits = \authorization respond -> do

6447− let authorized = verifyMaintenanceAuth authorization

6543+ authorized <- verifyMaintenanceAuth authorization

6448 if not authorized then yield (respond {ok: False {}, globalRequest: 0, ipRate: 0, sendRate: 0, federationRate: 0, authGlobal: 0, recvRate: 0, challengeRate: 0, pairRate: 0, lookupRate: 0, clientWriteRate: 0})

6544 if not authorized then yield (respond {ok: False {}, globalRequest: 0, ipRate: 0, sendRate: 0, federationRate: 0, authGlobal: 0, recvRate: 0, challengeRate: 0, pairRate: 0, lookupRate: 0, clientWriteRate: 0})

6449 else do

6545 else do

6450 result <- atomic do

6546 result <- atomic do

@@ -6464,7 +6560,7 @@ handleGetRateLimits = \authorization respond -> do

6464-- POST /maintenance/rate-limits: Update rate limits at runtime. 0 = reset to compile-time default.

6560-- POST /maintenance/rate-limits: Update rate limits at runtime. 0 = reset to compile-time default.

6465-- Negative values are clamped to 0. Changes take effect on the next rate check cycle.

6561-- Negative values are clamped to 0. Changes take effect on the next rate check cycle.

6466handleSetRateLimits = \globalRequest ipRate sendRate federationRate authGlobal recvRate challengeRate pairRate lookupRate clientWriteRate authorization respond -> do

6562handleSetRateLimits = \globalRequest ipRate sendRate federationRate authGlobal recvRate challengeRate pairRate lookupRate clientWriteRate authorization respond -> do

6467− let authorized = verifyMaintenanceAuth authorization

6563+ authorized <- verifyMaintenanceAuth authorization

6468 if not authorized then yield (respond {ok: False {}})

6564 if not authorized then yield (respond {ok: False {}})

6469 else do

6565 else do

6470 atomic do

6566 atomic do

@@ -6524,6 +6620,7 @@ requestPath = \req -> case req of

6524 UnallowPeer {} -> "POST /maintenance/unallow"

6620 UnallowPeer {} -> "POST /maintenance/unallow"

6525 ListAllowed {} -> "GET /maintenance/allowed"

6621 ListAllowed {} -> "GET /maintenance/allowed"

6526 Reconcile {} -> "POST /maintenance/reconcile"

6622 Reconcile {} -> "POST /maintenance/reconcile"

6623+ RotateSecret {} -> "POST /maintenance/rotate-secret"

6527 Health {} -> "GET /health"

6624 Health {} -> "GET /health"

6528 Stats {} -> "GET /stats"

6625 Stats {} -> "GET /stats"

6529 StatsPeers {} -> "GET /stats/peers"

6626 StatsPeers {} -> "GET /stats/peers"

@@ -6574,6 +6671,7 @@ requestClientIp = \req -> case req of

6574 UnallowPeer {} -> ""

6671 UnallowPeer {} -> ""

6575 ListAllowed {} -> ""

6672 ListAllowed {} -> ""

6576 Reconcile {} -> ""

6673 Reconcile {} -> ""

6674+ RotateSecret {} -> ""

6577 Health {} -> ""

6675 Health {} -> ""

6578 Stats {} -> ""

6676 Stats {} -> ""

6579 StatsPeers {} -> ""

6677 StatsPeers {} -> ""

@@ -6610,7 +6708,7 @@ serveInner = \req -> case req of

6610 Shutdown {authorization, respond} -> handleShutdown authorization respond

6708 Shutdown {authorization, respond} -> handleShutdown authorization respond

6611 Undrain {authorization, respond} -> handleUndrain authorization respond

6709 Undrain {authorization, respond} -> handleUndrain authorization respond

6612 Snapshot {authorization, respond} -> handleSnapshot authorization respond

6710 Snapshot {authorization, respond} -> handleSnapshot authorization respond

6613− Restore {version, checksum, presence, ackedIds, ackOverflow, deliveredIds, forwardFailures, forwards, forwardNonces, blockedPeers, allowedPeers, pendingAckCleanups, authorization, respond} -> handleRestore version checksum presence ackedIds ackOverflow deliveredIds forwardFailures forwards forwardNonces blockedPeers allowedPeers pendingAckCleanups authorization respond

6711+ Restore {version, checksum, presence, ackedIds, ackOverflow, deliveredIds, forwardFailures, forwards, forwardNonces, blockedPeers, allowedPeers, pendingAckCleanups, senderExpirations, authorization, respond} -> handleRestore version checksum presence ackedIds ackOverflow deliveredIds forwardFailures forwards forwardNonces blockedPeers allowedPeers pendingAckCleanups senderExpirations authorization respond

6614 BlockPeer {server, authorization, respond} -> handleBlockPeer server authorization respond

6712 BlockPeer {server, authorization, respond} -> handleBlockPeer server authorization respond

6615 UnblockPeer {server, authorization, respond} -> handleUnblockPeer server authorization respond

6713 UnblockPeer {server, authorization, respond} -> handleUnblockPeer server authorization respond

6616 ListBlocked {authorization, respond} -> handleListBlocked authorization respond

6714 ListBlocked {authorization, respond} -> handleListBlocked authorization respond

@@ -6627,6 +6725,7 @@ serveInner = \req -> case req of

6627 UnallowPeer {server, authorization, respond} -> handleUnallowPeer server authorization respond

6725 UnallowPeer {server, authorization, respond} -> handleUnallowPeer server authorization respond

6628 ListAllowed {authorization, respond} -> handleListAllowed authorization respond

6726 ListAllowed {authorization, respond} -> handleListAllowed authorization respond

6629 Reconcile {authorization, respond} -> handleReconcile authorization respond

6727 Reconcile {authorization, respond} -> handleReconcile authorization respond

6728+ RotateSecret {newSecret, authorization, respond} -> handleRotateSecret newSecret authorization respond

6630 Health {authorization, respond} -> handleHealth authorization respond

6729 Health {authorization, respond} -> handleHealth authorization respond

6631 Stats {authorization, respond} -> handleStats authorization respond

6730 Stats {authorization, respond} -> handleStats authorization respond

6632 StatsPeers {authorization, respond} -> handleStatsPeers authorization respond

6731 StatsPeers {authorization, respond} -> handleStatsPeers authorization respond

@@ -6649,7 +6748,9 @@ serve = \req -> do

6649 let ipSuffix = if ip != "" then " " ++ ip else ""

6748 let ipSuffix = if ip != "" then " " ++ ip else ""

6650 if path != ""

6749 if path != ""

6651 then logInfo ("[ACCESS] " ++ reqId ++ " " ++ path ++ " " ++ show latency ++ "ms" ++ ipSuffix)

6750 then logInfo ("[ACCESS] " ++ reqId ++ " " ++ path ++ " " ++ show latency ++ "ms" ++ ipSuffix)

6751+ else when (latency > slowRequestThreshold) (do

6652− else when (latency > slowRequestThreshold) (logInfo ("[SLOW] " ++ reqId ++ " poll " ++ show latency ++ "ms" ++ ipSuffix))

6752+ logInfo ("[SLOW] " ++ reqId ++ " poll " ++ show latency ++ "ms" ++ ipSuffix)

6753+ logEvent "warn" "slow_request" ("poll " ++ show latency ++ "ms"))

6653

6754

6654-- ============================================

6755-- ============================================

6655-- Entry Point

6756-- Entry Point

@@ -6822,8 +6923,9 @@ backgroundSnapshot = do

6822 bp <- *blockedPeers

6923 bp <- *blockedPeers

6823 ap <- *allowedPeers

6924 ap <- *allowedPeers

6824 pac <- *pendingAckCleanups

6925 pac <- *pendingAckCleanups

6926+ se <- *senderExpirations

6825 let exportCs = snapshotChecksum (count p) (count a) (count d) (count f) (count fn)

6927 let exportCs = snapshotChecksum (count p) (count a) (count d) (count f) (count fn)

6826− yield {version: snapshotVersion, checksum: exportCs, presence: p, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: f, forwardNonces: fn, blockedPeers: bp, allowedPeers: ap, pendingAckCleanups: pac}

6928+ yield {version: snapshotVersion, checksum: exportCs, presence: p, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: f, forwardNonces: fn, blockedPeers: bp, allowedPeers: ap, pendingAckCleanups: pac, senderExpirations: se}

6827 println ("[SNAPSHOT_DATA] " ++ toJson exportData)

6929 println ("[SNAPSHOT_DATA] " ++ toJson exportData)

6828 println "[SNAPSHOT_END]"

6930 println "[SNAPSHOT_END]"

6829 atomic do

6931 atomic do

@@ -6958,6 +7060,10 @@ main = do

6958 (logWarn "maintenanceSecret is empty — /maintenance/* and /stats endpoints are disabled")

7060 (logWarn "maintenanceSecret is empty — /maintenance/* and /stats endpoints are disabled")

6959 when (maintenanceSecret != "" && not (tooLong maintenanceSecret 15))

7061 when (maintenanceSecret != "" && not (tooLong maintenanceSecret 15))

6960 (logWarn "maintenanceSecret is shorter than 16 characters — use a longer secret for production deployments")

7062 (logWarn "maintenanceSecret is shorter than 16 characters — use a longer secret for production deployments")

7063+ when (maintenanceSecret != "" && not (tooLong maintenanceSecret 31))

7064+ (logWarn "maintenanceSecret is shorter than 32 characters — 32+ character secrets recommended for production (128+ bits of entropy)")

7065+ when (maintenanceSecret != "" && federationEnabled && not (tooLong maintenanceSecret 15))

7066+ (logError "SECURITY: maintenanceSecret is weak and federation is enabled — maintenance endpoints control drain, snapshot, and peer blocking. Use a secret of at least 32 characters.")

6961 when (matchesHostnameOrSubdomain serverHostN "localhost")

7067 when (matchesHostnameOrSubdomain serverHostN "localhost")

6962 (logWarn ("serverHost is " ++ serverHost ++ " — peers cannot reach this server for federation"))

7068 (logWarn ("serverHost is " ++ serverHost ++ " — peers cannot reach this server for federation"))

6963 when (federationEnabled && isBadServerName serverHostN)

7069 when (federationEnabled && isBadServerName serverHostN)