@@ -138,6 +138,11 @@ maxServerDowntime = 86400000 -- 24 hours; stop retrying a server after this dura
138-- Federation in-flight tracking: skip servers with outstanding requests older than this threshold
138-- Federation in-flight tracking: skip servers with outstanding requests older than this threshold
139federationFetchTimeoutMs : Int<Ms>
139federationFetchTimeoutMs : Int<Ms>
140federationFetchTimeoutMs = 30000 -- 30 seconds; consider server unresponsive if fetch hasn't returned
140federationFetchTimeoutMs = 30000 -- 30 seconds; consider server unresponsive if fetch hasn't returned
141+-- Verify callback latency threshold: warn when federation verify round-trips exceed this.
142+-- Shorter than federationFetchTimeoutMs because verify is a single lightweight POST
143+-- (no payload); slow verifies indicate network issues or overloaded peers.
144+federationVerifyTimeoutMs : Int<Ms>
145+federationVerifyTimeoutMs = 10000 -- 10 seconds
141maxFederationInFlight = 100
146maxFederationInFlight = 100
142-- Federation circuit breaker: pause all outbound federation when this many unique peer
147-- Federation circuit breaker: pause all outbound federation when this many unique peer
143-- servers have active (non-expired) failure records. Prevents wasted retries and log spam
148-- servers have active (non-expired) failure records. Prevents wasted retries and log spam
@@ -160,7 +165,19 @@ federationLatencyWindowMs : Int<Ms>
160federationLatencyWindowMs = 300000 -- 5 minutes; reset latency metrics after this interval
165federationLatencyWindowMs = 300000 -- 5 minutes; reset latency metrics after this interval
162-- Snapshot format version (increment on schema changes to prevent corrupt restores)
167-- Snapshot format version (increment on schema changes to prevent corrupt restores)
163−snapshotVersion = 5 -- bumped for allowedPeers field in snapshot
168+snapshotVersion = 6 -- bumped for checksum field in snapshot
170+-- Server build version: included in /health and /stats responses for deployment verification.
171+-- Update this string when deploying a new build so operators can confirm which version is running.
172+serverBuildVersion = "2026-04-25"
174+-- Snapshot checksum: deterministic hash of entry counts for integrity verification.
175+-- Uses multiplicative hash to distinguish permutations of the same counts
176+-- (e.g., {presence:1, ackedIds:0} vs {presence:0, ackedIds:1}).
177+-- Not cryptographic — catches truncation/corruption, not tampering.
178+snapshotChecksum : Int -> Int -> Int -> Int -> Int -> Int
179+snapshotChecksum = \presenceCount ackedCount deliveredCount forwardCount nonceCount ->
180+ presenceCount * 100003 + ackedCount * 100019 + deliveredCount * 100043 + forwardCount * 100049 + nonceCount * 100057
165-- Per-peer gossip event volume limiting (prevents flooding via large batches of unique pubkeys)
182-- Per-peer gossip event volume limiting (prevents flooding via large batches of unique pubkeys)
166maxGossipEventsPerServer = 500 -- max total gossip events per server per window
183maxGossipEventsPerServer = 500 -- max total gossip events per server per window
@@ -196,8 +213,11 @@ maxActivePolls = 5000
196maxWatchKeys = 50
213maxWatchKeys = 50
197presenceRefreshInterval : Int<Ms>
214presenceRefreshInterval : Int<Ms>
198presenceRefreshInterval = 300000 -- 5 minutes
215presenceRefreshInterval = 300000 -- 5 minutes
216+-- Refresh entries expiring within 1.5 refresh intervals. At 2x, entries with 8-10 min
217+-- remaining would be refreshed every cycle (wasteful gossip). 1.5x catches entries
218+-- likely to expire before the next refresh without over-refreshing healthy ones.
199presenceRefreshThreshold : Int<Ms>
219presenceRefreshThreshold : Int<Ms>
200−presenceRefreshThreshold = presenceRefreshInterval * 2 -- refresh entries expiring within this window
220+presenceRefreshThreshold = presenceRefreshInterval + presenceRefreshInterval / 2
201federationEnabled = True {}
221federationEnabled = True {}
202-- WARNING: DNS rebinding risk — federationUrl resolves DNS at fetch time, not at
222-- WARNING: DNS rebinding risk — federationUrl resolves DNS at fetch time, not at
203-- ServerName validation time. A hostname passing isBadServerName could rebind to a
223-- ServerName validation time. A hostname passing isBadServerName could rebind to a
@@ -734,6 +754,10 @@ type SenderExpiration = {senderKey: Text, messageId: MessageIdHex, toKey: Pubkey
734*rateOverrideFederationRate : Int
754*rateOverrideFederationRate : Int
735*rateOverrideAuthGlobal : Int
755*rateOverrideAuthGlobal : Int
736*rateOverrideRecvRate : Int
756*rateOverrideRecvRate : Int
757+*rateOverrideChallengeRate : Int
758+*rateOverridePairRate : Int
759+*rateOverrideLookupRate : Int
760+*rateOverrideClientWriteRate : Int
738-- Monotonic counter bumped on any presence change (gossip, auth, logout, prune, restore, drain)
762-- Monotonic counter bumped on any presence change (gossip, auth, logout, prune, restore, drain)
739-- Polls compare against this to skip expensive presence lookups when nothing changed.
763-- Polls compare against this to skip expensive presence lookups when nothing changed.
@@ -888,11 +912,12 @@ route MaintenanceApi where
888 -- Messages are excluded (ephemeral by design; clients handle re-delivery).
912 -- Messages are excluded (ephemeral by design; clients handle re-delivery).
889 -- Includes presence, acked/delivered IDs (dedup state), and peer failure records.
913 -- Includes presence, acked/delivered IDs (dedup state), and peer failure records.
890 GET /maintenance/snapshot headers {authorization: Text}
914 GET /maintenance/snapshot headers {authorization: Text}
891− -> {ok: Bool, version: Int, presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure], forwards: [ForwardEntry], forwardNonces: [ForwardNonce], blockedPeers: [Text], allowedPeers: [Text]}
915+ -> {ok: Bool, version: Int, checksum: Int, presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure], forwards: [ForwardEntry], forwardNonces: [ForwardNonce], blockedPeers: [Text], allowedPeers: [Text]}
892 = Snapshot
916 = Snapshot
893 -- Restore: import a previously-exported snapshot. Server must be in drain mode
917 -- Restore: import a previously-exported snapshot. Server must be in drain mode
894 -- to prevent conflicts with live state. Run /maintenance/undrain after restore.
918 -- to prevent conflicts with live state. Run /maintenance/undrain after restore.
919+ -- checksum: integrity check (0 = skip verification, for pre-v6 snapshots)
895− POST {version: Int, presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure], forwards: [ForwardEntry], forwardNonces: [ForwardNonce], blockedPeers: [Text], allowedPeers: [Text]} /maintenance/restore headers {authorization: Text}
920+ POST {version: Int, checksum: Int, presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure], forwards: [ForwardEntry], forwardNonces: [ForwardNonce], blockedPeers: [Text], allowedPeers: [Text]} /maintenance/restore headers {authorization: Text}
896 -> {ok: Bool, error: Text, presenceCount: Int, ackedIdsCount: Int, deliveredIdsCount: Int}
921 -> {ok: Bool, error: Text, presenceCount: Int, ackedIdsCount: Int, deliveredIdsCount: Int}
897 = Restore
922 = Restore
898 -- Block a federation peer (rejects all inbound/outbound federation with this server)
923 -- Block a federation peer (rejects all inbound/outbound federation with this server)
@@ -925,10 +950,10 @@ route MaintenanceApi where
925 = AdminPurgeMessages
950 = AdminPurgeMessages
926 -- Read current effective rate limits (compile-time defaults + runtime overrides)
951 -- Read current effective rate limits (compile-time defaults + runtime overrides)
927 GET /maintenance/rate-limits headers {authorization: Text}
952 GET /maintenance/rate-limits headers {authorization: Text}
928− -> {ok: Bool, globalRequest: Int, ipRate: Int, sendRate: Int, federationRate: Int, authGlobal: Int, recvRate: Int}
953+ -> {ok: Bool, globalRequest: Int, ipRate: Int, sendRate: Int, federationRate: Int, authGlobal: Int, recvRate: Int, challengeRate: Int, pairRate: Int, lookupRate: Int, clientWriteRate: Int}
929 = GetRateLimits
954 = GetRateLimits
930 -- Update rate limits at runtime (0 = reset to compile-time default)
955 -- Update rate limits at runtime (0 = reset to compile-time default)
931− POST {globalRequest: Int, ipRate: Int, sendRate: Int, federationRate: Int, authGlobal: Int, recvRate: Int} /maintenance/rate-limits headers {authorization: Text}
956+ POST {globalRequest: Int, ipRate: Int, sendRate: Int, federationRate: Int, authGlobal: Int, recvRate: Int, challengeRate: Int, pairRate: Int, lookupRate: Int, clientWriteRate: Int} /maintenance/rate-limits headers {authorization: Text}
932 -> {ok: Bool}
957 -> {ok: Bool}
933 = SetRateLimits
958 = SetRateLimits
934 -- Flush forward nonces (recovery from corruption or capacity issues)
959 -- Flush forward nonces (recovery from corruption or capacity issues)
@@ -947,9 +972,14 @@ route MaintenanceApi where
947 GET /maintenance/allowed headers {authorization: Text}
972 GET /maintenance/allowed headers {authorization: Text}
948 -> {peers: [Text], ok: Bool}
973 -> {peers: [Text], ok: Bool}
949 = ListAllowed
974 = ListAllowed
975+ -- Force counter reconciliation (message counts, presence, sessions).
976+ -- Returns drift values; non-zero means counters were corrected.
977+ POST {} /maintenance/reconcile headers {authorization: Text}
978+ -> {ok: Bool, messageDrift: Int, presenceDrift: Int, sessionDrift: Int}
951route HealthApi where
981route HealthApi where
952− GET /health headers {authorization: Text} -> {ok: Bool, pruneHealthy: Bool, heartbeatHealthy: Bool, capacityHealthy: Bool, federationHealthy: Bool, snapshotHealthy: Bool, watchdogHealthy: Bool, draining: Bool} = Health
982+ GET /health headers {authorization: Text} -> {ok: Bool, version: Text, pruneHealthy: Bool, heartbeatHealthy: Bool, capacityHealthy: Bool, federationHealthy: Bool, snapshotHealthy: Bool, watchdogHealthy: Bool, draining: Bool} = Health
954-- Returns counts of in-memory collections (requires maintenance secret)
984-- Returns counts of in-memory collections (requires maintenance secret)
955-- TODO: If Knot adds raw text response support, add a /metrics endpoint
985-- TODO: If Knot adds raw text response support, add a /metrics endpoint
@@ -959,7 +989,7 @@ route HealthApi where
959-- BREAKING CHANGE from flat format — existing scrapers must update field paths.
989-- BREAKING CHANGE from flat format — existing scrapers must update field paths.
960route StatsApi where
990route StatsApi where
961 GET /stats headers {authorization: Text}
991 GET /stats headers {authorization: Text}
962− -> {ok: Bool, capacity: {sessions: Int, sessionsMax: Int, sessionsPct: Int, messages: Int, messagesMax: Int, messagesPct: Int, totalMessageCount: Int, presence: Int, presenceMax: Int, presencePct: Int, forwards: Int, forwardsMax: Int, forwardsPct: Int, challenges: Int, challengesMax: Int, ackedIds: Int, ackedIdsMax: Int, ackOverflow: Int, ackOverflowMax: Int, deliveredIds: Int, deliveredIdsMax: Int, forwardNonces: Int, forwardNoncesMax: Int, activePolls: Int, activePollsMax: Int, recipientCounts: Int}, federation: {enabled: Bool, inFlight: Int, hungSlots: Int, avgLatencyMs: Int, maxLatencyMs: Int, callCount: Int, forwardFailures: Int, oldestForwardAge: Int, oldestAckAge: Int, recentGossip: Int, blockedPeers: Int, drainCompletedAt: Timestamp, forwardServers: Int, maxForwardQueueDepth: Int}, rates: {send: Int, challenge: Int, federation: Int, federationPair: Int, global: Int, ip: Int, lookup: Int, recv: Int, pair: Int, gossipEvent: Int, verifyIp: Int, uniqueKeyIp: Int, pendingAckCleanups: Int, senderExpirations: Int}, timing: {pruneCount: Int, pruneRunning: Timestamp, heartbeatCount: Int, uptime: Int, lastPruneAt: Timestamp, lastHeartbeatAt: Timestamp, draining: Bool, drainPhase: Int}}
992+ -> {ok: Bool, version: Text, capacity: {sessions: Int, sessionsMax: Int, sessionsPct: Int, messages: Int, messagesMax: Int, messagesPct: Int, totalMessageCount: Int, presence: Int, presenceMax: Int, presencePct: Int, forwards: Int, forwardsMax: Int, forwardsPct: Int, challenges: Int, challengesMax: Int, ackedIds: Int, ackedIdsMax: Int, ackOverflow: Int, ackOverflowMax: Int, deliveredIds: Int, deliveredIdsMax: Int, forwardNonces: Int, forwardNoncesMax: Int, activePolls: Int, activePollsMax: Int, recipientCounts: Int}, federation: {enabled: Bool, inFlight: Int, hungSlots: Int, avgLatencyMs: Int, maxLatencyMs: Int, callCount: Int, forwardFailures: Int, oldestForwardAge: Int, oldestAckAge: Int, recentGossip: Int, blockedPeers: Int, drainCompletedAt: Timestamp, forwardServers: Int, maxForwardQueueDepth: Int}, rates: {send: Int, challenge: Int, federation: Int, federationPair: Int, global: Int, ip: Int, lookup: Int, recv: Int, pair: Int, gossipEvent: Int, verifyIp: Int, uniqueKeyIp: Int, pendingAckCleanups: Int, senderExpirations: Int}, timing: {pruneCount: Int, pruneRunning: Timestamp, heartbeatCount: Int, uptime: Int, lastPruneAt: Timestamp, lastHeartbeatAt: Timestamp, draining: Bool, drainPhase: Int}}
964 GET /stats/peers headers {authorization: Text}
994 GET /stats/peers headers {authorization: Text}
965 -> {peers: [PeerStatus], ok: Bool}
995 -> {peers: [PeerStatus], ok: Bool}
@@ -1053,6 +1083,11 @@ sanitizeCorrelationId = \cid -> take maxCorrelationIdLen cid
1053isHexChar : Text -> Bool
1083isHexChar : Text -> Bool
1054isHexChar = \c -> contains c "0123456789abcdef"
1084isHexChar = \c -> contains c "0123456789abcdef"
1086+-- Validates that a string is composed of valid hex character pairs.
1087+-- Recursive with drop 2 per call; the recursive call is in tail position so
1088+-- Knot's TCO compiles it to a loop. Max depth is bounded by isValidBoundedHex's
1089+-- tooLong check: maxBlobLen/2 = 32768 frames for the largest input (BlobHex).
1090+-- If TCO is ever disabled, this would stack overflow on large blobs.
1056isValidHexPairs : Text -> Bool
1091isValidHexPairs : Text -> Bool
1057isValidHexPairs = \s ->
1092isValidHexPairs = \s ->
1058 if s == "" then True {}
1093 if s == "" then True {}
@@ -1269,6 +1304,10 @@ authenticateSession = \sessions token t ->
1269-- running safeCompare. Reduces safeCompare calls from O(bucket_size) to
1304-- running safeCompare. Reduces safeCompare calls from O(bucket_size) to
1270-- O(sessions_for_key_in_bucket) (~1-2 vs ~600). Safe in this context because
1305-- O(sessions_for_key_in_bucket) (~1-2 vs ~600). Safe in this context because
1271-- the pubkey was already verified in Phase 1.
1306-- the pubkey was already verified in Phase 1.
1307+-- Theoretical race: if the session is revoked between Phase 1 and Phase 3 and a
1308+-- new session with the same token is created for a different pubkey, the pre-filter
1309+-- would miss it. This requires guessing a 24-byte (192-bit) random token, which is
1310+-- computationally infeasible (2^-192 collision probability).
1272reauthenticateSession : [Session] -> Text -> PubkeyHex -> Timestamp -> Bool
1311reauthenticateSession : [Session] -> Text -> PubkeyHex -> Timestamp -> Bool
1273reauthenticateSession = \sessions token pubkey t ->
1312reauthenticateSession = \sessions token pubkey t ->
1274 any sessions (\s -> s.pubkey == pubkey && s.expiresAt > t && t - s.createdAt < maxSessionLifetime && safeCompare s.token token)
1313 any sessions (\s -> s.pubkey == pubkey && s.expiresAt > t && t - s.createdAt < maxSessionLifetime && safeCompare s.token token)
@@ -1329,7 +1368,12 @@ verifyFederationCallback = \fromServer authorization scope ->
1329 case extractToken authorization of
1368 case extractToken authorization of
1330 Nothing {} -> yield (False {})
1369 Nothing {} -> yield (False {})
1331 Just {value: token} -> do
1370 Just {value: token} -> do
1332 result <- fetchFederation fromServer (VerifyToken {token: token, scope: scope})
1372 result <- fetchFederation fromServer (VerifyToken {token: token, scope: scope})
1374+ let verifyLatency = endTime - startTime
1375+ when (verifyLatency > federationVerifyTimeoutMs)
1376+ (logWarn ("federation: slow verify callback to " ++ fromServer ++ " (" ++ show verifyLatency ++ "ms, threshold: " ++ show federationVerifyTimeoutMs ++ "ms)"))
1333 case result of
1377 case result of
1334 Err e -> do
1378 Err e -> do
1335 logWarn ("federation: verify callback to " ++ fromServer ++ " failed: " ++ show e)
1379 logWarn ("federation: verify callback to " ++ fromServer ++ " failed: " ++ show e)
@@ -1460,6 +1504,18 @@ filterActiveSessionKeys = \keys t ->
1460 remaining <- filterActiveSessionKeys rest t
1504 remaining <- filterActiveSessionKeys rest t
1461 if hasActive then yield (union [k] remaining) else yield remaining
1505 if hasActive then yield (union [k] remaining) else yield remaining
1507+-- Collect all sessions for a specific pubkey across all buckets without materializing the full
1508+-- session list. Each bucket is read independently (O(n/16) per bucket), avoiding the O(n)
1509+-- readAllSessions union. Used by handleAdminListSessions.
1510+collectSessionsForKey : PubkeyHex -> Int -> IO [Session]
1511+collectSessionsForKey = \pubkey b ->
1512+ if b > 15 then yield []
1514+ bucket <- readSessionBucket b
1515+ let matching = do s <- bucket; where s.pubkey == pubkey; yield s
1516+ rest <- collectSessionsForKey pubkey (b + 1)
1517+ yield (union matching rest)
1463-- Count sessions for a specific pubkey across all buckets (avoids materializing full session list).
1519-- Count sessions for a specific pubkey across all buckets (avoids materializing full session list).
1464-- Used in verify fast path to skip readAllSessions.
1520-- Used in verify fast path to skip readAllSessions.
1465countSessionsForKey : PubkeyHex -> IO Int
1521countSessionsForKey : PubkeyHex -> IO Int
@@ -1900,6 +1956,11 @@ pruneMessageBucketsConcurrent = \b t ->
1900-- at once, reducing lock contention during reconciliation. Each pubkey maps to exactly one
1956-- at once, reducing lock contention during reconciliation. Each pubkey maps to exactly one
1901-- bucket, so per-bucket recipient counts are disjoint — each bucket's counts are written
1957-- bucket, so per-bucket recipient counts are disjoint — each bucket's counts are written
1902-- to the corresponding rcBucket without overlap.
1958-- to the corresponding rcBucket without overlap.
1959+-- NOTE: TOCTOU window — messages can arrive between per-bucket reads, so the final
1960+-- totalMessageCount write may slightly overwrite concurrent updates from sendMessageCore.
1961+-- This is acceptable: (a) reconciliation runs on drift detection or every 10th prune cycle,
1962+-- (b) the next prune re-reconciles, (c) capacity checks on the hot path use per-bucket
1963+-- rcBucket counts (not totalMessageCount) so message acceptance isn't affected.
1903reconcileMessageCounts = \b totalAcc ->
1964reconcileMessageCounts = \b totalAcc ->
1904 if b > 15 then do
1965 if b > 15 then do
1905 atomic do
1966 atomic do
@@ -2009,6 +2070,27 @@ countAllRcEntries = \b ->
2009 rest <- countAllRcEntries (b + 1)
2070 rest <- countAllRcEntries (b + 1)
2010 yield (count bucket + rest)
2071 yield (count bucket + rest)
2073+-- Merge two sorted-by-count-desc lists, keeping only the top N by count.
2074+-- O(n+m) merge pass + O(result) truncation. Used by topRecipientsByBucket.
2075+mergeTopN : [RecipientCount] -> [RecipientCount] -> Int -> [RecipientCount]
2076+mergeTopN = \acc newItems limit ->
2077+ let combined = fold (\a c ->
2078+ if any a (\x -> x.pubkey == c.pubkey) then a
2079+ else union a [c]) acc newItems
2080+ in let sorted = sortBy (\c -> 0 - c.count) combined
2081+ in takeRelation limit sorted
2083+-- Top-N recipients by message count, scanning one rcBucket at a time.
2084+-- Avoids materializing all 16 rcBuckets into a single list (O(16 * bucket_size)
2085+-- vs O(total_recipients)). Each bucket's entries are merged into a running top-N.
2086+topRecipientsByBucket : Int -> Int -> [RecipientCount] -> IO [RecipientCount]
2087+topRecipientsByBucket = \b limit acc ->
2088+ if b > 15 then yield acc
2090+ bucket <- readRcBucket b
2091+ let merged = mergeTopN acc bucket limit
2092+ topRecipientsByBucket (b + 1) limit merged
2012-- Sum total message count from rcBuckets (cheap: sums .count fields, doesn't scan messages)
2094-- Sum total message count from rcBuckets (cheap: sums .count fields, doesn't scan messages)
2013sumRcBucketTotals : Int -> IO Int
2095sumRcBucketTotals : Int -> IO Int
2014sumRcBucketTotals = \b ->
2096sumRcBucketTotals = \b ->
@@ -2494,7 +2576,9 @@ clampTimestamp = \timestamp t -> clamp (t - messageMaxAge) t timestamp
2494nextSeqTs : Timestamp -> Timestamp -> Timestamp
2576nextSeqTs : Timestamp -> Timestamp -> Timestamp
2495nextSeqTs = \t seqCounter -> if t > seqCounter then t else seqCounter + 1
2577nextSeqTs = \t seqCounter -> if t > seqCounter then t else seqCounter + 1
2497−-- Deduplicate message IDs, preserving order, capped at maxMessageIds
2579+-- Deduplicate message IDs, preserving order, capped at maxMessageIds.
2580+-- O(n^2) due to `any acc` inside fold, but bounded by maxMessageIds = 100,
2581+-- so worst case is 10,000 comparisons (acceptable for the hot path).
2498dedupMessageIds : [MessageIdHex] -> [MessageIdHex]
2582dedupMessageIds : [MessageIdHex] -> [MessageIdHex]
2499dedupMessageIds = \messageIds ->
2583dedupMessageIds = \messageIds ->
2500 let capped = takeRelation maxMessageIds messageIds in
2584 let capped = takeRelation maxMessageIds messageIds in
@@ -3054,6 +3138,22 @@ effectiveRecvRate = do
3054 o <- *rateOverrideRecvRate
3138 o <- *rateOverrideRecvRate
3055 yield (if o > 0 then o else maxRecvRatePerKey)
3139 yield (if o > 0 then o else maxRecvRatePerKey)
3141+effectiveChallengeRate = do
3142+ o <- *rateOverrideChallengeRate
3143+ yield (if o > 0 then o else maxChallengeRatePerKey)
3145+effectivePairRate = do
3146+ o <- *rateOverridePairRate
3147+ yield (if o > 0 then o else maxPairRatePerPair)
3149+effectiveLookupRate = do
3150+ o <- *rateOverrideLookupRate
3151+ yield (if o > 0 then o else maxLookupRatePerKey)
3153+effectiveClientWriteRate = do
3154+ o <- *rateOverrideClientWriteRate
3155+ yield (if o > 0 then o else maxClientWriteRatePerKey)
3057checkFederationRate = \server t -> atomic do
3157checkFederationRate = \server t -> atomic do
3058 federationRates <- *federationRates
3158 federationRates <- *federationRates
3059 fedRate <- effectiveFederationRate
3159 fedRate <- effectiveFederationRate
@@ -3130,6 +3230,13 @@ checkPreAuth = \clientIp t -> do
3130 if ipLimited then yield "ip_rate_limited"
3230 if ipLimited then yield "ip_rate_limited"
3131 else yield ""
3231 else yield ""
3233+-- Federation auth wrapper. Intentionally does NOT call checkPreAuth because:
3234+-- (a) Soft drain (phase 1) blocks new client auth but must not block federation —
3235+-- peers need to deliver messages and acks to existing recipients during drain.
3236+-- (b) Federation has its own rate limiting via checkFederationRate (per-server)
3237+-- and checkGlobalRate (shared with client routes), so IP rate limiting is redundant.
3238+-- (c) Federation routes don't carry a ClientIp header (peer-to-peer, not client-facing),
3239+-- so IP-based rate limiting doesn't apply.
3133withFederationAuth = \authorization t respond fromServer scope requestTimestamp precondition clearOnAuth action -> do
3240withFederationAuth = \authorization t respond fromServer scope requestTimestamp precondition clearOnAuth action -> do
3134 -- Check blocklist and allowlist before spending resources on verification callback
3241 -- Check blocklist and allowlist before spending resources on verification callback
3135 blocked <- *blockedPeers
3242 blocked <- *blockedPeers
@@ -3598,6 +3705,11 @@ pruneAndRetry = \t -> do
3598 pss <- *prunesSinceSnapshot
3705 pss <- *prunesSinceSnapshot
3599 set *prunesSinceSnapshot = pss + 1
3706 set *prunesSinceSnapshot = pss + 1
3600 pr <- pruneExpired t
3707 pr <- pruneExpired t
3708+ -- Warn if forward nonces are near capacity (non-expired eviction creates a replay window).
3709+ -- Operators should increase maxForwardNonces or investigate high message volume.
3710+ nonceCount <- countAllNonces 0
3711+ when (nonceCount * 10 >= maxForwardNonces * 9)
3712+ (logWarn ("forward nonces at " ++ show (nonceCount * 100 / maxForwardNonces) ++ "% capacity (" ++ show nonceCount ++ "/" ++ show maxForwardNonces ++ ") — non-expired eviction may create replay windows, consider increasing maxForwardNonces"))
3601 -- Counter reconciliation: reconcile all counters every prune cycle.
3713 -- Counter reconciliation: reconcile all counters every prune cycle.
3602 -- Presence and session counters are cheap (just counting list lengths across buckets).
3714 -- Presence and session counters are cheap (just counting list lengths across buckets).
3603 -- Message reconciliation is heavier (reads all messages, rebuilds per-recipient counts)
3715 -- Message reconciliation is heavier (reads all messages, rebuilds per-recipient counts)
@@ -4056,7 +4168,8 @@ handleChallenge = \pubkey clientIp respond -> do
4056 then yield {accepted: False {}, rateLimited: False {}, retryAfterMs: 0, error: "capacity"}
4168 then yield {accepted: False {}, rateLimited: False {}, retryAfterMs: 0, error: "capacity"}
4058 challengeRates <- *challengeRates
4170 challengeRates <- *challengeRates
4171+ chalRate <- effectiveChallengeRate
4059− let rateResult = checkPubkeyRate challengeRates pubkey maxChallengeRatePerKey challengeRateWindowMs maxChallengeRateEntries t
4172+ let rateResult = checkPubkeyRate challengeRates pubkey chalRate challengeRateWindowMs maxChallengeRateEntries t
4060 if rateResult.limited
4173 if rateResult.limited
4061 then yield {accepted: False {}, rateLimited: True {}, retryAfterMs: rateResult.retryAfterMs, error: "rate_limited"}
4174 then yield {accepted: False {}, rateLimited: True {}, retryAfterMs: rateResult.retryAfterMs, error: "rate_limited"}
@@ -4082,7 +4195,8 @@ handleVerifyAuth = \pubkey challenge signature revokeOthers clientIp respond ->
4083 authRateLimited <- atomic do
4196 authRateLimited <- atomic do
4084 challengeRates <- *challengeRates
4197 challengeRates <- *challengeRates
4198+ chalRate <- effectiveChallengeRate
4085− let rateResult = checkPubkeyRate challengeRates pubkey maxChallengeRatePerKey challengeRateWindowMs maxChallengeRateEntries t
4199+ let rateResult = checkPubkeyRate challengeRates pubkey chalRate challengeRateWindowMs maxChallengeRateEntries t
4086 set *challengeRates = rateResult.entries
4200 set *challengeRates = rateResult.entries
4087 yield rateResult.limited
4201 yield rateResult.limited
4088 if authRateLimited then do
4202 if authRateLimited then do
@@ -4279,7 +4393,8 @@ sendMessageCore = \senderKey to encryptedBlob clampedTs effectiveMsgId t ->
4280 set *sendRates = rateResult.entries
4394 set *sendRates = rateResult.entries
4281 pairRates <- *pairRates
4395 pairRates <- *pairRates
4396+ pRate <- effectivePairRate
4282− let pairResult = checkPairRate pairRates senderKey to maxPairRatePerPair pairRateWindowMs maxPairRateEntries t
4397+ let pairResult = checkPairRate pairRates senderKey to pRate pairRateWindowMs maxPairRateEntries t
4283 set *pairRates = pairResult.entries
4398 set *pairRates = pairResult.entries
4284 if pairResult.limited then yield {status: MsRateLimited {}, remoteServers: [], retryAfterMs: pairResult.retryAfterMs}
4399 if pairResult.limited then yield {status: MsRateLimited {}, remoteServers: [], retryAfterMs: pairResult.retryAfterMs}
@@ -4447,7 +4562,8 @@ handleLookupPresence = \pubkey authorization clientIp respond -> do
4447 (yield {server: "", authorized: False {}, error: ""})
4562 (yield {server: "", authorized: False {}, error: ""})
4448 (\senderKey -> do
4563 (\senderKey -> do
4449 lookupRates <- *lookupRates
4564 lookupRates <- *lookupRates
4565+ lkRate <- effectiveLookupRate
4450− let rateResult = checkPubkeyRate lookupRates senderKey maxLookupRatePerKey lookupRateWindowMs maxLookupRateEntries t
4566+ let rateResult = checkPubkeyRate lookupRates senderKey lkRate lookupRateWindowMs maxLookupRateEntries t
4451 set *lookupRates = rateResult.entries
4567 set *lookupRates = rateResult.entries
4452 if rateResult.limited
4568 if rateResult.limited
4453 then yield {server: "", authorized: True {}, error: "rate_limited"}
4569 then yield {server: "", authorized: True {}, error: "rate_limited"}
@@ -4510,7 +4626,8 @@ handleMessageStatus = \to messageId authorization clientIp respond -> do
4510 -- Per-key rate limiting (reuses lookup rate limiter — both are info lookups)
4626 -- Per-key rate limiting (reuses lookup rate limiter — both are info lookups)
4511 statusRateLimited <- atomic do
4627 statusRateLimited <- atomic do
4512 lookupRates <- *lookupRates
4628 lookupRates <- *lookupRates
4629+ lkRate <- effectiveLookupRate
4513− let rateResult = checkPubkeyRate lookupRates senderKey maxLookupRatePerKey lookupRateWindowMs maxLookupRateEntries t
4630+ let rateResult = checkPubkeyRate lookupRates senderKey lkRate lookupRateWindowMs maxLookupRateEntries t
4514 set *lookupRates = rateResult.entries
4631 set *lookupRates = rateResult.entries
4515 yield rateResult.limited
4632 yield rateResult.limited
4516 if statusRateLimited then yield (respond {status: "", authorized: True {}})
4633 if statusRateLimited then yield (respond {status: "", authorized: True {}})
@@ -5022,7 +5139,8 @@ handleDrain = \phase authorization respond -> do
5022 fn <- readAllNonces
5139 fn <- readAllNonces
5023 bp <- *blockedPeers
5140 bp <- *blockedPeers
5024 ap <- *allowedPeers
5141 ap <- *allowedPeers
5142+ let drainCs = snapshotChecksum (count drainResult.localEntries) (count a) (count d) (count flushState.forwards) (count fn)
5025− yield {version: snapshotVersion, presence: drainResult.localEntries, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: flushState.forwards, forwardNonces: fn, blockedPeers: bp, allowedPeers: ap}
5143+ yield {version: snapshotVersion, checksum: drainCs, presence: drainResult.localEntries, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: flushState.forwards, forwardNonces: fn, blockedPeers: bp, allowedPeers: ap}
5026 println ("[SNAPSHOT_DATA] " ++ toJson drainSnapshot)
5144 println ("[SNAPSHOT_DATA] " ++ toJson drainSnapshot)
5027 atomic do set *prunesSinceSnapshot = 0
5145 atomic do set *prunesSinceSnapshot = 0
5028 logInfo ("drain: auto-snapshot exported (" ++ show (count drainResult.localEntries) ++ " presence, " ++ show (count flushState.forwards) ++ " forwards)")
5146 logInfo ("drain: auto-snapshot exported (" ++ show (count drainResult.localEntries) ++ " presence, " ++ show (count flushState.forwards) ++ " forwards)")
@@ -5107,6 +5225,28 @@ handlePrune = \authorization respond -> do
5107 when authorized (fork (pruneAndRetry t))
5225 when authorized (fork (pruneAndRetry t))
5108 yield (respond {ok: authorized})
5226 yield (respond {ok: authorized})
5228+-- POST /maintenance/reconcile: Force counter reconciliation without a full prune cycle.
5229+-- Returns drift values for messages, presence, and sessions. Non-zero drift means
5230+-- counters were corrected. Use when /stats shows suspicious counter values.
5231+handleReconcile = \authorization respond -> do
5232+ let authorized = verifyMaintenanceAuth authorization
5233+ if not authorized then yield (respond {ok: False {}, messageDrift: 0, presenceDrift: 0, sessionDrift: 0})
5235+ presDrift <- atomic do
5236+ actual <- countAllPresence
5237+ stored <- *totalPresenceCount
5238+ set *totalPresenceCount = actual
5239+ yield (actual - stored)
5240+ sessDrift <- atomic do
5241+ actual <- countAllSessions
5242+ stored <- *totalSessionCount
5243+ set *totalSessionCount = actual
5244+ yield (actual - stored)
5245+ msgDrift <- reconcileMessageCounts 0 0
5246+ when (presDrift != 0 || sessDrift != 0 || msgDrift != 0)
5247+ (logInfo ("reconcile: corrected drift — messages: " ++ show msgDrift ++ ", presence: " ++ show presDrift ++ ", sessions: " ++ show sessDrift))
5248+ yield (respond {ok: True {}, messageDrift: msgDrift, presenceDrift: presDrift, sessionDrift: sessDrift})
5110-- Compute health state and update the cached health values.
5250-- Compute health state and update the cached health values.
5111-- Called from both handleHealth (request-driven) and pruneAndRetry (background-driven)
5251-- Called from both handleHealth (request-driven) and pruneAndRetry (background-driven)
5112-- so the cache stays fresh even when /health is rate-limited under sustained monitoring load.
5252-- so the cache stays fresh even when /health is rate-limited under sustained monitoring load.
@@ -5176,7 +5316,7 @@ handleHealth = \authorization respond -> do
5176 cS <- *cachedHealthSnapshot
5316 cS <- *cachedHealthSnapshot
5177 cW <- *cachedHealthWatchdog
5317 cW <- *cachedHealthWatchdog
5178 dp <- *drainPhase
5318 dp <- *drainPhase
5179− yield {ok: cOk && dp == 0, pruneHealthy: cP, heartbeatHealthy: cH, capacityHealthy: cC, federationHealthy: cF, snapshotHealthy: cS, watchdogHealthy: cW, draining: dp > 0}
5319+ yield {ok: cOk && dp == 0, version: serverBuildVersion, pruneHealthy: cP, heartbeatHealthy: cH, capacityHealthy: cC, federationHealthy: cF, snapshotHealthy: cS, watchdogHealthy: cW, draining: dp > 0}
5180 yield (respond cached)
5320 yield (respond cached)
5182 let isAuthed = verifyMaintenanceAuth authorization
5322 let isAuthed = verifyMaintenanceAuth authorization
@@ -5185,14 +5325,14 @@ handleHealth = \authorization respond -> do
5185 -- but hide subsystem breakdown to prevent targeted load-correlation attacks.
5325 -- but hide subsystem breakdown to prevent targeted load-correlation attacks.
5186 -- Draining status is always visible (load balancers need it for routing decisions).
5326 -- Draining status is always visible (load balancers need it for routing decisions).
5187 if isAuthed
5327 if isAuthed
5188− then yield (respond {ok: allOk, pruneHealthy: result.pruneOk,
5328+ then yield (respond {ok: allOk, version: serverBuildVersion, pruneHealthy: result.pruneOk,
5189 heartbeatHealthy: result.heartbeatOk,
5329 heartbeatHealthy: result.heartbeatOk,
5190 capacityHealthy: result.capacityOk,
5330 capacityHealthy: result.capacityOk,
5191 federationHealthy: result.federationOk,
5331 federationHealthy: result.federationOk,
5192 snapshotHealthy: result.snapshotOk,
5332 snapshotHealthy: result.snapshotOk,
5193 watchdogHealthy: result.watchdogOk,
5333 watchdogHealthy: result.watchdogOk,
5194 draining: result.isDraining})
5334 draining: result.isDraining})
5195− else yield (respond {ok: allOk, pruneHealthy: True {},
5335+ else yield (respond {ok: allOk, version: serverBuildVersion, pruneHealthy: True {},
5196 heartbeatHealthy: True {},
5336 heartbeatHealthy: True {},
5197 capacityHealthy: True {},
5337 capacityHealthy: True {},
5198 federationHealthy: True {},
5338 federationHealthy: True {},
@@ -5200,7 +5340,7 @@ handleHealth = \authorization respond -> do
5200 watchdogHealthy: True {},
5340 watchdogHealthy: True {},
5201 draining: result.isDraining})
5341 draining: result.isDraining})
5203−zeroStats = {ok: False {}, capacity: {sessions: 0, sessionsMax: 0, sessionsPct: 0, messages: 0, messagesMax: 0, messagesPct: 0, totalMessageCount: 0, presence: 0, presenceMax: 0, presencePct: 0, forwards: 0, forwardsMax: 0, forwardsPct: 0, challenges: 0, challengesMax: 0, ackedIds: 0, ackedIdsMax: 0, ackOverflow: 0, ackOverflowMax: 0, deliveredIds: 0, deliveredIdsMax: 0, forwardNonces: 0, forwardNoncesMax: 0, activePolls: 0, activePollsMax: 0, recipientCounts: 0}, federation: {enabled: False {}, inFlight: 0, hungSlots: 0, avgLatencyMs: 0, maxLatencyMs: 0, callCount: 0, forwardFailures: 0, oldestForwardAge: 0, oldestAckAge: 0, recentGossip: 0, blockedPeers: 0, drainCompletedAt: 0, forwardServers: 0, maxForwardQueueDepth: 0}, rates: {send: 0, challenge: 0, federation: 0, federationPair: 0, global: 0, ip: 0, lookup: 0, recv: 0, pair: 0, gossipEvent: 0, verifyIp: 0, uniqueKeyIp: 0, pendingAckCleanups: 0, senderExpirations: 0}, timing: {pruneCount: 0, pruneRunning: 0, heartbeatCount: 0, uptime: 0, lastPruneAt: 0, lastHeartbeatAt: 0, draining: False {}, drainPhase: 0}}
5343+zeroStats = {ok: False {}, version: serverBuildVersion, capacity: {sessions: 0, sessionsMax: 0, sessionsPct: 0, messages: 0, messagesMax: 0, messagesPct: 0, totalMessageCount: 0, presence: 0, presenceMax: 0, presencePct: 0, forwards: 0, forwardsMax: 0, forwardsPct: 0, challenges: 0, challengesMax: 0, ackedIds: 0, ackedIdsMax: 0, ackOverflow: 0, ackOverflowMax: 0, deliveredIds: 0, deliveredIdsMax: 0, forwardNonces: 0, forwardNoncesMax: 0, activePolls: 0, activePollsMax: 0, recipientCounts: 0}, federation: {enabled: False {}, inFlight: 0, hungSlots: 0, avgLatencyMs: 0, maxLatencyMs: 0, callCount: 0, forwardFailures: 0, oldestForwardAge: 0, oldestAckAge: 0, recentGossip: 0, blockedPeers: 0, drainCompletedAt: 0, forwardServers: 0, maxForwardQueueDepth: 0}, rates: {send: 0, challenge: 0, federation: 0, federationPair: 0, global: 0, ip: 0, lookup: 0, recv: 0, pair: 0, gossipEvent: 0, verifyIp: 0, uniqueKeyIp: 0, pendingAckCleanups: 0, senderExpirations: 0}, timing: {pruneCount: 0, pruneRunning: 0, heartbeatCount: 0, uptime: 0, lastPruneAt: 0, lastHeartbeatAt: 0, draining: False {}, drainPhase: 0}}
5205handleStats = \authorization respond -> do
5345handleStats = \authorization respond -> do
5206 t <- now
5346 t <- now
@@ -5278,7 +5418,7 @@ handleStats = \authorization respond -> do
5278 let oldestAckAge = if oldestAckTs == 0 then 0 else t - oldestAckTs
5418 let oldestAckAge = if oldestAckTs == 0 then 0 else t - oldestAckTs
5279 let fwdServerCount = fwdStats.serverCount
5419 let fwdServerCount = fwdStats.serverCount
5280 let maxFwdDepth = fwdStats.maxDepth
5420 let maxFwdDepth = fwdStats.maxDepth
5281− yield (respond {ok: True {},
5421+ yield (respond {ok: True {}, version: serverBuildVersion,
5282 capacity: {sessions: sessions, sessionsMax: maxSessions, sessionsPct: sessPct, messages: messages, messagesMax: maxMessages, messagesPct: msgPct, totalMessageCount: messages, presence: presence, presenceMax: maxPresenceEntries, presencePct: presPct, forwards: fwds, forwardsMax: maxForwardsEntries, forwardsPct: fwdPct, challenges: count core.c, challengesMax: maxChallenges, ackedIds: core.aCount, ackedIdsMax: maxAckedIds, ackOverflow: count core.ao, ackOverflowMax: maxAckOverflow, deliveredIds: core.dCount, deliveredIdsMax: maxDeliveredIds, forwardNonces: core.fnCount, forwardNoncesMax: maxForwardNonces, activePolls: count core.ap, activePollsMax: maxActivePolls, recipientCounts: count core.rc},
5422 capacity: {sessions: sessions, sessionsMax: maxSessions, sessionsPct: sessPct, messages: messages, messagesMax: maxMessages, messagesPct: msgPct, totalMessageCount: messages, presence: presence, presenceMax: maxPresenceEntries, presencePct: presPct, forwards: fwds, forwardsMax: maxForwardsEntries, forwardsPct: fwdPct, challenges: count core.c, challengesMax: maxChallenges, ackedIds: core.aCount, ackedIdsMax: maxAckedIds, ackOverflow: count core.ao, ackOverflowMax: maxAckOverflow, deliveredIds: core.dCount, deliveredIdsMax: maxDeliveredIds, forwardNonces: core.fnCount, forwardNoncesMax: maxForwardNonces, activePolls: count core.ap, activePollsMax: maxActivePolls, recipientCounts: count core.rc},
5283 federation: {enabled: federationEnabled, inFlight: count core.fi, hungSlots: hungFedSlots, avgLatencyMs: fedAvg, maxLatencyMs: counters.flm, callCount: counters.flc, forwardFailures: count core.ff, oldestForwardAge: oldestFwdAge, oldestAckAge: oldestAckAge, recentGossip: count core.rg, blockedPeers: count core.bp, drainCompletedAt: counters.dca, forwardServers: fwdServerCount, maxForwardQueueDepth: maxFwdDepth},
5423 federation: {enabled: federationEnabled, inFlight: count core.fi, hungSlots: hungFedSlots, avgLatencyMs: fedAvg, maxLatencyMs: counters.flm, callCount: counters.flc, forwardFailures: count core.ff, oldestForwardAge: oldestFwdAge, oldestAckAge: oldestAckAge, recentGossip: count core.rg, blockedPeers: count core.bp, drainCompletedAt: counters.dca, forwardServers: fwdServerCount, maxForwardQueueDepth: maxFwdDepth},
5284 rates: {send: count counters.sr, challenge: count counters.cr, federation: count counters.fr, federationPair: count counters.fpr, global: counters.grc, ip: count counters.ir, lookup: count counters.lr, recv: count counters.rr, pair: count counters.prr, gossipEvent: count core.ger, verifyIp: count core.vir, uniqueKeyIp: count core.ukir, pendingAckCleanups: count core.pac, senderExpirations: count core.se},
5424 rates: {send: count counters.sr, challenge: count counters.cr, federation: count counters.fr, federationPair: count counters.fpr, global: counters.grc, ip: count counters.ir, lookup: count counters.lr, recv: count counters.rr, pair: count counters.prr, gossipEvent: count core.ger, verifyIp: count core.vir, uniqueKeyIp: count core.ukir, pendingAckCleanups: count core.pac, senderExpirations: count core.se},
@@ -5342,23 +5482,14 @@ handleTopRecipients = \limit authorization respond -> do
5342 let authorized = verifyMaintenanceAuth authorization
5482 let authorized = verifyMaintenanceAuth authorization
5343 if authorized
5483 if authorized
5345− -- Collect all recipient counts across 16 buckets and sort by count descending
5346− allRc <- collectAllRcEntries 0
5347− let sorted = sortBy (\c -> 0 - c.count) allRc
5348 let effectiveLimit = if limit <= 0 then 20 else clamp 1 100 limit
5485 let effectiveLimit = if limit <= 0 then 20 else clamp 1 100 limit
5486+ -- Per-bucket top-N merge: scans one rcBucket at a time, maintaining a running
5487+ -- top-N list. Avoids materializing all 16 buckets into a single list.
5349− let top = takeRelation effectiveLimit sorted
5488+ top <- topRecipientsByBucket 0 effectiveLimit []
5489+ totalRc <- countAllRcEntries 0
5350− yield (respond {recipients: top, ok: True {}, total: count allRc})
5490+ yield (respond {recipients: top, ok: True {}, total: totalRc})
5351 else yield (respond {recipients: [], ok: False {}, total: 0})
5491 else yield (respond {recipients: [], ok: False {}, total: 0})
5353−-- Collect RecipientCount entries from all 16 rc buckets
5354−collectAllRcEntries : Int -> IO [RecipientCount]
5355−collectAllRcEntries = \b ->
5356− if b > 15 then yield []
5358− bucket <- readRcBucket b
5359− rest <- collectAllRcEntries (b + 1)
5360− yield (union bucket rest)
5362-- GET /ready: Readiness probe — returns true once the server has completed startup.
5493-- GET /ready: Readiness probe — returns true once the server has completed startup.
5363-- Unlike /health, this is unauthenticated and doesn't check subsystem liveness.
5494-- Unlike /health, this is unauthenticated and doesn't check subsystem liveness.
5364-- Returns false until the first prune cycle completes (server fully initialized).
5495-- Returns false until the first prune cycle completes (server fully initialized).
@@ -5370,7 +5501,7 @@ handleReady = \respond -> do
5370 let isReady = lastPrune > 0 && lastPrune >= sa
5501 let isReady = lastPrune > 0 && lastPrune >= sa
5371 yield (respond {ready: isReady})
5502 yield (respond {ready: isReady})
5373−emptySnapshot = {ok: False {}, version: 0, presence: [], ackedIds: [], ackOverflow: [], deliveredIds: [], forwardFailures: [], forwards: [], forwardNonces: [], blockedPeers: [], allowedPeers: []}
5504+emptySnapshot = {ok: False {}, version: 0, checksum: 0, presence: [], ackedIds: [], ackOverflow: [], deliveredIds: [], forwardFailures: [], forwards: [], forwardNonces: [], blockedPeers: [], allowedPeers: []}
5375-- GET /maintenance/snapshot: Export federation metadata for persistence.
5506-- GET /maintenance/snapshot: Export federation metadata for persistence.
5376-- Reads all dedup state (ackedIds, deliveredIds), presence, peer failure records,
5507-- Reads all dedup state (ackedIds, deliveredIds), presence, peer failure records,
@@ -5395,25 +5526,32 @@ handleSnapshot = \authorization respond -> do
5395 atomic do
5526 atomic do
5396 set *lastSnapshotExportAt = t
5527 set *lastSnapshotExportAt = t
5397 set *prunesSinceSnapshot = 0
5528 set *prunesSinceSnapshot = 0
5529+ let cs = snapshotChecksum (count result.presence) (count result.ackedIds) (count result.deliveredIds) (count result.forwards) (count result.forwardNonces)
5398− logInfo ("snapshot: exported " ++ show (count result.presence) ++ " presence, " ++ show (count result.ackedIds) ++ " ackedIds, " ++ show (count result.deliveredIds) ++ " deliveredIds, " ++ show (count result.forwards) ++ " forwards, " ++ show (count result.forwardNonces) ++ " forwardNonces, " ++ show (count result.blockedPeers) ++ " blockedPeers, " ++ show (count result.allowedPeers) ++ " allowedPeers")
5530+ logInfo ("snapshot: exported " ++ show (count result.presence) ++ " presence, " ++ show (count result.ackedIds) ++ " ackedIds, " ++ show (count result.deliveredIds) ++ " deliveredIds, " ++ show (count result.forwards) ++ " forwards, " ++ show (count result.forwardNonces) ++ " forwardNonces, " ++ show (count result.blockedPeers) ++ " blockedPeers, " ++ show (count result.allowedPeers) ++ " allowedPeers, checksum=" ++ show cs)
5399− yield (respond {ok: True {}, version: snapshotVersion, presence: result.presence, ackedIds: result.ackedIds, ackOverflow: result.ackOverflow, deliveredIds: result.deliveredIds, forwardFailures: result.forwardFailures, forwards: result.forwards, forwardNonces: result.forwardNonces, blockedPeers: result.blockedPeers, allowedPeers: result.allowedPeers})
5531+ yield (respond {ok: True {}, version: snapshotVersion, checksum: cs, presence: result.presence, ackedIds: result.ackedIds, ackOverflow: result.ackOverflow, deliveredIds: result.deliveredIds, forwardFailures: result.forwardFailures, forwards: result.forwards, forwardNonces: result.forwardNonces, blockedPeers: result.blockedPeers, allowedPeers: result.allowedPeers})
5401-- POST /maintenance/restore: Import a previously-exported snapshot.
5533-- POST /maintenance/restore: Import a previously-exported snapshot.
5402-- Requires drain mode to prevent conflicts with live state. Merges imported data
5534-- Requires drain mode to prevent conflicts with live state. Merges imported data
5403-- with existing state (dedup entries are unioned, presence is merged by freshness).
5535-- with existing state (dedup entries are unioned, presence is merged by freshness).
5404-- After restore, call /maintenance/undrain to resume, then /maintenance/prune to
5536-- After restore, call /maintenance/undrain to resume, then /maintenance/prune to
5405-- trigger federation forward creation from messages + restored presence.
5537-- trigger federation forward creation from messages + restored presence.
5406−handleRestore = \version presenceData ackedIdsData ackOverflowData deliveredIdsData forwardFailuresData forwardsData forwardNoncesData blockedPeersData allowedPeersData authorization respond -> do
5538+handleRestore = \version checksumIn presenceData ackedIdsData ackOverflowData deliveredIdsData forwardFailuresData forwardsData forwardNoncesData blockedPeersData allowedPeersData authorization respond -> do
5407 let authorized = verifyMaintenanceAuth authorization
5539 let authorized = verifyMaintenanceAuth authorization
5408 if not authorized then yield (respond {ok: False {}, error: "unauthorized", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})
5540 if not authorized then yield (respond {ok: False {}, error: "unauthorized", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})
5410 -- Version 0 = pre-versioning snapshot, version 1 = pre-forwardNonces, version 2 = pre-blockedPeers,
5542 -- Version 0 = pre-versioning snapshot, version 1 = pre-forwardNonces, version 2 = pre-blockedPeers,
5411− -- version 3 = pre-ForwardEntry changes, version 4 = pre-allowedPeers
5543+ -- version 3 = pre-ForwardEntry changes, version 4 = pre-allowedPeers, version 5 = pre-checksum
5412 -- (accept all for backwards compatibility; missing fields default to empty list or 0)
5544 -- (accept all for backwards compatibility; missing fields default to empty list or 0)
5413− if version != 0 && version != 1 && version != 2 && version != 3 && version != 4 && version != snapshotVersion then do
5545+ if version != 0 && version != 1 && version != 2 && version != 3 && version != 4 && version != 5 && version != snapshotVersion then do
5414 logError ("restore: incompatible snapshot version " ++ show version ++ " (expected " ++ show snapshotVersion ++ ")")
5546 logError ("restore: incompatible snapshot version " ++ show version ++ " (expected " ++ show snapshotVersion ++ ")")
5415 yield (respond {ok: False {}, error: "incompatible_version", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})
5547 yield (respond {ok: False {}, error: "incompatible_version", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})
5549+ -- Checksum verification (skip for pre-v6 snapshots where checksum defaults to 0)
5550+ let expectedCs = snapshotChecksum (count presenceData) (count ackedIdsData) (count deliveredIdsData) (count forwardsData) (count forwardNoncesData)
5551+ if checksumIn != 0 && checksumIn != expectedCs then do
5552+ logError ("restore: checksum mismatch (got " ++ show checksumIn ++ ", expected " ++ show expectedCs ++ ") — snapshot may be truncated or corrupted")
5553+ yield (respond {ok: False {}, error: "checksum_mismatch", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})
5417 t <- now
5555 t <- now
5418 dp <- *drainPhase
5556 dp <- *drainPhase
5419 if dp < 2 then yield (respond {ok: False {}, error: "server_must_be_draining", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})
5557 if dp < 2 then yield (respond {ok: False {}, error: "server_must_be_draining", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})
@@ -5504,7 +5642,8 @@ handleAckMessages = \messageIds authorization clientIp respond -> do
5504 -- Uses clientWriteRates (separate from lookupRates) so ack bursts don't starve lookups
5642 -- Uses clientWriteRates (separate from lookupRates) so ack bursts don't starve lookups
5505 ackRateLimited <- atomic do
5643 ackRateLimited <- atomic do
5506 cwr <- *clientWriteRates
5644 cwr <- *clientWriteRates
5645+ cwRate <- effectiveClientWriteRate
5507− let rateResult = checkPubkeyRate cwr pubkey maxClientWriteRatePerKey clientWriteRateWindowMs maxClientWriteRateEntries t
5646+ let rateResult = checkPubkeyRate cwr pubkey cwRate clientWriteRateWindowMs maxClientWriteRateEntries t
5508 set *clientWriteRates = rateResult.entries
5647 set *clientWriteRates = rateResult.entries
5509 yield rateResult.limited
5648 yield rateResult.limited
5510 if ackRateLimited then yield (respond {ok: False {}, authorized: True {}, ackedCount: 0, error: "rate_limited"})
5649 if ackRateLimited then yield (respond {ok: False {}, authorized: True {}, ackedCount: 0, error: "rate_limited"})
@@ -5554,7 +5693,8 @@ handleListMessages = \offset authorization clientIp respond -> do
5554 (\pubkey -> do
5693 (\pubkey -> do
5555 listRateLimited <- atomic do
5694 listRateLimited <- atomic do
5556 lookupRates <- *lookupRates
5695 lookupRates <- *lookupRates
5696+ lkRate <- effectiveLookupRate
5557− let rateResult = checkPubkeyRate lookupRates pubkey maxLookupRatePerKey lookupRateWindowMs maxLookupRateEntries t
5697+ let rateResult = checkPubkeyRate lookupRates pubkey lkRate lookupRateWindowMs maxLookupRateEntries t
5558 set *lookupRates = rateResult.entries
5698 set *lookupRates = rateResult.entries
5559 yield rateResult.limited
5699 yield rateResult.limited
5560 if listRateLimited then yield (respond {messages: [], authorized: True {}, count: 0, error: "rate_limited"})
5700 if listRateLimited then yield (respond {messages: [], authorized: True {}, count: 0, error: "rate_limited"})
@@ -5590,7 +5730,8 @@ handleDeleteMessages = \messageIds authorization clientIp respond -> do
5590 -- Uses clientWriteRates (separate from lookupRates) so delete bursts don't starve lookups
5730 -- Uses clientWriteRates (separate from lookupRates) so delete bursts don't starve lookups
5591 deleteRateLimited <- atomic do
5731 deleteRateLimited <- atomic do
5592 cwr <- *clientWriteRates
5732 cwr <- *clientWriteRates
5733+ cwRate <- effectiveClientWriteRate
5593− let rateResult = checkPubkeyRate cwr pubkey maxClientWriteRatePerKey clientWriteRateWindowMs maxClientWriteRateEntries t
5734+ let rateResult = checkPubkeyRate cwr pubkey cwRate clientWriteRateWindowMs maxClientWriteRateEntries t
5594 set *clientWriteRates = rateResult.entries
5735 set *clientWriteRates = rateResult.entries
5595 yield rateResult.limited
5736 yield rateResult.limited
5596 if deleteRateLimited then yield (respond {ok: False {}, authorized: True {}, deletedCount: 0, error: "rate_limited"})
5737 if deleteRateLimited then yield (respond {ok: False {}, authorized: True {}, deletedCount: 0, error: "rate_limited"})
@@ -5647,7 +5788,8 @@ handleCancelMessage = \messageId to authorization clientIp respond -> do
5647 -- Uses clientWriteRates (like ack/delete) so cancel bursts don't starve lookups
5788 -- Uses clientWriteRates (like ack/delete) so cancel bursts don't starve lookups
5648 cancelRateLimited <- atomic do
5789 cancelRateLimited <- atomic do
5649 cwr <- *clientWriteRates
5790 cwr <- *clientWriteRates
5791+ cwRate <- effectiveClientWriteRate
5650− let rateResult = checkPubkeyRate cwr senderKey maxClientWriteRatePerKey clientWriteRateWindowMs maxClientWriteRateEntries t
5792+ let rateResult = checkPubkeyRate cwr senderKey cwRate clientWriteRateWindowMs maxClientWriteRateEntries t
5651 set *clientWriteRates = rateResult.entries
5793 set *clientWriteRates = rateResult.entries
5652 yield rateResult.limited
5794 yield rateResult.limited
5653 if cancelRateLimited then yield (respond {ok: False {}, authorized: True {}, cancelled: False {}, error: "rate_limited"})
5795 if cancelRateLimited then yield (respond {ok: False {}, authorized: True {}, cancelled: False {}, error: "rate_limited"})
@@ -5871,19 +6013,18 @@ handlePingPeer = \server authorization respond -> do
5871 Ok r -> yield (respond {ok: r.ok, latencyMs: latency, error: r.error})
6013 Ok r -> yield (respond {ok: r.ok, latencyMs: latency, error: r.error})
5873-- GET /maintenance/sessions/{pubkey}: List sessions for a specific pubkey.
6015-- GET /maintenance/sessions/{pubkey}: List sessions for a specific pubkey.
5874−-- Returns masked tokens (first 8 chars only) for identification without exposing full tokens.
6016+-- Uses per-bucket scan (collectSessionsForKey) instead of readAllSessions to avoid
6017+-- materializing all 16 session buckets into one list just to filter by pubkey.
5875handleAdminListSessions = \pubkey authorization respond -> do
6018handleAdminListSessions = \pubkey authorization respond -> do
5876 let authorized = verifyMaintenanceAuth authorization
6019 let authorized = verifyMaintenanceAuth authorization
5877 if not authorized then yield (respond {sessions: [], count: 0, ok: False {}})
6020 if not authorized then yield (respond {sessions: [], count: 0, ok: False {}})
5879− result <- atomic do
6022+ keySessions <- collectSessionsForKey pubkey 0
5880− allSessions <- readAllSessions
5881− -- Return only metadata (IP, timestamps) — no token fragments. Even a 4-char prefix
6023+ -- Return only metadata (IP, timestamps) — no token fragments. Even a 4-char prefix
5882− -- (16 bits) narrows brute-force search space and can be correlated with token bucket
6024+ -- (16 bits) narrows brute-force search space and can be correlated with token bucket
5883− -- indices. Sessions are identifiable by (ip, createdAt) pair without token exposure.
6025+ -- indices. Sessions are identifiable by (ip, createdAt) pair without token exposure.
5884− let keySessions = do s <- allSessions; where s.pubkey == pubkey; yield {token: "***", expiresAt: s.expiresAt, ip: s.ip, createdAt: s.createdAt}
6026+ let masked = do s <- keySessions; yield {token: "***", expiresAt: s.expiresAt, ip: s.ip, createdAt: s.createdAt}
5885− yield {sessions: keySessions, total: count keySessions}
5886− yield (respond {sessions: result.sessions, count: result.total, ok: True {}})
6027+ yield (respond {sessions: masked, count: count masked, ok: True {}})
5888-- POST /maintenance/sessions/revoke-all: Revoke all sessions for a pubkey (abuse response).
6029-- POST /maintenance/sessions/revoke-all: Revoke all sessions for a pubkey (abuse response).
5889-- Also clears local presence and gossips offline if federation is enabled.
6030-- Also clears local presence and gossips offline if federation is enabled.
@@ -5979,7 +6120,7 @@ handleFlushNonces = \authorization respond -> do
5979-- GET /maintenance/rate-limits: Return current effective rate limits.
6120-- GET /maintenance/rate-limits: Return current effective rate limits.
5980handleGetRateLimits = \authorization respond -> do
6121handleGetRateLimits = \authorization respond -> do
5981 let authorized = verifyMaintenanceAuth authorization
6122 let authorized = verifyMaintenanceAuth authorization
5982− if not authorized then yield (respond {ok: False {}, globalRequest: 0, ipRate: 0, sendRate: 0, federationRate: 0, authGlobal: 0, recvRate: 0})
6123+ if not authorized then yield (respond {ok: False {}, globalRequest: 0, ipRate: 0, sendRate: 0, federationRate: 0, authGlobal: 0, recvRate: 0, challengeRate: 0, pairRate: 0, lookupRate: 0, clientWriteRate: 0})
5984 result <- atomic do
6125 result <- atomic do
5985 gr <- effectiveGlobalRequestRate
6126 gr <- effectiveGlobalRequestRate
@@ -5988,12 +6129,16 @@ handleGetRateLimits = \authorization respond -> do
5988 fr <- effectiveFederationRate
6129 fr <- effectiveFederationRate
5989 ar <- effectiveAuthGlobalRate
6130 ar <- effectiveAuthGlobalRate
5990 rr <- effectiveRecvRate
6131 rr <- effectiveRecvRate
6132+ cr <- effectiveChallengeRate
6133+ pr <- effectivePairRate
6134+ lr <- effectiveLookupRate
6135+ cwr <- effectiveClientWriteRate
5991− yield {globalRequest: gr, ipRate: ir, sendRate: sr, federationRate: fr, authGlobal: ar, recvRate: rr}
6136+ yield {globalRequest: gr, ipRate: ir, sendRate: sr, federationRate: fr, authGlobal: ar, recvRate: rr, challengeRate: cr, pairRate: pr, lookupRate: lr, clientWriteRate: cwr}
5992− yield (respond {ok: True {}, globalRequest: result.globalRequest, ipRate: result.ipRate, sendRate: result.sendRate, federationRate: result.federationRate, authGlobal: result.authGlobal, recvRate: result.recvRate})
6137+ yield (respond {ok: True {}, globalRequest: result.globalRequest, ipRate: result.ipRate, sendRate: result.sendRate, federationRate: result.federationRate, authGlobal: result.authGlobal, recvRate: result.recvRate, challengeRate: result.challengeRate, pairRate: result.pairRate, lookupRate: result.lookupRate, clientWriteRate: result.clientWriteRate})
5994-- POST /maintenance/rate-limits: Update rate limits at runtime. 0 = reset to compile-time default.
6139-- POST /maintenance/rate-limits: Update rate limits at runtime. 0 = reset to compile-time default.
5995-- Negative values are clamped to 0. Changes take effect on the next rate check cycle.
6140-- Negative values are clamped to 0. Changes take effect on the next rate check cycle.
5996−handleSetRateLimits = \globalRequest ipRate sendRate federationRate authGlobal recvRate authorization respond -> do
6141+handleSetRateLimits = \globalRequest ipRate sendRate federationRate authGlobal recvRate challengeRate pairRate lookupRate clientWriteRate authorization respond -> do
5997 let authorized = verifyMaintenanceAuth authorization
6142 let authorized = verifyMaintenanceAuth authorization
5998 if not authorized then yield (respond {ok: False {}})
6143 if not authorized then yield (respond {ok: False {}})
@@ -6004,7 +6149,11 @@ handleSetRateLimits = \globalRequest ipRate sendRate federationRate authGlobal r
6004 set *rateOverrideFederationRate = max 0 federationRate
6149 set *rateOverrideFederationRate = max 0 federationRate
6005 set *rateOverrideAuthGlobal = max 0 authGlobal
6150 set *rateOverrideAuthGlobal = max 0 authGlobal
6006 set *rateOverrideRecvRate = max 0 recvRate
6151 set *rateOverrideRecvRate = max 0 recvRate
6152+ set *rateOverrideChallengeRate = max 0 challengeRate
6153+ set *rateOverridePairRate = max 0 pairRate
6154+ set *rateOverrideLookupRate = max 0 lookupRate
6155+ set *rateOverrideClientWriteRate = max 0 clientWriteRate
6007− logInfo ("rate-limits: updated — global=" ++ show globalRequest ++ " ip=" ++ show ipRate ++ " send=" ++ show sendRate ++ " federation=" ++ show federationRate ++ " auth=" ++ show authGlobal ++ " recv=" ++ show recvRate ++ " (0 = default)")
6156+ logInfo ("rate-limits: updated — global=" ++ show globalRequest ++ " ip=" ++ show ipRate ++ " send=" ++ show sendRate ++ " federation=" ++ show federationRate ++ " auth=" ++ show authGlobal ++ " recv=" ++ show recvRate ++ " challenge=" ++ show challengeRate ++ " pair=" ++ show pairRate ++ " lookup=" ++ show lookupRate ++ " clientWrite=" ++ show clientWriteRate ++ " (0 = default)")
6008 yield (respond {ok: True {}})
6157 yield (respond {ok: True {}})
6010-- Extract request path for access logging (excludes poll to reduce log volume)
6159-- Extract request path for access logging (excludes poll to reduce log volume)
@@ -6047,6 +6196,7 @@ requestPath = \req -> case req of
6047 AllowPeer {} -> "POST /maintenance/allow"
6196 AllowPeer {} -> "POST /maintenance/allow"
6048 UnallowPeer {} -> "POST /maintenance/unallow"
6197 UnallowPeer {} -> "POST /maintenance/unallow"
6049 ListAllowed {} -> "GET /maintenance/allowed"
6198 ListAllowed {} -> "GET /maintenance/allowed"
6199+ Reconcile {} -> "POST /maintenance/reconcile"
6050 Health {} -> "GET /health"
6200 Health {} -> "GET /health"
6051 Stats {} -> "GET /stats"
6201 Stats {} -> "GET /stats"
6052 StatsPeers {} -> "GET /stats/peers"
6202 StatsPeers {} -> "GET /stats/peers"
@@ -6094,6 +6244,7 @@ requestClientIp = \req -> case req of
6094 AllowPeer {} -> ""
6244 AllowPeer {} -> ""
6095 UnallowPeer {} -> ""
6245 UnallowPeer {} -> ""
6096 ListAllowed {} -> ""
6246 ListAllowed {} -> ""
6097 Health {} -> ""
6248 Health {} -> ""
6098 Stats {} -> ""
6249 Stats {} -> ""
6099 StatsPeers {} -> ""
6250 StatsPeers {} -> ""
@@ -6130,7 +6281,7 @@ serveInner = \req -> case req of
6130 Shutdown {authorization, respond} -> handleShutdown authorization respond
6281 Shutdown {authorization, respond} -> handleShutdown authorization respond
6131 Undrain {authorization, respond} -> handleUndrain authorization respond
6282 Undrain {authorization, respond} -> handleUndrain authorization respond
6132 Snapshot {authorization, respond} -> handleSnapshot authorization respond
6283 Snapshot {authorization, respond} -> handleSnapshot authorization respond
6133− Restore {version, presence, ackedIds, ackOverflow, deliveredIds, forwardFailures, forwards, forwardNonces, blockedPeers, allowedPeers, authorization, respond} -> handleRestore version presence ackedIds ackOverflow deliveredIds forwardFailures forwards forwardNonces blockedPeers allowedPeers authorization respond
6284+ Restore {version, checksum, presence, ackedIds, ackOverflow, deliveredIds, forwardFailures, forwards, forwardNonces, blockedPeers, allowedPeers, authorization, respond} -> handleRestore version checksum presence ackedIds ackOverflow deliveredIds forwardFailures forwards forwardNonces blockedPeers allowedPeers authorization respond
6134 BlockPeer {server, authorization, respond} -> handleBlockPeer server authorization respond
6285 BlockPeer {server, authorization, respond} -> handleBlockPeer server authorization respond
6135 UnblockPeer {server, authorization, respond} -> handleUnblockPeer server authorization respond
6286 UnblockPeer {server, authorization, respond} -> handleUnblockPeer server authorization respond
6136 ListBlocked {authorization, respond} -> handleListBlocked authorization respond
6287 ListBlocked {authorization, respond} -> handleListBlocked authorization respond
@@ -6139,11 +6290,12 @@ serveInner = \req -> case req of
6139 AdminRevokeSessions {pubkey, authorization, respond} -> handleAdminRevokeSessions pubkey authorization respond
6290 AdminRevokeSessions {pubkey, authorization, respond} -> handleAdminRevokeSessions pubkey authorization respond
6140 AdminPurgeMessages {pubkey, authorization, respond} -> handleAdminPurgeMessages pubkey authorization respond
6291 AdminPurgeMessages {pubkey, authorization, respond} -> handleAdminPurgeMessages pubkey authorization respond
6141 GetRateLimits {authorization, respond} -> handleGetRateLimits authorization respond
6292 GetRateLimits {authorization, respond} -> handleGetRateLimits authorization respond
6142− SetRateLimits {globalRequest, ipRate, sendRate, federationRate, authGlobal, recvRate, authorization, respond} -> handleSetRateLimits globalRequest ipRate sendRate federationRate authGlobal recvRate authorization respond
6293+ SetRateLimits {globalRequest, ipRate, sendRate, federationRate, authGlobal, recvRate, challengeRate, pairRate, lookupRate, clientWriteRate, authorization, respond} -> handleSetRateLimits globalRequest ipRate sendRate federationRate authGlobal recvRate challengeRate pairRate lookupRate clientWriteRate authorization respond
6143 FlushNonces {authorization, respond} -> handleFlushNonces authorization respond
6294 FlushNonces {authorization, respond} -> handleFlushNonces authorization respond
6144 AllowPeer {server, authorization, respond} -> handleAllowPeer server authorization respond
6295 AllowPeer {server, authorization, respond} -> handleAllowPeer server authorization respond
6145 UnallowPeer {server, authorization, respond} -> handleUnallowPeer server authorization respond
6296 UnallowPeer {server, authorization, respond} -> handleUnallowPeer server authorization respond
6146 ListAllowed {authorization, respond} -> handleListAllowed authorization respond
6297 ListAllowed {authorization, respond} -> handleListAllowed authorization respond
6298+ Reconcile {authorization, respond} -> handleReconcile authorization respond
6147 Health {authorization, respond} -> handleHealth authorization respond
6299 Health {authorization, respond} -> handleHealth authorization respond
6148 Stats {authorization, respond} -> handleStats authorization respond
6300 Stats {authorization, respond} -> handleStats authorization respond
6149 StatsPeers {authorization, respond} -> handleStatsPeers authorization respond
6301 StatsPeers {authorization, respond} -> handleStatsPeers authorization respond
@@ -6330,7 +6482,8 @@ backgroundSnapshot = do
6330 fn <- readAllNonces
6482 fn <- readAllNonces
6331 bp <- *blockedPeers
6483 bp <- *blockedPeers
6332 ap <- *allowedPeers
6484 ap <- *allowedPeers
6485+ let exportCs = snapshotChecksum (count p) (count a) (count d) (count f) (count fn)
6333− yield {version: snapshotVersion, presence: p, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: f, forwardNonces: fn, blockedPeers: bp, allowedPeers: ap}
6486+ yield {version: snapshotVersion, checksum: exportCs, presence: p, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: f, forwardNonces: fn, blockedPeers: bp, allowedPeers: ap}
6334 println ("[SNAPSHOT_DATA] " ++ toJson exportData)
6487 println ("[SNAPSHOT_DATA] " ++ toJson exportData)
6335 atomic do
6488 atomic do
6336 set *lastAutoSnapshotExportAt = t
6489 set *lastAutoSnapshotExportAt = t