@@ -136,7 +136,9 @@ maxRecentFederationRequests = 1000
136136maxForwardNonces = 100000
137137forwardNonceRetention : Int<Ms>
138138forwardNonceRetention = messageMaxAge * 2 -- 60 days; extends 30 days beyond message dedup
139−-- (reconciliationFrequency removed: message counters now reconcile every prune cycle)
139+-- Message reconciliation: full scan runs every Nth prune cycle or when cheap drift check detects mismatch.
140+-- Cheap check (sum rcBucket totals vs totalMessageCount) runs every cycle; full scan only when needed.
141+reconciliationFullCycleInterval = 10 -- full reconciliation every 10th prune cycle
140142
141143-- Federation latency metric window
142144federationLatencyWindowMs : Int<Ms>
@@ -163,6 +165,11 @@ backgroundPruneInterval : Int<Ms>
163165backgroundPruneInterval = 30000 -- 30 seconds; also retries failed forwards/acks
164166autoSnapshotInterval : Int<Ms>
165167autoSnapshotInterval = 300000 -- 5 minutes; periodic state summary to stdout (0 to disable)
168+-- Full snapshot auto-export interval: emit [SNAPSHOT_DATA] JSON to stdout for sidecar capture.
169+-- 0 to disable. When enabled, a crash loses at most this interval of federation state.
170+-- Only emits when there's meaningful state to persist (presence/forwards/ackedIds > 0).
171+autoSnapshotExportInterval : Int<Ms>
172+autoSnapshotExportInterval = 3600000 -- 1 hour; full state export to stdout (0 to disable)
166173snapshotStalenessWarning : Int<Ms>
167174snapshotStalenessWarning = 3600000 -- 1 hour; warn if no snapshot exported via /maintenance/snapshot
168175pollTimeoutMs : Int<Ms>
@@ -577,6 +584,9 @@ type BatchResult = {messageId: OptionalMessageId, status: MessageStatus}
577584-- Tracks when /maintenance/snapshot was last exported (for staleness warnings)
578585*lastSnapshotExportAt : Timestamp
579586
587+-- Tracks when the last auto-export [SNAPSHOT_DATA] was emitted to stdout
588+*lastAutoSnapshotExportAt : Timestamp
589+
580590-- Per-peer gossip event volume rate limiting
581591*gossipEventRates : [GossipEventRateEntry]
582592
@@ -594,6 +604,15 @@ type BatchResult = {messageId: OptionalMessageId, status: MessageStatus}
594604-- (key-rotation abuse bypassing per-key rate limits). Pruned with other rate limiters.
595605*uniqueKeyIpRates : [UniqueKeyIpEntry]
596606
607+-- Runtime-configurable rate limit overrides (0 = use compile-time default).
608+-- Adjusted via /maintenance/rate-limits POST endpoint without recompile+restart.
609+*rateOverrideGlobalRequest : Int
610+*rateOverrideIpRate : Int
611+*rateOverrideSendRate : Int
612+*rateOverrideFederationRate : Int
613+*rateOverrideAuthGlobal : Int
614+*rateOverrideRecvRate : Int
615+
597616-- Monotonic counter bumped on any presence change (gossip, auth, logout, prune, restore, drain)
598617-- Polls compare against this to skip expensive presence lookups when nothing changed.
599618*presenceVersion : Timestamp
@@ -601,6 +620,16 @@ type BatchResult = {messageId: OptionalMessageId, status: MessageStatus}
601620-- Last time the background watchdog loop ran (for health check liveness detection)
602621*lastWatchdogAt : Timestamp
603622
623+-- Cached health result: returned when /health is rate-limited instead of false all-healthy.
624+-- Updated on every successful (non-rate-limited) health check.
625+*cachedHealthOk : Bool
626+*cachedHealthPrune : Bool
627+*cachedHealthHeartbeat : Bool
628+*cachedHealthCapacity : Bool
629+*cachedHealthFederation : Bool
630+*cachedHealthSnapshot : Bool
631+*cachedHealthWatchdog : Bool
632+
604633-- ============================================
605634-- Routes
606635-- ============================================
@@ -662,18 +691,20 @@ route ClientApi where
662691 = LookupPresence
663692
664693-- Server-to-server federation API (authenticated via domain callback verification)
694+-- correlationId: optional trace ID for cross-server request tracing (empty = not provided).
695+-- Peers that don't support it send "" — logged when present but not enforced.
665696route FederationApi where
666697 /federation
667698 -- Receive presence gossip events from a peer server
668− POST {events: [GossipEvent], hopsRemaining: Int, fromServer: ServerName, requestTimestamp: Timestamp} /gossip headers {authorization: Text}
699+ POST {events: [GossipEvent], hopsRemaining: Int, fromServer: ServerName, requestTimestamp: Timestamp, correlationId: Text} /gossip headers {authorization: Text}
669700 -> {ok: Bool, error: Text}
670701 = RecvGossip
671702 -- Receive a forwarded message from a peer server on behalf of a sender
672− POST {toKey: PubkeyHex, encryptedBlob: BlobHex, timestamp: Timestamp, originServer: ServerName, messageId: MessageIdHex, fromServer: ServerName, requestTimestamp: Timestamp} /forward headers {authorization: Text}
703+ POST {toKey: PubkeyHex, encryptedBlob: BlobHex, timestamp: Timestamp, originServer: ServerName, messageId: MessageIdHex, fromServer: ServerName, requestTimestamp: Timestamp, correlationId: Text} /forward headers {authorization: Text}
673704 -> {ok: Bool, error: Text}
674705 = ForwardMessage
675706 -- Acknowledge delivery of messages back to the origin server
676− POST {messageIds: [MessageIdHex], originServer: ServerName, fromServer: ServerName, requestTimestamp: Timestamp} /ack headers {authorization: Text}
707+ POST {messageIds: [MessageIdHex], originServer: ServerName, fromServer: ServerName, requestTimestamp: Timestamp, correlationId: Text} /ack headers {authorization: Text}
677708 -> {ok: Bool, error: Text, ackedCount: Int}
678709 = FederationAck
679710
@@ -728,6 +759,14 @@ route MaintenanceApi where
728759 POST {pubkey: PubkeyHex} /maintenance/sessions/revoke-all headers {authorization: Text}
729760 -> {ok: Bool, revokedCount: Int}
730761 = AdminRevokeSessions
762+ -- Read current effective rate limits (compile-time defaults + runtime overrides)
763+ GET /maintenance/rate-limits headers {authorization: Text}
764+ -> {ok: Bool, globalRequest: Int, ipRate: Int, sendRate: Int, federationRate: Int, authGlobal: Int, recvRate: Int}
765+ = GetRateLimits
766+ -- Update rate limits at runtime (0 = reset to compile-time default)
767+ POST {globalRequest: Int, ipRate: Int, sendRate: Int, federationRate: Int, authGlobal: Int, recvRate: Int} /maintenance/rate-limits headers {authorization: Text}
768+ -> {ok: Bool}
769+ = SetRateLimits
731770
732771route HealthApi where
733772 GET /health headers {authorization: Text} -> {ok: Bool, pruneHealthy: Bool, heartbeatHealthy: Bool, capacityHealthy: Bool, federationHealthy: Bool, snapshotHealthy: Bool, watchdogHealthy: Bool} = Health
@@ -1343,6 +1382,18 @@ readSelectedPresenceBuckets = \buckets ->
13431382 remaining <- readSelectedPresenceBuckets rest
13441383 yield (union entries remaining)
13451384
1385+-- Write presence entries to only the specified bucket indices. Entries are filtered
1386+-- by pubkeyBucket so each bucket gets only its entries. Unspecified buckets are untouched.
1387+writeSelectedPresenceBuckets : [Int] -> [PresenceEntry] -> IO ()
1388+writeSelectedPresenceBuckets = \buckets entries ->
1389+ case headOf buckets of
1390+ Nothing {} -> yield {}
1391+ Just {value: b} -> do
1392+ let rest = do x <- buckets; where x != b; yield x
1393+ let bucketEntries = do e <- entries; where pubkeyBucket e.pubkey == b; yield e
1394+ writePresenceBucket b bucketEntries
1395+ writeSelectedPresenceBuckets rest entries
1396+
13461397-- Single-pass partitioning: each entry's bucket is computed once (vs 16x in the list-comprehension approach)
13471398writeAllPresence : [PresenceEntry] -> IO ()
13481399writeAllPresence = \entries -> do
@@ -1655,6 +1706,16 @@ countAllRcEntries = \b ->
16551706 rest <- countAllRcEntries (b + 1)
16561707 yield (count bucket + rest)
16571708
1709+-- Sum total message count from rcBuckets (cheap: sums .count fields, doesn't scan messages)
1710+sumRcBucketTotals : Int -> IO Int
1711+sumRcBucketTotals = \b ->
1712+ if b > 15 then yield 0
1713+ else do
1714+ bucket <- readRcBucket b
1715+ let bucketTotal = fold (\acc c -> acc + c.count) 0 bucket
1716+ rest <- sumRcBucketTotals (b + 1)
1717+ yield (bucketTotal + rest)
1718+
16581719-- Check message dedup only (without counting — count comes from rcBuckets)
16591720checkMessageDup : [Message] -> Text -> PubkeyHex -> Bool
16601721checkMessageDup = \messages messageId toKey ->
@@ -1867,6 +1928,21 @@ evictSmallest = \items n getKey ->
18671928 let rest = evictSmallest r.items (n - 1) getKey in
18681929 {kept: rest.kept, evicted: union [c] rest.evicted}
18691930
1931+-- Single-pass eviction by timestamp cutoff: keeps items where getKey >= cutoff.
1932+-- O(n) vs O(n*k) for evictSmallest. Used for forward nonce eviction on the hot path.
1933+evictByCutoff = \items cutoff getKey ->
1934+ do x <- items; where getKey x >= cutoff; yield x
1935+
1936+-- Evict nonces with fallback: first try expired-nonce eviction (O(n)), then if still
1937+-- over capacity, evict oldest 1% via evictSmallest (small k keeps O(n*k) bounded).
1938+-- Minimizes replay window: expired nonces are removed first (no window), and the
1939+-- 1% fallback creates a window of only ~1000 entries vs ~10000 with the old 10%.
1940+evictNonces = \items maxCount retention t ->
1941+ let afterExpired = evictByCutoff items (t - retention) (\n -> n.seenAt) in
1942+ if count afterExpired <= maxCount then afterExpired
1943+ else let evictCount = max 1 (count afterExpired / 100) in
1944+ (evictSmallest afterExpired evictCount (\n -> n.seenAt)).kept
1945+
18701946-- Enforce per-key capacity: if a key has >= maxPerKey items, evict the oldest for that key.
18711947-- Also prunes expired items.
18721948capPerKey = \items matchKey maxPerKey getExpiry t ->
@@ -2147,10 +2223,11 @@ sendBatchedGossipToServer = \server allEvents hops ->
21472223 forEachBatch allEvents maxGossipEvents (\batch -> do
21482224 t <- now
21492225 auth <- generateFederationAuth "gossip"
2226+ corrId <- genTokenN 1
21502227 result <- fetchFederation server (RecvGossip {
21512228 authorization: auth,
21522229 events: batch, hopsRemaining: hops, fromServer: serverHostN,
2153− requestTimestamp: t})
2230+ requestTimestamp: t, correlationId: corrId})
21542231 case result of
21552232 Err e -> logWarn ("federation: gossip to " ++ server ++ " failed: " ++ show e)
21562233 Ok _ -> yield {})
@@ -2190,21 +2267,48 @@ mergeGossipPresence = \presence deduped trustedOffline fromServer t ->
21902267 withinCapacity: capResult.withinCapacity}
21912268
21922269
2270+-- Effective rate limit readers: return runtime override if set (> 0), else compile-time default
2271+effectiveGlobalRequestRate = do
2272+ o <- *rateOverrideGlobalRequest
2273+ yield (if o > 0 then o else maxGlobalRequestRate)
2274+
2275+effectiveIpRate = do
2276+ o <- *rateOverrideIpRate
2277+ yield (if o > 0 then o else maxIpRatePerIp)
2278+
2279+effectiveSendRate = do
2280+ o <- *rateOverrideSendRate
2281+ yield (if o > 0 then o else maxSendRatePerKey)
2282+
2283+effectiveFederationRate = do
2284+ o <- *rateOverrideFederationRate
2285+ yield (if o > 0 then o else maxFederationRatePerServer)
2286+
2287+effectiveAuthGlobalRate = do
2288+ o <- *rateOverrideAuthGlobal
2289+ yield (if o > 0 then o else maxAuthGlobalRate)
2290+
2291+effectiveRecvRate = do
2292+ o <- *rateOverrideRecvRate
2293+ yield (if o > 0 then o else maxRecvRatePerKey)
2294+
21932295checkFederationRate = \server t -> atomic do
21942296 federationRates <- *federationRates
2195− let result = checkServerRate federationRates server maxFederationRatePerServer federationRateWindowMs maxFederationRateEntries t
2297+ fedRate <- effectiveFederationRate
2298+ let result = checkServerRate federationRates server fedRate federationRateWindowMs maxFederationRateEntries t
21962299 set *federationRates = result.entries
21972300 yield result.limited
21982301
21992302checkGlobalRate = \t -> atomic do
22002303 windowStart <- *globalRateWindowStart
22012304 rateCount <- *globalRateCount
2305+ effectiveRate <- effectiveGlobalRequestRate
22022306 if t - windowStart >= globalRateWindowMs
22032307 then do
22042308 set *globalRateWindowStart = t
22052309 set *globalRateCount = 1
22062310 yield (False {})
2207− else if rateCount >= maxGlobalRequestRate
2311+ else if rateCount >= effectiveRate
22082312 then yield (True {})
22092313 else do
22102314 set *globalRateCount = rateCount + 1
@@ -2214,12 +2318,13 @@ checkGlobalRate = \t -> atomic do
22142318checkAuthGlobalRate = \t -> atomic do
22152319 windowStart <- *authGlobalRateWindowStart
22162320 rateCount <- *authGlobalRateCount
2321+ authRate <- effectiveAuthGlobalRate
22172322 if t - windowStart >= authGlobalRateWindowMs
22182323 then do
22192324 set *authGlobalRateWindowStart = t
22202325 set *authGlobalRateCount = 1
22212326 yield (False {})
2222− else if rateCount >= maxAuthGlobalRate
2327+ else if rateCount >= authRate
22232328 then yield (True {})
22242329 else do
22252330 set *authGlobalRateCount = rateCount + 1
@@ -2245,7 +2350,8 @@ checkClientIpRate = \ip t ->
22452350 if ip == "" then yield (False {})
22462351 else atomic do
22472352 ipRates <- *ipRates
2248− let result = checkIpRate ipRates ip maxIpRatePerIp ipRateWindowMs maxIpRateEntries t
2353+ ipRate <- effectiveIpRate
2354+ let result = checkIpRate ipRates ip ipRate ipRateWindowMs maxIpRateEntries t
22492355 set *ipRates = result.entries
22502356 yield result.limited
22512357
@@ -2426,11 +2532,12 @@ retryForwardsForServer = \entries remaining ->
24262532 let rest = do x <- entries; where x.messageId != f.messageId; yield x
24272533 t <- now
24282534 auth <- generateFederationAuth "forward"
2535+ corrId <- genTokenN 1
24292536 result <- fetchFederation f.toServer (ForwardMessage {
24302537 toKey: f.toKey, encryptedBlob: f.encryptedBlob,
24312538 timestamp: f.timestamp, originServer: f.originServer, messageId: f.messageId,
24322539 fromServer: serverHostN, requestTimestamp: t,
2433− authorization: auth})
2540+ authorization: auth, correlationId: corrId})
24342541 case result of
24352542 Err e -> do
24362543 logWarn ("federation: forward to " ++ f.toServer ++ " failed for " ++ f.messageId ++ ": " ++ show e)
@@ -2493,11 +2600,12 @@ forwardQueuedMessages = \targets t -> do
24932600fetchFederationAck = \origin messageIds -> do
24942601 t <- now
24952602 auth <- generateFederationAuth "ack"
2603+ corrId <- genTokenN 1
24962604 fetchFederation origin (FederationAck {
24972605 authorization: auth,
24982606 messageIds: messageIds,
24992607 originServer: origin, fromServer: serverHostN,
2500− requestTimestamp: t})
2608+ requestTimestamp: t, correlationId: corrId})
25012609
25022610sendBatchedAcksToServer : [{messageId: MessageIdHex, toKey: PubkeyHex, originServer: ServerName, | a}] -> ServerName -> IO {network} {}
25032611sendBatchedAcksToServer = \serverAcks server ->
@@ -2679,10 +2787,20 @@ pruneAndRetry = \t -> do
26792787 yield (actual - stored)
26802788 when (presDrift != 0 || sessDrift != 0)
26812789 (logInfo ("reconciliation: corrected drift — presence: " ++ show presDrift ++ ", sessions: " ++ show sessDrift))
2682− -- Message reconciliation: per-bucket reads reduce lock contention vs reading all 16 at once
2683− msgDrift <- reconcileMessageCounts 0 0
2684− when (msgDrift != 0)
2685− (logInfo ("reconciliation: corrected message drift: " ++ show msgDrift))
2790+ -- Message reconciliation: cheap check first (sum rcBucket .count fields vs totalMessageCount).
2791+ -- Full scan only when drift detected or every Nth cycle (reconciliationFullCycleInterval).
2792+ pc <- *pruneCount
2793+ let isFullCycle = pc > 0 && (pc - (pc / reconciliationFullCycleInterval) * reconciliationFullCycleInterval) == 0
2794+ cheapMsgDrift <- atomic do
2795+ rcTotal <- sumRcBucketTotals 0
2796+ stored <- *totalMessageCount
2797+ yield (rcTotal - stored)
2798+ if cheapMsgDrift != 0 || isFullCycle
2799+ then do
2800+ msgDrift <- reconcileMessageCounts 0 0
2801+ when (msgDrift != 0)
2802+ (logInfo ("reconciliation: corrected message drift: " ++ show msgDrift))
2803+ else yield {}
26862804 pendingForwards <- atomic do f <- *forwards; yield f
26872805 ackState <- atomic do
26882806 ackedIds2 <- *ackedIds
@@ -3270,7 +3388,8 @@ sendMessageCore = \senderKey to encryptedBlob clampedTs effectiveMsgId t ->
32703388 if isDup then yield {status: MsDuplicate {}, remoteServers: [], retryAfterMs: 0}
32713389 else do
32723390 sendRates <- *sendRates
3273− let rateResult = checkPubkeyRate sendRates senderKey maxSendRatePerKey sendRateWindowMs maxSendRateEntries t
3391+ sendRate <- effectiveSendRate
3392+ let rateResult = checkPubkeyRate sendRates senderKey sendRate sendRateWindowMs maxSendRateEntries t
32743393 if rateResult.limited then yield {status: MsRateLimited {}, remoteServers: [], retryAfterMs: rateResult.retryAfterMs}
32753394 else do
32763395 set *sendRates = rateResult.entries
@@ -3280,7 +3399,8 @@ sendMessageCore = \senderKey to encryptedBlob clampedTs effectiveMsgId t ->
32803399 if pairResult.limited then yield {status: MsRateLimited {}, remoteServers: [], retryAfterMs: pairResult.retryAfterMs}
32813400 else do
32823401 recvRates <- *recvRates
3283− let recvResult = checkPubkeyRate recvRates to maxRecvRatePerKey recvRateWindowMs maxRecvRateEntries t
3402+ recvRate <- effectiveRecvRate
3403+ let recvResult = checkPubkeyRate recvRates to recvRate recvRateWindowMs maxRecvRateEntries t
32843404 if recvResult.limited then yield {status: MsRateLimited {}, remoteServers: [], retryAfterMs: recvResult.retryAfterMs}
32853405 else do
32863406 set *recvRates = recvResult.entries
@@ -3307,20 +3427,23 @@ sendMessageCore = \senderKey to encryptedBlob clampedTs effectiveMsgId t ->
33073427 -- would miss locally-originated messages entirely.
33083428 fwdNonces <- *forwardNonces
33093429 let updatedNonces = union fwdNonces [{messageId: effectiveMsgId, toKey: to, originServer: "", seenAt: t}]
3430+ -- O(n) cutoff eviction: remove expired nonces first, avoiding O(n*k) evictSmallest on hot path
33103431 set *forwardNonces = if count updatedNonces > maxForwardNonces
3311− then let evictCount = max 1 (count updatedNonces / 10) in
3312− (evictSmallest updatedNonces evictCount (\n -> n.seenAt)).kept
3432+ then evictNonces updatedNonces maxForwardNonces forwardNonceRetention t
33133433 else updatedNonces
33143434 let toPresence = presenceForKey recipientPresence to t
33153435 let localOnline = hasLocalPresence toPresence
33163436 let remoteServers = collectPeerServers toPresence (\p -> p.server)
3317− when (federationEnabled && count remoteServers > 0) (do
3437+ let forwardsEnqueued = if federationEnabled && count remoteServers > 0 then do
33183438 forwards <- *forwards
33193439 let newFwds = do s <- remoteServers; where none forwards (\f -> f.messageId == effectiveMsgId && f.toServer == s); yield {messageId: effectiveMsgId, toServer: s, createdAt: t, toKey: to, encryptedBlob: encryptedBlob, timestamp: clampedTs, originServer: serverHostN, retries: 0}
33203440 let cappedFwds = takeRelation (maxForwardsEntries - count forwards) newFwds
3321− set *forwards = union forwards cappedFwds)
3441+ set *forwards = union forwards cappedFwds
3442+ -- Return whether any forwards were actually enqueued (vs dropped by capacity cap)
3443+ yield (count cappedFwds > 0)
3444+ else yield (False {})
33223445 let status = if localOnline then MsDelivered {}
3323− else if federationEnabled && count remoteServers > 0 then MsFederated {}
3446+ else if federationEnabled && forwardsEnqueued then MsFederated {}
33243447 else MsQueued {}
33253448 yield {status: status, remoteServers: remoteServers, retryAfterMs: 0}
33263449
@@ -3334,11 +3457,12 @@ forkSendForwards = \effectiveMsgId to encryptedBlob clampedTs remoteServers ->
33343457 when stillPending (do
33353458 fwdT <- now
33363459 fwdAuth <- generateFederationAuth "forward"
3460+ fwdCorrId <- genTokenN 1
33373461 fwdResult <- fetchFederation server (ForwardMessage {
33383462 toKey: to, encryptedBlob: encryptedBlob,
33393463 timestamp: clampedTs, originServer: serverHostN, messageId: effectiveMsgId,
33403464 fromServer: serverHostN, requestTimestamp: fwdT,
3341− authorization: fwdAuth})
3465+ authorization: fwdAuth, correlationId: fwdCorrId})
33423466 case fwdResult of
33433467 Err e -> do
33443468 logWarn ("federation: forward to " ++ server ++ " failed for " ++ effectiveMsgId ++ ": " ++ show e)
@@ -3363,11 +3487,12 @@ sendBatchForwardsSequential = \msgs server ->
33633487 when stillPending (do
33643488 fwdT <- now
33653489 fwdAuth <- generateFederationAuth "forward"
3490+ fwdCorrId <- genTokenN 1
33663491 fwdResult <- fetchFederation server (ForwardMessage {
33673492 toKey: r.toKey, encryptedBlob: r.blob,
33683493 timestamp: r.ts, originServer: serverHostN, messageId: r.messageId,
33693494 fromServer: serverHostN, requestTimestamp: fwdT,
3370− authorization: fwdAuth})
3495+ authorization: fwdAuth, correlationId: fwdCorrId})
33713496 case fwdResult of
33723497 Err e -> do
33733498 logWarn ("federation: forward to " ++ server ++ " failed for " ++ r.messageId ++ ": " ++ show e)
@@ -3635,9 +3760,11 @@ handleVerifyToken = \token scope clientIp respond -> do
36353760-- Validates events, merges into presence table (last-write-wins), and re-gossips
36363761-- to other known peers with decremented hop count. Triggers message forwarding
36373762-- when a remote key comes online.
3638−handleRecvGossip = \events hopsRemaining fromServer requestTimestamp authorization respond -> do
3763+handleRecvGossip = \events hopsRemaining fromServer requestTimestamp correlationId authorization respond -> do
36393764 t <- now
36403765 let cappedHops = clamp 0 maxGossipHops hopsRemaining
3766+ let corrId = if correlationId != "" then correlationId else ""
3767+ when (corrId != "") (logInfo ("[CORR] " ++ corrId ++ " gossip from " ++ fromServer))
36413768 withFederationAuth authorization t respond fromServer "gossip" requestTimestamp (cappedHops >= 1 && not (isLocalServer fromServer) && count events <= maxGossipEvents) (do
36423769 -- Per-peer gossip event volume limiting: counts total events (not just requests) per server.
36433770 -- A single gossip request carries up to maxGossipEvents entries; without this, a peer
@@ -3682,18 +3809,28 @@ handleRecvGossip = \events hopsRemaining fromServer requestTimestamp authorizati
36823809 yield (do e <- firstParty; where any localPresence (\p -> p.pubkey == e.pubkey && isLocalServer p.server && p.expiresAt > t); yield e.pubkey)
36833810 when (count localClashes > 0)
36843811 (logWarn ("gossip: " ++ fromServer ++ " claims " ++ show (count localClashes) ++ " locally-authenticated key(s) — ignoring for local routing"))
3812+ -- Per-bucket gossip merge: read only presence buckets affected by this gossip batch
3813+ -- instead of all 16. All entries for a pubkey hash to the same bucket, so per-bucket
3814+ -- merge is equivalent to global merge. Reduces lock scope for small gossip batches.
3815+ let allGossipEntries = union deduped trustedOffline
3816+ let affectedBuckets = uniqueBucketsFor allGossipEntries (\e -> e.pubkey)
36853817 gossipResult <- atomic do
3686− allPresence <- readAllPresence
3818+ selectedPresence <- readSelectedPresenceBuckets affectedBuckets
36873819 tpc <- *totalPresenceCount
36883820 -- Pre-filter: when near capacity, reject gossip entries for pubkeys not already in presence.
36893821 -- This prevents a malicious peer from spiking presence above maxPresenceEntries via large batches.
36903822 let capacityDeduped = if tpc >= maxPresenceEntries
3691− then do d <- deduped; where any allPresence (\p -> p.pubkey == d.pubkey); yield d
3823+ then do d <- deduped; where any selectedPresence (\p -> p.pubkey == d.pubkey); yield d
36923824 else deduped
3693− let gm = mergeGossipPresence allPresence capacityDeduped trustedOffline fromServer t
3694− writeAllPresence gm.finalPresence
3695− set *totalPresenceCount = count gm.finalPresence
3696− set *knownPeerServers = gm.gossipPeerServers
3825+ let gm = mergeGossipPresence selectedPresence capacityDeduped trustedOffline fromServer t
3826+ -- Write back only affected buckets (not writeAllPresence, which would zero unaffected buckets)
3827+ writeSelectedPresenceBuckets affectedBuckets gm.finalPresence
3828+ let delta = count gm.finalPresence - count selectedPresence
3829+ set *totalPresenceCount = max 0 (tpc + delta)
3830+ -- Merge new peer servers with existing known peers (selected buckets may not contain all peers)
3831+ knownPeers <- *knownPeerServers
3832+ let newPeers = gm.gossipPeerServers
3833+ set *knownPeerServers = fold (\acc s -> if any acc (\a -> a == s) then acc else union acc [s]) knownPeers newPeers
36973834 set *presenceVersion = t
36983835 yield {gm: gm, preFiltered: count deduped - count capacityDeduped}
36993836 let effectiveEvents = union (effectiveGossipEvents filteredOnline gossipResult.gm.appliedKeys (GossipOnline {})) (effectiveGossipEvents filteredOffline gossipResult.gm.appliedKeys (GossipOffline {}))
@@ -3732,8 +3869,9 @@ handleRecvGossip = \events hopsRemaining fromServer requestTimestamp authorizati
37323869-- (fromServer != originServer) where an intermediate server re-forwards because
37333870-- the recipient moved. Cascade is bounded by dedup (same messageId won't be stored
37343871-- twice) and forwardCandidates (won't re-forward to the same server).
3735−handleForwardMessage = \toKey encryptedBlob timestamp originServer messageId fromServer requestTimestamp authorization respond -> do
3872+handleForwardMessage = \toKey encryptedBlob timestamp originServer messageId fromServer requestTimestamp correlationId authorization respond -> do
37363873 t <- now
3874+ when (correlationId != "") (logInfo ("[CORR] " ++ correlationId ++ " forward " ++ messageId ++ " from " ++ fromServer))
37373875 withFederationAuth authorization t respond fromServer "forward" requestTimestamp (not (isLocalServer originServer) && not (isLocalServer fromServer)) (do
37383876 when (fromServer != originServer)
37393877 (logInfo ("forward: relay from " ++ fromServer ++ " (origin: " ++ originServer ++ ") for " ++ toKey))
@@ -3811,13 +3949,9 @@ handleForwardMessage = \toKey encryptedBlob timestamp originServer messageId fro
38113949 -- Record forward nonce for replay protection beyond message/ack dedup lifetime
38123950 fwdNonces <- *forwardNonces
38133951 let updatedNonces = union fwdNonces [{messageId: messageId, toKey: toKey, originServer: originServer, seenAt: t}]
3814− -- Global age-based eviction: evict oldest nonces regardless of origin.
3815− -- Previous per-originServer partitioning could be gamed by a relay peer
3816− -- forging varied originServer values to spread nonces across partitions
3817− -- and crowd out legitimate entries.
3952+ -- Evict expired nonces first (O(n)), then 1% LRU fallback to minimize replay window
38183953 set *forwardNonces = if count updatedNonces > maxForwardNonces
3819− then let evictCount = max 1 (count updatedNonces / 10) in
3820− (evictSmallest updatedNonces evictCount (\n -> n.seenAt)).kept
3954+ then evictNonces updatedNonces maxForwardNonces forwardNonceRetention t
38213955 else updatedNonces
38223956 yield {forward: ForwardStored {}, globalCapReached: False {}, recvRateLimited: False {}, noPresence: False {}}
38233957 if result.noPresence then do
@@ -3839,8 +3973,9 @@ handleForwardMessage = \toKey encryptedBlob timestamp originServer messageId fro
38393973-- POST /federation/ack: Peer server confirms delivery of messages we originated.
38403974-- Removes corresponding forward entries. Also cleans up the original message if the
38413975-- recipient has no local presence and no other pending forwards for that message.
3842−handleFederationAck = \messageIds originServer fromServer requestTimestamp authorization respond -> do
3976+handleFederationAck = \messageIds originServer fromServer requestTimestamp correlationId authorization respond -> do
38433977 t <- now
3978+ when (correlationId != "") (logInfo ("[CORR] " ++ correlationId ++ " ack from " ++ fromServer))
38443979 -- Wrap respond to add ackedCount default for withFederationAuth error paths
38453980 let ackRespond = \r -> respond {ok: r.ok, error: r.error, ackedCount: 0}
38463981 withFederationAuth authorization t ackRespond fromServer "ack" requestTimestamp (isLocalServer originServer && not (isLocalServer fromServer)) (do
@@ -3973,7 +4108,18 @@ handleHealth = \authorization respond -> do
39734108 -- Health checks exempt from global rate limiting (monitoring traffic shouldn't starve real requests)
39744109 -- but have their own dedicated rate limiter to prevent DDoS on /health from saturating reads.
39754110 healthLimited <- checkHealthRate t
3976− if healthLimited then yield (respond {ok: True {}, pruneHealthy: True {}, heartbeatHealthy: True {}, capacityHealthy: True {}, federationHealthy: True {}, snapshotHealthy: True {}, watchdogHealthy: True {}})
4111+ if healthLimited then do
4112+ -- Return cached last-known health (prevents DDoS on /health from masking real failures)
4113+ cached <- atomic do
4114+ cOk <- *cachedHealthOk
4115+ cP <- *cachedHealthPrune
4116+ cH <- *cachedHealthHeartbeat
4117+ cC <- *cachedHealthCapacity
4118+ cF <- *cachedHealthFederation
4119+ cS <- *cachedHealthSnapshot
4120+ cW <- *cachedHealthWatchdog
4121+ yield {ok: cOk, pruneHealthy: cP, heartbeatHealthy: cH, capacityHealthy: cC, federationHealthy: cF, snapshotHealthy: cS, watchdogHealthy: cW}
4122+ yield (respond cached)
39774123 else do
39784124 let isAuthed = verifyMaintenanceAuth authorization
39794125 result <- atomic do
@@ -4016,6 +4162,15 @@ handleHealth = \authorization respond -> do
40164162 snapshotOk: snapOk,
40174163 watchdogOk: wdOk}
40184164 let allOk = result.pruneOk && result.heartbeatOk && result.capacityOk && result.federationOk && result.snapshotOk && result.watchdogOk
4165+ -- Cache health result for rate-limited responses
4166+ atomic do
4167+ set *cachedHealthOk = allOk
4168+ set *cachedHealthPrune = result.pruneOk
4169+ set *cachedHealthHeartbeat = result.heartbeatOk
4170+ set *cachedHealthCapacity = result.capacityOk
4171+ set *cachedHealthFederation = result.federationOk
4172+ set *cachedHealthSnapshot = result.snapshotOk
4173+ set *cachedHealthWatchdog = result.watchdogOk
40194174 -- Unauthenticated: return real ok boolean so load balancers can route around unhealthy nodes,
40204175 -- but hide subsystem breakdown to prevent targeted load-correlation attacks.
40214176 if isAuthed
@@ -4372,7 +4527,12 @@ handleSendBatch = \messages authorization clientIp respond -> do
43724527 -- Fork federation forwarding: one forked task per target server (not per message).
43734528 -- Within each task, forwards are sent sequentially to avoid per-message fork overhead.
43744529 let forwardTargets = do r <- results; where count r.remoteServers > 0; yield r
4375− let allServers = collectPeerServers forwardTargets (\r -> case headOf r.remoteServers of Nothing {} -> ""; Just {value: s} -> s)
4530+ -- Flatten all remote servers from all results (previous code used headOf, missing 2nd+ servers per message)
4531+ let allServers = fold (\acc r ->
4532+ fold (\innerAcc s ->
4533+ if isLocalServer s then innerAcc
4534+ else if any innerAcc (\a -> a == s) then innerAcc
4535+ else union innerAcc [s]) acc r.remoteServers) [] forwardTargets
43764536 fork (when (federationEnabled && count forwardTargets > 0)
43774537 (sendToEach allServers (\server -> do
43784538 let serverMsgs = do r <- forwardTargets; where any r.remoteServers (\s -> s == server); yield r
@@ -4461,10 +4621,11 @@ handlePingPeer = \server authorization respond -> do
44614621 else do
44624622 startTime <- now
44634623 auth <- generateFederationAuth "gossip"
4624+ pingCorrId <- genTokenN 1
44644625 result <- fetchFederation server (RecvGossip {
44654626 authorization: auth,
44664627 events: [], hopsRemaining: 1, fromServer: serverHostN,
4467− requestTimestamp: startTime})
4628+ requestTimestamp: startTime, correlationId: pingCorrId})
44684629 endTime <- now
44694630 let latency = endTime - startTime
44704631 case result of
@@ -4516,6 +4677,37 @@ handleAdminRevokeSessions = \pubkey authorization respond -> do
45164677 (fork (sendBatchedGossip result.peerServers [{eventType: GossipOffline {}, pubkey: pubkey, server: serverHostN, timestamp: t, ttl: offlineGossipMinTtl}] gossipOriginHops))
45174678 yield (respond {ok: True {}, revokedCount: result.revokedCount})
45184679
4680+-- GET /maintenance/rate-limits: Return current effective rate limits.
4681+handleGetRateLimits = \authorization respond -> do
4682+ let authorized = verifyMaintenanceAuth authorization
4683+ if not authorized then yield (respond {ok: False {}, globalRequest: 0, ipRate: 0, sendRate: 0, federationRate: 0, authGlobal: 0, recvRate: 0})
4684+ else do
4685+ result <- atomic do
4686+ gr <- effectiveGlobalRequestRate
4687+ ir <- effectiveIpRate
4688+ sr <- effectiveSendRate
4689+ fr <- effectiveFederationRate
4690+ ar <- effectiveAuthGlobalRate
4691+ rr <- effectiveRecvRate
4692+ yield {globalRequest: gr, ipRate: ir, sendRate: sr, federationRate: fr, authGlobal: ar, recvRate: rr}
4693+ yield (respond {ok: True {}, globalRequest: result.globalRequest, ipRate: result.ipRate, sendRate: result.sendRate, federationRate: result.federationRate, authGlobal: result.authGlobal, recvRate: result.recvRate})
4694+
4695+-- POST /maintenance/rate-limits: Update rate limits at runtime. 0 = reset to compile-time default.
4696+-- Negative values are clamped to 0. Changes take effect on the next rate check cycle.
4697+handleSetRateLimits = \globalRequest ipRate sendRate federationRate authGlobal recvRate authorization respond -> do
4698+ let authorized = verifyMaintenanceAuth authorization
4699+ if not authorized then yield (respond {ok: False {}})
4700+ else do
4701+ atomic do
4702+ set *rateOverrideGlobalRequest = max 0 globalRequest
4703+ set *rateOverrideIpRate = max 0 ipRate
4704+ set *rateOverrideSendRate = max 0 sendRate
4705+ set *rateOverrideFederationRate = max 0 federationRate
4706+ set *rateOverrideAuthGlobal = max 0 authGlobal
4707+ set *rateOverrideRecvRate = max 0 recvRate
4708+ logInfo ("rate-limits: updated — global=" ++ show globalRequest ++ " ip=" ++ show ipRate ++ " send=" ++ show sendRate ++ " federation=" ++ show federationRate ++ " auth=" ++ show authGlobal ++ " recv=" ++ show recvRate ++ " (0 = default)")
4709+ yield (respond {ok: True {}})
4710+
45194711-- Extract request path for access logging (excludes poll to reduce log volume)
45204712requestPath = \req -> case req of
45214713 ReqChallenge {} -> "POST /auth/challenge"
@@ -4545,6 +4737,8 @@ requestPath = \req -> case req of
45454737 PingPeer {} -> "POST /maintenance/ping"
45464738 AdminListSessions {} -> "GET /maintenance/sessions"
45474739 AdminRevokeSessions {} -> "POST /maintenance/sessions/revoke-all"
4740+ GetRateLimits {} -> "GET /maintenance/rate-limits"
4741+ SetRateLimits {} -> "POST /maintenance/rate-limits"
45484742 Health {} -> "GET /health"
45494743 Stats {} -> "GET /stats"
45504744 StatsPeers {} -> "GET /stats/peers"
@@ -4579,6 +4773,8 @@ requestClientIp = \req -> case req of
45794773 PingPeer {} -> ""
45804774 AdminListSessions {} -> ""
45814775 AdminRevokeSessions {} -> ""
4776+ GetRateLimits {} -> ""
4777+ SetRateLimits {} -> ""
45824778 Health {} -> ""
45834779 Stats {} -> ""
45844780 StatsPeers {} -> ""
@@ -4601,9 +4797,9 @@ serveInner = \req -> case req of
46014797 ListMessages {authorization, clientIp, respond} -> handleListMessages authorization clientIp respond
46024798 DeleteMessages {messageIds, authorization, clientIp, respond} -> handleDeleteMessages messageIds authorization clientIp respond
46034799 LookupPresence {pubkey, authorization, clientIp, respond} -> handleLookupPresence pubkey authorization clientIp respond
4604− RecvGossip {events, hopsRemaining, fromServer, requestTimestamp, authorization, respond} -> handleRecvGossip events hopsRemaining fromServer requestTimestamp authorization respond
4605− ForwardMessage {toKey, encryptedBlob, timestamp, originServer, messageId, fromServer, requestTimestamp, authorization, respond} -> handleForwardMessage toKey encryptedBlob timestamp originServer messageId fromServer requestTimestamp authorization respond
4606− FederationAck {messageIds, originServer, fromServer, requestTimestamp, authorization, respond} -> handleFederationAck messageIds originServer fromServer requestTimestamp authorization respond
4800+ RecvGossip {events, hopsRemaining, fromServer, requestTimestamp, correlationId, authorization, respond} -> handleRecvGossip events hopsRemaining fromServer requestTimestamp correlationId authorization respond
4801+ ForwardMessage {toKey, encryptedBlob, timestamp, originServer, messageId, fromServer, requestTimestamp, correlationId, authorization, respond} -> handleForwardMessage toKey encryptedBlob timestamp originServer messageId fromServer requestTimestamp correlationId authorization respond
4802+ FederationAck {messageIds, originServer, fromServer, requestTimestamp, correlationId, authorization, respond} -> handleFederationAck messageIds originServer fromServer requestTimestamp correlationId authorization respond
46074803 VerifyToken {token, scope, clientIp, respond} -> handleVerifyToken token scope clientIp respond
46084804 Prune {authorization, respond} -> handlePrune authorization respond
46094805 Drain {authorization, respond} -> handleDrain authorization respond
@@ -4616,6 +4812,8 @@ serveInner = \req -> case req of
46164812 PingPeer {server, authorization, respond} -> handlePingPeer server authorization respond
46174813 AdminListSessions {pubkey, authorization, respond} -> handleAdminListSessions pubkey authorization respond
46184814 AdminRevokeSessions {pubkey, authorization, respond} -> handleAdminRevokeSessions pubkey authorization respond
4815+ GetRateLimits {authorization, respond} -> handleGetRateLimits authorization respond
4816+ SetRateLimits {globalRequest, ipRate, sendRate, federationRate, authGlobal, recvRate, authorization, respond} -> handleSetRateLimits globalRequest ipRate sendRate federationRate authGlobal recvRate authorization respond
46194817 Health {authorization, respond} -> handleHealth authorization respond
46204818 Stats {authorization, respond} -> handleStats authorization respond
46214819 StatsPeers {authorization, respond} -> handleStatsPeers authorization respond
@@ -4661,13 +4859,24 @@ backgroundPrune = do
46614859 backgroundPrune
46624860 else do
46634861 t <- now
4664− -- Detect backward clock jump: if seqCounter is far ahead of now, reset to prevent
4665− -- all messages getting unreachable receivedAt values for an extended period
4862+ -- Detect clock jumps in both directions:
4863+ -- Backward: seqCounter ahead of now (clock corrected back after forward jump)
4864+ -- Forward: now jumped far ahead of last prune (VM suspend, NTP step)
4865+ -- Forward jumps cause inflated receivedAt timestamps; clients with those cursors
4866+ -- miss new messages until real time catches up. Log a warning so operators can
4867+ -- advise affected clients to reset their cursor (generation change handles this).
46664868 atomic do
46674869 sc <- *seqCounter
4870+ lp <- *lastPrunedAt
46684871 when (sc > t + clockSkewTolerance) (do
46694872 logWarn ("clock jump detected — seqCounter " ++ show (sc - t) ++ "ms ahead of now, resetting")
4670− set *seqCounter = t)
4873+ set *seqCounter = t
4874+ -- Bump server generation so clients reset their cursors (avoids stuck polls
4875+ -- from messages with inflated receivedAt timestamps during the forward jump)
4876+ set *serverGeneration = t)
4877+ -- Forward jump detection: now is far ahead of last prune (more than 10x interval)
4878+ when (lp > 0 && t - lp > backgroundPruneInterval * 10) (do
4879+ logWarn ("clock forward jump detected — " ++ show (t - lp) ++ "ms since last prune (expected ~" ++ show backgroundPruneInterval ++ "ms), messages during this period may have inflated timestamps"))
46714880 -- Heartbeat watchdog: bump pollWakeup if heartbeat loop appears dead
46724881 sa <- *startedAt
46734882 lastHb <- *lastHeartbeatAt
@@ -4737,9 +4946,8 @@ pollHeartbeat = do
47374946
47384947-- Background loop: periodically logs a state summary to stdout.
47394948-- Operators can capture stdout to detect state drift; use /maintenance/snapshot for full export.
4740−-- The drain handler now auto-exports a [SNAPSHOT_DATA] JSON line for log-shipping pipelines,
4741−-- but a crash outside drain still loses state. Mitigate with frequent /maintenance/snapshot calls
4742−-- or a sidecar that watches for [SNAPSHOT_DATA] lines and writes them to disk.
4949+-- When autoSnapshotExportInterval > 0, periodically emits [SNAPSHOT_DATA] JSON to stdout
4950+-- for sidecar capture, so a crash outside drain loses at most one export interval of state.
47434951-- TODO: If Knot adds file I/O primitives, auto-export snapshots to a file
47444952-- (e.g., /var/lib/skrepka/snapshot.json) in addition to logging.
47454953backgroundSnapshot = do
@@ -4758,14 +4966,36 @@ backgroundSnapshot = do
47584966 pac <- *pendingAckCleanups
47594967 yield {messages: tmc, sessions: sc, presence: pc, forwards: count f, ackedIds: count a, ackOverflow: count ao, deliveredIds: count d, forwardFailures: count ff, pendingAckCleanups: count pac}
47604968 logInfo ("snapshot: messages=" ++ show result.messages ++ " sessions=" ++ show result.sessions ++ " presence=" ++ show result.presence ++ " forwards=" ++ show result.forwards ++ " ackedIds=" ++ show result.ackedIds ++ " ackOverflow=" ++ show result.ackOverflow ++ " deliveredIds=" ++ show result.deliveredIds ++ " forwardFailures=" ++ show result.forwardFailures ++ " pendingAckCleanups=" ++ show result.pendingAckCleanups)
4761− -- Warn if federation state hasn't been exported recently (will be lost on restart)
47624969 t <- now
4970+ -- Periodic full snapshot auto-export: emit [SNAPSHOT_DATA] JSON to stdout so a
4971+ -- sidecar process can persist it. Only when there's meaningful state to save.
4972+ when (autoSnapshotExportInterval > 0) (do
4973+ lastAutoExport <- *lastAutoSnapshotExportAt
4974+ let hasState = result.presence > 0 || result.forwards > 0 || result.ackedIds > 0 || result.deliveredIds > 0
4975+ when (hasState && (lastAutoExport == 0 || t - lastAutoExport >= autoSnapshotExportInterval)) (do
4976+ exportData <- atomic do
4977+ p <- readAllPresence
4978+ a <- *ackedIds
4979+ ao <- *ackOverflow
4980+ d <- *deliveredIds
4981+ ff <- *forwardFailures
4982+ f <- *forwards
4983+ fn <- *forwardNonces
4984+ bp <- *blockedPeers
4985+ yield {version: snapshotVersion, presence: p, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: f, forwardNonces: fn, blockedPeers: bp}
4986+ println ("[SNAPSHOT_DATA] " ++ toJson exportData)
4987+ atomic do set *lastAutoSnapshotExportAt = t
4988+ logInfo ("snapshot: auto-exported full state (" ++ show result.presence ++ " presence, " ++ show result.forwards ++ " forwards)")))
4989+ -- Warn if federation state hasn't been exported recently (will be lost on restart)
47634990 lastExport <- *lastSnapshotExportAt
4991+ lastAutoExport <- *lastAutoSnapshotExportAt
4992+ -- Consider auto-exports as valid exports for staleness check
4993+ let effectiveLastExport = max lastExport lastAutoExport
47644994 when (maintenanceSecret != "" && snapshotStalenessWarning > 0
4765− && (lastExport == 0 || t - lastExport > snapshotStalenessWarning)
4995+ && (effectiveLastExport == 0 || t - effectiveLastExport > snapshotStalenessWarning)
47664996 && (result.ackedIds > 0 || result.deliveredIds > 0 || result.forwards > 0 || result.presence > 0))
47674997 (do
4768− logError ("no snapshot exported in " ++ show (if lastExport == 0 then 0 else (t - lastExport) / 1000) ++ "s — federation state will be lost on restart (presence=" ++ show result.presence ++ " forwards=" ++ show result.forwards ++ " ackedIds=" ++ show result.ackedIds ++ " deliveredIds=" ++ show result.deliveredIds ++ ")")
4998+ logError ("no snapshot exported in " ++ show (if effectiveLastExport == 0 then 0 else (t - effectiveLastExport) / 1000) ++ "s — federation state will be lost on restart (presence=" ++ show result.presence ++ " forwards=" ++ show result.forwards ++ " ackedIds=" ++ show result.ackedIds ++ " deliveredIds=" ++ show result.deliveredIds ++ ")")
47694999 logError "ACTION REQUIRED: run GET /maintenance/snapshot to persist federation state before restart")
47705000 else yield {}
47715001 backgroundSnapshot
@@ -4821,7 +5051,7 @@ startupProbeNext = \peers probed ->
48215051 result <- fetchFederation server (RecvGossip {
48225052 authorization: auth,
48235053 events: [], hopsRemaining: 1, fromServer: serverHostN,
4824− requestTimestamp: startTime})
5054+ requestTimestamp: startTime, correlationId: ""})
48255055 endTime <- now
48265056 let latency = endTime - startTime
48275057 case result of
@@ -4905,6 +5135,8 @@ main = do
49055135 (logInfo (" circuit breaker: " ++ show federationCircuitBreakerThreshold ++ " failing peers"))
49065136 when (autoSnapshotInterval > 0)
49075137 (logInfo (" auto-snapshot: " ++ show autoSnapshotInterval ++ "ms"))
5138+ when (autoSnapshotExportInterval > 0)
5139+ (logInfo (" auto-snapshot export: " ++ show autoSnapshotExportInterval ++ "ms (full [SNAPSHOT_DATA] to stdout)"))
49085140 fork backgroundPrune
49095141 fork pollHeartbeat
49105142 fork backgroundWatchdog