@@ -185,7 +185,7 @@ federationLatencyWindowMs = 300000 -- 5 minutes; reset latency metrics after thi
185185
186186-- Server build version: included in /health and /stats responses for deployment verification.
187187-- Update this string when deploying a new build so operators can confirm which version is running.
188−serverBuildVersion = "2026-04-26d"
188+serverBuildVersion = "2026-04-26e"
189189
190190-- Federation protocol version: included in gossip/forward/ack requests.
191191-- Peers log warnings when they receive requests from older versions.
@@ -1160,7 +1160,7 @@ route HealthApi where
11601160-- BREAKING CHANGE from flat format — existing scrapers must update field paths.
11611161route StatsApi where
11621162 GET /stats headers {authorization: Text}
1163− -> {ok: Bool, version: Text, capacity: {sessions: Int, sessionsMax: Int, sessionsPct: Int, messages: Int, messagesMax: Int, messagesPct: Int, totalMessageCount: Int, presence: Int, presenceMax: Int, presencePct: Int, forwards: Int, forwardsMax: Int, forwardsPct: Int, challenges: Int, challengesMax: Int, ackedIds: Int, ackedIdsMax: Int, ackOverflow: Int, ackOverflowMax: Int, deliveredIds: Int, deliveredIdsMax: Int, forwardNonces: Int, forwardNoncesMax: Int, activePolls: Int, activePollsMax: Int, recipientCounts: Int}, federation: {enabled: Bool, inFlight: Int, hungSlots: Int, avgLatencyMs: Int, maxLatencyMs: Int, callCount: Int, slowVerifies: Int, verifyFailures: Int, forwardFailures: Int, oldestForwardAge: Int, oldestAckAge: Int, recentGossip: Int, blockedPeers: Int, drainCompletedAt: Timestamp, forwardServers: Int, maxForwardQueueDepth: Int, topFailingPeers: [{server: Text, failures: Int}]}, rates: {send: Int, challenge: Int, federation: Int, federationPair: Int, global: Int, ip: Int, lookup: Int, recv: Int, pair: Int, gossipEvent: Int, verifyIp: Int, uniqueKeyIp: Int, pendingAckCleanups: Int, senderExpirations: Int, senderDeliveries: Int, rejectedGlobal: Int, rejectedIp: Int, rejectedAuth: Int, rejectedPerKey: Int, rejectedFederation: Int}, timing: {pruneCount: Int, pruneRunning: Timestamp, heartbeatCount: Int, uptime: Int, lastPruneAt: Timestamp, lastHeartbeatAt: Timestamp, draining: Bool, drainPhase: Int, totalEntries: Int, activeRequests: Int, peakActiveRequests: Int, pollAvgMs: Int, pollMaxMs: Int, pollCount: Int, sendAvgMs: Int, sendMaxMs: Int, sendCount: Int}}
1163+ -> {ok: Bool, version: Text, capacity: {sessions: Int, sessionsMax: Int, sessionsPct: Int, messages: Int, messagesMax: Int, messagesPct: Int, totalMessageCount: Int, presence: Int, presenceMax: Int, presencePct: Int, forwards: Int, forwardsMax: Int, forwardsPct: Int, challenges: Int, challengesMax: Int, ackedIds: Int, ackedIdsMax: Int, ackOverflow: Int, ackOverflowMax: Int, deliveredIds: Int, deliveredIdsMax: Int, forwardNonces: Int, forwardNoncesMax: Int, activePolls: Int, activePollsMax: Int, recipientCounts: Int}, federation: {enabled: Bool, inFlight: Int, hungSlots: Int, avgLatencyMs: Int, maxLatencyMs: Int, callCount: Int, slowVerifies: Int, verifyFailures: Int, forwardFailures: Int, oldestForwardAge: Int, oldestAckAge: Int, recentGossip: Int, blockedPeers: Int, drainCompletedAt: Timestamp, forwardServers: Int, maxForwardQueueDepth: Int, topFailingPeers: [{server: Text, failures: Int}], pendingCancels: Int}, rates: {send: Int, challenge: Int, federation: Int, federationPair: Int, global: Int, ip: Int, lookup: Int, recv: Int, pair: Int, gossipEvent: Int, verifyIp: Int, uniqueKeyIp: Int, pendingAckCleanups: Int, senderExpirations: Int, senderDeliveries: Int, rejectedGlobal: Int, rejectedIp: Int, rejectedAuth: Int, rejectedPerKey: Int, rejectedFederation: Int}, timing: {pruneCount: Int, pruneRunning: Timestamp, heartbeatCount: Int, uptime: Int, lastPruneAt: Timestamp, lastHeartbeatAt: Timestamp, draining: Bool, drainPhase: Int, totalEntries: Int, activeRequests: Int, peakActiveRequests: Int, pollAvgMs: Int, pollMaxMs: Int, pollCount: Int, sendAvgMs: Int, sendMaxMs: Int, sendCount: Int}}
11641164 = Stats
11651165 GET /stats/peers headers {authorization: Text}
11661166 -> {peers: [PeerStatus], ok: Bool}
@@ -5524,9 +5524,15 @@ sendMessageCore = \senderKey to encryptedBlob clampedTs effectiveMsgId expiresIn
55245524 else updatedNonces
55255525 -- Log LRU nonce eviction: evicted nonces create a bounded replay window
55265526 -- where messages with those nonces could be re-accepted until natural expiry.
5527+ -- oldestRetainedAge: age of the oldest surviving nonce (new dedup coverage floor).
5528+ -- If > messageMaxAge, evicted nonces were in the extended-protection zone only
5529+ -- (message/acked/delivered dedup still covers the gap). If <= messageMaxAge,
5530+ -- some evicted nonces were still within primary dedup coverage (lower risk).
55275531 let lruEvicted = count updatedNonces - count evictedNonces
5528− when (lruEvicted > 0)
5529− (logKV "warn" "nonce_lru_eviction" ["bucket=" ++ show nb, "evicted=" ++ show lruEvicted, "capacity=" ++ show maxNoncesPerBucket])
5532+ when (lruEvicted > 0) (do
5533+ let oldestRetainedAge = case minBy evictedNonces (\n -> n.seenAt) of Nothing {} -> 0; Just {value: n} -> t - n.seenAt
5534+ let level = if oldestRetainedAge < messageMaxAge then "error" else "warn"
5535+ logKV level "nonce_lru_eviction" ["bucket=" ++ show nb, "evicted=" ++ show lruEvicted, "capacity=" ++ show maxNoncesPerBucket, "oldestRetainedMs=" ++ show oldestRetainedAge])
55305536 writeNonceBucket nb evictedNonces
55315537 let toPresence = presenceForKey recipientPresence to t
55325538 let localOnline = hasLocalPresence toPresence
@@ -5582,40 +5588,94 @@ forkSendForwards = \effectiveMsgId to encryptedBlob clampedTs remoteServers ->
55825588 else
55835589 logKV "warn" "federation_forward_rejected" ["server=" ++ server, "messageId=" ++ effectiveMsgId, "error=" ++ response.error]))))
55845590
5585−-- Send all batch forwards for one server sequentially (no per-message fork overhead).
5586−-- Used by handleSendBatch; reduces fork count from O(messages * servers) to O(servers).
5587−-- Uses pre-computed fwdExpiresAt/fwdPriority from the batch result to avoid re-reading
5588−-- the forward bucket for those values; only checks if the forward entry still exists
5589−-- (it could be removed by a concurrent poll/ack between creation and this forked send).
5591+-- Send all batch forwards for one server using FederationBatchForward (1 HTTP round-trip).
5592+-- Used by handleSendBatch; reduces both fork count (O(servers) not O(messages * servers))
5593+-- and HTTP round-trips (1 per server not 1 per message).
5594+-- Falls back to individual ForwardMessage if the peer rejects the batch (old peer without
5595+-- batch support), same as retryForwardsForServer.
5596+-- Pre-filters to still-pending forwards: concurrent poll/ack between message creation
5597+-- and this forked send may have already removed the forward entry.
55905598sendBatchForwardsSequential = \msgs server ->
5599+ if count msgs == 0 then yield {}
5600+ else do
5601+ -- Filter to messages still pending in forward buckets
5602+ pending <- filterPendingBatchForwards msgs server []
5603+ if count pending == 0 then yield {}
5604+ else do
5605+ t <- now
5606+ auth <- generateFederationAuth "forward-batch" server
5607+ corrId <- genTokenN 1
5608+ let batchPayload = do r <- pending; yield {toKey: r.toKey, encryptedBlob: r.blob, timestamp: r.ts, originServer: serverHostN, messageId: r.messageId, expiresAt: r.fwdExpiresAt, priority: r.fwdPriority}
5609+ result <- fetchFederation server (FederationBatchForward {
5610+ forwards: batchPayload, fromServer: serverHostN,
5611+ requestTimestamp: t, authorization: auth, correlationId: corrId, protocolVersion: federationProtocolVersion})
5612+ case result of
5613+ Err e -> do
5614+ logKV "warn" "federation_batch_forward_failed" ["server=" ++ server, "batchSize=" ++ show (count pending), "error=" ++ show e]
5615+ recordFailure server
5616+ Ok response ->
5617+ if response.ok then do
5618+ clearFailure server
5619+ processBatchForwardResults pending response.results server
5620+ else do
5621+ logKV "warn" "federation_batch_forward_rejected" ["server=" ++ server, "error=" ++ response.error]
5622+ -- Fall back to individual forwards (old peer without /federation/forward-batch)
5623+ sendBatchForwardsFallback pending server
5624+
5625+-- Filter batch messages to those still pending in forward buckets.
5626+-- Concurrent poll/ack may have removed the forward entry between creation and send.
5627+filterPendingBatchForwards = \msgs server acc ->
55915628 case headOf msgs of
5592− Nothing {} -> yield {}
5629+ Nothing {} -> yield acc
55935630 Just {value: r} -> do
55945631 let rest = do m <- msgs; where m.messageId != r.messageId; yield m
55955632 stillPending <- atomic do
55965633 fwdBkt <- readFwdBucket (pubkeyBucket r.toKey)
55975634 yield (any fwdBkt (\f -> f.messageId == r.messageId && f.toServer == server))
5598− when stillPending (do
5599− fwdT <- now
5600− fwdAuth <- generateFederationAuth ("forward:" ++ r.messageId) server
5601− fwdCorrId <- genTokenN 1
5602− fwdResult <- fetchFederation server (ForwardMessage {
5603− toKey: r.toKey, encryptedBlob: r.blob,
5604− timestamp: r.ts, originServer: serverHostN, messageId: r.messageId,
5605− fromServer: serverHostN, requestTimestamp: fwdT,
5606− authorization: fwdAuth, correlationId: fwdCorrId,
5607− expiresAt: r.fwdExpiresAt, priority: r.fwdPriority, protocolVersion: federationProtocolVersion})
5608− case fwdResult of
5609− Err e -> do
5610− logKV "warn" "federation_forward_failed" ["server=" ++ server, "messageId=" ++ r.messageId, "error=" ++ show e]
5611− recordFailure server
5612− Ok response ->
5613− if response.ok then do
5614− removeForward r.messageId server r.toKey
5615− clearFailure server
5616− else
5617− logKV "warn" "federation_forward_rejected" ["server=" ++ server, "messageId=" ++ r.messageId, "error=" ++ response.error])
5618− sendBatchForwardsSequential rest server
5635+ filterPendingBatchForwards rest server (if stillPending then union acc [r] else acc)
5636+
5637+-- Process per-message results from a batch forward response (initial send path).
5638+-- Removes succeeded forward entries; logs rejections (no retry increment on initial send —
5639+-- the background prune cycle handles retries with proper backoff).
5640+processBatchForwardResults = \pending results server ->
5641+ case headOf pending of
5642+ Nothing {} -> yield {}
5643+ Just {value: r} -> do
5644+ let rest = do m <- pending; where m.messageId != r.messageId; yield m
5645+ let msgResult = headOf (do res <- results; where res.messageId == r.messageId; yield res)
5646+ case msgResult of
5647+ Just {value: res} ->
5648+ when res.ok (removeForward r.messageId server r.toKey)
5649+ Nothing {} -> yield {}
5650+ processBatchForwardResults rest results server
5651+
5652+-- Fallback: send forwards individually when batch forward is rejected (old peer).
5653+-- Same logic as the original sequential path but only runs when batch fails.
5654+sendBatchForwardsFallback = \msgs server ->
5655+ case headOf msgs of
5656+ Nothing {} -> yield {}
5657+ Just {value: r} -> do
5658+ let rest = do m <- msgs; where m.messageId != r.messageId; yield m
5659+ fwdT <- now
5660+ fwdAuth <- generateFederationAuth ("forward:" ++ r.messageId) server
5661+ fwdCorrId <- genTokenN 1
5662+ fwdResult <- fetchFederation server (ForwardMessage {
5663+ toKey: r.toKey, encryptedBlob: r.blob,
5664+ timestamp: r.ts, originServer: serverHostN, messageId: r.messageId,
5665+ fromServer: serverHostN, requestTimestamp: fwdT,
5666+ authorization: fwdAuth, correlationId: fwdCorrId,
5667+ expiresAt: r.fwdExpiresAt, priority: r.fwdPriority, protocolVersion: federationProtocolVersion})
5668+ case fwdResult of
5669+ Err e -> do
5670+ logKV "warn" "federation_forward_failed" ["server=" ++ server, "messageId=" ++ r.messageId, "error=" ++ show e]
5671+ recordFailure server
5672+ Ok response ->
5673+ if response.ok then do
5674+ removeForward r.messageId server r.toKey
5675+ clearFailure server
5676+ else
5677+ logKV "warn" "federation_forward_rejected" ["server=" ++ server, "messageId=" ++ r.messageId, "error=" ++ response.error]
5678+ sendBatchForwardsFallback rest server
56195679
56205680-- POST /messages: Send an encrypted message to a recipient.
56215681-- Deduplicates by messageId across messages, ackedIds, and deliveredIds.
@@ -6122,8 +6182,10 @@ handleForwardMessage = \toKey encryptedBlob timestamp originServer messageId fro
61226182 then evictNonces updatedNonces maxNoncesPerBucket forwardNonceRetention t
61236183 else updatedNonces
61246184 let lruEvicted = count updatedNonces - count evictedNonces
6125− when (lruEvicted > 0)
6126− (logKV "warn" "nonce_lru_eviction" ["bucket=" ++ show bkt, "evicted=" ++ show lruEvicted, "capacity=" ++ show maxNoncesPerBucket])
6185+ when (lruEvicted > 0) (do
6186+ let oldestRetainedAge = case minBy evictedNonces (\n -> n.seenAt) of Nothing {} -> 0; Just {value: n} -> t - n.seenAt
6187+ let level = if oldestRetainedAge < messageMaxAge then "error" else "warn"
6188+ logKV level "nonce_lru_eviction" ["bucket=" ++ show bkt, "evicted=" ++ show lruEvicted, "capacity=" ++ show maxNoncesPerBucket, "oldestRetainedMs=" ++ show oldestRetainedAge])
61276189 writeNonceBucket bkt evictedNonces
61286190 yield {forward: ForwardStored {}, globalCapReached: False {}, recvRateLimited: False {}, noPresence: False {}}
61296191 if result.noPresence then do
@@ -6785,20 +6847,24 @@ computeAndCacheHealth = \t -> do
67856847 tmc <- *totalMessageCount
67866848 lastFedSuccess <- *lastFederationSuccessAt
67876849 forwardFailures <- *forwardFailures
6850+ pendingCancels <- *pendingFederationCancels
67886851 pruneRun <- *pruneRunning
67896852 dp <- *drainPhase
67906853 let warmup = t - sa < healthPruneMaxAge
67916854 let pruneStuck = pruneRun > 0 && t - pruneRun > backgroundPruneInterval * 5
6855+ let cancelCount = count pendingCancels
67926856 let capacityOk = tmc * 10 <= maxMessages * 9
67936857 && presenceCount * 10 <= maxPresenceEntries * 9
67946858 && forwardCount * 10 <= maxForwardsEntries * 9
67956859 && sessionCount * 10 <= maxSessions * 9
67966860 && count activePolls * 10 <= maxActivePolls * 9
6861+ && cancelCount * 10 <= maxPendingCancels * 9
67976862 -- 75% capacity early warning (fires before the 90% health check failure)
67986863 let capacityWarn = tmc * 4 >= maxMessages * 3
67996864 || presenceCount * 4 >= maxPresenceEntries * 3
68006865 || forwardCount * 4 >= maxForwardsEntries * 3
68016866 || sessionCount * 4 >= maxSessions * 3
6867+ || cancelCount * 4 >= maxPendingCancels * 3
68026868 let fedOk = not federationEnabled || warmup
68036869 || count forwardFailures == 0
68046870 || (lastFedSuccess > 0 && t - lastFedSuccess < backgroundPruneInterval * 5)
@@ -6888,7 +6954,7 @@ handleHealth = \authorization respond -> do
68886954 draining: result.isDraining,
68896955 degradedPeers: 0})
68906956
6891−zeroStats = {ok: False {}, version: serverBuildVersion, capacity: {sessions: 0, sessionsMax: 0, sessionsPct: 0, messages: 0, messagesMax: 0, messagesPct: 0, totalMessageCount: 0, presence: 0, presenceMax: 0, presencePct: 0, forwards: 0, forwardsMax: 0, forwardsPct: 0, challenges: 0, challengesMax: 0, ackedIds: 0, ackedIdsMax: 0, ackOverflow: 0, ackOverflowMax: 0, deliveredIds: 0, deliveredIdsMax: 0, forwardNonces: 0, forwardNoncesMax: 0, activePolls: 0, activePollsMax: 0, recipientCounts: 0}, federation: {enabled: False {}, inFlight: 0, hungSlots: 0, avgLatencyMs: 0, maxLatencyMs: 0, callCount: 0, slowVerifies: 0, verifyFailures: 0, forwardFailures: 0, oldestForwardAge: 0, oldestAckAge: 0, recentGossip: 0, blockedPeers: 0, drainCompletedAt: 0, forwardServers: 0, maxForwardQueueDepth: 0, topFailingPeers: []}, rates: {send: 0, challenge: 0, federation: 0, federationPair: 0, global: 0, ip: 0, lookup: 0, recv: 0, pair: 0, gossipEvent: 0, verifyIp: 0, uniqueKeyIp: 0, pendingAckCleanups: 0, senderExpirations: 0, senderDeliveries: 0, rejectedGlobal: 0, rejectedIp: 0, rejectedAuth: 0, rejectedPerKey: 0, rejectedFederation: 0}, timing: {pruneCount: 0, pruneRunning: 0, heartbeatCount: 0, uptime: 0, lastPruneAt: 0, lastHeartbeatAt: 0, draining: False {}, drainPhase: 0, totalEntries: 0, activeRequests: 0, peakActiveRequests: 0, pollAvgMs: 0, pollMaxMs: 0, pollCount: 0, sendAvgMs: 0, sendMaxMs: 0, sendCount: 0}}
6957+zeroStats = {ok: False {}, version: serverBuildVersion, capacity: {sessions: 0, sessionsMax: 0, sessionsPct: 0, messages: 0, messagesMax: 0, messagesPct: 0, totalMessageCount: 0, presence: 0, presenceMax: 0, presencePct: 0, forwards: 0, forwardsMax: 0, forwardsPct: 0, challenges: 0, challengesMax: 0, ackedIds: 0, ackedIdsMax: 0, ackOverflow: 0, ackOverflowMax: 0, deliveredIds: 0, deliveredIdsMax: 0, forwardNonces: 0, forwardNoncesMax: 0, activePolls: 0, activePollsMax: 0, recipientCounts: 0}, federation: {enabled: False {}, inFlight: 0, hungSlots: 0, avgLatencyMs: 0, maxLatencyMs: 0, callCount: 0, slowVerifies: 0, verifyFailures: 0, forwardFailures: 0, oldestForwardAge: 0, oldestAckAge: 0, recentGossip: 0, blockedPeers: 0, drainCompletedAt: 0, forwardServers: 0, maxForwardQueueDepth: 0, topFailingPeers: [], pendingCancels: 0}, rates: {send: 0, challenge: 0, federation: 0, federationPair: 0, global: 0, ip: 0, lookup: 0, recv: 0, pair: 0, gossipEvent: 0, verifyIp: 0, uniqueKeyIp: 0, pendingAckCleanups: 0, senderExpirations: 0, senderDeliveries: 0, rejectedGlobal: 0, rejectedIp: 0, rejectedAuth: 0, rejectedPerKey: 0, rejectedFederation: 0}, timing: {pruneCount: 0, pruneRunning: 0, heartbeatCount: 0, uptime: 0, lastPruneAt: 0, lastHeartbeatAt: 0, draining: False {}, drainPhase: 0, totalEntries: 0, activeRequests: 0, peakActiveRequests: 0, pollAvgMs: 0, pollMaxMs: 0, pollCount: 0, sendAvgMs: 0, sendMaxMs: 0, sendCount: 0}}
68926958
68936959handleStats = \authorization respond -> do
68946960 t <- now
@@ -6926,7 +6992,8 @@ handleStats = \authorization respond -> do
69266992 ukir <- *uniqueKeyIpRates
69276993 se <- *senderExpirations
69286994 sdl <- *senderDeliveries
6929− yield {sc: sc, pc: pc, c: c, aCount: aCount, ao: ao, dCount: dCount, ff: ff, ap: ap, tmc: tmc, rc: rc, fi: fi, rg: rg, fnCount: fnCount, pac: pac, bp: bp, ger: ger, vir: vir, ukir: ukir, se: se, sdl: sdl}
6995+ pfc <- *pendingFederationCancels
6996+ yield {sc: sc, pc: pc, c: c, aCount: aCount, ao: ao, dCount: dCount, ff: ff, ap: ap, tmc: tmc, rc: rc, fi: fi, rg: rg, fnCount: fnCount, pac: pac, bp: bp, ger: ger, vir: vir, ukir: ukir, se: se, sdl: sdl, pfc: pfc}
69306997 counters <- atomic do
69316998 sr <- *sendRates
69326999 cr <- *challengeRates
@@ -6989,7 +7056,7 @@ handleStats = \authorization respond -> do
69897056 let sendAvg = if counters.cslc > 0 then counters.cslt / counters.cslc else 0
69907057 yield (respond {ok: True {}, version: serverBuildVersion,
69917058 capacity: {sessions: sessions, sessionsMax: maxSessions, sessionsPct: sessPct, messages: messages, messagesMax: maxMessages, messagesPct: msgPct, totalMessageCount: messages, presence: presence, presenceMax: maxPresenceEntries, presencePct: presPct, forwards: fwds, forwardsMax: maxForwardsEntries, forwardsPct: fwdPct, challenges: count core.c, challengesMax: maxChallenges, ackedIds: core.aCount, ackedIdsMax: maxAckedIds, ackOverflow: count core.ao, ackOverflowMax: maxAckOverflow, deliveredIds: core.dCount, deliveredIdsMax: maxDeliveredIds, forwardNonces: core.fnCount, forwardNoncesMax: maxForwardNonces, activePolls: count core.ap, activePollsMax: maxActivePolls, recipientCounts: count core.rc},
6992− federation: {enabled: federationEnabled, inFlight: count core.fi, hungSlots: hungFedSlots, avgLatencyMs: fedAvg, maxLatencyMs: counters.flm, callCount: counters.flc, slowVerifies: counters.svc, verifyFailures: counters.vfc, forwardFailures: count core.ff, oldestForwardAge: oldestFwdAge, oldestAckAge: oldestAckAge, recentGossip: count core.rg, blockedPeers: count core.bp, drainCompletedAt: counters.dca, forwardServers: fwdServerCount, maxForwardQueueDepth: maxFwdDepth, topFailingPeers: topFailingPeers},
7059+ federation: {enabled: federationEnabled, inFlight: count core.fi, hungSlots: hungFedSlots, avgLatencyMs: fedAvg, maxLatencyMs: counters.flm, callCount: counters.flc, slowVerifies: counters.svc, verifyFailures: counters.vfc, forwardFailures: count core.ff, oldestForwardAge: oldestFwdAge, oldestAckAge: oldestAckAge, recentGossip: count core.rg, blockedPeers: count core.bp, drainCompletedAt: counters.dca, forwardServers: fwdServerCount, maxForwardQueueDepth: maxFwdDepth, topFailingPeers: topFailingPeers, pendingCancels: count core.pfc},
69937060 rates: {send: count counters.sr, challenge: count counters.cr, federation: count counters.fr, federationPair: count counters.fpr, global: counters.grc, ip: count counters.ir, lookup: count counters.lr, recv: count counters.rr, pair: count counters.prr, gossipEvent: count core.ger, verifyIp: count core.vir, uniqueKeyIp: count core.ukir, pendingAckCleanups: count core.pac, senderExpirations: count core.se, senderDeliveries: count core.sdl, rejectedGlobal: counters.rlg, rejectedIp: counters.rli, rejectedAuth: counters.rla, rejectedPerKey: counters.rlk, rejectedFederation: counters.rlf},
69947061 timing: {pruneCount: counters.pc, pruneRunning: counters.pr, heartbeatCount: counters.hc, uptime: t - counters.sa, lastPruneAt: counters.lp, lastHeartbeatAt: counters.lh, draining: counters.dr > 0, drainPhase: counters.dr, totalEntries: sessions + messages + presence + fwds + count core.c + core.aCount + count core.ao + core.dCount + core.fnCount + count core.ap + count core.fi, activeRequests: counters.arq, peakActiveRequests: counters.parq, pollAvgMs: pollAvg, pollMaxMs: counters.cplm, pollCount: counters.cplc, sendAvgMs: sendAvg, sendMaxMs: counters.cslm, sendCount: counters.cslc}})
69957062 else yield (respond zeroStats)
@@ -8386,8 +8453,11 @@ backgroundMetaWatchdog = do
83868453 fork backgroundWatchdog)
83878454 backgroundMetaWatchdog
83888455
8389−-- Startup connectivity probe: send an empty gossip to each known peer and log results.
8390−-- Surfaces DNS, firewall, and TLS issues before the first real gossip cycle.
8456+-- Startup connectivity probe: two-phase check for each peer.
8457+-- Phase 1: unauthenticated probe tests TCP/TLS/HTTP connectivity.
8458+-- Phase 2: authenticated probe tests the verify callback round-trip — the critical
8459+-- federation auth path. Catches misconfigured reverse proxies that pass HTTP but block
8460+-- the /federation/verify callback, which would cause all federation requests to fail.
83918461-- Probes up to 10 peers sequentially (not forked) to avoid overwhelming the network.
83928462startupProbePeers = do
83938463 atomic do set *startupProbeActive = True {}
@@ -8402,10 +8472,7 @@ startupProbeNext = \peers probed ->
84028472 Just {value: server} -> do
84038473 let rest = do p <- peers; where p != server; yield p
84048474 startTime <- now
8405− -- Unauthenticated probe: tests TCP/TLS/HTTP connectivity without generating
8406− -- federation auth tokens. Peers reject with "unauthorized" (expected), which
8407− -- still confirms reachability. Avoids leaking single-use tokens to potentially
8408− -- untrusted peers during startup (DNS spoofing window before first real gossip).
8475+ -- Phase 1: unauthenticated probe (TCP/TLS/HTTP connectivity)
84098476 result <- fetchFederation server (RecvGossip {
84108477 authorization: "",
84118478 events: [], hopsRemaining: 1, fromServer: serverHostN,
@@ -8414,7 +8481,25 @@ startupProbeNext = \peers probed ->
84148481 let latency = endTime - startTime
84158482 case result of
84168483 Err e -> logWarn ("startup: peer " ++ server ++ " unreachable (" ++ show e ++ ") — check DNS/firewall/TLS")
8417− Ok r -> logInfo ("startup: peer " ++ server ++ " reachable (" ++ show latency ++ "ms)")
8484+ Ok r -> do
8485+ logInfo ("startup: peer " ++ server ++ " reachable (" ++ show latency ++ "ms)")
8486+ -- Phase 2: authenticated probe (verify callback round-trip).
8487+ -- Sends an empty gossip with a real auth token. The peer will call back to
8488+ -- /federation/verify on this server to confirm the token, testing the full
8489+ -- federation auth path in both directions.
8490+ authStart <- now
8491+ auth <- generateFederationAuth "gossip" server
8492+ authResult <- fetchFederation server (RecvGossip {
8493+ authorization: auth,
8494+ events: [], hopsRemaining: 1, fromServer: serverHostN,
8495+ requestTimestamp: authStart, correlationId: "", protocolVersion: federationProtocolVersion})
8496+ authEnd <- now
8497+ let authLatency = authEnd - authStart
8498+ case authResult of
8499+ Err e -> logWarn ("startup: peer " ++ server ++ " verify callback failed (" ++ show e ++ ") — check reverse proxy allows /federation/verify")
8500+ Ok ar -> if ar.ok
8501+ then logInfo ("startup: peer " ++ server ++ " verify callback OK (" ++ show authLatency ++ "ms)")
8502+ else logWarn ("startup: peer " ++ server ++ " verify callback rejected: " ++ ar.error ++ " — check /federation/verify is reachable from peer")
84188503 startupProbeNext rest (probed + 1)
84198504
84208505main = do