@@ -101,6 +101,7 @@ maxForwardRetriesPerServer = 100
101101maxForwardRetries = 10 -- max retries per individual forward entry before dropping
102102maxAckRetries = 10 -- max retries per ack entry before dropping to deliveredIds
103103maxGossipHops = 5 -- max hops a gossip event can propagate across servers
104+maxThirdPartyGossipHops = 2 -- max re-broadcast hops for relayed (third-party) gossip events
104105gossipOriginHops = 2 -- hops set when this server originates a gossip event
105106gossipDedupWindow : Int<Ms>
106107gossipDedupWindow = 60000 -- suppress re-broadcast of identical gossip events within this window
@@ -140,6 +141,8 @@ backgroundPruneInterval : Int<Ms>
140141backgroundPruneInterval = 30000 -- 30 seconds; also retries failed forwards/acks
141142autoSnapshotInterval : Int<Ms>
142143autoSnapshotInterval = 300000 -- 5 minutes; periodic state summary to stdout (0 to disable)
144+snapshotStalenessWarning : Int<Ms>
145+snapshotStalenessWarning = 3600000 -- 1 hour; warn if no snapshot exported via /maintenance/snapshot
143146pollTimeoutMs : Int<Ms>
144147pollTimeoutMs = 25000 -- long-poll heartbeat interval (how long a poll blocks before returning empty)
145148maxPollsPerKey = 3 -- max concurrent long-poll connections per pubkey
@@ -166,6 +169,11 @@ maxLookupRatePerKey = 5 -- max lookups per key per window
166169lookupRateWindowMs : Int<Ms>
167170lookupRateWindowMs = 1000
168171maxLookupRateEntries = 1000
172+-- Federation verify rate limiting (per-IP, tighter than general IP rate)
173+maxVerifyRatePerIp = 5 -- max verify requests per IP per window
174+verifyRateWindowMs : Int<Ms>
175+verifyRateWindowMs = 1000
176+maxVerifyRateEntries = 1000
169177-- Unacked message aging (messages pruned after this regardless of ack status)
170178unackedMessageMaxAge : Int<Ms>
171179unackedMessageMaxAge = 604800000 -- 7 days
@@ -485,6 +493,12 @@ type RecentFederationRequest = {server: Text, timestamp: Timestamp, seenAt: Time
485493-- Forward nonce dedup: (messageId, toKey, seenAt) for replay protection beyond message lifetime
486494*forwardNonces : [{messageId: MessageIdHex, toKey: PubkeyHex, seenAt: Timestamp}]
487495
496+-- Federation verify per-IP rate limiting (tighter than general IP rate)
497+*verifyIpRates : [IpRateEntry]
498+
499+-- Tracks when /maintenance/snapshot was last exported (for staleness warnings)
500+*lastSnapshotExportAt : Timestamp
501+
488502-- ============================================
489503-- Routes
490504-- ============================================
@@ -565,11 +579,11 @@ route MaintenanceApi where
565579 -- Messages are excluded (ephemeral by design; clients handle re-delivery).
566580 -- Includes presence, acked/delivered IDs (dedup state), and peer failure records.
567581 GET /maintenance/snapshot headers {authorization: Text}
568− -> {ok: Bool, presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure]}
582+ -> {ok: Bool, presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure], forwards: [ForwardEntry]}
569583 = Snapshot
570584 -- Restore: import a previously-exported snapshot. Server must be in drain mode
571585 -- to prevent conflicts with live state. Run /maintenance/undrain after restore.
572− POST {presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure]} /maintenance/restore headers {authorization: Text}
586+ POST {presence: [PresenceEntry], ackedIds: [AckedEntry], ackOverflow: [AckedEntry], deliveredIds: [DeliveredEntry], forwardFailures: [ForwardFailure], forwards: [ForwardEntry]} /maintenance/restore headers {authorization: Text}
573587 -> {ok: Bool, error: Text, presenceCount: Int, ackedIdsCount: Int, deliveredIdsCount: Int}
574588 = Restore
575589
@@ -579,7 +593,7 @@ route HealthApi where
579593-- Returns counts of in-memory collections (requires maintenance secret)
580594route StatsApi where
581595 GET /stats headers {authorization: Text}
582− -> {sessions: Int, sessionsMax: Int, messages: Int, messagesMax: Int, presence: Int, presenceMax: Int, forwards: Int, forwardsMax: Int, challenges: Int, challengesMax: Int, ackedIds: Int, ackedIdsMax: Int, ackOverflow: Int, ackOverflowMax: Int, deliveredIds: Int, deliveredIdsMax: Int, forwardFailures: Int, activePolls: Int, activePollsMax: Int, sendRates: Int, challengeRates: Int, federationRates: Int, globalRateCount: Int, ipRates: Int, lookupRates: Int, recvRates: Int, pairRates: Int, federationPairRates: Int, pruneCount: Int, pruneRunning: Timestamp, heartbeatCount: Int, uptime: Int, lastPruneAt: Timestamp, lastHeartbeatAt: Timestamp, draining: Bool, federationEnabled: Bool, recentGossip: Int, totalMessageCount: Int, recipientCounts: Int, federationInFlight: Int, messagesPct: Int, presencePct: Int, forwardsPct: Int, sessionsPct: Int, federationAvgLatencyMs: Int, federationMaxLatencyMs: Int, federationCallCount: Int, drainCompletedAt: Timestamp, ok: Bool}
596+ -> {sessions: Int, sessionsMax: Int, messages: Int, messagesMax: Int, presence: Int, presenceMax: Int, forwards: Int, forwardsMax: Int, challenges: Int, challengesMax: Int, ackedIds: Int, ackedIdsMax: Int, ackOverflow: Int, ackOverflowMax: Int, deliveredIds: Int, deliveredIdsMax: Int, forwardFailures: Int, activePolls: Int, activePollsMax: Int, sendRates: Int, challengeRates: Int, federationRates: Int, globalRateCount: Int, ipRates: Int, lookupRates: Int, recvRates: Int, pairRates: Int, federationPairRates: Int, pruneCount: Int, pruneRunning: Timestamp, heartbeatCount: Int, uptime: Int, lastPruneAt: Timestamp, lastHeartbeatAt: Timestamp, draining: Bool, federationEnabled: Bool, recentGossip: Int, totalMessageCount: Int, recipientCounts: Int, federationInFlight: Int, messagesPct: Int, presencePct: Int, forwardsPct: Int, sessionsPct: Int, federationAvgLatencyMs: Int, federationMaxLatencyMs: Int, federationCallCount: Int, drainCompletedAt: Timestamp, oldestForwardAge: Int, oldestAckAge: Int, ok: Bool}
583597 = Stats
584598 GET /stats/peers headers {authorization: Text}
585599 -> {peers: [PeerStatus], ok: Bool}
@@ -1648,6 +1662,8 @@ pruneExpired = \t -> do
16481662 set *ipRates = do r <- ipRates; where t - r.windowStart < ipRateWindowMs; yield r
16491663 lookupRates <- *lookupRates
16501664 set *lookupRates = do r <- lookupRates; where t - r.windowStart < lookupRateWindowMs; yield r
1665+ verifyIpRates <- *verifyIpRates
1666+ set *verifyIpRates = do r <- verifyIpRates; where t - r.windowStart < verifyRateWindowMs; yield r
16511667 atomic do
16521668 activePolls <- *activePolls
16531669 set *activePolls = do p <- activePolls; where t - p.startedAt < pollTimeoutMs * 3; yield p
@@ -3040,12 +3056,27 @@ handleVerifyToken = \token scope clientIp respond -> do
30403056 ipLimited <- checkIpRate clientIp t
30413057 if ipLimited then yield (respond {ok: False {}})
30423058 else do
3059+ -- Dedicated verify rate limit (tighter than general IP rate to limit token probing)
3060+ verifyLimited <- if clientIp == "" then yield (False {})
3061+ else atomic do
3062+ verifyIpRates <- *verifyIpRates
3063+ let result = checkRateWindow verifyIpRates (\r -> r.ip == clientIp)
3064+ (\ts -> {ip: clientIp, windowStart: ts, count: 1})
3065+ (\r -> {ip: r.ip, windowStart: r.windowStart, count: r.count + 1})
3066+ maxVerifyRatePerIp verifyRateWindowMs maxVerifyRateEntries t
3067+ set *verifyIpRates = result.entries
3068+ yield result.limited
3069+ if verifyLimited then do
3070+ println ("federation/verify: rate limited for IP " ++ clientIp)
3071+ yield (respond {ok: False {}})
3072+ else do
30433073 found <- atomic do
30443074 pending <- *pendingVerifications
30453075 -- Scope is required: callers must specify the operation type (gossip/forward/ack)
30463076 let match = any pending (\p -> safeCompare p.token token && p.scope == scope && t - p.createdAt < verificationTokenExpiry)
30473077 when match (set *pendingVerifications = do p <- pending; where not (safeCompare p.token token); yield p)
30483078 yield match
3079+ when (not found) (println ("federation/verify: failed attempt from " ++ clientIp ++ " (scope: " ++ scope ++ ")"))
30493080 yield (respond {ok: found})
30503081
30513082-- POST /federation/gossip: Receive presence gossip from a peer server.
@@ -3090,8 +3121,15 @@ handleRecvGossip = \events hopsRemaining fromServer requestTimestamp authorizati
30903121 yield fresh
30913122 let newlyOnlineRemote = newlyOnlineRemoteKeys filteredOnline gossipResult.gm.appliedKeys
30923123 fork (do
3093− when (cappedHops > 1 && federationEnabled && count freshEvents > 0) (
3094− sendBatchedGossip (gossipTargets gossipResult.gm.gossipPeerServers fromServer) freshEvents (cappedHops - 1))
3124+ when (cappedHops > 1 && federationEnabled && count freshEvents > 0) (do
3125+ let targets = gossipTargets gossipResult.gm.gossipPeerServers fromServer
3126+ -- Split first-party (from the sender) vs third-party (relayed) events.
3127+ -- Third-party events get fewer hops to limit amplification from inflated hopsRemaining.
3128+ let firstPartyEvents = do e <- freshEvents; where e.server == fromServer; yield e
3129+ let thirdPartyEvents = do e <- freshEvents; where e.server != fromServer; yield e
3130+ when (count firstPartyEvents > 0) (sendBatchedGossip targets firstPartyEvents (cappedHops - 1))
3131+ let thirdPartyHops = min (cappedHops - 1) maxThirdPartyGossipHops
3132+ when (thirdPartyHops > 1 && count thirdPartyEvents > 0) (sendBatchedGossip targets thirdPartyEvents thirdPartyHops))
30953133 when (federationEnabled && count newlyOnlineRemote > 0)
30963134 (forwardQueuedMessages newlyOnlineRemote t))
30973135 yield (respond {ok: True {}, error: ""}))
@@ -3106,6 +3144,17 @@ handleForwardMessage = \toKey encryptedBlob timestamp originServer messageId fro
31063144 withFederationAuth authorization t respond fromServer "forward" requestTimestamp (not (isLocalServer originServer) && not (isLocalServer fromServer)) (do
31073145 when (fromServer != originServer)
31083146 (println ("forward: relay from " ++ fromServer ++ " (origin: " ++ originServer ++ ") for " ++ toKey))
3147+ -- Relay origin verification: for relay forwards, verify originServer is a known peer
3148+ -- to prevent ack routing (and metadata leakage) to arbitrary servers.
3149+ originOk <- if fromServer == originServer then yield (True {})
3150+ else do
3151+ knownPeers <- *knownPeerServers
3152+ yield (any knownPeers (\p -> p == originServer))
3153+ if not originOk
3154+ then do
3155+ println ("forward: rejected relay with unknown origin " ++ originServer ++ " from " ++ fromServer)
3156+ yield (respond {ok: False {}, error: "unknown_origin"})
3157+ else do
31093158 -- Replay protection: reject forwards with timestamps older than messageMaxAge.
31103159 -- Legitimate queued messages have timestamps within this window. Replayed captures
31113160 -- become stale once dedup state (messages + ackedIds + deliveredIds) is pruned (~30 days).
@@ -3313,7 +3362,7 @@ handleHealth = \respond -> do
33133362 capacityHealthy: result.capacityOk,
33143363 federationHealthy: result.federationOk})
33153364
3316−zeroStats = {sessions: 0, sessionsMax: 0, messages: 0, messagesMax: 0, presence: 0, presenceMax: 0, forwards: 0, forwardsMax: 0, challenges: 0, challengesMax: 0, ackedIds: 0, ackedIdsMax: 0, ackOverflow: 0, ackOverflowMax: 0, deliveredIds: 0, deliveredIdsMax: 0, forwardFailures: 0, activePolls: 0, activePollsMax: 0, sendRates: 0, challengeRates: 0, federationRates: 0, globalRateCount: 0, ipRates: 0, lookupRates: 0, recvRates: 0, pairRates: 0, federationPairRates: 0, pruneCount: 0, pruneRunning: 0, heartbeatCount: 0, uptime: 0, lastPruneAt: 0, lastHeartbeatAt: 0, draining: False {}, federationEnabled: False {}, recentGossip: 0, totalMessageCount: 0, recipientCounts: 0, federationInFlight: 0, messagesPct: 0, presencePct: 0, forwardsPct: 0, sessionsPct: 0, federationAvgLatencyMs: 0, federationMaxLatencyMs: 0, federationCallCount: 0, drainCompletedAt: 0, ok: False {}}
3365+zeroStats = {sessions: 0, sessionsMax: 0, messages: 0, messagesMax: 0, presence: 0, presenceMax: 0, forwards: 0, forwardsMax: 0, challenges: 0, challengesMax: 0, ackedIds: 0, ackedIdsMax: 0, ackOverflow: 0, ackOverflowMax: 0, deliveredIds: 0, deliveredIdsMax: 0, forwardFailures: 0, activePolls: 0, activePollsMax: 0, sendRates: 0, challengeRates: 0, federationRates: 0, globalRateCount: 0, ipRates: 0, lookupRates: 0, recvRates: 0, pairRates: 0, federationPairRates: 0, pruneCount: 0, pruneRunning: 0, heartbeatCount: 0, uptime: 0, lastPruneAt: 0, lastHeartbeatAt: 0, draining: False {}, federationEnabled: False {}, recentGossip: 0, totalMessageCount: 0, recipientCounts: 0, federationInFlight: 0, messagesPct: 0, presencePct: 0, forwardsPct: 0, sessionsPct: 0, federationAvgLatencyMs: 0, federationMaxLatencyMs: 0, federationCallCount: 0, drainCompletedAt: 0, oldestForwardAge: 0, oldestAckAge: 0, ok: False {}}
33173366
33183367handleStats = \authorization respond -> do
33193368 t <- now
@@ -3368,7 +3417,9 @@ handleStats = \authorization respond -> do
33683417 let fwdPct = if maxForwardsEntries > 0 then result.forwards * 100 / maxForwardsEntries else 0
33693418 let sessPct = if maxSessions > 0 then result.sessions * 100 / maxSessions else 0
33703419 let fedAvg = if result.federationLatencyCount > 0 then result.federationLatencyTotal / result.federationLatencyCount else 0
3371− yield (respond {sessions: result.sessions, sessionsMax: maxSessions, messages: result.messages, messagesMax: maxMessages, presence: result.presence, presenceMax: maxPresenceEntries, forwards: result.forwards, forwardsMax: maxForwardsEntries, challenges: result.challenges, challengesMax: maxChallenges, ackedIds: result.ackedIds, ackedIdsMax: maxAckedIds, ackOverflow: result.ackOverflow, ackOverflowMax: maxAckOverflow, deliveredIds: result.deliveredIds, deliveredIdsMax: maxDeliveredIds, forwardFailures: result.forwardFailures, activePolls: result.activePolls, activePollsMax: maxActivePolls, sendRates: result.sendRates, challengeRates: result.challengeRates, federationRates: result.federationRates, globalRateCount: result.globalRateCount, ipRates: result.ipRates, lookupRates: result.lookupRates, recvRates: result.recvRates, pairRates: result.pairRates, federationPairRates: result.federationPairRates, pruneCount: result.pruneCount, pruneRunning: result.pruneRunning, heartbeatCount: result.heartbeatCount, uptime: result.uptime, lastPruneAt: result.lastPruneAt, lastHeartbeatAt: result.lastHeartbeatAt, draining: result.draining, federationEnabled: federationEnabled, recentGossip: result.recentGossip, totalMessageCount: result.totalMessageCount, recipientCounts: result.recipientCounts, federationInFlight: result.federationInFlight, messagesPct: msgPct, presencePct: presPct, forwardsPct: fwdPct, sessionsPct: sessPct, federationAvgLatencyMs: fedAvg, federationMaxLatencyMs: result.federationLatencyMax, federationCallCount: result.federationLatencyCount, drainCompletedAt: result.drainCompletedAt, ok: True {}})
3420+ let oldestFwdAge = case minBy core.f (\x -> x.createdAt) of Nothing {} -> 0; Just {value: x} -> t - x.createdAt
3421+ let oldestAckAge = case minBy core.a (\x -> x.ackedAt) of Nothing {} -> 0; Just {value: x} -> t - x.ackedAt
3422+ yield (respond {sessions: result.sessions, sessionsMax: maxSessions, messages: result.messages, messagesMax: maxMessages, presence: result.presence, presenceMax: maxPresenceEntries, forwards: result.forwards, forwardsMax: maxForwardsEntries, challenges: result.challenges, challengesMax: maxChallenges, ackedIds: result.ackedIds, ackedIdsMax: maxAckedIds, ackOverflow: result.ackOverflow, ackOverflowMax: maxAckOverflow, deliveredIds: result.deliveredIds, deliveredIdsMax: maxDeliveredIds, forwardFailures: result.forwardFailures, activePolls: result.activePolls, activePollsMax: maxActivePolls, sendRates: result.sendRates, challengeRates: result.challengeRates, federationRates: result.federationRates, globalRateCount: result.globalRateCount, ipRates: result.ipRates, lookupRates: result.lookupRates, recvRates: result.recvRates, pairRates: result.pairRates, federationPairRates: result.federationPairRates, pruneCount: result.pruneCount, pruneRunning: result.pruneRunning, heartbeatCount: result.heartbeatCount, uptime: result.uptime, lastPruneAt: result.lastPruneAt, lastHeartbeatAt: result.lastHeartbeatAt, draining: result.draining, federationEnabled: federationEnabled, recentGossip: result.recentGossip, totalMessageCount: result.totalMessageCount, recipientCounts: result.recipientCounts, federationInFlight: result.federationInFlight, messagesPct: msgPct, presencePct: presPct, forwardsPct: fwdPct, sessionsPct: sessPct, federationAvgLatencyMs: fedAvg, federationMaxLatencyMs: result.federationLatencyMax, federationCallCount: result.federationLatencyCount, drainCompletedAt: result.drainCompletedAt, oldestForwardAge: oldestFwdAge, oldestAckAge: oldestAckAge, ok: True {}})
33723423 else yield (respond zeroStats)
33733424
33743425handleStatsPeers = \authorization respond -> do
@@ -3408,33 +3459,35 @@ handleStatsKey = \pubkey authorization respond -> do
34083459 yield (respond {messages: result.messages, presence: result.presence, sessions: result.sessions, forwards: result.forwards, activePolls: result.activePolls, ok: True {}})
34093460 else yield (respond {messages: 0, presence: [], sessions: 0, forwards: 0, activePolls: 0, ok: False {}})
34103461
3411−emptySnapshot = {ok: False {}, presence: [], ackedIds: [], ackOverflow: [], deliveredIds: [], forwardFailures: []}
3462+emptySnapshot = {ok: False {}, presence: [], ackedIds: [], ackOverflow: [], deliveredIds: [], forwardFailures: [], forwards: []}
34123463
34133464-- GET /maintenance/snapshot: Export federation metadata for persistence.
3414−-- Reads all dedup state (ackedIds, deliveredIds), presence, and peer failure records
3415−-- in a single atomic snapshot. Messages and forwards are excluded — messages are
3416−-- ephemeral (clients re-send unconfirmed), and forwards are re-created from
3417−-- messages + presence after restore by running a prune cycle.
3465+-- Reads all dedup state (ackedIds, deliveredIds), presence, peer failure records,
3466+-- and pending forwards in a single atomic snapshot. Forward retry counters are
3467+-- preserved so they survive restarts (prevents re-trying permanently-failing forwards).
34183468handleSnapshot = \authorization respond -> do
34193469 let authorized = verifyMaintenanceAuth authorization
34203470 if not authorized then yield (respond emptySnapshot)
34213471 else do
3472+ t <- now
34223473 result <- atomic do
34233474 p <- readAllPresence
34243475 a <- *ackedIds
34253476 ao <- *ackOverflow
34263477 d <- *deliveredIds
34273478 ff <- *forwardFailures
3428− yield {presence: p, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff}
3429− println ("snapshot: exported " ++ show (count result.presence) ++ " presence, " ++ show (count result.ackedIds) ++ " ackedIds, " ++ show (count result.deliveredIds) ++ " deliveredIds")
3430− yield (respond {ok: True {}, presence: result.presence, ackedIds: result.ackedIds, ackOverflow: result.ackOverflow, deliveredIds: result.deliveredIds, forwardFailures: result.forwardFailures})
3479+ f <- *forwards
3480+ yield {presence: p, ackedIds: a, ackOverflow: ao, deliveredIds: d, forwardFailures: ff, forwards: f}
3481+ atomic do set *lastSnapshotExportAt = t
3482+ println ("snapshot: exported " ++ show (count result.presence) ++ " presence, " ++ show (count result.ackedIds) ++ " ackedIds, " ++ show (count result.deliveredIds) ++ " deliveredIds, " ++ show (count result.forwards) ++ " forwards")
3483+ yield (respond {ok: True {}, presence: result.presence, ackedIds: result.ackedIds, ackOverflow: result.ackOverflow, deliveredIds: result.deliveredIds, forwardFailures: result.forwardFailures, forwards: result.forwards})
34313484
34323485-- POST /maintenance/restore: Import a previously-exported snapshot.
34333486-- Requires drain mode to prevent conflicts with live state. Merges imported data
34343487-- with existing state (dedup entries are unioned, presence is merged by freshness).
34353488-- After restore, call /maintenance/undrain to resume, then /maintenance/prune to
34363489-- trigger federation forward creation from messages + restored presence.
3437−handleRestore = \presenceData ackedIdsData ackOverflowData deliveredIdsData forwardFailuresData authorization respond -> do
3490+handleRestore = \presenceData ackedIdsData ackOverflowData deliveredIdsData forwardFailuresData forwardsData authorization respond -> do
34383491 let authorized = verifyMaintenanceAuth authorization
34393492 if not authorized then yield (respond {ok: False {}, error: "unauthorized", presenceCount: 0, ackedIdsCount: 0, deliveredIdsCount: 0})
34403493 else do
@@ -3467,6 +3520,11 @@ handleRestore = \presenceData ackedIdsData ackOverflowData deliveredIdsData forw
34673520 let mergedFailures = mergeByKey (\a b -> a.server == b.server) (\x -> x.failedAt) forwardFailures forwardFailuresData
34683521 let cappedFailures = takeRelation maxForwardFailures mergedFailures
34693522 set *forwardFailures = cappedFailures
3523+ -- Restore forward entries (dedup by messageId + toServer, keep fresher on conflict)
3524+ forwards <- *forwards
3525+ let mergedForwards = mergeByKey (\a b -> a.messageId == b.messageId && a.toServer == b.toServer) (\x -> x.createdAt) forwards forwardsData
3526+ let cappedForwards = takeRelation maxForwardsEntries mergedForwards
3527+ set *forwards = cappedForwards
34703528 yield {presenceCount: count cappedPresence, ackedIdsCount: count cappedAcked, deliveredIdsCount: count cappedDelivered}
34713529 println ("restore: imported " ++ show result.presenceCount ++ " presence, " ++ show result.ackedIdsCount ++ " ackedIds, " ++ show result.deliveredIdsCount ++ " deliveredIds")
34723530 yield (respond {ok: True {}, error: "", presenceCount: result.presenceCount, ackedIdsCount: result.ackedIdsCount, deliveredIdsCount: result.deliveredIdsCount})
@@ -3488,7 +3546,7 @@ serve = \req -> case req of
34883546 Drain {authorization, respond} -> handleDrain authorization respond
34893547 Undrain {authorization, respond} -> handleUndrain authorization respond
34903548 Snapshot {authorization, respond} -> handleSnapshot authorization respond
3491− Restore {presence, ackedIds, ackOverflow, deliveredIds, forwardFailures, authorization, respond} -> handleRestore presence ackedIds ackOverflow deliveredIds forwardFailures authorization respond
3549+ Restore {presence, ackedIds, ackOverflow, deliveredIds, forwardFailures, forwards, authorization, respond} -> handleRestore presence ackedIds ackOverflow deliveredIds forwardFailures forwards authorization respond
34923550 Health {respond} -> handleHealth respond
34933551 Stats {authorization, respond} -> handleStats authorization respond
34943552 StatsPeers {authorization, respond} -> handleStatsPeers authorization respond
@@ -3529,8 +3587,10 @@ backgroundPrune = do
35293587 pc <- *pruneCount
35303588 set *pruneCount = pc + 1
35313589 yield {canRun: True {}, recovered: running > 0, gen: newGen, stuckSince: running}
3532− when status.recovered
3533− (println ("backgroundPrune: previous cycle stuck for " ++ show (t - status.stuckSince) ++ "ms, forcing recovery"))
3590+ when status.recovered (do
3591+ println ("backgroundPrune: previous cycle stuck for " ++ show (t - status.stuckSince) ++ "ms, forcing recovery")
3592+ -- Reset fork counter: the stuck fork leaked its slot, so reclaim it to prevent permanent lockout
3593+ atomic do set *activePruneForks = 0)
35343594 if status.canRun
35353595 then do
35363596 -- Guard against unbounded concurrent prune forks from repeated stuck recovery
@@ -3591,6 +3651,13 @@ backgroundSnapshot = do
35913651 ff <- *forwardFailures
35923652 yield {messages: tmc, sessions: sc, presence: pc, forwards: count f, ackedIds: count a, ackOverflow: count ao, deliveredIds: count d, forwardFailures: count ff}
35933653 println ("snapshot: messages=" ++ show result.messages ++ " sessions=" ++ show result.sessions ++ " presence=" ++ show result.presence ++ " forwards=" ++ show result.forwards ++ " ackedIds=" ++ show result.ackedIds ++ " ackOverflow=" ++ show result.ackOverflow ++ " deliveredIds=" ++ show result.deliveredIds ++ " forwardFailures=" ++ show result.forwardFailures)
3654+ -- Warn if federation state hasn't been exported recently (will be lost on restart)
3655+ t <- now
3656+ lastExport <- *lastSnapshotExportAt
3657+ when (maintenanceSecret != "" && snapshotStalenessWarning > 0
3658+ && (lastExport == 0 || t - lastExport > snapshotStalenessWarning)
3659+ && (result.ackedIds > 0 || result.deliveredIds > 0 || result.forwards > 0 || result.presence > 0))
3660+ (println ("WARNING: no snapshot exported in " ++ show (if lastExport == 0 then 0 else (t - lastExport) / 1000) ++ "s — federation state will be lost on restart (presence=" ++ show result.presence ++ " forwards=" ++ show result.forwards ++ " ackedIds=" ++ show result.ackedIds ++ " deliveredIds=" ++ show result.deliveredIds ++ ")"))
35943661 else yield {}
35953662 backgroundSnapshot
35963663
@@ -3600,7 +3667,7 @@ main = do
36003667 && pollTimeoutMs > 0 && challengeExpiry > 0 && sessionExpiry > 0
36013668 && maxPresenceTtl > 0 && maxForwardsEntries > 0 && maxAckedIds > 0
36023669 && maxForwardRetries > 0 && maxGossipHops > 0 && maxPollsPerKey > 0
3603− && gossipOriginHops > 0
3670+ && gossipOriginHops > 0 && maxThirdPartyGossipHops > 0
36043671 && verificationTokenExpiry > 0
36053672 && maxAckRetries > 0 && maxPairRatePerPair > 0 && pairRateWindowMs > 0
36063673 && maxFederationPairRatePerPair > 0 && federationPairRateWindowMs > 0
@@ -3622,12 +3689,13 @@ main = do
36223689 && federationFetchTimeoutMs <= backgroundPruneInterval
36233690 && maxAckOverflow <= maxAckedIds
36243691 && gossipOriginHops <= maxGossipHops
3692+ && maxThirdPartyGossipHops <= maxGossipHops
36253693 && verificationTokenExpiry <= federationFetchTimeoutMs
36263694 && forwardNonceRetention >= messageMaxAge
36273695 if not configOk
36283696 then println "FATAL: invalid configuration — capacity limits and intervals must be > 0, server not started"
36293697 else if not configRelationsOk
3630− then println "FATAL: invalid configuration — relationship invariants violated (check: unackedMessageMaxAge <= messageMaxAge, challengeExpiry < sessionExpiry, retryBackoffBase <= retryBackoffMax, backgroundPruneInterval < healthPruneMaxAge, pollTimeoutMs < sessionExpiry, gossipDedupWindow < presenceRefreshInterval, federationFetchTimeoutMs <= backgroundPruneInterval, maxAckOverflow <= maxAckedIds, gossipOriginHops <= maxGossipHops, verificationTokenExpiry <= federationFetchTimeoutMs, forwardNonceRetention >= messageMaxAge, per-key limits <= global limits), server not started"
3698+ then println "FATAL: invalid configuration — relationship invariants violated (check: unackedMessageMaxAge <= messageMaxAge, challengeExpiry < sessionExpiry, retryBackoffBase <= retryBackoffMax, backgroundPruneInterval < healthPruneMaxAge, pollTimeoutMs < sessionExpiry, gossipDedupWindow < presenceRefreshInterval, federationFetchTimeoutMs <= backgroundPruneInterval, maxAckOverflow <= maxAckedIds, gossipOriginHops <= maxGossipHops, maxThirdPartyGossipHops <= maxGossipHops, verificationTokenExpiry <= federationFetchTimeoutMs, forwardNonceRetention >= messageMaxAge, per-key limits <= global limits), server not started"
36313699 else do
36323700 t <- now
36333701 atomic do