@@ -852,8 +852,8 @@ maxPeerLatencyEntries = 100
852852-- Per-sender drop counters: tracks how many delivery receipts / expiration notifications
853853-- were evicted before the sender could poll them. Keyed by senderKey so each sender sees
854854-- only their own drops (not a global counter that conflates all senders). Drained on poll.
855−*droppedDeliveryReceipts : [{senderKey: Text, count: Int}]
856−*droppedExpirationNotifs : [{senderKey: Text, count: Int}]
855+*droppedDeliveryReceipts : [{senderKey: Text, count: Int, lastUpdated: Timestamp}]
856+*droppedExpirationNotifs : [{senderKey: Text, count: Int, lastUpdated: Timestamp}]
857857
858858-- Runtime maintenance secret: when non-empty, used instead of compile-time maintenanceSecret.
859859-- Set via /maintenance/rotate-secret to enable zero-downtime secret rotation.
@@ -915,6 +915,9 @@ maxPeerLatencyEntries = 100
915915*cachedHealthForwardQueue : Bool
916916-- Cached count of federation peers with active failure records (degraded connectivity)
917917*cachedHealthDegradedPeers : Int
918+-- Cached count of destination servers with stale forward queues (updated during prune cycle).
919+-- Used by health checks to avoid scanning all 16 forward buckets on every /health request.
920+*cachedForwardStaleCount : Int
918921
919922-- ============================================
920923-- Routes
@@ -1273,17 +1276,24 @@ max = \a b -> if a > b then a else b
12731276
12741277-- Per-sender drop counter helpers: increment a per-key counter and drain it for a specific key.
12751278-- Used to track per-sender delivery receipt / expiration notification drops.
1276−incrementDropCounter = \entries key n ->
1279+-- LRU eviction: when at capacity, evict the entry least recently updated (oldest lastUpdated).
1280+-- Previous strategy evicted by lowest count, which could discard active senders with few drops.
1281+incrementDropCounter = \entries key n t ->
12771282 if any entries (\e -> e.senderKey == key)
1278− then do e <- entries; yield (if e.senderKey == key then {senderKey: e.senderKey, count: e.count + n} else e)
1283+ then do e <- entries; yield (if e.senderKey == key then {senderKey: e.senderKey, count: e.count + n, lastUpdated: t} else e)
12791284 else if count entries >= maxDroppedCountEntries
1280− then union (evictOneOldest entries (\e -> e.count)) [{senderKey: key, count: n}]
1281− else union entries [{senderKey: key, count: n}]
1285+ then union (evictOneOldest entries (\e -> e.lastUpdated)) [{senderKey: key, count: n, lastUpdated: t}]
1286+ else union entries [{senderKey: key, count: n, lastUpdated: t}]
12821287
12831288drainDropCounter = \entries key ->
12841289 case headOf (do e <- entries; where e.senderKey == key; yield e) of
12851290 Nothing {} -> {count: 0, remaining: entries}
12861291 Just {value: e} -> {count: e.count, remaining: do x <- entries; where x.senderKey != key; yield x}
1292+-- NOTE: fold always traverses the full list even after finding the first element.
1293+-- This is O(n) where O(1) suffices, but Knot lacks list destructuring (case xs of
1294+-- (x : rest) -> ...) that would allow short-circuit. Used pervasively on hot paths
1295+-- (sendMessageCore, handleForwardMessage, gossip). If Knot adds list pattern matching
1296+-- or a built-in head/uncons primitive, replace this to eliminate the O(n) overhead.
12871297headOf = \items -> fold (\acc x -> case acc of
12881298 Nothing {} -> Just {value: x}
12891299 _ -> acc) Nothing {} items
@@ -1812,6 +1822,7 @@ hasSessionForKey = \pubkey b ->
18121822 else hasSessionForKey pubkey (b + 1)
18131823
18141824-- Check if a pubkey has any non-expired session across all buckets (short-circuits on first match).
1825+-- Fast-path: returns False immediately when totalSessionCount is 0, avoiding 16 bucket reads.
18151826hasActiveSessionForKey : PubkeyHex -> Timestamp -> Int -> IO Bool
18161827hasActiveSessionForKey = \pubkey t b ->
18171828 if b > 15 then yield (False {})
@@ -1822,9 +1833,12 @@ hasActiveSessionForKey = \pubkey t b ->
18221833
18231834-- Filter a list of pubkeys to those with active (non-expired) sessions.
18241835-- Checks each key independently (short-circuits per key), avoids materializing all sessions.
1836+-- Fast-path: returns empty list when no sessions exist (totalSessionCount == 0).
18251837filterActiveSessionKeys : [PubkeyHex] -> Timestamp -> IO [PubkeyHex]
18261838filterActiveSessionKeys = \keys t ->
1827− case headOf keys of
1839+ totalSessions <- *totalSessionCount
1840+ if totalSessions == 0 then yield []
1841+ else case headOf keys of
18281842 Nothing {} -> yield []
18291843 Just {value: k} -> do
18301844 let rest = do x <- keys; where x != k; yield x
@@ -2074,7 +2088,11 @@ writeSelectedPresenceBuckets = \buckets entries ->
20742088 writePresenceBucket b bucketEntries
20752089 writeSelectedPresenceBuckets rest entries
20762090
2077−-- Single-pass partitioning: each entry's bucket is computed once (vs 16x in the list-comprehension approach)
2091+-- Single-pass partitioning: each entry's bucket is computed once (vs 16x in the list-comprehension approach).
2092+-- The per-element 16-field record allocation is unavoidable without Knot-level indexed arrays:
2093+-- the alternative (16 list-comprehension passes, each computing pubkeyBucket) is O(16n) bucket
2094+-- computations vs O(n) here, and sortBy-based splitting is O(n log n) vs O(n) here.
2095+-- This pattern is optimal given Knot's current data structure primitives.
20782096writeAllPresence : [PresenceEntry] -> IO ()
20792097writeAllPresence = \entries -> do
20802098 let partitioned = fold (\acc p ->
@@ -2295,16 +2313,16 @@ pruneMessageBucket = \b t -> atomic do
22952313 let eviction = evictSmallest combined evictCount (\e -> e.expiredAt)
22962314 set *senderExpirations = eviction.kept
22972315 den <- *droppedExpirationNotifs
2298− let denAfterCap = fold (\acc ev -> incrementDropCounter acc ev.senderKey 1) den eviction.evicted
2316+ let denAfterCap = fold (\acc ev -> incrementDropCounter acc ev.senderKey 1 t) den eviction.evicted
22992317 -- Also record per-key drops for senders that hit maxSenderExpirationsPerKey
23002318 let perKeyDroppedEntries = do e <- localExpired; where none perKeyCapped (\c -> c.messageId == e.messageId && c.toKey == e.toKey); yield e
2301− set *droppedExpirationNotifs = fold (\acc e -> incrementDropCounter acc e.senderKey 1) denAfterCap perKeyDroppedEntries
2319+ set *droppedExpirationNotifs = fold (\acc e -> incrementDropCounter acc e.senderKey 1 t) denAfterCap perKeyDroppedEntries
23022320 else do
23032321 set *senderExpirations = combined
23042322 when (perKeyDropped > 0) (do
23052323 den <- *droppedExpirationNotifs
23062324 let perKeyDroppedEntries = do e <- localExpired; where none perKeyCapped (\c -> c.messageId == e.messageId && c.toKey == e.toKey); yield e
2307− set *droppedExpirationNotifs = fold (\acc e -> incrementDropCounter acc e.senderKey 1) den perKeyDroppedEntries))
2325+ set *droppedExpirationNotifs = fold (\acc e -> incrementDropCounter acc e.senderKey 1 t) den perKeyDroppedEntries))
23082326 else yield {}
23092327
23102328-- Prune all 16 message buckets with concurrency: fork first 15, run last in foreground.
@@ -3486,6 +3504,10 @@ pruneExpired = \t -> do
34863504 -- with sendMessageCore which writes to forward buckets on the hot path.
34873505 fwdSurvivorCount <- pruneAllFwdBuckets 0 retainCutoff federationEnabled 0
34883506 atomic do set *totalForwardCount = fwdSurvivorCount
3507+ -- Cache forward queue stale count after pruning (forward buckets are clean now).
3508+ -- Health checks read this cached value instead of scanning all 16 buckets each time.
3509+ fwdStale <- countStaleForwardServers 0 t []
3510+ atomic do set *cachedForwardStaleCount = fwdStale
34893511 -- AckedIds: per-bucket pruning (each bucket in its own atomic block to reduce lock
34903512 -- contention with poll-ack paths that write to ackedIds concurrently).
34913513 pruneAllAckedBuckets 0 retainCutoff
@@ -3662,10 +3684,16 @@ broadcastOfflineGossip = \t peerServers expiredLocal ->
36623684-- Build re-broadcast gossip events from validated entries. The seenBy list includes
36633685-- the fromServer (who sent us these events) and ourselves (serverHostN), so downstream
36643686-- peers that already have this gossip via either path won't receive it again.
3665−effectiveGossipEvents = \entries appliedKeys eventType seenBy -> do
3666− e <- entries
3667− where any appliedKeys (\a -> samePubkeyServer a e)
3668− yield {eventType: eventType, pubkey: e.pubkey, server: e.server, timestamp: e.timestamp, ttl: e.ttl, seenBy: seenBy}
3687+-- Optimization: pre-extract unique pubkeys from appliedKeys for a fast-path rejection.
3688+-- Entries whose pubkey doesn't appear in any appliedKey skip the full samePubkeyServer
3689+-- check (two-field comparison). Helps when entries contain many pubkeys not in appliedKeys.
3690+effectiveGossipEvents = \entries appliedKeys eventType seenBy ->
3691+ let appliedPubkeys = fold (\acc a -> if any acc (\p -> p == a.pubkey) then acc else union [a.pubkey] acc) [] appliedKeys
3692+ in do
3693+ e <- entries
3694+ where any appliedPubkeys (\pk -> pk == e.pubkey)
3695+ where any appliedKeys (\a -> samePubkeyServer a e)
3696+ yield {eventType: eventType, pubkey: e.pubkey, server: e.server, timestamp: e.timestamp, ttl: e.ttl, seenBy: seenBy}
36693697
36703698mergeGossipPresence = \presence deduped trustedOffline fromServer t ->
36713699 let trustedDeduped = trustedGossipEntries deduped presence fromServer t in
@@ -4864,10 +4892,10 @@ ackPolledMessages = \pubkey cursor t -> do
48644892 let eviction = evictSmallest combined evictCount (\d -> d.deliveredAt)
48654893 set *senderDeliveries = eviction.kept
48664894 -- Track per-sender drop counts for all evicted entries
4867− let dropCounts = fold (\acc e -> incrementDropCounter acc e.senderKey 1) [] eviction.evicted
4895+ let dropCounts = fold (\acc e -> incrementDropCounter acc e.senderKey 1 t) [] eviction.evicted
48684896 when (count dropCounts > 0) (do
48694897 ddr <- *droppedDeliveryReceipts
4870− set *droppedDeliveryReceipts = fold (\acc dc -> incrementDropCounter acc dc.senderKey dc.count) ddr dropCounts)
4898+ set *droppedDeliveryReceipts = fold (\acc dc -> incrementDropCounter acc dc.senderKey dc.count t) ddr dropCounts)
48714899 else set *senderDeliveries = combined)
48724900 yield {pendingAcks: ackedFederated, evictedEntries: eviction.evicted}
48734901 else yield {pendingAcks: [], evictedEntries: []}
@@ -5103,9 +5131,12 @@ trustedOfflineEvents = \events fromServer ->
51035131gossipTargets = \peerServers fromServer ->
51045132 do s <- peerServers; where s != fromServer; yield s
51055133
5106−-- Remote keys that just came online (triggers forwarding of queued messages)
5134+-- Remote keys that just came online (triggers forwarding of queued messages).
5135+-- Pre-extracts unique pubkeys from appliedKeys for fast-path rejection (same optimization
5136+-- as effectiveGossipEvents: skip full samePubkeyServer check when pubkey doesn't match).
51075137newlyOnlineRemoteKeys = \events appliedKeys ->
5108− do e <- events; where any appliedKeys (\a -> samePubkeyServer a e); where not (isLocalServer e.server); yield {pubkey: e.pubkey, server: e.server}
5138+ let appliedPubkeys = fold (\acc a -> if any acc (\p -> p == a.pubkey) then acc else union [a.pubkey] acc) [] appliedKeys
5139+ in do e <- events; where any appliedPubkeys (\pk -> pk == e.pubkey); where any appliedKeys (\a -> samePubkeyServer a e); where not (isLocalServer e.server); yield {pubkey: e.pubkey, server: e.server}
51095140
51105141-- Trust gossip entries if they're first-party (server == fromServer) or updating
51115142-- an existing (pubkey, server) presence entry. Third-party gossip cannot create
@@ -6032,7 +6063,8 @@ handleForwardMessage = \toKey encryptedBlob timestamp originServer messageId fro
60326063 then yield {forward: ForwardRejected {}, globalCapReached: False {}, recvRateLimited: False {}, noPresence: False {}}
60336064 else do
60346065 recvRates <- *recvRates
6035− let recvResult = checkPubkeyRate recvRates toKey maxRecvRatePerKey recvRateWindowMs maxRecvRateEntries t
6066+ recvRate <- effectiveRecvRate
6067+ let recvResult = checkPubkeyRate recvRates toKey recvRate recvRateWindowMs maxRecvRateEntries t
60366068 if recvResult.limited
60376069 then yield {forward: ForwardRejected {}, globalCapReached: False {}, recvRateLimited: True {}, noPresence: False {}}
60386070 else do
@@ -6114,7 +6146,7 @@ fedDeliveryReceipts = \bucketIds forwards t -> case headOf bucketIds of
61146146 Nothing {} -> yield {}
61156147 Just {value: ev} -> do
61166148 ddr <- *droppedDeliveryReceipts
6117− set *droppedDeliveryReceipts = incrementDropCounter ddr ev.senderKey 1
6149+ set *droppedDeliveryReceipts = incrementDropCounter ddr ev.senderKey 1 t
61186150 else set *senderDeliveries = combined)
61196151 fedDeliveryReceipts rest forwards t
61206152
@@ -6745,10 +6777,11 @@ computeAndCacheHealth = \t -> do
67456777 let nonceOk = nonceCount * 10 <= maxForwardNonces * 9
67466778 -- Count peers with active failure records (degraded federation connectivity)
67476779 let degradedPeerCount = count (do f <- forwardFailures; where t - f.firstFailedAt < maxServerDowntime; yield f)
6748− -- Forward queue age health: per-server awareness. Only flag unhealthy when
6749− -- multiple destination servers have stale queues (oldest forward > healthForwardQueueMaxAge).
6750− -- A single stuck peer shouldn't mark the server unhealthy — it's an isolated issue.
6751− fwdStaleCount <- countStaleForwardServers 0 t []
6780+ -- Forward queue age health: use cached stale count from the last prune cycle
6781+ -- (updated every ~30s) instead of scanning all 16 forward buckets per health check.
6782+ -- Staleness of up to 30 seconds is acceptable for health monitoring; stale forwards
6783+ -- take >24 minutes (healthForwardQueueMaxAge) to appear, so they're always caught.
6784+ fwdStaleCount <- *cachedForwardStaleCount
67526785 let fwdQueueOk = fwdStaleCount < healthForwardQueueMinStaleServers
67536786 yield {pruneOk: warmup || (t - lastPrune < healthPruneMaxAge && not pruneStuck),
67546787 heartbeatOk: warmup || t - lastHb < healthHeartbeatMaxAge,
@@ -7139,7 +7172,7 @@ handleAckMessages = \messageIds authorization clientIp respond -> do
71397172 Nothing {} -> yield {}
71407173 Just {value: ev} -> do
71417174 ddr <- *droppedDeliveryReceipts
7142− set *droppedDeliveryReceipts = incrementDropCounter ddr ev.senderKey 1
7175+ set *droppedDeliveryReceipts = incrementDropCounter ddr ev.senderKey 1 t
71437176 else set *senderDeliveries = combined)
71447177 yield {ackedCount: ackCount, pendingAcks: ackedFederated, evictedEntries: eviction.evicted}
71457178 else yield {ackedCount: 0, pendingAcks: [], evictedEntries: []}
@@ -7243,7 +7276,7 @@ handleDeleteMessages = \messageIds authorization clientIp respond -> do
72437276 Nothing {} -> yield {}
72447277 Just {value: ev} -> do
72457278 ddr <- *droppedDeliveryReceipts
7246− set *droppedDeliveryReceipts = incrementDropCounter ddr ev.senderKey 1
7279+ set *droppedDeliveryReceipts = incrementDropCounter ddr ev.senderKey 1 t
72477280 else set *senderDeliveries = combinedDel)
72487281 if count ackedFederated > 0 then do
72497282 ackedBkt <- readAckedBucket b
@@ -8421,8 +8454,11 @@ main = do
84218454 atomic do
84228455 set *startedAt = t
84238456 set *serverGeneration = t
8457+ -- Empty maintenanceSecret is the recommended deployment path: use /maintenance/rotate-secret
8458+ -- at startup via deployment scripts for zero-knowledge secret management. Log at INFO level
8459+ -- since operators following the recommended path shouldn't see warnings on every startup.
84248460 when (maintenanceSecret == "")
8425− (logWarn "maintenanceSecret is empty — /maintenance/* and /stats endpoints are disabled")
8461+ (logInfo "maintenanceSecret is empty — use /maintenance/rotate-secret at startup to enable /maintenance/* and /stats endpoints")
84268462 when (maintenanceSecret != "" && not (tooLong maintenanceSecret 15))
84278463 (logWarn "maintenanceSecret is shorter than 16 characters — use a longer secret for production deployments")
84288464 when (maintenanceSecret != "" && not (tooLong maintenanceSecret 31))