0f0fc65 · skrepka · git.iko.soy

git.iko.soy/skrepka

Improve server security and observability: federation global rate limit, challenge brute-force protection, graceful drain endpoint, duplicate status, stats enrichment

0f0fc65iko6 months ago

server.knot+65−21

compare with master

unifiedsplit

@@ -129,7 +129,7 @@ type ValidGossipEntry = {pubkey: PubkeyHex, server: ServerName, expiresAt: Times

129-- Status returned to the sender after SendMessage

130-- Delivered: recipient is online locally; Federated: forwarded to remote server;

131-- Queued: stored but recipient location unknown; Unauthorized/Rejected: not accepted

132−data MessageStatus = MsDelivered {} | MsFederated {} | MsQueued {} | MsUnauthorized {} | MsRejected {} | MsRateLimited {}

132+data MessageStatus = MsDelivered {} | MsFederated {} | MsQueued {} | MsUnauthorized {} | MsRejected {} | MsRateLimited {} | MsDuplicate {}

133

134impl ToJSON MessageStatus where

135 toJson ms = toJson (case ms of

@@ -138,7 +138,8 @@ impl ToJSON MessageStatus where

138 MsQueued {} -> "queued"

139 MsUnauthorized {} -> "unauthorized"

140 MsRejected {} -> "rejected"

141− MsRateLimited {} -> "rate_limited")

141+ MsRateLimited {} -> "rate_limited"

142+ MsDuplicate {} -> "duplicate")

142

143

143-- Internal delivery status (same as MessageStatus but without error variants)

144-- Internal delivery status (same as MessageStatus but without error variants)

144data DeliveryStatus = DsDelivered {} | DsFederated {} | DsQueued {}

145data DeliveryStatus = DsDelivered {} | DsFederated {} | DsQueued {}

@@ -258,6 +259,11 @@ data ForwardResult = ForwardDuplicate {} | ForwardRejected {} | ForwardStored {}

258

259

259*globalRateWindowStart : Timestamp

260*globalRateWindowStart : Timestamp

260

261

262+-- Background task cycle counters (exposed via /stats for health monitoring)

263+*pruneCount : Int

264+

265+*heartbeatCount : Int

266+

261-- ============================================

267-- ============================================

262-- Routes

268-- Routes

263-- ============================================

269-- ============================================

@@ -312,6 +318,9 @@ route MaintenanceApi where

312 POST {} /maintenance/prune headers {authorization: Text}

318 POST {} /maintenance/prune headers {authorization: Text}

313 -> {ok: Bool}

319 -> {ok: Bool}

314 = Prune

320 = Prune

321+ POST {} /maintenance/drain headers {authorization: Text}

322+ -> {ok: Bool, drained: Int}

323+ = Drain

315

324

316route HealthApi where

325route HealthApi where

317 GET /health -> {ok: Bool} = Health

326 GET /health -> {ok: Bool} = Health

@@ -319,7 +328,7 @@ route HealthApi where

319-- Returns counts of in-memory collections (requires federation or maintenance secret)

328-- Returns counts of in-memory collections (requires federation or maintenance secret)

320route StatsApi where

329route StatsApi where

321 GET /stats headers {authorization: Text}

330 GET /stats headers {authorization: Text}

322− -> {sessions: Int, messages: Int, presence: Int, forwards: Int, challenges: Int, ackedIds: Int, ackOverflow: Int, deliveredIds: Int, forwardFailures: Int, activePolls: Int, ok: Bool}

331+ -> {sessions: Int, messages: Int, presence: Int, forwards: Int, challenges: Int, ackedIds: Int, ackOverflow: Int, deliveredIds: Int, forwardFailures: Int, activePolls: Int, sendRates: Int, challengeRates: Int, federationRates: Int, pruneCount: Int, heartbeatCount: Int, ok: Bool}

323 = Stats

332 = Stats

324

333

@@ -901,13 +910,20 @@ withFederationAuth = \authorization t respond fromServer precondition action ->

901 maybePruneAndBroadcast t

910 maybePruneAndBroadcast t

902 yield (respond {ok: False {}})

911 yield (respond {ok: False {}})

903 else do

912 else do

904− rateLimited <- checkFederationRate fromServer t

913+ globalLimited <- checkGlobalRate t

905− if rateLimited

914+ if globalLimited

906 then do

915 then do

907− println ("federation: rate limited (" ++ show maxFederationRatePerServer ++ "/s per server), rejecting for " ++ fromServer)

916+ println ("federation: global rate limited (" ++ show maxGlobalRequestRate ++ "/s), rejecting for " ++ fromServer)

908 maybePruneAndBroadcast t

917 maybePruneAndBroadcast t

909 yield (respond {ok: False {}})

918 yield (respond {ok: False {}})

919+ else do

920+ rateLimited <- checkFederationRate fromServer t

921+ if rateLimited

922+ then do

923+ println ("federation: rate limited (" ++ show maxFederationRatePerServer ++ "/s per server), rejecting for " ++ fromServer)

924+ maybePruneAndBroadcast t

925+ yield (respond {ok: False {}})

910− else action

926+ else action

911

927

912withSessionAuth = \authorization t onUnauth onAuth -> do

928withSessionAuth = \authorization t onUnauth onAuth -> do

913 sessions <- *sessions

929 sessions <- *sessions

@@ -1458,16 +1474,13 @@ verifyAuthInner = \pubkey challenge signature respond t -> do

1458 let sigValid = verify (bytesFromHex pubkey) (textToBytes challenge) (bytesFromHex signature)

1474 let sigValid = verify (bytesFromHex pubkey) (textToBytes challenge) (bytesFromHex signature)

1459 validation <- atomic do

1475 validation <- atomic do

1460 pr <- maybePrune t

1476 pr <- maybePrune t

1477+ challenges <- *challenges

1478+ let hasMatch = challengeMatch challenges pubkey challenge t

1479+ -- Consume challenge on any verify attempt (prevents signature brute-force)

1480+ when hasMatch (set *challenges = challengesExcluding challenges pubkey challenge)

1461− if not sigValid

1481+ if not hasMatch || not sigValid

1462 then yield {pr: pr, valid: False {}}

1482 then yield {pr: pr, valid: False {}}

1463− else do

1464− challenges <- *challenges

1465− let hasMatch = challengeMatch challenges pubkey challenge t

1466− if not hasMatch

1467− then yield {pr: pr, valid: False {}}

1468− else do

1469− set *challenges = challengesExcluding challenges pubkey challenge

1470− yield {pr: pr, valid: True {}}

1483+ else yield {pr: pr, valid: True {}}

1471 if validation.valid

1484 if validation.valid

1472 then do

1485 then do

1473 token <- genToken

1486 token <- genToken

@@ -1561,7 +1574,7 @@ handleSendMessage = \to encryptedBlob timestamp messageId authorization respond

1561 -- Create forward entries atomically with message storage (prevents race with ack)

1574 -- Create forward entries atomically with message storage (prevents race with ack)

1562 when (federationEnabled && count remoteServers > 0) (do

1575 when (federationEnabled && count remoteServers > 0) (do

1563 forwards <- *forwards

1576 forwards <- *forwards

1564− let newFwds = do s <- remoteServers; yield {messageId: effectiveMsgId, toServer: s, createdAt: t, toKey: to, encryptedBlob: encryptedBlob, timestamp: clampedTs}

1577+ let newFwds = do s <- remoteServers; where none forwards (\f -> f.messageId == effectiveMsgId && f.toServer == s); yield {messageId: effectiveMsgId, toServer: s, createdAt: t, toKey: to, encryptedBlob: encryptedBlob, timestamp: clampedTs}

1565 let cappedFwds = takeRelation (maxForwardsEntries - count forwards) newFwds

1578 let cappedFwds = takeRelation (maxForwardsEntries - count forwards) newFwds

1566 set *forwards = union forwards cappedFwds)

1579 set *forwards = union forwards cappedFwds)

1567 yield {pr: pr, send: SendDone {delivery: delivery, remoteServers: remoteServers}})

1580 yield {pr: pr, send: SendDone {delivery: delivery, remoteServers: remoteServers}})

@@ -1585,7 +1598,7 @@ handleSendMessage = \to encryptedBlob timestamp messageId authorization respond

1585 _ -> forkBroadcastOffline t result.pr

1598 _ -> forkBroadcastOffline t result.pr

1586 yield (case result.send of

1599 yield (case result.send of

1587 SendFailed {status} -> respond {status: status, messageId: ""}

1600 SendFailed {status} -> respond {status: status, messageId: ""}

1588− SendDuplicate {} -> respond {status: MsDelivered {}, messageId: effectiveMsgId}

1601+ SendDuplicate {} -> respond {status: MsDuplicate {}, messageId: effectiveMsgId}

1589 SendDone {delivery} -> respond {status: deliveryToMessageStatus delivery, messageId: effectiveMsgId})

1602 SendDone {delivery} -> respond {status: deliveryToMessageStatus delivery, messageId: effectiveMsgId})

1590

1603

1591-- GET /lookup/:pubkey: Look up the best known server for a pubkey.

1604-- GET /lookup/:pubkey: Look up the best known server for a pubkey.

@@ -1697,6 +1710,26 @@ handleFederationAck = \messageIds originServer fromServer authorization respond

1697 forkBroadcastOffline t pruneResult

1710 forkBroadcastOffline t pruneResult

1698 yield (respond {ok: True {}}))

1711 yield (respond {ok: True {}}))

1699

1712

1713+-- POST /maintenance/drain: Broadcast offline gossip for all local keys (graceful shutdown).

1714+-- Call before stopping the server so peers learn keys went offline immediately.

1715+-- Gossip is sent via fork; allow a few seconds after response before process exit.

1716+handleDrain = \authorization respond -> do

1717+ t <- now

1718+ let authorized = verifyMaintenanceAuth authorization

1719+ if not authorized then yield (respond {ok: False {}, drained: 0})

1720+ else do

1721+ drainResult <- atomic do

1722+ presence <- *presence

1723+ let localEntries = do p <- presence; where isLocalServer p.server && p.expiresAt > t; yield p

1724+ set *presence = do p <- presence; where not (isLocalServer p.server) || p.expiresAt <= t; yield p

1725+ yield {localEntries: localEntries, peerServers: collectPeerServers presence (\p -> p.server)}

1726+ let drainedCount = count drainResult.localEntries

1727+ when (federationEnabled && drainedCount > 0 && count drainResult.peerServers > 0) (do

1728+ let gossipEvents = do p <- drainResult.localEntries; yield {eventType: GossipOffline {}, pubkey: p.pubkey, server: serverHostN, timestamp: t, ttl: offlineGossipMinTtl}

1729+ sendBatchedGossip drainResult.peerServers gossipEvents gossipOriginHops)

1730+ println ("drain: broadcast offline for " ++ show drainedCount ++ " local keys")

1731+ yield (respond {ok: True {}, drained: drainedCount})

1732+

1700-- POST /maintenance/prune: Trigger a full maintenance cycle (prune + retry all pending ops).

1733-- POST /maintenance/prune: Trigger a full maintenance cycle (prune + retry all pending ops).

1701-- Accepts either federation or maintenance secret.

1734-- Accepts either federation or maintenance secret.

1702handlePrune = \authorization respond -> do

1735handlePrune = \authorization respond -> do

@@ -1722,9 +1755,14 @@ handleStats = \authorization respond -> do

1722 d <- *deliveredIds

1755 d <- *deliveredIds

1723 ff <- *forwardFailures

1756 ff <- *forwardFailures

1724 ap <- *activePolls

1757 ap <- *activePolls

1758+ sr <- *sendRates

1759+ cr <- *challengeRates

1760+ fr <- *federationRates

1761+ pc <- *pruneCount

1762+ hc <- *heartbeatCount

1725− yield {sessions: count s, messages: count m, presence: count p, forwards: count f, challenges: count c, ackedIds: count a, ackOverflow: count ao, deliveredIds: count d, forwardFailures: count ff, activePolls: count ap}

1763+ yield {sessions: count s, messages: count m, presence: count p, forwards: count f, challenges: count c, ackedIds: count a, ackOverflow: count ao, deliveredIds: count d, forwardFailures: count ff, activePolls: count ap, sendRates: count sr, challengeRates: count cr, federationRates: count fr, pruneCount: pc, heartbeatCount: hc}

1726− yield (respond {sessions: result.sessions, messages: result.messages, presence: result.presence, forwards: result.forwards, challenges: result.challenges, ackedIds: result.ackedIds, ackOverflow: result.ackOverflow, deliveredIds: result.deliveredIds, forwardFailures: result.forwardFailures, activePolls: result.activePolls, ok: True {}})

1764+ yield (respond {sessions: result.sessions, messages: result.messages, presence: result.presence, forwards: result.forwards, challenges: result.challenges, ackedIds: result.ackedIds, ackOverflow: result.ackOverflow, deliveredIds: result.deliveredIds, forwardFailures: result.forwardFailures, activePolls: result.activePolls, sendRates: result.sendRates, challengeRates: result.challengeRates, federationRates: result.federationRates, pruneCount: result.pruneCount, heartbeatCount: result.heartbeatCount, ok: True {}})

1727− else yield (respond {sessions: 0, messages: 0, presence: 0, forwards: 0, challenges: 0, ackedIds: 0, ackOverflow: 0, deliveredIds: 0, forwardFailures: 0, activePolls: 0, ok: False {}})

1765+ else yield (respond {sessions: 0, messages: 0, presence: 0, forwards: 0, challenges: 0, ackedIds: 0, ackOverflow: 0, deliveredIds: 0, forwardFailures: 0, activePolls: 0, sendRates: 0, challengeRates: 0, federationRates: 0, pruneCount: 0, heartbeatCount: 0, ok: False {}})

1728

1766

1729serve = \req -> case req of

1767serve = \req -> case req of

1730 ReqChallenge {pubkey, respond} -> handleChallenge pubkey respond

1768 ReqChallenge {pubkey, respond} -> handleChallenge pubkey respond

@@ -1736,6 +1774,7 @@ serve = \req -> case req of

1736 ForwardMessage {toKey, encryptedBlob, timestamp, originServer, messageId, fromServer, authorization, respond} -> handleForwardMessage toKey encryptedBlob timestamp originServer messageId fromServer authorization respond

1774 ForwardMessage {toKey, encryptedBlob, timestamp, originServer, messageId, fromServer, authorization, respond} -> handleForwardMessage toKey encryptedBlob timestamp originServer messageId fromServer authorization respond

1737 FederationAck {messageIds, originServer, fromServer, authorization, respond} -> handleFederationAck messageIds originServer fromServer authorization respond

1775 FederationAck {messageIds, originServer, fromServer, authorization, respond} -> handleFederationAck messageIds originServer fromServer authorization respond

1738 Prune {authorization, respond} -> handlePrune authorization respond

1776 Prune {authorization, respond} -> handlePrune authorization respond

1777+ Drain {authorization, respond} -> handleDrain authorization respond

1739 Health {respond} -> handleHealth respond

1778 Health {respond} -> handleHealth respond

1740 Stats {authorization, respond} -> handleStats authorization respond

1779 Stats {authorization, respond} -> handleStats authorization respond

1741

1780

@@ -1749,6 +1788,9 @@ backgroundPrune = do

1749 jitter <- randomInt 5000

1788 jitter <- randomInt 5000

1750 sleep (backgroundPruneInterval + jitter)

1789 sleep (backgroundPruneInterval + jitter)

1751 t <- now

1790 t <- now

1791+ atomic do

1792+ pc <- *pruneCount

1793+ set *pruneCount = pc + 1

1752 fork (pruneAndRetry t)

1794 fork (pruneAndRetry t)

1753 backgroundPrune

1795 backgroundPrune

1754

1796

@@ -1760,6 +1802,8 @@ pollHeartbeat = do

1760 t <- now

1802 t <- now

1761 atomic do

1803 atomic do

1762 set *pollWakeup = t

1804 set *pollWakeup = t

1805+ hc <- *heartbeatCount

1806+ set *heartbeatCount = hc + 1

1763 pollHeartbeat

1807 pollHeartbeat

1764

1808

1765main = do

1809main = do