From 181a3753f4aa3cac75093183f6bf361e138f1cc2 Mon Sep 17 00:00:00 2001
From: Stefan Filip <sfilip@fb.com>
Date: Thu, 25 Mar 2021 21:10:09 -0700
Subject: [PATCH] segmented_changelog: add jitter to periodic reload

Summary:
In production, all repos are instantiated roughly the same time so all reload
processes are started roughly the same time. Reload makes a bunch of requests
and could potentially cause load. Jitter spreads out the load of the reload.
Avoiding the load spike will make overall server behavior more predictable.

Reviewed By: krallin

Differential Revision: D27280117

fbshipit-source-id: 0727af2e7f231a5b6c948424022788a8e7071f82
---
 .../segmented_changelog/src/periodic_reload.rs         | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/eden/mononoke/segmented_changelog/src/periodic_reload.rs b/eden/mononoke/segmented_changelog/src/periodic_reload.rs
index ad441d9a6e..94634cb2f7 100644
--- a/eden/mononoke/segmented_changelog/src/periodic_reload.rs
+++ b/eden/mononoke/segmented_changelog/src/periodic_reload.rs
@@ -13,8 +13,8 @@ use anyhow::Result;
 use arc_swap::ArcSwap;
 use async_trait::async_trait;
 use futures::future::{BoxFuture, FutureExt};
+use rand::Rng;
 use tokio::sync::Notify;
-use tokio::time::Instant;
 
 use futures_ext::future::{spawn_controlled, ControlledHandle};
 
@@ -46,8 +46,12 @@ impl PeriodicReloadSegmentedChangelog {
             let my_sc = Arc::clone(&sc);
             let my_notify = Arc::clone(&update_notify);
             async move {
-                let start = Instant::now() + period;
-                let mut interval = tokio::time::interval_at(start, period);
+                // jitter is here so not all repos try to reload at the same time
+                // 10% of the period seems good enough
+                let jitter = rand::thread_rng().gen_range(Duration::from_secs(0), period / 10);
+                tokio::time::delay_for(period + jitter).await;
+
+                let mut interval = tokio::time::interval(period);
                 loop {
                     interval.tick().await;
                     match load_fn().await {