Add retries to sqlblob_gc mark

Summary:
In prod, I see errors that imply a master move or a one-off DB
failure, like:
```
Error: While executing InsertGeneration query

Caused by:
    Mysql Query failed: Failed The MySQL server is running with the --read-only
    (super) option so it cannot execute this statement.
```
and
```
Error: While executing InsertGeneration query

Caused by:
    Mysql Query failed: TimedOut [7002](MySQL Client) Query timed out (no rows seen, took 10000 ms)
```
Teach garbage marking to retry after a random delay,

Reviewed By: StanislavGlebik

Differential Revision: D25974428

fbshipit-source-id: d905ea2c9da19de79b1b8322193831774c3f1ce9

eden/mononoke/cmds/sqlblob_gc/subcommand_mark.rs

@@ -5,9 +5,9 @@
  * GNU General Public License version 2.
  */
 
-use std::{ops::Range, sync::Arc};
+use std::{ops::Range, sync::Arc, time::Duration};
 
-use anyhow::Result;
+use anyhow::{anyhow, Result};
 use clap::{App, ArgMatches, SubCommand};
 use fbinit::FacebookInit;
 use futures::{
@@ -15,18 +15,32 @@ use futures::{
     sink::SinkExt,
     stream::{StreamExt, TryStreamExt},
 };
+use rand::{thread_rng, Rng};
 use slog::Logger;
+use tokio::time::delay_for;
 
 use sqlblob::Sqlblob;
 
 pub const MARK_SAFE: &str = "mark";
+const MIN_RETRY_DELAY: Duration = Duration::from_millis(10);
+const MAX_RETRY_DELAY: Duration = Duration::from_millis(100);
+const RETRIES: usize = 3;
 
 pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
     SubCommand::with_name(MARK_SAFE).about("mark referenced blobs as not safe to delete")
 }
 
 async fn handle_one_key(key: String, store: Arc<Sqlblob>) -> Result<()> {
-    store.set_generation(&key).await
+    for _retry in 0..RETRIES {
+        let res = store.set_generation(&key).await;
+        if res.is_ok() {
+            return res;
+        }
+        let delay = thread_rng().gen_range(MIN_RETRY_DELAY, MAX_RETRY_DELAY);
+        eprintln!("Failure {:#?} - delaying for {:#?}", res, delay);
+        delay_for(delay).await;
+    }
+    Err(anyhow!("Did not make an attempt to handle {}", &key))
 }
 
 pub async fn subcommand_mark<'a>(