mononoke/blobstore_healer: add more Context to errors

Summary:
Being told `Input/output error: Connection refused (os error 111)` isn't very
helpful when things are broken. However, being told:

```
Execution error: While waiting for replication

Caused by:
    0: While fetching repliction lag for altoona
    1: Input/output error: Connection refused (os error 111)
```

Is nicer.

Reviewed By: farnz

Differential Revision: D21063120

fbshipit-source-id: 1408b9eca025b120790a95d336895d2f50be3d5d

eden/mononoke/cmds/blobstore_healer/main.rs

@@ -12,7 +12,7 @@ mod dummy;
 mod healer;
 mod replication_lag;
 
-use anyhow::{bail, format_err, Error, Result};
+use anyhow::{bail, format_err, Context, Error, Result};
 use blobstore::Blobstore;
 use blobstore_factory::{make_blobstore, BlobstoreOptions, ReadOnlyStorage};
 use blobstore_sync_queue::{BlobstoreSyncQueue, SqlBlobstoreSyncQueue};
@@ -125,7 +125,8 @@ async fn maybe_schedule_healer_for_storage(
         mysql_options,
         readonly_storage.0,
     )
-    .await?;
+    .await
+    .context("While opening sync queue")?;
 
     let sync_queue: Arc<dyn BlobstoreSyncQueue> = if dry_run {
         let logger = ctx.logger().new(o!("sync_queue" => ""));
@@ -233,14 +234,17 @@ async fn schedule_healing(
             }
         }
 
-        wait_for_replication(ctx.logger(), conns.as_ref()).await?;
+        wait_for_replication(ctx.logger(), conns.as_ref())
+            .await
+            .context("While waiting for replication")?;
 
         let now = DateTime::now().into_chrono();
         let healing_deadline = DateTime::new(now - heal_min_age);
         let last_batch_was_full_size = multiplex_healer
             .heal(ctx.clone(), healing_deadline)
             .compat()
-            .await?;
+            .await
+            .context("While healing")?;
 
         // if last batch read was not full,  wait at least 1 second, to avoid busy looping as don't
         // want to hammer the database with thousands of reads a second.

eden/mononoke/cmds/blobstore_healer/replication_lag.rs

@@ -5,7 +5,7 @@
  * GNU General Public License version 2.
  */
 
-use anyhow::{format_err, Error};
+use anyhow::{format_err, Context, Error};
 use async_trait::async_trait;
 use futures::compat::Future01CompatExt;
 use futures::future;
@@ -73,12 +73,16 @@ async fn get_max_replication_lag<'a, C: Laggable>(
     conns: &'a [(String, C)],
 ) -> Result<Option<(&'a str, u64)>, Error> {
     let futs = conns.iter().map(|(region, conn)| async move {
-        let lag = conn.get_lag_secs().await?.ok_or_else(|| {
-            format_err!(
-                "Could not fetch db replication lag for {}. Failing to avoid overloading db",
-                region
-            )
-        })?;
+        let lag = conn
+            .get_lag_secs()
+            .await
+            .with_context(|| format!("While fetching replication lag for {}", region))?
+            .ok_or_else(|| {
+                format_err!(
+                    "Could not fetch db replication lag for {}. Failing to avoid overloading db",
+                    region
+                )
+            })?;
 
         Result::<_, Error>::Ok((region, lag))
     });