tools/admin: add blobstore-unlink command

Summary:
Copy the `blobstore-unlink` command into the new admin tool.

The blobstore-unlink command only works on SQLBlob, so keep it as a separate
command for now.  In the future we can integrate it with the main blobstore
command.

Reviewed By: kris1319

Differential Revision: D33791722

fbshipit-source-id: 83afc8d87dc1b96e91b7e96bdb1adf0106a822b9

eden/mononoke/cmdlib/mononoke_app/src/app.rs

@@ -10,6 +10,7 @@ use std::sync::Arc;
 
 use anyhow::{anyhow, Context, Result};
 use blobstore::Blobstore;
+use cached_config::ConfigStore;
 use clap::{ArgMatches, Error as ClapError, FromArgMatches};
 use context::CoreContext;
 use environment::MononokeEnvironment;
@@ -29,6 +30,7 @@ use tokio::runtime::Handle;
 use crate::args::{ConfigArgs, RepoArg, RepoArgs, RepoBlobstoreArgs};
 
 pub struct MononokeApp {
+    pub fb: FacebookInit,
     args: ArgMatches,
     env: Arc<MononokeEnvironment>,
     storage_configs: StorageConfigs,
@@ -38,7 +40,7 @@ pub struct MononokeApp {
 
 impl MononokeApp {
     pub(crate) fn new(
-        _fb: FacebookInit,
+        fb: FacebookInit,
         args: ArgMatches,
         env: MononokeEnvironment,
     ) -> Result<Self> {
@@ -52,6 +54,7 @@ impl MononokeApp {
         let repo_factory = RepoFactory::new(env.clone(), &repo_configs.common);
 
         Ok(MononokeApp {
+            fb,
             args,
             env,
             storage_configs,
@@ -91,6 +94,11 @@ impl MononokeApp {
         self.env.runtime.handle()
     }
 
+    /// The config store for this app.
+    pub fn config_store(&self) -> &ConfigStore {
+        &self.env.config_store
+    }
+
     /// The repo configs for this app.
     pub fn repo_configs(&self) -> &RepoConfigs {
         &self.repo_configs

eden/mononoke/tests/integration/test-newadmin-blobstore.t

@@ -8,7 +8,7 @@
   $ . "${TEST_FIXTURES}/library.sh"
 
 setup configuration
-  $ default_setup_blobimport "blob_files"
+  $ default_setup_blobimport "blob_sqlite"
   hg repo
   o  C [draft;rev=2;26805aba1e60]
   │
@@ -25,6 +25,17 @@ Check we can upload and fetch an arbitrary blob.
   $ mononoke_newadmin blobstore -R repo fetch -q somekey -o "$TESTTMP/fetched_value"
   $ diff "$TESTTMP/value" "$TESTTMP/fetched_value"
 
+Test we can unlink a blob
+
+NOTE: The blobstore-unlink command currently only works for sqlblob, and
+doesn't construct the blobstore in the usual way, so we need to give the full
+key.
+
+  $ mononoke_newadmin blobstore-unlink -R repo repo0000.somekey
+  Unlinking key repo0000.somekey
+  $ mononoke_newadmin blobstore -R repo fetch -q somekey -o "$TESTTMP/fetched_value_unlinked"
+  No blob exists for somekey
+
 Examine some of the data
   $ mononoke_newadmin blobstore -R repo fetch changeset.blake2.9feb8ddd3e8eddcfa3a4913b57df7842bedf84b8ea3b7b3fcb14c6424aa81fec
   Key: changeset.blake2.9feb8ddd3e8eddcfa3a4913b57df7842bedf84b8ea3b7b3fcb14c6424aa81fec

eden/mononoke/tools/admin/Cargo.toml

@@ -11,8 +11,10 @@ license = "GPLv2+"
 anyhow = "1.0.51"
 blobrepo = { version = "0.1.0", path = "../../blobrepo" }
 blobstore = { version = "0.1.0", path = "../../blobstore" }
+blobstore_factory = { version = "0.1.0", path = "../../blobstore/factory" }
 bookmarks = { version = "0.1.0", path = "../../bookmarks" }
 bytes = { version = "1.1", features = ["serde"] }
+cached_config = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
 chrono = { version = "0.4", features = ["clock", "serde", "std"], default-features = false }
 clap = { version = "3.0.9", features = ["derive", "regex", "unicode", "wrap_help"] }
 cmdlib_scrubbing = { version = "0.1.0", path = "../../cmdlib/scrubbing" }
@@ -22,6 +24,7 @@ git_types = { version = "0.1.0", path = "../../git/git_types" }
 heck = "0.3.1"
 hex = "0.4.3"
 mercurial_types = { version = "0.1.0", path = "../../mercurial/types" }
+metaconfig_types = { version = "0.1.0", path = "../../metaconfig/types" }
 mononoke_app = { version = "0.1.0", path = "../../cmdlib/mononoke_app" }
 mononoke_types = { version = "0.1.0", path = "../../mononoke_types" }
 regex = "1.5.4"

eden/mononoke/tools/admin/src/commands/blobstore_unlink.rs

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * This software may be used and distributed according to the terms of the
+ * GNU General Public License version 2.
+ */
+
+use std::io::Write;
+use std::sync::Arc;
+
+use anyhow::{format_err, Context, Error, Result};
+use blobstore::BlobstoreWithLink;
+use blobstore_factory::{make_sql_blobstore, BlobstoreOptions, ReadOnlyStorage};
+use cached_config::ConfigStore;
+use clap::Parser;
+use fbinit::FacebookInit;
+use metaconfig_types::{BlobConfig, BlobstoreId, StorageConfig};
+use mononoke_app::args::RepoArgs;
+use mononoke_app::MononokeApp;
+
+/// Directly access blobstore keys
+#[derive(Parser)]
+pub struct CommandArgs {
+    #[clap(flatten)]
+    repo_args: RepoArgs,
+
+    /// If the repo's blobstore is multiplexed, use this inner blobstore
+    #[clap(long)]
+    inner_blobstore_id: Option<u64>,
+
+    /// Key of the blob to unlink
+    key: String,
+}
+
+fn remove_wrapper_blobconfigs(mut blob_config: BlobConfig) -> BlobConfig {
+    // Pack is a wrapper store - remove it
+    while let BlobConfig::Pack { ref blobconfig, .. } = blob_config {
+        blob_config = BlobConfig::clone(blobconfig);
+    }
+    blob_config
+}
+
+fn get_blobconfig(blob_config: BlobConfig, inner_blobstore_id: Option<u64>) -> Result<BlobConfig> {
+    match inner_blobstore_id {
+        None => Ok(blob_config),
+        Some(inner_blobstore_id) => match blob_config {
+            BlobConfig::Multiplexed { blobstores, .. } => {
+                let seeked_id = BlobstoreId::new(inner_blobstore_id);
+                blobstores
+                    .into_iter()
+                    .find_map(|(blobstore_id, _, blobstore)| {
+                        if blobstore_id == seeked_id {
+                            Some(remove_wrapper_blobconfigs(blobstore))
+                        } else {
+                            None
+                        }
+                    })
+                    .ok_or_else(|| {
+                        format_err!("could not find a blobstore with id {}", inner_blobstore_id)
+                    })
+            }
+            _ => Err(format_err!(
+                "inner-blobstore-id supplied but blobstore is not multiplexed"
+            )),
+        },
+    }
+}
+
+async fn get_blobstore(
+    fb: FacebookInit,
+    storage_config: StorageConfig,
+    inner_blobstore_id: Option<u64>,
+    readonly_storage: ReadOnlyStorage,
+    blobstore_options: &BlobstoreOptions,
+    config_store: &ConfigStore,
+) -> Result<Arc<dyn BlobstoreWithLink>, Error> {
+    let blobconfig = get_blobconfig(storage_config.blobstore, inner_blobstore_id)?;
+
+    // TODO: Do this for all blobstores that can support unlink, not just SQLBlob
+    let sql_blob = make_sql_blobstore(
+        fb,
+        blobconfig,
+        readonly_storage,
+        blobstore_options,
+        config_store,
+    )
+    .await?;
+
+    Ok(Arc::new(sql_blob) as Arc<dyn BlobstoreWithLink>)
+}
+
+pub async fn run(app: MononokeApp, args: CommandArgs) -> Result<()> {
+    let ctx = app.new_context();
+
+    let (_repo_name, repo_config) = app.repo_config(&args.repo_args)?;
+    let blobstore = get_blobstore(
+        app.fb,
+        repo_config.storage_config,
+        args.inner_blobstore_id,
+        app.environment().readonly_storage,
+        &app.environment().blobstore_options,
+        app.config_store(),
+    )
+    .await?;
+
+    writeln!(std::io::stdout(), "Unlinking key {}", args.key)?;
+
+    blobstore
+        .unlink(&ctx, &args.key)
+        .await
+        .context("Failed to unlink blob")?;
+
+    Ok(())
+}

eden/mononoke/tools/admin/src/commands/mod.rs

@@ -45,6 +45,7 @@ macro_rules! commands {
 
 commands! {
     mod blobstore;
+    mod blobstore_unlink;
     mod list_repos;
     mod repo_info;
 }