From d1d3224ba12250003e036a96736b5c0baf70b382 Mon Sep 17 00:00:00 2001 From: Arun Kulshreshtha Date: Thu, 9 Jul 2020 13:06:18 -0700 Subject: [PATCH] revisionstore: use new EdenAPI crate Summary: Update the `revisionstore` and `backingstore` crates to use the new EdenAPI crate. Reviewed By: quark-zju Differential Revision: D22378330 fbshipit-source-id: 989f34827b744ff4b4ac0aa10d004f03dbe9058f --- eden/scm/lib/backingstore/Cargo.toml | 2 +- eden/scm/lib/backingstore/src/backingstore.rs | 19 ++-- eden/scm/lib/edenapi/src/api.rs | 2 +- eden/scm/lib/edenapi/src/lib.rs | 2 +- eden/scm/lib/edenapi/src/name.rs | 9 ++ eden/scm/lib/edenapi/src/response.rs | 2 +- eden/scm/lib/http_client/src/stats.rs | 2 +- eden/scm/lib/revisionstore/Cargo.toml | 3 +- eden/scm/lib/revisionstore/src/edenapi.rs | 32 ++++--- eden/scm/lib/revisionstore/src/testutil.rs | 86 +++++++++---------- 10 files changed, 91 insertions(+), 68 deletions(-) diff --git a/eden/scm/lib/backingstore/Cargo.toml b/eden/scm/lib/backingstore/Cargo.toml index ba2c4856a3..f93382a559 100644 --- a/eden/scm/lib/backingstore/Cargo.toml +++ b/eden/scm/lib/backingstore/Cargo.toml @@ -6,7 +6,7 @@ edition = "2018" [dependencies] configparser = { path = "../configparser" } -edenapi-old = { path = "../edenapi/old" } +edenapi = { path = "../edenapi" } log = "0.4" manifest = { path = "../manifest" } manifest-tree = { path = "../manifest-tree" } diff --git a/eden/scm/lib/backingstore/src/backingstore.rs b/eden/scm/lib/backingstore/src/backingstore.rs index fa5cd3924b..09204e5cc7 100644 --- a/eden/scm/lib/backingstore/src/backingstore.rs +++ b/eden/scm/lib/backingstore/src/backingstore.rs @@ -8,10 +8,10 @@ use crate::remotestore::FakeRemoteStore; use crate::treecontentstore::TreeContentStore; use crate::utils::key_from_path_node_slice; -use anyhow::Result; +use anyhow::{Context, Result}; use configparser::config::ConfigSet; use configparser::hg::ConfigSetHgExt; -use edenapi_old::{EdenApi, EdenApiCurlClient}; +use edenapi::{Builder as EdenApiBuilder, EdenApi, RepoName}; use log::warn; use manifest::{List, Manifest}; use manifest_tree::TreeManifest; @@ -36,6 +36,11 @@ impl BackingStore { config.load_user(); config.load_hgrc(hg.join("hgrc"), "repository"); + let repo_name: RepoName = config + .get_opt("remotefilelog", "reponame") + .context("Invalid repo name")? + .context("No repo name specified in remotefilelog.reponame")?; + let store_path = hg.join("store"); let mut blobstore = ContentStoreBuilder::new(&config).local_path(&store_path); let treestore = ContentStoreBuilder::new(&config) @@ -51,11 +56,13 @@ impl BackingStore { } let (blobstore, treestore) = if use_edenapi { - let edenapi_config = edenapi_old::Config::from_hg_config(&config)?; - let edenapi = EdenApiCurlClient::new(edenapi_config)?; + let edenapi = EdenApiBuilder::from_config(&config)?.build()?; let edenapi: Arc = Arc::new(edenapi); - let fileremotestore = Arc::new(EdenApiHgIdRemoteStore::filestore(edenapi.clone())); - let treeremotestore = Arc::new(EdenApiHgIdRemoteStore::treestore(edenapi)); + let fileremotestore = Arc::new(EdenApiHgIdRemoteStore::filestore( + repo_name.clone(), + edenapi.clone(), + )); + let treeremotestore = Arc::new(EdenApiHgIdRemoteStore::treestore(repo_name, edenapi)); ( blobstore.remotestore(fileremotestore).build()?, diff --git a/eden/scm/lib/edenapi/src/api.rs b/eden/scm/lib/edenapi/src/api.rs index 4782c97ba9..f1d5ea5e38 100644 --- a/eden/scm/lib/edenapi/src/api.rs +++ b/eden/scm/lib/edenapi/src/api.rs @@ -18,7 +18,7 @@ use crate::response::{Fetch, ResponseMeta}; pub type ProgressCallback = Box; #[async_trait] -pub trait EdenApi: Send + 'static { +pub trait EdenApi: Send + Sync + 'static { async fn health(&self) -> Result; async fn files( diff --git a/eden/scm/lib/edenapi/src/lib.rs b/eden/scm/lib/edenapi/src/lib.rs index 29ee2c489f..511351d786 100644 --- a/eden/scm/lib/edenapi/src/lib.rs +++ b/eden/scm/lib/edenapi/src/lib.rs @@ -24,4 +24,4 @@ pub use crate::name::RepoName; pub use crate::response::{BlockingFetch, Entries, Fetch, ResponseMeta}; // Re-export for convenience. -pub use http_client::Progress; +pub use http_client::{Progress, Stats}; diff --git a/eden/scm/lib/edenapi/src/name.rs b/eden/scm/lib/edenapi/src/name.rs index d1a8db5455..3d48fe0299 100644 --- a/eden/scm/lib/edenapi/src/name.rs +++ b/eden/scm/lib/edenapi/src/name.rs @@ -8,8 +8,11 @@ use std::fmt; use std::str::FromStr; +use anyhow::Error; use ascii::{AsAsciiStr, AsciiString}; +use configparser::hg::FromConfigValue; + use crate::errors::EdenApiError; /// A valididated repo name. @@ -49,6 +52,12 @@ impl fmt::Display for RepoName { } } +impl FromConfigValue for RepoName { + fn try_from_str(s: &str) -> Result { + Ok(s.parse()?) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/eden/scm/lib/edenapi/src/response.rs b/eden/scm/lib/edenapi/src/response.rs index e7aca592ee..d9bc69acc6 100644 --- a/eden/scm/lib/edenapi/src/response.rs +++ b/eden/scm/lib/edenapi/src/response.rs @@ -76,7 +76,7 @@ impl BlockingFetch { } /// Metadata extracted from the headers of an individual HTTP response. -#[derive(Debug)] +#[derive(Debug, Default)] pub struct ResponseMeta { pub version: Version, pub status: StatusCode, diff --git a/eden/scm/lib/http_client/src/stats.rs b/eden/scm/lib/http_client/src/stats.rs index e415618af9..3d95972293 100644 --- a/eden/scm/lib/http_client/src/stats.rs +++ b/eden/scm/lib/http_client/src/stats.rs @@ -7,7 +7,7 @@ use std::{fmt, time::Duration}; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct Stats { pub downloaded: usize, pub uploaded: usize, diff --git a/eden/scm/lib/revisionstore/Cargo.toml b/eden/scm/lib/revisionstore/Cargo.toml index 2afb651fd7..d9cebc47c4 100644 --- a/eden/scm/lib/revisionstore/Cargo.toml +++ b/eden/scm/lib/revisionstore/Cargo.toml @@ -11,10 +11,11 @@ for-tests = [] [dependencies] anyhow = "1.0.20" +async-trait = "0.1" bytes = { version = "0.5", features = ["serde"] } byteorder = "1.2.7" configparser = { path = "../configparser" } -edenapi-old = { path = "../edenapi/old" } +edenapi = { path = "../edenapi" } edenapi_types = { path = "../edenapi/types" } futures = "0.3" hex = "0.4" diff --git a/eden/scm/lib/revisionstore/src/edenapi.rs b/eden/scm/lib/revisionstore/src/edenapi.rs index 0348941836..8e21b6e251 100644 --- a/eden/scm/lib/revisionstore/src/edenapi.rs +++ b/eden/scm/lib/revisionstore/src/edenapi.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use anyhow::Result; -use edenapi_old::EdenApi; +use edenapi::{EdenApi, EdenApiBlocking, RepoName}; use types::Key; use crate::{ @@ -30,20 +30,23 @@ enum EdenApiHgIdRemoteStoreKind { /// the `HgIdDataStore` methods will always fetch data from the network. pub struct EdenApiHgIdRemoteStore { edenapi: Arc, + repo: RepoName, kind: EdenApiHgIdRemoteStoreKind, } impl EdenApiHgIdRemoteStore { - pub fn filestore(edenapi: Arc) -> Self { + pub fn filestore(repo: RepoName, edenapi: Arc) -> Self { Self { edenapi, + repo, kind: EdenApiHgIdRemoteStoreKind::File, } } - pub fn treestore(edenapi: Arc) -> Self { + pub fn treestore(repo: RepoName, edenapi: Arc) -> Self { Self { edenapi, + repo, kind: EdenApiHgIdRemoteStoreKind::Tree, } } @@ -76,6 +79,7 @@ struct EdenApiRemoteDataStore { impl RemoteDataStore for EdenApiRemoteDataStore { fn prefetch(&self, keys: &[StoreKey]) -> Result<()> { let edenapi = &self.edenapi; + let repo = edenapi.repo.clone(); let keys = keys .iter() @@ -84,13 +88,13 @@ impl RemoteDataStore for EdenApiRemoteDataStore { StoreKey::Content(_, _) => None, }) .collect::>(); - let (entries, _) = match edenapi.kind { - EdenApiHgIdRemoteStoreKind::File => edenapi.edenapi.get_files(keys, None)?, - EdenApiHgIdRemoteStoreKind::Tree => edenapi.edenapi.get_trees(keys, None)?, + let response = match edenapi.kind { + EdenApiHgIdRemoteStoreKind::File => edenapi.edenapi.files_blocking(repo, keys, None)?, + EdenApiHgIdRemoteStoreKind::Tree => edenapi.edenapi.trees_blocking(repo, keys, None)?, }; - for entry in entries { - let key = entry.0.clone(); - let data = entry.1; + for entry in response.entries { + let key = entry.key().clone(); + let data = entry.data()?; let metadata = Metadata { size: Some(data.len() as u64), flags: None, @@ -157,7 +161,10 @@ mod tests { let mut map = HashMap::new(); map.insert(k.clone(), d.data.clone()); - let edenapi = Arc::new(EdenApiHgIdRemoteStore::filestore(fake_edenapi(map))); + let edenapi = Arc::new(EdenApiHgIdRemoteStore::filestore( + "repo".parse()?, + fake_edenapi(map), + )); let remotestore = edenapi.datastore(store.clone()); @@ -176,7 +183,10 @@ mod tests { let store = Arc::new(IndexedLogHgIdDataStore::new(&tmp)?); let map = HashMap::new(); - let edenapi = Arc::new(EdenApiHgIdRemoteStore::filestore(fake_edenapi(map))); + let edenapi = Arc::new(EdenApiHgIdRemoteStore::filestore( + "repo".parse()?, + fake_edenapi(map), + )); let remotestore = edenapi.datastore(store); diff --git a/eden/scm/lib/revisionstore/src/testutil.rs b/eden/scm/lib/revisionstore/src/testutil.rs index e0631e1006..7e937927be 100644 --- a/eden/scm/lib/revisionstore/src/testutil.rs +++ b/eden/scm/lib/revisionstore/src/testutil.rs @@ -5,15 +5,17 @@ * GNU General Public License version 2. */ -use std::{collections::HashMap, path::Path, sync::Arc, time::Duration}; +use std::{collections::HashMap, path::Path, sync::Arc}; -use anyhow::{Error, Result}; +use anyhow::{Context, Error, Result}; +use async_trait::async_trait; use bytes::Bytes; +use futures::prelude::*; use configparser::config::ConfigSet; -use edenapi_old::{ApiResult, DownloadStats, EdenApi, ProgressFn}; -use edenapi_types::HistoryEntry; -use types::{HgId, Key, NodeInfo, RepoPathBuf}; +use edenapi::{EdenApi, EdenApiError, Fetch, ProgressCallback, RepoName, ResponseMeta, Stats}; +use edenapi_types::{DataEntry, HistoryEntry}; +use types::{HgId, Key, NodeInfo, Parents, RepoPathBuf}; use crate::{ datastore::{Delta, HgIdDataStore, HgIdMutableDeltaStore, Metadata, RemoteDataStore}, @@ -182,71 +184,65 @@ impl FakeEdenApi { pub fn new(map: HashMap) -> FakeEdenApi { Self { map } } - - fn fake_downloadstats(&self) -> DownloadStats { - DownloadStats { - downloaded: 0, - uploaded: 0, - requests: 0, - time: Duration::from_secs(0), - latency: Duration::from_secs(0), - } - } } +#[async_trait] impl EdenApi for FakeEdenApi { - fn health_check(&self) -> ApiResult<()> { - Ok(()) + async fn health(&self) -> Result { + Ok(ResponseMeta::default()) } - fn hostname(&self) -> ApiResult { - Ok("test".to_string()) - } - - fn get_files( + async fn files( &self, + _repo: RepoName, keys: Vec, - _progress: Option, - ) -> ApiResult<(Box>, DownloadStats)> { - let stats = self.fake_downloadstats(); - let iter = keys + _progress: Option, + ) -> Result, EdenApiError> { + let entries = keys .into_iter() .map(|key| { - self.map - .get(&key) - .ok_or_else(|| "Not found".into()) - .map(|data| (key, data.clone())) + let data = self.map.get(&key).context("Not found")?.clone(); + let parents = Parents::default(); + Ok(DataEntry::new(key, data, parents)) }) - .collect::>>()?; - Ok((Box::new(iter.into_iter()), stats)) + .collect::>(); + + Ok(Fetch { + meta: vec![ResponseMeta::default()], + entries: Box::pin(stream::iter(entries)), + stats: Box::pin(future::ok(Stats::default())), + }) } - fn get_history( + async fn history( &self, + _repo: RepoName, _keys: Vec, - _max_depth: Option, - _progress: Option, - ) -> ApiResult<(Box>, DownloadStats)> { - unreachable!(); + _length: Option, + _progress: Option, + ) -> Result, EdenApiError> { + unimplemented!() } - fn get_trees( + async fn trees( &self, + _repo: RepoName, _keys: Vec, - _progress: Option, - ) -> ApiResult<(Box>, DownloadStats)> { - unreachable!(); + _progress: Option, + ) -> Result, EdenApiError> { + unimplemented!() } - fn prefetch_trees( + async fn complete_trees( &self, + _repo: RepoName, _rootdir: RepoPathBuf, _mfnodes: Vec, _basemfnodes: Vec, _depth: Option, - _progress: Option, - ) -> ApiResult<(Box>, DownloadStats)> { - unreachable!(); + _progress: Option, + ) -> Result, EdenApiError> { + unimplemented!() } }