scmstore: update edenapi scmstore tests to use new scmstore

Summary:
Update unit tests in `revisionstore::edenapi::data` to use new scmstore. There's not really a wrapper to exercise anymore for edenapi specifically, so it's probably better to just make these `scmstore` unit tests instead of edenapi (or indexedlogdatastore as in the next change)-specific.

For ease of unit testing, make fetch_logger optional and introduce `empty` constructor.

Reviewed By: kulshrax

Differential Revision: D29397495

fbshipit-source-id: d7ef0df16cf83a2506606c55c78fcbfa684904d7

eden/scm/lib/revisionstore/src/edenapi/data.rs

@@ -157,6 +157,7 @@ mod tests {
         indexedlogdatastore::{IndexedLogDataStoreType, IndexedLogHgIdDataStore},
         localstore::ExtStoredPolicy,
         remotestore::HgIdRemoteStore,
+        scmstore::specialized::{FileAttributes, FileStore, TreeStore},
         testutil::*,
     };
 
@@ -166,63 +167,33 @@ mod tests {
         let k = key("a", "def6f29d7b61f9cb70b2f14f79cd5c43c38e21b2");
         let d = delta("1234", None, k.clone());
         let files = hashmap! { k.clone() => d.data.clone() };
-        let trees = HashMap::new();
 
-        let client = FakeEdenApi::new().files(files).trees(trees).into_arc();
-        let remote_files = EdenApiRemoteStore::<File>::new("repo", client.clone(), None);
-        let remote_trees = EdenApiRemoteStore::<Tree>::new("repo", client, None);
+        let client = FakeEdenApi::new().files(files).into_arc();
+        let remote_files = EdenApiRemoteStore::<File>::new("repo", client, None);
+
+        // Set up local cache store to write received data.
+        let mut store = FileStore::empty();
 
-        // Set up local mutable store to write received data.
         let tmp = TempDir::new()?;
-        let local = Arc::new(IndexedLogHgIdDataStore::new(
+        let cache = Arc::new(IndexedLogHgIdDataStore::new(
             &tmp,
             ExtStoredPolicy::Ignore,
             &ConfigSet::new(),
             IndexedLogDataStoreType::Shared,
         )?);
-
-        // Set up `EdenApiDataStore<File>`.
-        let edenapi_files = remote_files.datastore(local.clone());
+        store.indexedlog_cache = Some(cache.clone());
+        store.edenapi = Some(remote_files);
 
         // Attempt fetch.
-        let k = StoreKey::hgid(k);
-        let data = edenapi_files.get(k.clone())?;
-        let meta = edenapi_files.get_meta(k.clone())?;
-        assert_eq!(data, StoreResult::Found(d.data.as_ref().to_vec()));
-        assert_eq!(
-            meta,
-            StoreResult::Found(Metadata {
-                size: Some(d.data.len() as u64),
-                flags: None
-            })
-        );
+        let mut fetched = store
+            .fetch(std::iter::once(k.clone()), FileAttributes::CONTENT)
+            .single()?
+            .expect("key not found");
+        assert_eq!(fetched.file_content()?.to_vec(), d.data.as_ref().to_vec());
 
         // Check that data was written to the local store.
-        let data = local.get(k.clone())?;
-        let meta = local.get_meta(k.clone())?;
-        assert_eq!(data, StoreResult::Found(d.data.as_ref().to_vec()));
-        assert_eq!(
-            meta,
-            StoreResult::Found(Metadata {
-                size: Some(d.data.len() as u64),
-                flags: None
-            })
-        );
-
-        // Using the same mock client, set up a store for trees.
-        // Need to use a new local store since otherwise the key
-        // would still be present locally from the previous fetch.
-        let tmp = TempDir::new()?;
-        let local = Arc::new(IndexedLogHgIdDataStore::new(
-            &tmp,
-            ExtStoredPolicy::Ignore,
-            &ConfigSet::new(),
-            IndexedLogDataStoreType::Shared,
-        )?);
-        let edenapi_trees = remote_trees.datastore(local.clone());
-
-        // Check that the same key cannot be accessed via the tree store.
-        assert_eq!(edenapi_trees.get(k.clone())?, StoreResult::NotFound(k));
+        let mut fetched = cache.get_entry(k.clone())?.expect("key not found");
+        assert_eq!(fetched.content()?.to_vec(), d.data.as_ref().to_vec());
 
         Ok(())
     }
@@ -232,88 +203,54 @@ mod tests {
         // Set up mocked EdenAPI file and tree stores.
         let k = key("a", "def6f29d7b61f9cb70b2f14f79cd5c43c38e21b2");
         let d = delta("1234", None, k.clone());
-        let files = HashMap::new();
         let trees = hashmap! { k.clone() => d.data.clone() };
 
-        let client = FakeEdenApi::new().files(files).trees(trees).into_arc();
-        let remote_files = EdenApiRemoteStore::<File>::new("repo", client.clone(), None);
+        let client = FakeEdenApi::new().trees(trees).into_arc();
         let remote_trees = EdenApiRemoteStore::<Tree>::new("repo", client, None);
 
-        // Set up local mutable store to write received data.
+        // Set up local cache store to write received data.
+        let mut store = TreeStore::empty();
+
         let tmp = TempDir::new()?;
-        let local = Arc::new(IndexedLogHgIdDataStore::new(
+        let cache = Arc::new(IndexedLogHgIdDataStore::new(
             &tmp,
             ExtStoredPolicy::Ignore,
             &ConfigSet::new(),
             IndexedLogDataStoreType::Shared,
         )?);
-
-        // Set up `EdenApiDataStore<Tree>`.
-        let edenapi_trees = remote_trees.datastore(local.clone());
+        store.indexedlog_cache = Some(cache.clone());
+        store.edenapi = Some(remote_trees);
 
         // Attempt fetch.
-        let k = StoreKey::hgid(k);
-        let data = edenapi_trees.get(k.clone())?;
-        let meta = edenapi_trees.get_meta(k.clone())?;
-        assert_eq!(data, StoreResult::Found(d.data.as_ref().to_vec()));
-        assert_eq!(
-            meta,
-            StoreResult::Found(Metadata {
-                size: None,
-                flags: None
-            })
-        );
+        let mut fetched = store
+            .fetch_batch(std::iter::once(k.clone()))?
+            .complete
+            .pop()
+            .expect("key not found");
+        assert_eq!(fetched.content()?.to_vec(), d.data.as_ref().to_vec());
 
         // Check that data was written to the local store.
-        let data = local.get(k.clone())?;
-        let meta = local.get_meta(k.clone())?;
-        assert_eq!(data, StoreResult::Found(d.data.as_ref().to_vec()));
-        assert_eq!(
-            meta,
-            StoreResult::Found(Metadata {
-                size: None,
-                flags: None
-            })
-        );
-
-        // Using the same mock client, set up a store for files.
-        // Need to use a new local store since otherwise the key
-        // would still be present locally from the previous fetch.
-        let tmp = TempDir::new()?;
-        let local = Arc::new(IndexedLogHgIdDataStore::new(
-            &tmp,
-            ExtStoredPolicy::Ignore,
-            &ConfigSet::new(),
-            IndexedLogDataStoreType::Shared,
-        )?);
-        let edenapi_files = remote_files.datastore(local);
-
-        // Check that the same key cannot be accessed via the file store.
-        assert_eq!(edenapi_files.get(k.clone())?, StoreResult::NotFound(k));
+        let mut fetched = cache.get_entry(k.clone())?.expect("key not found");
+        assert_eq!(fetched.content()?.to_vec(), d.data.as_ref().to_vec());
 
         Ok(())
     }
 
     #[test]
-    fn test_missing() -> Result<()> {
-        // Set up empty EdenApi remote store.
+    fn test_not_found() -> Result<()> {
         let client = FakeEdenApi::new().into_arc();
-        let remote = EdenApiRemoteStore::<File>::new("repo", client, None);
+        let remote_trees = EdenApiRemoteStore::<Tree>::new("repo", client, None);
 
-        // Set up local mutable store.
-        let tmp = TempDir::new()?;
-        let store = Arc::new(IndexedLogHgIdDataStore::new(
-            &tmp,
-            ExtStoredPolicy::Ignore,
-            &ConfigSet::new(),
-            IndexedLogDataStoreType::Shared,
-        )?);
+        // Set up local cache store to write received data.
+        let mut store = TreeStore::empty();
+        store.edenapi = Some(remote_trees);
 
-        // Set up `EdenApiDataStore`.
-        let edenapi = remote.datastore(store.clone());
+        let k = key("a", "def6f29d7b61f9cb70b2f14f79cd5c43c38e21b2");
 
-        let k = StoreKey::hgid(key("a", "1"));
-        assert_eq!(edenapi.get(k.clone())?, StoreResult::NotFound(k));
+        // Attempt fetch.
+        let mut fetched = store.fetch_batch(std::iter::once(k.clone()))?;
+        assert_eq!(fetched.complete.len(), 0);
+        assert_eq!(fetched.incomplete, vec![k]);
 
         Ok(())
     }

eden/scm/lib/revisionstore/src/scmstore/specialized/builder.rs

@@ -320,7 +320,7 @@ impl<'a> FileStoreBuilder<'a> {
             .get_opt::<String>("remotefilelog", "undesiredfileregex")?
             .map(|s| Regex::new(&s))
             .transpose()?;
-        let fetch_logger = Arc::new(FetchLogger::new(logging_regex));
+        let fetch_logger = Some(Arc::new(FetchLogger::new(logging_regex)));
 
         Ok(FileStore {
             extstored_policy,

eden/scm/lib/revisionstore/src/scmstore/specialized/file.rs

@@ -45,7 +45,7 @@ pub struct FileStore {
     pub(crate) cache_to_memcache: bool,
 
     // Record remote fetches
-    pub(crate) fetch_logger: Arc<FetchLogger>,
+    pub(crate) fetch_logger: Option<Arc<FetchLogger>>,
 
     // Local-only stores
     pub(crate) indexedlog_local: Option<Arc<IndexedLogHgIdDataStore>>,
@@ -156,7 +156,7 @@ pub struct FileStoreFetch {
 
 impl FileStoreFetch {
     /// Return the list of keys which could not be fetched, or any errors encountered
-    fn missing(mut self) -> Result<Vec<Key>> {
+    pub fn missing(mut self) -> Result<Vec<Key>> {
         if let Some(err) = self.other_errors.pop() {
             return Err(err).into();
         }
@@ -173,7 +173,7 @@ impl FileStoreFetch {
     }
 
     /// Return the single requested file if found, or any errors encountered
-    fn single(mut self) -> Result<Option<StoreFile>> {
+    pub fn single(mut self) -> Result<Option<StoreFile>> {
         if let Some(err) = self.other_errors.pop() {
             return Err(err).into();
         }
@@ -429,6 +429,33 @@ impl FileStore {
     pub fn fallbacks(&self) -> Arc<ContentStoreFallbacks> {
         self.fallbacks.clone()
     }
+
+    pub fn empty() -> Self {
+        FileStore {
+            extstored_policy: ExtStoredPolicy::Ignore,
+            lfs_threshold_bytes: None,
+
+            indexedlog_local: None,
+            lfs_local: None,
+
+            indexedlog_cache: None,
+            lfs_cache: None,
+            cache_to_local_cache: true,
+
+            memcache: None,
+            cache_to_memcache: true,
+
+            edenapi: None,
+            lfs_remote: None,
+
+            contentstore: None,
+            fallbacks: Arc::new(ContentStoreFallbacks::new()),
+            fetch_logger: None,
+
+            aux_local: None,
+            aux_cache: None,
+        }
+    }
 }
 
 impl LegacyStore for FileStore {
@@ -468,7 +495,11 @@ impl LegacyStore for FileStore {
     }
 
     fn get_logged_fetches(&self) -> HashSet<RepoPathBuf> {
-        let mut seen = self.fetch_logger.take_seen();
+        let mut seen = self
+            .fetch_logger
+            .as_ref()
+            .map(|fl| fl.take_seen())
+            .unwrap_or_default();
         if let Some(contentstore) = self.contentstore.as_ref() {
             seen.extend(contentstore.get_logged_fetches());
         }
@@ -886,7 +917,7 @@ pub struct FetchState {
     computed_aux_data: HashMap<Key, LocalStoreType>,
 
     /// Tracks remote fetches which match a specific regex
-    fetch_logger: Arc<FetchLogger>,
+    fetch_logger: Option<Arc<FetchLogger>>,
 
     /// Track ContentStore Fallbacks
     fallbacks: Arc<ContentStoreFallbacks>,
@@ -1147,7 +1178,9 @@ impl FetchState {
         if pending.is_empty() {
             return Ok(());
         }
-        self.fetch_logger.report_keys(pending.iter());
+        self.fetch_logger
+            .as_ref()
+            .map(|fl| fl.report_keys(pending.iter()));
 
         for res in store.get_data_iter(&pending)?.into_iter() {
             match res {
@@ -1185,7 +1218,9 @@ impl FetchState {
         if pending.is_empty() {
             return Ok(());
         }
-        self.fetch_logger.report_keys(pending.iter());
+        self.fetch_logger
+            .as_ref()
+            .map(|fl| fl.report_keys(pending.iter()));
 
         for entry in store.files_blocking(pending, None)?.entries.into_iter() {
             self.found_edenapi(entry);
@@ -1214,7 +1249,9 @@ impl FetchState {
         if pending.is_empty() {
             return Ok(());
         }
-        self.fetch_logger.report_keys(self.lfs_pointers.keys());
+        self.fetch_logger
+            .as_ref()
+            .map(|fl| fl.report_keys(self.lfs_pointers.keys()));
 
         // Fetch & write to local LFS stores
         store.batch_fetch(&pending, {

eden/scm/lib/revisionstore/src/scmstore/specialized/tree.rs

@@ -218,6 +218,22 @@ impl TreeStore {
             contentstore: None,
         }))
     }
+
+    pub fn empty() -> Self {
+        TreeStore {
+            indexedlog_local: None,
+
+            indexedlog_cache: None,
+            cache_to_local_cache: true,
+
+            memcache: None,
+            cache_to_memcache: true,
+
+            edenapi: None,
+
+            contentstore: None,
+        }
+    }
 }
 
 impl HgIdDataStore for TreeStore {