edenapi: add attribute support to EdenApi trait

Summary:
Adds a new method, `files_attrs`, to the `EdenApi` trait, which allows the caller to specify per-key attributes.

This method is intended to be temporary, and should later be unified with `files`.

Implement `files_attrs` in `FakeEdenApi`, and implement a placeholder method in EagerRepo.

Reviewed By: DurhamG

Differential Revision: D29635233

fbshipit-source-id: d0773927939527d799918139e4abba5ea3b5efca

eden/scm/lib/eagerepo/src/api.rs

@@ -26,6 +26,7 @@ use edenapi::types::CommitLocationToHashResponse;
 use edenapi::types::CommitRevlogData;
 use edenapi::types::FileContent;
 use edenapi::types::FileEntry;
+use edenapi::types::FileSpec;
 use edenapi::types::HgFilenodeData;
 use edenapi::types::HgId;
 use edenapi::types::HgMutationEntryContent;
@@ -94,6 +95,36 @@ impl EdenApi for EagerRepo {
         Ok(convert_to_fetch(values))
     }
 
+    async fn files_attrs(
+        &self,
+        _repo: String,
+        reqs: Vec<FileSpec>,
+        _progress: Option<ProgressCallback>,
+    ) -> edenapi::Result<Fetch<FileEntry>> {
+        debug!("files {}", debug_spec_list(&reqs));
+        let mut values = Vec::with_capacity(reqs.len());
+        for spec in reqs {
+            let key = spec.key;
+            let id = key.hgid;
+            let data = self.get_sha1_blob_for_api(id)?;
+            let (p1, p2) = extract_p1_p2(&data);
+            let parents = Parents::new(p1, p2);
+            // TODO(meyer): Actually implement aux data here.
+            let entry = FileEntry {
+                key,
+                parents,
+                // PERF: to_vec().into() converts minibytes::Bytes to bytes::Bytes.
+                content: Some(FileContent {
+                    hg_file_blob: extract_body(&data).to_vec().into(),
+                    metadata: Default::default(),
+                }),
+                aux_data: None,
+            };
+            values.push(Ok(entry));
+        }
+        Ok(convert_to_fetch(values))
+    }
+
     async fn history(
         &self,
         _repo: String,
@@ -617,6 +648,10 @@ fn debug_key_list(keys: &[Key]) -> String {
     debug_list(keys, |k| k.hgid.to_hex())
 }
 
+fn debug_spec_list(reqs: &[FileSpec]) -> String {
+    debug_list(reqs, |s| s.key.hgid.to_hex())
+}
+
 fn debug_hgid_list(ids: &[HgId]) -> String {
     debug_list(ids, |i| i.to_hex())
 }

eden/scm/lib/edenapi/src/blocking.rs

@@ -8,8 +8,8 @@
 use async_runtime::block_on;
 use edenapi_types::{
     BookmarkEntry, CloneData, CommitHashToLocationResponse, CommitLocationToHashRequest,
-    CommitLocationToHashResponse, CommitRevlogData, EdenApiServerError, FileEntry, HistoryEntry,
-    TreeAttributes, TreeEntry,
+    CommitLocationToHashResponse, CommitRevlogData, EdenApiServerError, FileEntry, FileSpec,
+    HistoryEntry, TreeAttributes, TreeEntry,
 };
 use types::{HgId, Key, RepoPathBuf};
 
@@ -31,6 +31,15 @@ pub trait EdenApiBlocking: EdenApi {
         BlockingFetch::from_async(self.files(repo, keys, progress))
     }
 
+    fn files_attrs_blocking(
+        &self,
+        repo: String,
+        reqs: Vec<FileSpec>,
+        progress: Option<ProgressCallback>,
+    ) -> Result<BlockingFetch<FileEntry>, EdenApiError> {
+        BlockingFetch::from_async(self.files_attrs(repo, reqs, progress))
+    }
+
     fn history_blocking(
         &self,
         repo: String,

eden/scm/lib/edenapi/src/client.rs

@@ -37,7 +37,7 @@ use edenapi_types::{
     CommitHashToLocationRequestBatch, CommitHashToLocationResponse, CommitLocationToHashRequest,
     CommitLocationToHashRequestBatch, CommitLocationToHashResponse, CommitRevlogData,
     CommitRevlogDataRequest, CompleteTreeRequest, EdenApiServerError, FileEntry, FileRequest,
-    HgFilenodeData, HgMutationEntryContent, HistoryEntry, HistoryRequest, LookupRequest,
+    FileSpec, HgFilenodeData, HgMutationEntryContent, HistoryEntry, HistoryRequest, LookupRequest,
     LookupResponse, ServerError, ToApi, ToWire, TreeAttributes, TreeEntry, TreeRequest,
     UploadHgChangeset, UploadHgChangesetsRequest, UploadHgChangesetsResponse,
     UploadHgFilenodeRequest, UploadHgFilenodeResponse, UploadToken, UploadTreeEntry,
@@ -345,6 +345,32 @@ impl EdenApi for Client {
         Ok(self.fetch::<WireFileEntry>(requests, progress).await?)
     }
 
+    async fn files_attrs(
+        &self,
+        repo: String,
+        reqs: Vec<FileSpec>,
+        progress: Option<ProgressCallback>,
+    ) -> Result<Fetch<FileEntry>, EdenApiError> {
+        let msg = format!("Requesting attributes for {} file(s)", reqs.len());
+        tracing::info!("{}", &msg);
+        if self.config.debug {
+            eprintln!("{}", &msg);
+        }
+
+        if reqs.is_empty() {
+            return Ok(Fetch::empty());
+        }
+
+        let url = self.url(paths::FILES, Some(&repo))?;
+        let requests = self.prepare(&url, reqs, self.config.max_files, |reqs| {
+            let req = FileRequest { reqs, keys: vec![] };
+            self.log_request(&req, "files");
+            req.to_wire()
+        })?;
+
+        Ok(self.fetch::<WireFileEntry>(requests, progress).await?)
+    }
+
     async fn history(
         &self,
         repo: String,

eden/scm/lib/edenapi/trait/src/api.rs

@@ -15,7 +15,7 @@ use edenapi_types::CommitKnownResponse;
 use edenapi_types::{
     AnyFileContentId, AnyId, BookmarkEntry, CloneData, CommitHashToLocationResponse,
     CommitLocationToHashRequest, CommitLocationToHashResponse, CommitRevlogData,
-    EdenApiServerError, FileEntry, HgFilenodeData, HgMutationEntryContent, HistoryEntry,
+    EdenApiServerError, FileEntry, FileSpec, HgFilenodeData, HgMutationEntryContent, HistoryEntry,
     LookupResponse, TreeAttributes, TreeEntry, UploadHgChangeset, UploadHgChangesetsResponse,
     UploadHgFilenodeResponse, UploadToken, UploadTreeEntry, UploadTreeResponse,
 };
@@ -39,6 +39,13 @@ pub trait EdenApi: Send + Sync + 'static {
         progress: Option<ProgressCallback>,
     ) -> Result<Fetch<FileEntry>, EdenApiError>;
 
+    async fn files_attrs(
+        &self,
+        repo: String,
+        reqs: Vec<FileSpec>,
+        progress: Option<ProgressCallback>,
+    ) -> Result<Fetch<FileEntry>, EdenApiError>;
+
     async fn history(
         &self,
         repo: String,
@@ -198,6 +205,17 @@ impl EdenApi for Arc<dyn EdenApi> {
             .await
     }
 
+    async fn files_attrs(
+        &self,
+        repo: String,
+        reqs: Vec<FileSpec>,
+        progress: Option<ProgressCallback>,
+    ) -> Result<Fetch<FileEntry>, EdenApiError> {
+        <Arc<dyn EdenApi> as Borrow<dyn EdenApi>>::borrow(self)
+            .files_attrs(repo, reqs, progress)
+            .await
+    }
+
     async fn history(
         &self,
         repo: String,

eden/scm/lib/edenapi/types/src/file.rs

@@ -174,6 +174,10 @@ impl FileEntry {
     pub fn parents(&self) -> &Parents {
         &self.parents
     }
+
+    pub fn aux_data(&self) -> Option<&FileAuxData> {
+        self.aux_data.as_ref()
+    }
 }
 
 #[cfg(any(test, feature = "for-tests"))]

eden/scm/lib/revisionstore/src/edenapi/mod.rs

@@ -11,7 +11,7 @@ use std::sync::Arc;
 use async_trait::async_trait;
 
 use edenapi::{BlockingFetch, EdenApi, EdenApiBlocking, EdenApiError, Fetch, ProgressCallback};
-use edenapi_types::{EdenApiServerError, FileEntry, TreeAttributes, TreeEntry};
+use edenapi_types::{EdenApiServerError, FileEntry, FileSpec, TreeAttributes, TreeEntry};
 use progress::{NullProgressFactory, ProgressFactory};
 use types::Key;
 
@@ -130,6 +130,15 @@ impl EdenApiFileStore {
         self.client
             .files_blocking(self.repo.clone(), keys, progress)
     }
+
+    pub fn files_attrs_blocking(
+        &self,
+        reqs: Vec<FileSpec>,
+        progress: Option<ProgressCallback>,
+    ) -> Result<BlockingFetch<FileEntry>, EdenApiError> {
+        self.client
+            .files_attrs_blocking(self.repo.clone(), reqs, progress)
+    }
 }
 
 impl EdenApiTreeStore {

eden/scm/lib/revisionstore/src/scmstore/file.rs

@@ -18,7 +18,7 @@ use parking_lot::{Mutex, RwLock};
 use serde::{Deserialize, Serialize};
 use tracing::instrument;
 
-use edenapi_types::{ContentId, FileEntry, Sha1};
+use edenapi_types::{ContentId, FileAuxData as EdenApiFileAuxData, FileEntry, Sha1};
 
 use minibytes::Bytes;
 use types::{HgId, Key, RepoPathBuf, Sha256};
@@ -731,6 +731,28 @@ impl From<FileAuxData> for AuxDataEntry {
     }
 }
 
+impl From<EdenApiFileAuxData> for FileAuxData {
+    fn from(v: EdenApiFileAuxData) -> Self {
+        FileAuxData {
+            total_size: v.total_size,
+            content_id: v.content_id,
+            content_sha1: v.sha1,
+            content_sha256: Sha256::from_byte_array(v.sha256.0),
+        }
+    }
+}
+
+impl From<FileAuxData> for EdenApiFileAuxData {
+    fn from(v: FileAuxData) -> Self {
+        EdenApiFileAuxData {
+            total_size: v.total_size,
+            content_id: v.content_id,
+            sha1: v.content_sha1,
+            sha256: v.content_sha256.into_inner().into(),
+        }
+    }
+}
+
 #[derive(Debug)]
 pub struct StoreFile {
     // TODO(meyer): We'll probably eventually need a better "canonical lazy file" abstraction, since EdenApi FileEntry won't always carry content
@@ -914,7 +936,7 @@ impl Sub for FileAttributes {
 /// A minimal file enum that simply wraps the possible underlying file types,
 /// with no processing (so Entry might have the wrong Key.path, etc.)
 #[derive(Debug)]
-enum LazyFile {
+pub(crate) enum LazyFile {
     /// A response from calling into the legacy storage API
     ContentStore(Bytes, Metadata),
 
@@ -945,7 +967,7 @@ impl LazyFile {
 
     /// Compute's the aux data associated with this file from the content.
     #[instrument(level = "debug", skip(self))]
-    fn aux_data(&mut self) -> Result<FileAuxData> {
+    pub(crate) fn aux_data(&mut self) -> Result<FileAuxData> {
         // TODO(meyer): Implement the rest of the aux data fields
         Ok(if let LazyFile::Lfs(content, ref ptr) = self {
             FileAuxData {

eden/scm/lib/revisionstore/src/scmstore/mod.rs

@@ -7,7 +7,7 @@
 
 pub use self::{
     builder::{FileStoreBuilder, TreeStoreBuilder},
-    file::{ContentStoreFallbacks, FileAttributes, FileStore, StoreFile},
+    file::{ContentStoreFallbacks, FileAttributes, FileAuxData, FileStore, StoreFile},
     tree::TreeStore,
     util::file_to_async_key_stream,
 };

eden/scm/lib/revisionstore/src/testutil.rs

@@ -14,10 +14,10 @@ use edenapi::{EdenApi, EdenApiError, Fetch, ProgressCallback, ResponseMeta, Stat
 use edenapi_types::{
     AnyFileContentId, AnyId, BookmarkEntry, CloneData, CommitHashToLocationResponse,
     CommitLocationToHashRequest, CommitLocationToHashResponse, CommitRevlogData,
-    EdenApiServerError, FileContent, FileEntry, HgFilenodeData, HgMutationEntryContent,
-    HistoryEntry, LookupResponse, TreeAttributes, TreeEntry, UploadHgChangeset,
-    UploadHgChangesetsResponse, UploadHgFilenodeResponse, UploadToken, UploadTreeEntry,
-    UploadTreeResponse,
+    EdenApiServerError, FileAttributes, FileAuxData, FileContent, FileEntry, FileSpec,
+    HgFilenodeData, HgMutationEntryContent, HistoryEntry, LookupResponse, TreeAttributes,
+    TreeEntry, UploadHgChangeset, UploadHgChangesetsResponse, UploadHgFilenodeResponse,
+    UploadToken, UploadTreeEntry, UploadTreeResponse,
 };
 use futures::prelude::*;
 use minibytes::Bytes;
@@ -30,6 +30,7 @@ use crate::{
     historystore::{HgIdHistoryStore, HgIdMutableHistoryStore, RemoteHistoryStore},
     localstore::LocalStore,
     remotestore::HgIdRemoteStore,
+    scmstore::file::LazyFile,
     types::StoreKey,
 };
 
@@ -236,22 +237,36 @@ impl FakeEdenApi {
 
     fn get_files(
         map: &HashMap<Key, (Bytes, Option<u64>)>,
-        keys: Vec<Key>,
+        reqs: impl Iterator<Item = FileSpec>,
     ) -> Result<Fetch<FileEntry>, EdenApiError> {
-        let entries = keys
-            .into_iter()
-            .filter_map(|key| {
-                let (data, flags) = map.get(&key)?.clone();
+        let entries = reqs
+            .filter_map(|spec| {
                 let parents = Parents::default();
+                let mut entry = FileEntry::new(spec.key.clone(), parents);
+
+                let (data, flags) = map.get(&spec.key)?.clone();
                 let metadata = Metadata {
                     flags,
                     size: Some(data.len() as u64),
                 };
                 let data = data.to_vec().into();
-                Some(Ok(FileEntry::new(key, parents).with_content(FileContent {
+                let content = FileContent {
                     hg_file_blob: data,
                     metadata,
-                })))
+                };
+
+                if spec.attrs.aux_data {
+                    // TODO(meyer): Compute aux data directly.
+                    let mut file = LazyFile::EdenApi(entry.clone().with_content(content.clone()));
+                    let aux = file.aux_data().ok()?;
+                    entry = entry.with_aux_data(aux.into());
+                }
+
+                if spec.attrs.content {
+                    entry = entry.with_content(content);
+                }
+
+                Some(Ok(entry))
             })
             .collect::<Vec<_>>();
 
@@ -294,7 +309,25 @@ impl EdenApi for FakeEdenApi {
         keys: Vec<Key>,
         _progress: Option<ProgressCallback>,
     ) -> Result<Fetch<FileEntry>, EdenApiError> {
-        Self::get_files(&self.files, keys)
+        Self::get_files(
+            &self.files,
+            keys.into_iter().map(|key| FileSpec {
+                key,
+                attrs: FileAttributes {
+                    content: true,
+                    aux_data: false,
+                },
+            }),
+        )
+    }
+
+    async fn files_attrs(
+        &self,
+        _repo: String,
+        reqs: Vec<FileSpec>,
+        _progress: Option<ProgressCallback>,
+    ) -> Result<Fetch<FileEntry>, EdenApiError> {
+        Self::get_files(&self.files, reqs.into_iter())
     }
 
     async fn history(