checkout: introduce file system tests

Summary: This diff contains basic test setup for checkout tests - we compare transition between two trees without dirty changes

Reviewed By: quark-zju

Differential Revision: D26359502

fbshipit-source-id: ef670c944200bae1652863c91ada92c6fecce4ac

eden/scm/lib/checkout/Cargo.toml

@@ -13,3 +13,9 @@ vfs = { path = "../vfs" }
 anyhow = "1.0"
 futures = { version = "0.3.5", features = ["async-await", "compat"] }
 tokio = { version = "1", features = ["full", "test-util"] }
+
+[dev-dependencies]
+manifest-tree = { path = "../manifest-tree", features = ["for-tests"] }
+pathmatcher = { path = "../pathmatcher" }
+tempfile = "3.1"
+walkdir = "2.2.9"

eden/scm/lib/checkout/src/lib.rs

@@ -147,11 +147,7 @@ impl CheckoutPlan {
                     StoreResult::NotFound(key) => bail!("Key {:?} not found in data store", key),
                 };
                 let path = action.path;
-                let flag = match action.file_type {
-                    FileType::Regular => None,
-                    FileType::Executable => Some(UpdateFlag::Executable),
-                    FileType::Symlink => Some(UpdateFlag::Symlink),
-                };
+                let flag = type_to_flag(&action.file_type);
 
                 Self::write_file(vfs, stats, path, data, flag).await
             });
@@ -267,6 +263,15 @@ impl CheckoutPlan {
     }
 }
 
+// todo: possibly migrate VFS api to use FileType?
+fn type_to_flag(ft: &FileType) -> Option<UpdateFlag> {
+    match ft {
+        FileType::Regular => None,
+        FileType::Executable => Some(UpdateFlag::Executable),
+        FileType::Symlink => Some(UpdateFlag::Symlink),
+    }
+}
+
 impl UpdateContentAction {
     pub fn new(item: DiffEntry, meta: FileMetadata) -> Self {
         Self {
@@ -276,3 +281,233 @@ impl UpdateContentAction {
         }
     }
 }
+
+#[cfg(test)]
+// todo - consider moving some of this code to vfs / separate test create
+// todo parallel execution for the test
+mod test {
+    use super::*;
+    use anyhow::ensure;
+    use anyhow::Context;
+    use manifest_tree::testutil::make_tree_manifest_from_meta;
+    use manifest_tree::Diff;
+    use pathmatcher::AlwaysMatcher;
+    use std::collections::HashMap;
+    use std::path::Path;
+    use tempfile::TempDir;
+    use walkdir::{DirEntry, WalkDir};
+
+    #[tokio::test]
+    async fn test_basic_checkout() -> Result<()> {
+        // Pattern - lowercase_path_[hgid!=1]_[flags!=normal]
+        let a = (rp("A"), FileMetadata::regular(hgid(1)));
+        let a_2 = (rp("A"), FileMetadata::regular(hgid(2)));
+        let a_e = (rp("A"), FileMetadata::executable(hgid(1)));
+        let a_s = (rp("A"), FileMetadata::symlink(hgid(1)));
+        let b = (rp("B"), FileMetadata::regular(hgid(1)));
+        let ab = (rp("A/B"), FileMetadata::regular(hgid(1)));
+        let cd = (rp("C/D"), FileMetadata::regular(hgid(1)));
+
+        // update file
+        assert_checkout(&[a.clone()], &[a_2.clone()]).await?;
+        // mv file
+        assert_checkout(&[a.clone()], &[b.clone()]).await?;
+        // add / rm file
+        assert_checkout_symmetrical(&[a.clone()], &[a.clone(), b.clone()]).await?;
+        // regular<->exec
+        assert_checkout_symmetrical(&[a.clone()], &[a_e.clone()]).await?;
+        // regular->symlink
+        assert_checkout(&[a.clone()], &[a_s.clone()]).await?;
+        // symlink->regular - todo - does not currently work
+        // assert_checkout(vec![a_s.clone()], vec![a.clone()]).await?;
+        // dir <-> file with the same name
+        assert_checkout_symmetrical(&[ab.clone()], &[a.clone()]).await?;
+        // mv file between dirs
+        assert_checkout(&[ab.clone()], &[cd.clone()]).await?;
+
+        Ok(())
+    }
+
+    fn rp(p: &str) -> RepoPathBuf {
+        RepoPathBuf::from_string(p.to_string()).unwrap()
+    }
+
+    fn hgid(p: u8) -> HgId {
+        let mut r = HgId::default().into_byte_array();
+        r[0] = p;
+        HgId::from_byte_array(r)
+    }
+
+    async fn assert_checkout_symmetrical(
+        a: &[(RepoPathBuf, FileMetadata)],
+        b: &[(RepoPathBuf, FileMetadata)],
+    ) -> Result<()> {
+        assert_checkout(a, b).await?;
+        assert_checkout(b, a).await
+    }
+
+    async fn assert_checkout(
+        from: &[(RepoPathBuf, FileMetadata)],
+        to: &[(RepoPathBuf, FileMetadata)],
+    ) -> Result<()> {
+        let tempdir = tempfile::tempdir()?;
+        if let Err(e) = assert_checkout_impl(from, to, &tempdir).await {
+            eprintln!("===");
+            eprintln!("Failed transitioning from tree");
+            print_tree(&from);
+            eprintln!("To tree");
+            print_tree(&to);
+            eprintln!("===");
+            eprintln!(
+                "Working directory: {} (not deleted)",
+                tempdir.into_path().display()
+            );
+            return Err(e);
+        }
+        Ok(())
+    }
+
+    async fn assert_checkout_impl(
+        from: &[(RepoPathBuf, FileMetadata)],
+        to: &[(RepoPathBuf, FileMetadata)],
+        tempdir: &TempDir,
+    ) -> Result<()> {
+        let vfs = VFS::new(tempdir.path().to_path_buf())?;
+        roll_out_fs(&vfs, from)?;
+
+        let matcher = AlwaysMatcher::new();
+        let left_tree = make_tree_manifest_from_meta(from.iter().cloned());
+        let right_tree = make_tree_manifest_from_meta(to.iter().cloned());
+        let diff = Diff::new(&left_tree, &right_tree, &matcher);
+        let plan = CheckoutPlan::from_diff(diff).context("Plan construction failed")?;
+
+        // Use clean vfs for test
+        let vfs = VFS::new(tempdir.path().to_path_buf())?;
+        plan.apply_stream(&vfs, dummy_fs)
+            .await
+            .context("Plan execution failed")?;
+
+        assert_fs(tempdir.path(), to)
+    }
+
+    fn print_tree(t: &[(RepoPathBuf, FileMetadata)]) {
+        for (path, meta) in t {
+            eprintln!("{} [{:?}]", path, meta);
+        }
+    }
+
+    fn roll_out_fs(vfs: &VFS, files: &[(RepoPathBuf, FileMetadata)]) -> Result<()> {
+        for (path, meta) in files {
+            let flag = type_to_flag(&meta.file_type);
+            let data = hgid_file(&meta.hgid);
+            vfs.write(path.as_repo_path(), &data.into(), flag)?;
+        }
+        Ok(())
+    }
+
+    fn assert_fs(root: &Path, expected: &[(RepoPathBuf, FileMetadata)]) -> Result<()> {
+        let mut expected: HashMap<_, _> = expected.iter().cloned().collect();
+        for dir in WalkDir::new(root).into_iter() {
+            let dir = dir?;
+            if dir.file_type().is_dir() {
+                // todo check is not empty
+                continue;
+            }
+            let rel_path = dir.path().strip_prefix(root)?;
+            let rel_path = into_repo_path(rel_path.to_string_lossy().into_owned());
+            let rel_path = RepoPathBuf::from_string(rel_path)?;
+            let expected_meta = if let Some(m) = expected.remove(&rel_path) {
+                m
+            } else {
+                bail!("Checkout created unexpected file {}", rel_path);
+            };
+            assert_metadata(&expected_meta, &dir)?;
+        }
+        if !expected.is_empty() {
+            bail!(
+                "Some files are not present after checkout: {:?}",
+                expected.keys().collect::<Vec<_>>()
+            );
+        }
+        Ok(())
+    }
+
+    #[cfg(not(windows))]
+    fn into_repo_path(path: String) -> String {
+        path
+    }
+
+    #[cfg(windows)]
+    fn into_repo_path(path: String) -> String {
+        path.replace("\\", "/")
+    }
+
+    fn assert_metadata(expected: &FileMetadata, actual: &DirEntry) -> Result<()> {
+        match expected.file_type {
+            FileType::Regular => assert_regular(actual),
+            FileType::Executable => assert_exec(actual),
+            FileType::Symlink => assert_symlink(actual),
+        }
+    }
+
+    // When compiling on unknown platform will get function not defined compile error and will need to address it
+
+    #[cfg(unix)] // This is where PermissionsExt is defined
+    fn assert_regular(actual: &DirEntry) -> Result<()> {
+        use std::os::unix::fs::PermissionsExt;
+        let meta = actual.metadata()?;
+        ensure!(
+            meta.permissions().mode() & 0o111 == 0,
+            "Expected {} to be a regular file, actual mode {:#o}",
+            actual.path().display(),
+            meta.permissions().mode()
+        );
+        Ok(())
+    }
+
+    #[cfg(unix)]
+    fn assert_exec(actual: &DirEntry) -> Result<()> {
+        use std::os::unix::fs::PermissionsExt;
+        let meta = actual.metadata()?;
+        ensure!(
+            meta.permissions().mode() & 0o111 != 0,
+            "Expected {} to be a executable file, actual mode {:#o}",
+            actual.path().display(),
+            meta.permissions().mode()
+        );
+        Ok(())
+    }
+
+    #[cfg(unix)]
+    fn assert_symlink(actual: &DirEntry) -> Result<()> {
+        ensure!(
+            actual.path_is_symlink(),
+            "Expected {} to be a symlink",
+            actual.path().display()
+        );
+        Ok(())
+    }
+
+    #[cfg(windows)]
+    fn assert_regular(_actual: &DirEntry) -> Result<()> {
+        Ok(())
+    }
+
+    #[cfg(windows)]
+    fn assert_exec(_actual: &DirEntry) -> Result<()> {
+        Ok(())
+    }
+
+    #[cfg(windows)]
+    fn assert_symlink(_actual: &DirEntry) -> Result<()> {
+        Ok(())
+    }
+
+    fn dummy_fs(v: Vec<Key>) -> Result<impl Stream<Item = Result<StoreResult<Vec<u8>>>>> {
+        Ok(stream::iter(v).map(|key| Ok(StoreResult::Found(hgid_file(&key.hgid)))))
+    }
+
+    fn hgid_file(hgid: &HgId) -> Vec<u8> {
+        hgid.to_string().into_bytes()
+    }
+}

eden/scm/lib/manifest-tree/src/testutil.rs

@@ -14,7 +14,7 @@ use parking_lot::{Mutex, RwLock};
 use manifest::{testutil::*, Manifest};
 use types::{testutil::*, HgId, Key, RepoPath, RepoPathBuf};
 
-use crate::{TreeManifest, TreeStore};
+use crate::{FileMetadata, TreeManifest, TreeStore};
 
 pub fn make_tree_manifest<'a>(
     paths: impl IntoIterator<Item = &'a (&'a str, &'a str)>,
@@ -27,6 +27,16 @@ pub fn make_tree_manifest<'a>(
     tree
 }
 
+pub fn make_tree_manifest_from_meta(
+    paths: impl IntoIterator<Item = (RepoPathBuf, FileMetadata)>,
+) -> TreeManifest {
+    let mut tree = TreeManifest::ephemeral(Arc::new(TestStore::new()));
+    for (path, meta) in paths {
+        tree.insert(path, meta).unwrap();
+    }
+    tree
+}
+
 /// An in memory `Store` implementation backed by HashMaps. Primarily intended for tests.
 pub struct TestStore {
     entries: RwLock<HashMap<RepoPathBuf, HashMap<HgId, Bytes>>>,