Rename deleted_files_manifest to deleted_manifest

Summary:
It was always inconsistent that DM were called deleted manifest in some places and deleted files manifest in others.

As I moved them to v2, I tried to more consistently using deleted manifest everywhere, so I'm now renaming the directory where most of the code is stored to use this more correct name.

Reviewed By: kris1319

Differential Revision: D36482057

fbshipit-source-id: 8f99afc40e0a90e4dab9da7b368d2f5b300e567b
This commit is contained in:
Yan Soares Couto 2022-06-06 07:15:28 -07:00 committed by Facebook GitHub Bot
parent b556f6a067
commit 148b95ecc3
34 changed files with 48 additions and 53 deletions

View File

@ -271,7 +271,7 @@ members = [
"derived_data/blame",
"derived_data/changeset_info",
"derived_data/changeset_info/if",
"derived_data/deleted_files_manifest",
"derived_data/deleted_manifest",
"derived_data/derived_generation",
"derived_data/fastlog",
"derived_data/filenodes",

View File

@ -39,7 +39,7 @@ cmdlib = { version = "0.1.0", path = "../cmdlib" }
context = { version = "0.1.0", path = "../server/context" }
copy_utils = { version = "0.1.0", path = "../common/copy_utils" }
cross_repo_sync = { version = "0.1.0", path = "../commit_rewriting/cross_repo_sync" }
deleted_files_manifest = { version = "0.1.0", path = "../derived_data/deleted_files_manifest" }
deleted_manifest = { version = "0.1.0", path = "../derived_data/deleted_manifest" }
derived_data = { version = "0.1.0", path = "../derived_data" }
derived_data_manager = { version = "0.1.0", path = "../derived_data/manager" }
derived_data_utils = { version = "0.1.0", path = "../derived_data/utils" }

View File

@ -28,7 +28,7 @@ clap-old = { package = "clap", version = "2.33" }
cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
cmdlib = { version = "0.1.0", path = "../cmdlib" }
context = { version = "0.1.0", path = "../server/context" }
deleted_files_manifest = { version = "0.1.0", path = "../derived_data/deleted_files_manifest" }
deleted_manifest = { version = "0.1.0", path = "../derived_data/deleted_manifest" }
derived_data = { version = "0.1.0", path = "../derived_data" }
derived_data_manager = { version = "0.1.0", path = "../derived_data/manager" }
derived_data_utils = { version = "0.1.0", path = "../derived_data/utils" }

View File

@ -15,7 +15,7 @@ path = "benchmark_large_directory.rs"
anyhow = "1.0.56"
blobrepo = { version = "0.1.0", path = "../../blobrepo" }
context = { version = "0.1.0", path = "../../server/context" }
deleted_files_manifest = { version = "0.1.0", path = "../../derived_data/deleted_files_manifest" }
deleted_manifest = { version = "0.1.0", path = "../../derived_data/deleted_manifest" }
derived_data = { version = "0.1.0", path = "../../derived_data" }
derived_data_manager = { version = "0.1.0", path = "../../derived_data/manager" }
fbinit = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }

View File

@ -10,14 +10,14 @@
//! remove some of those files at random.
//!
//! It then benchmarks deriving one of the derived data types (fsnodes,
//! unodes, skeleton manifest or deleted files manifests) for those commits.
//! unodes, skeleton manifest or deleted manifests) for those commits.
use std::collections::BTreeSet;
use anyhow::Result;
use blobrepo::BlobRepo;
use context::CoreContext;
use deleted_files_manifest::{RootDeletedManifestIdCommon, RootDeletedManifestV2Id};
use deleted_manifest::{RootDeletedManifestIdCommon, RootDeletedManifestV2Id};
use derived_data::BonsaiDerived;
use derived_data_manager::BonsaiDerivable as NewBonsaiDerivable;
use fbinit::FacebookInit;

View File

@ -19,7 +19,7 @@ bookmarks_types = { version = "0.1.0", path = "../bookmarks_types" }
changeset_info = { version = "0.1.0", path = "../../derived_data/changeset_info" }
cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
context = { version = "0.1.0", path = "../../server/context" }
deleted_files_manifest = { version = "0.1.0", path = "../../derived_data/deleted_files_manifest" }
deleted_manifest = { version = "0.1.0", path = "../../derived_data/deleted_manifest" }
derived_data_filenodes = { version = "0.1.0", path = "../../derived_data/filenodes" }
derived_data_manager = { version = "0.1.0", path = "../../derived_data/manager" }
facet = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }

View File

@ -22,7 +22,7 @@ use bookmarks_types::{Bookmark, BookmarkKind, BookmarkPagination, BookmarkPrefix
use changeset_info::ChangesetInfo;
use cloned::cloned;
use context::{CoreContext, SessionClass};
use deleted_files_manifest::RootDeletedManifestV2Id;
use deleted_manifest::RootDeletedManifestV2Id;
use derived_data_filenodes::FilenodesOnlyPublic;
use derived_data_manager::BonsaiDerivable as NewBonsaiDerivable;
use fastlog::RootFastlog;

View File

@ -18,7 +18,7 @@ use cmdlib::{
helpers,
};
use context::CoreContext;
use deleted_files_manifest::{DeletedManifestOps, RootDeletedManifestV2Id};
use deleted_manifest::{DeletedManifestOps, RootDeletedManifestV2Id};
use derived_data::BonsaiDerived;
use fbinit::FacebookInit;
use futures::{compat::Stream01CompatExt, future, StreamExt, TryStreamExt};
@ -50,10 +50,10 @@ pub fn build_subcommand<'a, 'b>() -> App<'a, 'b> {
.default_value("");
SubCommand::with_name(DELETED_MANIFEST)
.about("derive, inspect and verify deleted files manifest")
.about("derive, inspect and verify deleted manifest")
.subcommand(
SubCommand::with_name(COMMAND_MANIFEST)
.about("recursively list all deleted files manifest entries under the given path")
.about("recursively list all deleted manifest entries under the given path")
.arg(csid_arg.clone())
.arg(path_arg.clone()),
)
@ -135,10 +135,7 @@ async fn subcommand_manifest(
prefix: Option<MPath>,
) -> Result<(), Error> {
let root_manifest = RootDeletedManifestV2Id::derive(&ctx, &repo, cs_id).await?;
debug!(
ctx.logger(),
"ROOT Deleted Files Manifest V2 {:?}", root_manifest,
);
debug!(ctx.logger(), "ROOT Deleted Manifest V2 {:?}", root_manifest,);
let mut entries: Vec<_> = root_manifest
.find_entries(&ctx, repo.blobstore(), Some(PathOrPrefix::Prefix(prefix)))
.try_collect()

View File

@ -10,7 +10,7 @@ use blame::{fetch_content_for_blame, BlameRoot};
use blobrepo::BlobRepo;
use blobstore::Loadable;
use context::CoreContext;
use deleted_files_manifest::RootDeletedManifestV2Id;
use deleted_manifest::RootDeletedManifestV2Id;
use derived_data::BonsaiDerived;
use fastlog::RootFastlog;
use fsnodes::{prefetch_content_metadata, RootFsnodeId};

View File

@ -1,7 +1,7 @@
# @generated by autocargo
[package]
name = "deleted_files_manifest"
name = "deleted_manifest"
version = "0.1.0"
authors = ["Facebook"]
edition = "2021"

View File

@ -31,11 +31,11 @@ use unodes::RootUnodeManifestId;
use crate::mapping::RootDeletedManifestIdCommon;
/// Derives deleted files manifest for bonsai changeset `cs_id` given parent deleted files
/// Derives deleted manifest for bonsai changeset `cs_id` given parent deleted files
/// manifests and the changes associated with the changeset. Parent deleted manifests should be
/// constructed for each parent of the given changeset.
///
/// Deleted files manifest is a recursive data structure that starts with a root manifest and
/// Deleted manifest is a recursive data structure that starts with a root manifest and
/// points to the other manifests. Each node may represent either deleted directoty or deleted file.
/// Both directory's and file's manifest can have subentries, if a file has subentries it means
/// that this path was a directory earlier, then was deleted and reincarnated as a file.
@ -49,7 +49,7 @@ use crate::mapping::RootDeletedManifestIdCommon;
/// recursively starting from the root of parent manifest.
///
/// 1. If no files were deleted or created on the current path or any subpaths
/// - if there was corresponding deleted files manifest, reuse it;
/// - if there was corresponding deleted manifest, reuse it;
/// - otherwise, there is no need to create a new node.
/// 2. If no change ends on the current path BUT there are creations/deletions on the subpaths,
/// recurse to the parent subentries and the current subpaths' changes
@ -162,7 +162,7 @@ impl<Manifest: DeletedManifestCommon> DeletedManifestDeriver<Manifest> {
match entry {
(None, _) => {
return Err(anyhow!(concat!(
"Failed to create deleted files manifest: ",
"Failed to create deleted manifest: ",
"subentry must have a path"
)));
}
@ -713,12 +713,12 @@ impl<Root: RootDeletedManifestIdCommon> RootDeletedManifestDeriver<Root> {
changeset_id: ChangesetId,
) -> Result<Option<Root>, Error> {
let key = Root::format_key(derivation_ctx, changeset_id);
Ok(derivation_ctx
derivation_ctx
.blobstore()
.get(ctx, &key)
.await?
.map(TryInto::try_into)
.transpose()?)
.transpose()
}
}

View File

@ -207,7 +207,7 @@ pub trait DeletedManifestOps: RootDeletedManifestIdCommon {
Ok(None)
}
/// List all Deleted Files Manifest paths recursively that were deleted and match specified paths
/// List all Deleted Manifest paths recursively that were deleted and match specified paths
/// and/or prefixes.
///
fn find_entries<'a>(
@ -332,7 +332,7 @@ pub trait DeletedManifestOps: RootDeletedManifestIdCommon {
}
}
/// List all Deleted files manifest entries recursively, that represent deleted paths.
/// List all Deleted manifest entries recursively, that represent deleted paths.
///
fn list_all_entries<'a>(
&self,

View File

@ -74,7 +74,7 @@ pub(crate) async fn linear_test<Root: RootDeletedManifestIdCommon>(fb: FacebookI
let repo: BlobRepo = build_repo::<Root>(fb).unwrap();
let ctx = CoreContext::test_mock(fb);
// create parent deleted files manifest
// create parent deleted manifest
let (bcs_id_1, mf_id_1) = {
let file_changes = btreemap! {
"file.txt" => Some("1\n"),
@ -549,7 +549,7 @@ pub(crate) async fn test_find_entries<Root: RootDeletedManifestIdCommon>(fb: Fac
let repo: BlobRepo = build_repo::<Root>(fb).unwrap();
let ctx = CoreContext::test_mock(fb);
// create parent deleted files manifest
// create parent deleted manifest
let (bcs_id_1, mf_id_1) = {
let file_changes = btreemap! {
"file.txt" => Some("1\n"),
@ -663,7 +663,7 @@ pub(crate) async fn test_list_all_entries<Root: RootDeletedManifestIdCommon>(fb:
let repo: BlobRepo = build_repo::<Root>(fb).unwrap();
let ctx = CoreContext::test_mock(fb);
// create parent deleted files manifest
// create parent deleted manifest
let (bcs_id_1, mf_id_1) = {
let file_changes = btreemap! {
"file.txt" => Some("1\n"),

View File

@ -18,7 +18,7 @@ blobstore = { version = "0.1.0", path = "../../blobstore" }
changeset_fetcher = { version = "0.1.0", path = "../../blobrepo/changeset_fetcher" }
cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
context = { version = "0.1.0", path = "../../server/context" }
deleted_files_manifest = { version = "0.1.0", path = "../deleted_files_manifest" }
deleted_manifest = { version = "0.1.0", path = "../deleted_manifest" }
derived_data = { version = "0.1.0", path = ".." }
derived_data_manager = { version = "0.1.0", path = "../manager" }
derived_data_service_if = { version = "0.1.0", path = "../remote/if" }

View File

@ -12,7 +12,7 @@ use blobstore::{Loadable, LoadableError};
use changeset_fetcher::{ArcChangesetFetcher, ChangesetFetcher};
use cloned::cloned;
use context::CoreContext;
use deleted_files_manifest::{DeletedManifestOps, PathState, RootDeletedManifestV2Id};
use deleted_manifest::{DeletedManifestOps, PathState, RootDeletedManifestV2Id};
use derived_data::{BonsaiDerived, DeriveError};
use futures::{
future,

View File

@ -21,7 +21,7 @@ changeset_info = { version = "0.1.0", path = "../changeset_info" }
changesets = { version = "0.1.0", path = "../../changesets" }
cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
context = { version = "0.1.0", path = "../../server/context" }
deleted_files_manifest = { version = "0.1.0", path = "../deleted_files_manifest" }
deleted_manifest = { version = "0.1.0", path = "../deleted_manifest" }
derived_data = { version = "0.1.0", path = ".." }
derived_data_filenodes = { version = "0.1.0", path = "../filenodes" }
derived_data_manager = { version = "0.1.0", path = "../manager" }

View File

@ -14,7 +14,7 @@ use changeset_info::ChangesetInfo;
use changesets::ChangesetsArc;
use cloned::cloned;
use context::CoreContext;
use deleted_files_manifest::RootDeletedManifestV2Id;
use deleted_manifest::RootDeletedManifestV2Id;
use derived_data::DerivedDataTypesConfig;
use derived_data_filenodes::FilenodesOnlyPublic;
use derived_data_manager::{

View File

@ -28,7 +28,7 @@ chrono = { version = "0.4", features = ["clock", "serde", "std"], default-featur
cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
context = { version = "0.1.0", path = "../server/context" }
cross_repo_sync = { version = "0.1.0", path = "../commit_rewriting/cross_repo_sync" }
deleted_files_manifest = { version = "0.1.0", path = "../derived_data/deleted_files_manifest" }
deleted_manifest = { version = "0.1.0", path = "../derived_data/deleted_manifest" }
derived_data = { version = "0.1.0", path = "../derived_data" }
derived_data_manager = { version = "0.1.0", path = "../derived_data/manager" }
edenapi_types = { version = "0.1.0", path = "../../scm/lib/edenapi/types" }

View File

@ -19,9 +19,7 @@ use changesets::ChangesetsRef;
use chrono::{DateTime, FixedOffset};
use cloned::cloned;
use context::{CoreContext, PerfCounterType};
use deleted_files_manifest::{
DeletedManifestOps, RootDeletedManifestIdCommon, RootDeletedManifestV2Id,
};
use deleted_manifest::{DeletedManifestOps, RootDeletedManifestIdCommon, RootDeletedManifestV2Id};
use derived_data::BonsaiDerived;
use derived_data_manager::BonsaiDerivable;
use fsnodes::RootFsnodeId;

View File

@ -19,7 +19,7 @@ use bytes::Bytes;
use changeset_info::ChangesetInfo;
use cloned::cloned;
use context::CoreContext;
use deleted_files_manifest::{DeletedManifestOps, RootDeletedManifestIdCommon};
use deleted_manifest::{DeletedManifestOps, RootDeletedManifestIdCommon};
use derived_data::BonsaiDerived;
use fastlog::{
list_file_history, CsAndPath, FastlogError, FollowMutableFileHistory, HistoryAcrossDeletions,

View File

@ -21,13 +21,13 @@ use crate::thrift;
use crate::typed_hash::{BlobstoreKey, ChangesetId, DeletedManifestV2Context, DeletedManifestV2Id};
use crate::MPathElement;
/// Deleted Files Manifest is a data structure that tracks deleted files and commits where they
/// Deleted Manifest is a data structure that tracks deleted files and commits where they
/// were deleted. This manifest was designed in addition to Unodes to make following file history
/// across deletions possible.
///
/// Both directories and files are represented by the same data structure, which consists of:
/// * optional<linknode>: if set, a changeset where this path was deleted
/// * subentries: a map from base name to the deleted files manifest for this path
/// * subentries: a map from base name to the deleted manifest for this path
/// Even though the manifest tracks only deleted paths, it will still have entries for the
/// existing directories where files were deleted. Optional field `linknode` indicates whether the
/// path still exists (not set) or it was deleted.
@ -36,7 +36,7 @@ use crate::MPathElement;
///
/// Why the manifest has same data structure for files and directories?
///
/// Deleted files manifest doesn't differ files from directories, because any file path can be
/// Deleted manifest doesn't differ files from directories, because any file path can be
/// reincarnated after the deletion as a directory and vice versa. The manifest doesn't need
/// to know whether the path is a directory or a file, the only important information is "if the
/// path was deleted, which changeset did it?"
@ -50,9 +50,9 @@ use crate::MPathElement;
/// However, if one of such files is recreated as a directory, we anyway create a new entry for it:
/// manifest entries are immutable.
///
/// How we derive deleted files manifest?
/// How we derive deleted manifest?
///
/// Assuming we have a computed deleted files manifests for all the current commits, for a new
/// Assuming we have a computed deleted manifests for all the current commits, for a new
/// changeset:
/// 1. For each deleted file create a new manifest entry with a linknode to the changeset, where
/// file was deleted.
@ -83,7 +83,7 @@ use crate::MPathElement;
/// | |
/// o F :
///
/// 1. Check deleted files manifest from node B: the file was deleted with linknode to B.
/// 1. Check deleted manifest from node B: the file was deleted with linknode to B.
/// 2. Need to consider all ancestors of B:
/// * check unodes for the parents: if the file exists,
/// traverse the history

View File

@ -106,7 +106,7 @@ pub struct FileUnodeId(Blake2);
#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Debug, Hash)]
pub struct ManifestUnodeId(Blake2);
/// An identifier for a deleted files manifest v2
/// An identifier for a deleted manifest v2
#[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Debug, Hash)]
pub struct DeletedManifestV2Id(Blake2);

View File

@ -24,7 +24,7 @@ changesets = { version = "0.1.0", path = "../../changesets" }
changesets_impl = { version = "0.1.0", path = "../../changesets/changesets_impl" }
context = { version = "0.1.0", path = "../../server/context" }
dbbookmarks = { version = "0.1.0", path = "../../bookmarks/dbbookmarks" }
deleted_files_manifest = { version = "0.1.0", path = "../../derived_data/deleted_files_manifest" }
deleted_manifest = { version = "0.1.0", path = "../../derived_data/deleted_manifest" }
derived_data_filenodes = { version = "0.1.0", path = "../../derived_data/filenodes" }
derived_data_manager = { version = "0.1.0", path = "../../derived_data/manager" }
ephemeral_blobstore = { version = "0.1.0", path = "../../blobstore/ephemeral_blobstore" }

View File

@ -26,7 +26,7 @@ use changesets::ArcChangesets;
use changesets_impl::SqlChangesetsBuilder;
use context::CoreContext;
use dbbookmarks::{ArcSqlBookmarks, SqlBookmarksBuilder};
use deleted_files_manifest::RootDeletedManifestV2Id;
use deleted_manifest::RootDeletedManifestV2Id;
use derived_data_filenodes::FilenodesOnlyPublic;
use derived_data_manager::BonsaiDerivable;
use ephemeral_blobstore::{ArcRepoEphemeralStore, RepoEphemeralStore};

View File

@ -126,14 +126,14 @@ count-objects, deep walk across blame
Seen,Loaded: 16,16
* Type:Walked,Checks,Children Blame:3,* Bookmark:1,* Changeset:3,* UnodeFile:3,* UnodeManifest:3,* UnodeMapping:3,* (glob)
count-objects, shallow walk across deleted files manifest
count-objects, shallow walk across deleted manifest
$ mononoke_walker -L sizing scrub -q -b master_bookmark -I shallow -i bonsai -i derived_deleted_manifest -X ChangesetToFileContent 2>&1 | strip_glog
Walking edge types [BookmarkToChangeset, ChangesetToDeletedManifestV2Mapping, DeletedManifestV2MappingToRootDeletedManifestV2, DeletedManifestV2ToDeletedManifestV2Child]
Walking node types [Bookmark, Changeset, DeletedManifestV2, DeletedManifestV2Mapping]
Seen,Loaded: 4,4
* Type:Walked,Checks,Children * DeletedManifestV2:1,* DeletedManifestV2Mapping:1,* (glob)
count-objects, deep walk across deleted files manifest
count-objects, deep walk across deleted manifest
$ mononoke_walker -L sizing scrub -q -b master_bookmark -I deep -i bonsai -i derived_deleted_manifest 2>&1 | strip_glog
Walking edge types [BookmarkToChangeset, ChangesetToBonsaiParent, ChangesetToDeletedManifestV2Mapping, DeletedManifestV2MappingToRootDeletedManifestV2, DeletedManifestV2ToDeletedManifestV2Child, DeletedManifestV2ToLinkedChangeset]
Walking node types [Bookmark, Changeset, DeletedManifestV2, DeletedManifestV2Mapping]

View File

@ -16,7 +16,7 @@ changeset_info = { version = "0.1.0", path = "../../derived_data/changeset_info"
changesets_creation = { version = "0.1.0", path = "../../changesets/changesets_creation" }
clap = { version = "=3.1.8", features = ["derive", "regex", "unicode", "wrap_help"] }
context = { version = "0.1.0", path = "../../server/context" }
deleted_files_manifest = { version = "0.1.0", path = "../../derived_data/deleted_files_manifest" }
deleted_manifest = { version = "0.1.0", path = "../../derived_data/deleted_manifest" }
derived_data_filenodes = { version = "0.1.0", path = "../../derived_data/filenodes" }
derived_data_manager = { version = "0.1.0", path = "../../derived_data/manager" }
facet = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }

View File

@ -42,7 +42,7 @@ use bookmarks::{BookmarkName, BookmarkUpdateReason};
use changeset_info::ChangesetInfo;
use clap::Parser;
use context::CoreContext;
use deleted_files_manifest::RootDeletedManifestV2Id;
use deleted_manifest::RootDeletedManifestV2Id;
use derived_data_filenodes::FilenodesOnlyPublic;
use derived_data_manager::BatchDeriveOptions;
use derived_data_manager::BonsaiDerivable;

View File

@ -32,7 +32,7 @@ cmdlib = { version = "0.1.0", path = "../cmdlib" }
cmdlib_scrubbing = { version = "0.1.0", path = "../cmdlib/scrubbing" }
context = { version = "0.1.0", path = "../server/context" }
dashmap = { version = "4.0.2", features = ["rayon", "serde"] }
deleted_files_manifest = { version = "0.1.0", path = "../derived_data/deleted_files_manifest" }
deleted_manifest = { version = "0.1.0", path = "../derived_data/deleted_manifest" }
derive_more = "0.99.3"
derived_data = { version = "0.1.0", path = "../derived_data" }
derived_data_filenodes = { version = "0.1.0", path = "../derived_data/filenodes" }

View File

@ -14,7 +14,7 @@ use blobstore_factory::SqlTierInfo;
use bookmarks::BookmarkName;
use changeset_info::ChangesetInfo;
use context::CoreContext;
use deleted_files_manifest::RootDeletedManifestV2Id;
use deleted_manifest::RootDeletedManifestV2Id;
use derived_data_filenodes::FilenodesOnlyPublic;
use derived_data_manager::BonsaiDerivable as NewBonsaiDerivable;
use fastlog::{unode_entry_to_fastlog_batch_key, RootFastlog};

View File

@ -30,7 +30,7 @@ use bounded_traversal::limited_by_key_shardable;
use changeset_info::ChangesetInfo;
use cloned::cloned;
use context::CoreContext;
use deleted_files_manifest::{RootDeletedManifestIdCommon, RootDeletedManifestV2Id};
use deleted_manifest::{RootDeletedManifestIdCommon, RootDeletedManifestV2Id};
use derived_data::BonsaiDerived;
use derived_data_filenodes::FilenodesOnlyPublic;
use derived_data_manager::BonsaiDerivable as NewBonsaiDerivable;