facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-10 00:45:18 +03:00
Code Issues Projects Releases Wiki Activity

mononoke: add SkeletonManifest to walker

Browse Source 
Summary: Add them to the walker so we can scrub and inspect them

Reviewed By: StanislavGlebik

Differential Revision: D25124144

fbshipit-source-id: 5df1ca6e48d18d3d55f68905e93fd75fbae92adb
This commit is contained in:
Alex Hornby 2020-11-26 10:10:15 -08:00 committed by Facebook GitHub Bot
parent 077b90de0d
commit 49ead437dd
8 changed files with 88 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
eden/mononoke/tests/integration/test-walker-count-objects.t
View File

@ -158,17 +158,17 @@ count-objects, deep walk across deleted files manifest
count-objects, shallow walk across skeleton manifest
$ mononoke_walker --readonly-storage scrub -q --bookmark master_bookmark -I shallow -i bonsai -i derived_skeleton_manifests -X ChangesetToFileContent 2>&1 | strip_glog
Walking roots * (glob)
Walking edge types [BookmarkToChangeset, ChangesetToSkeletonManifestMapping]
Walking node types [Bookmark, Changeset, SkeletonManifestMapping]
Final count: (3, 3)
Walking edge types [BookmarkToChangeset, ChangesetToSkeletonManifestMapping, SkeletonManifestMappingToRootSkeletonManifest, SkeletonManifestToSkeletonManifestChild]
Walking node types [Bookmark, Changeset, SkeletonManifest, SkeletonManifestMapping]
Final count: (4, 4)
Bytes/s,* (glob)
* Type:Walked,Checks,Children Bookmark:1,1,1 Changeset:1,* SkeletonManifestMapping:1,* (glob)
* Type:Walked,Checks,Children Bookmark:1,1,1 Changeset:1,* SkeletonManifest:1,* SkeletonManifestMapping:1,* (glob)
count-objects, deep walk across skeleton manifest
$ mononoke_walker --readonly-storage scrub -q --bookmark master_bookmark -I deep -i bonsai -i derived_skeleton_manifests 2>&1 | strip_glog
Walking roots * (glob)
Walking edge types [BookmarkToChangeset, ChangesetToBonsaiParent, ChangesetToSkeletonManifestMapping]
Walking node types [Bookmark, Changeset, SkeletonManifestMapping]
Final count: (7, 7)
Walking edge types [BookmarkToChangeset, ChangesetToBonsaiParent, ChangesetToSkeletonManifestMapping, SkeletonManifestMappingToRootSkeletonManifest, SkeletonManifestToSkeletonManifestChild]
Walking node types [Bookmark, Changeset, SkeletonManifest, SkeletonManifestMapping]
Final count: (10, 10)
Bytes/s,* (glob)
* Type:Walked,Checks,Children Bookmark:1,1,1 Changeset:3,* SkeletonManifestMapping:3,* (glob)
* Type:Walked,Checks,Children Bookmark:1,1,1 Changeset:3,* SkeletonManifest:3,* SkeletonManifestMapping:3,* (glob)

1
eden/mononoke/walker/src/corpus.rs
View File

@ -174,6 +174,7 @@ fn dump_with_extension(node_type: NodeType) -> bool {
NodeType::DeletedManifestMapping => false,
NodeType::Fsnode => false,
NodeType::FsnodeMapping => false,
NodeType::SkeletonManifest => false,
NodeType::SkeletonManifestMapping => false,
NodeType::UnodeFile => false,
NodeType::UnodeManifest => false,

14
eden/mononoke/walker/src/graph.rs
View File

@ -28,6 +28,7 @@ use mononoke_types::{
blame::Blame,
deleted_files_manifest::DeletedManifest,
fsnode::Fsnode,
skeleton_manifest::SkeletonManifest,
unode::{FileUnode, ManifestUnode},
BlameId, BonsaiChangeset, ChangesetId, ContentId, ContentMetadata, DeletedManifestId,
FileUnodeId, FsnodeId, MPath, MPathHash, ManifestUnodeId, MononokeId, SkeletonManifestId,
@ -233,6 +234,7 @@ create_graph!(
DeletedManifestMapping,
Fsnode,
FsnodeMapping,
SkeletonManifest,
SkeletonManifestMapping,
UnodeFile,
UnodeManifest,
@ -327,7 +329,12 @@ create_graph!(
[ChildFsnode(Fsnode), FileContent]
),
(FsnodeMapping, ChangesetId, [RootFsnode(Fsnode)]),
(SkeletonManifestMapping, ChangesetId, []),
(
SkeletonManifest,
SkeletonManifestId,
[SkeletonManifestChild(SkeletonManifest)]
),
(SkeletonManifestMapping, ChangesetId, [RootSkeletonManifest(SkeletonManifest)]),
(
UnodeFile,
PathKey<UnodeKey<FileUnodeId>>,
@ -378,6 +385,7 @@ impl NodeType {
NodeType::DeletedManifestMapping => Some(RootDeletedManifestId::NAME),
NodeType::Fsnode => Some(RootFsnodeId::NAME),
NodeType::FsnodeMapping => Some(RootFsnodeId::NAME),
NodeType::SkeletonManifest => Some(RootSkeletonManifestId::NAME),
NodeType::SkeletonManifestMapping => Some(RootSkeletonManifestId::NAME),
NodeType::UnodeFile => Some(RootUnodeManifestId::NAME),
NodeType::UnodeManifest => Some(RootUnodeManifestId::NAME),
@ -535,6 +543,7 @@ pub enum NodeData {
DeletedManifestMapping(Option<DeletedManifestId>),
Fsnode(Fsnode),
FsnodeMapping(Option<FsnodeId>),
SkeletonManifest(Option<SkeletonManifest>),
SkeletonManifestMapping(Option<SkeletonManifestId>),
UnodeFile(FileUnode),
UnodeManifest(ManifestUnode),
@ -569,6 +578,7 @@ impl Node {
Node::DeletedManifestMapping(k) => k.blobstore_key(),
Node::Fsnode(PathKey { id, path: _ }) => id.blobstore_key(),
Node::FsnodeMapping(k) => k.blobstore_key(),
Node::SkeletonManifest(k) => k.blobstore_key(),
Node::SkeletonManifestMapping(k) => k.blobstore_key(),
Node::UnodeFile(PathKey { id, path: _ }) => id.blobstore_key(),
Node::UnodeManifest(PathKey { id, path: _ }) => id.blobstore_key(),
@ -603,6 +613,7 @@ impl Node {
Node::DeletedManifestMapping(_) => None,
Node::Fsnode(PathKey { id: _, path }) => Some(&path),
Node::FsnodeMapping(_) => None,
Node::SkeletonManifest(_) => None,
Node::SkeletonManifestMapping(_) => None,
Node::UnodeFile(PathKey { id: _, path }) => Some(&path),
Node::UnodeManifest(PathKey { id: _, path }) => Some(&path),
@ -638,6 +649,7 @@ impl Node {
Node::DeletedManifestMapping(k) => Some(k.sampling_fingerprint()),
Node::Fsnode(PathKey { id, path: _ }) => Some(id.sampling_fingerprint()),
Node::FsnodeMapping(k) => Some(k.sampling_fingerprint()),
Node::SkeletonManifest(k) => Some(k.sampling_fingerprint()),
Node::SkeletonManifestMapping(k) => Some(k.sampling_fingerprint()),
Node::UnodeFile(PathKey { id, path: _ }) => Some(id.sampling_fingerprint()),
Node::UnodeManifest(PathKey { id, path: _ }) => Some(id.sampling_fingerprint()),

7
eden/mononoke/walker/src/parse_node.rs
View File

@ -323,6 +323,13 @@ mod tests {
.get_type()
);
}
NodeType::SkeletonManifest => {
assert_eq!(
node_type,
&parse_node(&format!("SkeletonManifest{}{}", NODE_SEP, SAMPLE_BLAKE2))?
.get_type()
);
}
NodeType::SkeletonManifestMapping => {
assert_eq!(
node_type,

1
eden/mononoke/walker/src/sampling.rs
View File

@ -120,6 +120,7 @@ fn filter_repo_path(node_type: NodeType, path: Option<&'_ WrappedPath>) -> Optio
NodeType::DeletedManifestMapping => None,
NodeType::Fsnode => path,
NodeType::FsnodeMapping => None,
NodeType::SkeletonManifest => path,
NodeType::SkeletonManifestMapping => None,
NodeType::UnodeFile => path,
NodeType::UnodeManifest => path,

4
eden/mononoke/walker/src/setup.rs
View File

@ -183,6 +183,8 @@ const DEEP_INCLUDE_EDGE_TYPES: &[EdgeType] = &[
EdgeType::FsnodeMappingToRootFsnode,
EdgeType::FsnodeToChildFsnode,
EdgeType::FsnodeToFileContent,
EdgeType::SkeletonManifestMappingToRootSkeletonManifest,
EdgeType::SkeletonManifestToSkeletonManifestChild,
EdgeType::UnodeFileToBlame,
EdgeType::UnodeFileToFileContent,
EdgeType::UnodeFileToLinkedChangeset,
@ -228,6 +230,8 @@ const SHALLOW_INCLUDE_EDGE_TYPES: &[EdgeType] = &[
EdgeType::FsnodeToChildFsnode,
EdgeType::FsnodeToFileContent,
EdgeType::FsnodeMappingToRootFsnode,
EdgeType::SkeletonManifestMappingToRootSkeletonManifest,
EdgeType::SkeletonManifestToSkeletonManifestChild,
EdgeType::UnodeFileToBlame,
EdgeType::UnodeFileToFileContent,
EdgeType::UnodeManifestToUnodeFileChild,

4
eden/mononoke/walker/src/state.rs
View File

@ -18,6 +18,7 @@ use futures::future::TryFutureExt;
use mercurial_types::{HgChangesetId, HgFileNodeId, HgManifestId};
use mononoke_types::{
ChangesetId, ContentId, DeletedManifestId, FileUnodeId, FsnodeId, MPathHash, ManifestUnodeId,
SkeletonManifestId,
};
use phases::{Phase, Phases};
use std::{
@ -178,6 +179,7 @@ pub struct WalkState {
visited_deleted_manifest_mapping: StateMap<InternedId<ChangesetId>>,
visited_fsnode: StateMap<(InternedId<Option<MPathHash>>, InternedId<FsnodeId>)>,
visited_fsnode_mapping: StateMap<InternedId<ChangesetId>>,
visited_skeleton_manifest: StateMap<SkeletonManifestId>,
visited_skeleton_manifest_mapping: StateMap<InternedId<ChangesetId>>,
visited_unode_file: StateMap<(InternedId<Option<MPathHash>>, UnodeInterned<FileUnodeId>)>,
visited_unode_manifest: StateMap<(
@ -232,6 +234,7 @@ impl WalkState {
visited_deleted_manifest_mapping: StateMap::with_hasher(fac.clone()),
visited_fsnode: StateMap::with_hasher(fac.clone()),
visited_fsnode_mapping: StateMap::with_hasher(fac.clone()),
visited_skeleton_manifest: StateMap::with_hasher(fac.clone()),
visited_skeleton_manifest_mapping: StateMap::with_hasher(fac.clone()),
visited_unode_file: StateMap::with_hasher(fac.clone()),
visited_unode_manifest: StateMap::with_hasher(fac.clone()),
@ -463,6 +466,7 @@ impl VisitOne for WalkState {
true
}
}
Node::SkeletonManifest(id) => self.record(&self.visited_skeleton_manifest, &id),
Node::SkeletonManifestMapping(bcs_id) => {
if let Some(id) = self.bcs_ids.get(bcs_id) {
!self.visited_skeleton_manifest_mapping.contains_key(&id) // Does not insert, see record_resolved_visit

53
eden/mononoke/walker/src/walk.rs
View File

@ -38,8 +38,9 @@ use manifest::{Entry, Manifest};
use mercurial_derived_data::MappedHgChangesetId;
use mercurial_types::{FileBytes, HgChangesetId, HgFileNodeId, HgManifestId, RepoPath};
use mononoke_types::{
blame::BlameMaybeRejected, fsnode::FsnodeEntry, unode::UnodeEntry, BlameId, ChangesetId,
ContentId, DeletedManifestId, FileUnodeId, FsnodeId, MPath, ManifestUnodeId,
blame::BlameMaybeRejected, fsnode::FsnodeEntry, skeleton_manifest::SkeletonManifestEntry,
unode::UnodeEntry, BlameId, ChangesetId, ContentId, DeletedManifestId, FileUnodeId, FsnodeId,
MPath, ManifestUnodeId, SkeletonManifestId,
};
use phases::{HeadsFetcher, Phase, Phases};
use scuba_ext::ScubaSampleBuilder;
@ -1064,6 +1065,42 @@ async fn deleted_manifest_mapping_step<V: VisitOne>(
}
}
async fn skeleton_manifest_step<V: VisitOne>(
ctx: &CoreContext,
repo: &BlobRepo,
checker: &Checker<V>,
manifest_id: &SkeletonManifestId,
path: Option<&WrappedPath>,
) -> Result<StepOutput, Error> {
let manifest = manifest_id.load(ctx.clone(), &repo.get_blobstore()).await?;
let mut edges = vec![];
for (child_path, entry) in manifest.list() {
match entry {
SkeletonManifestEntry::Directory(subdir) => {
checker.add_edge_with_path(
&mut edges,
EdgeType::SkeletonManifestToSkeletonManifestChild,
|| Node::SkeletonManifest(*subdir.id()),
|| {
path.map(|p| {
WrappedPath::from(MPath::join_element_opt(p.as_ref(), Some(child_path)))
})
},
);
}
SkeletonManifestEntry::File => {}
}
}
Ok(StepOutput(
checker.step_data(NodeType::SkeletonManifest, || {
NodeData::SkeletonManifest(Some(manifest))
}),
edges,
))
}
async fn skeleton_manifest_mapping_step<V: VisitOne>(
ctx: &CoreContext,
repo: &BlobRepo,
@ -1075,7 +1112,14 @@ async fn skeleton_manifest_mapping_step<V: VisitOne>(
maybe_derived::<RootSkeletonManifestId>(ctx, repo, bcs_id, enable_derive).await?;
if let Some(root_manifest_id) = root_manifest_id {
let edges = vec![];
let mut edges = vec![];
checker.add_edge_with_path(
&mut edges,
EdgeType::SkeletonManifestMappingToRootSkeletonManifest,
|| Node::SkeletonManifest(*root_manifest_id.skeleton_manifest_id()),
|| Some(WrappedPath::Root),
);
Ok(StepOutput(
checker.step_data(NodeType::SkeletonManifestMapping, || {
NodeData::SkeletonManifestMapping(Some(*root_manifest_id.skeleton_manifest_id()))
@ -1423,6 +1467,9 @@ where
Node::FsnodeMapping(bcs_id) => {
bonsai_to_fsnode_mapping_step(&ctx, &repo, &checker, bcs_id, enable_derive).await
}
Node::SkeletonManifest(id) => {
skeleton_manifest_step(&ctx, &repo, &checker, &id, walk_item.path.as_ref()).await
}
Node::SkeletonManifestMapping(bcs_id) => {
skeleton_manifest_mapping_step(&ctx, &repo, &checker, bcs_id, enable_derive).await
}