2016-08-23 01:45:06 +03:00
|
|
|
// treemanifest.h - c++ declarations of a tree manifest
|
|
|
|
//
|
|
|
|
// Copyright 2016 Facebook, Inc.
|
|
|
|
//
|
|
|
|
// This software may be used and distributed according to the terms of the
|
|
|
|
// GNU General Public License version 2 or any later version.
|
|
|
|
//
|
|
|
|
// no-check-code
|
|
|
|
|
|
|
|
#ifndef REMOTEFILELOG_TREEMANIFEST_H
|
|
|
|
#define REMOTEFILELOG_TREEMANIFEST_H
|
|
|
|
|
2016-08-30 02:19:52 +03:00
|
|
|
#include "pythonutil.h"
|
|
|
|
|
2016-08-23 01:45:06 +03:00
|
|
|
#include <string>
|
|
|
|
#include <vector>
|
|
|
|
|
|
|
|
#include "manifest_fetcher.h"
|
|
|
|
|
[ctree] replace _treemanifest_find with treemanifest_get
Summary: treemanifest_find is now a generic method to traverse a tree (like tree_path in cfastmanifest). It manages the tree by adding and removing intermediate nodes (if requested). Unlike _treemanifest_find, it uses lower-level APIs such as findChild and addChild, which uses std::list::iterator to avoid duplication of work. Previously, we would need to traverse the entire list of children to find a child, and then if it's not found, traverse the entire list again to properly locate it. Now we just find the exact location where the child is or should be, and then save that.
Test Plan: `PYTHONPATH=~/work/mercurial/facebook-hg-rpms/remotefilelog:~/work/mercurial/facebook-hg-rpms/fb-hgext/:~/work/mercurial/facebook-hg-rpms/remotenames/:~/work/mercurial/facebook-hg-rpms/lz4revlog/ /opt/local/bin/python2.7 ~/work/mercurial/facebook-hg-rpms/hg-crew/hg --config extensions.perftest=~/work/mercurial/facebook-hg-rpms/remotefilelog/tests/perftest.py testtree --kind flat,ctree,fast --test fulliter,diff,find --build "master~5::master"`
Reviewers: #fastmanifest, durham
Reviewed By: durham
Subscribers: mjpieters, durham, mitrandir
Differential Revision: https://phabricator.intern.facebook.com/D3772947
Signature: t1:3772947:1472758787:8e8597c30f2cf1512ac7195bfa44ed62c201deb2
2016-09-06 22:09:42 +03:00
|
|
|
enum FindResult {
|
|
|
|
FIND_PATH_OK,
|
|
|
|
FIND_PATH_NOT_FOUND,
|
|
|
|
FIND_PATH_CONFLICT,
|
|
|
|
FIND_PATH_WTF,
|
|
|
|
};
|
|
|
|
|
2016-09-10 01:22:42 +03:00
|
|
|
enum SetResult {
|
|
|
|
SET_OK,
|
|
|
|
SET_CONFLICT,
|
|
|
|
SET_WTF,
|
|
|
|
};
|
|
|
|
|
[ctree] replace _treemanifest_find with treemanifest_get
Summary: treemanifest_find is now a generic method to traverse a tree (like tree_path in cfastmanifest). It manages the tree by adding and removing intermediate nodes (if requested). Unlike _treemanifest_find, it uses lower-level APIs such as findChild and addChild, which uses std::list::iterator to avoid duplication of work. Previously, we would need to traverse the entire list of children to find a child, and then if it's not found, traverse the entire list again to properly locate it. Now we just find the exact location where the child is or should be, and then save that.
Test Plan: `PYTHONPATH=~/work/mercurial/facebook-hg-rpms/remotefilelog:~/work/mercurial/facebook-hg-rpms/fb-hgext/:~/work/mercurial/facebook-hg-rpms/remotenames/:~/work/mercurial/facebook-hg-rpms/lz4revlog/ /opt/local/bin/python2.7 ~/work/mercurial/facebook-hg-rpms/hg-crew/hg --config extensions.perftest=~/work/mercurial/facebook-hg-rpms/remotefilelog/tests/perftest.py testtree --kind flat,ctree,fast --test fulliter,diff,find --build "master~5::master"`
Reviewers: #fastmanifest, durham
Reviewed By: durham
Subscribers: mjpieters, durham, mitrandir
Differential Revision: https://phabricator.intern.facebook.com/D3772947
Signature: t1:3772947:1472758787:8e8597c30f2cf1512ac7195bfa44ed62c201deb2
2016-09-06 22:09:42 +03:00
|
|
|
enum FindMode {
|
|
|
|
// walks the tree and searches for a leaf node. if the path cannot be found,
|
|
|
|
// exit with `FIND_PATH_NOT_FOUND`.
|
2016-09-06 22:10:35 +03:00
|
|
|
BASIC_WALK,
|
[ctree] replace _treemanifest_find with treemanifest_get
Summary: treemanifest_find is now a generic method to traverse a tree (like tree_path in cfastmanifest). It manages the tree by adding and removing intermediate nodes (if requested). Unlike _treemanifest_find, it uses lower-level APIs such as findChild and addChild, which uses std::list::iterator to avoid duplication of work. Previously, we would need to traverse the entire list of children to find a child, and then if it's not found, traverse the entire list again to properly locate it. Now we just find the exact location where the child is or should be, and then save that.
Test Plan: `PYTHONPATH=~/work/mercurial/facebook-hg-rpms/remotefilelog:~/work/mercurial/facebook-hg-rpms/fb-hgext/:~/work/mercurial/facebook-hg-rpms/remotenames/:~/work/mercurial/facebook-hg-rpms/lz4revlog/ /opt/local/bin/python2.7 ~/work/mercurial/facebook-hg-rpms/hg-crew/hg --config extensions.perftest=~/work/mercurial/facebook-hg-rpms/remotefilelog/tests/perftest.py testtree --kind flat,ctree,fast --test fulliter,diff,find --build "master~5::master"`
Reviewers: #fastmanifest, durham
Reviewed By: durham
Subscribers: mjpieters, durham, mitrandir
Differential Revision: https://phabricator.intern.facebook.com/D3772947
Signature: t1:3772947:1472758787:8e8597c30f2cf1512ac7195bfa44ed62c201deb2
2016-09-06 22:09:42 +03:00
|
|
|
|
|
|
|
// walks the tree. if the intermediate paths cannot be found, create them.
|
|
|
|
// if a leaf node exists where an intermediate path node needs to be
|
|
|
|
// created, then return `FIND_PATH_CONFLICT`.
|
2016-09-06 22:10:35 +03:00
|
|
|
CREATE_IF_MISSING,
|
[ctree] replace _treemanifest_find with treemanifest_get
Summary: treemanifest_find is now a generic method to traverse a tree (like tree_path in cfastmanifest). It manages the tree by adding and removing intermediate nodes (if requested). Unlike _treemanifest_find, it uses lower-level APIs such as findChild and addChild, which uses std::list::iterator to avoid duplication of work. Previously, we would need to traverse the entire list of children to find a child, and then if it's not found, traverse the entire list again to properly locate it. Now we just find the exact location where the child is or should be, and then save that.
Test Plan: `PYTHONPATH=~/work/mercurial/facebook-hg-rpms/remotefilelog:~/work/mercurial/facebook-hg-rpms/fb-hgext/:~/work/mercurial/facebook-hg-rpms/remotenames/:~/work/mercurial/facebook-hg-rpms/lz4revlog/ /opt/local/bin/python2.7 ~/work/mercurial/facebook-hg-rpms/hg-crew/hg --config extensions.perftest=~/work/mercurial/facebook-hg-rpms/remotefilelog/tests/perftest.py testtree --kind flat,ctree,fast --test fulliter,diff,find --build "master~5::master"`
Reviewers: #fastmanifest, durham
Reviewed By: durham
Subscribers: mjpieters, durham, mitrandir
Differential Revision: https://phabricator.intern.facebook.com/D3772947
Signature: t1:3772947:1472758787:8e8597c30f2cf1512ac7195bfa44ed62c201deb2
2016-09-06 22:09:42 +03:00
|
|
|
|
|
|
|
// walks the tree. if the path cannot be found, exit with
|
|
|
|
// `FIND_PATH_NOT_FOUND`. if the operation is successful, then check
|
|
|
|
// intermediate nodes to ensure that they still have children. any nodes
|
|
|
|
// that do not should be removed.
|
2016-09-06 22:10:35 +03:00
|
|
|
REMOVE_EMPTY_IMPLICIT_NODES,
|
[ctree] replace _treemanifest_find with treemanifest_get
Summary: treemanifest_find is now a generic method to traverse a tree (like tree_path in cfastmanifest). It manages the tree by adding and removing intermediate nodes (if requested). Unlike _treemanifest_find, it uses lower-level APIs such as findChild and addChild, which uses std::list::iterator to avoid duplication of work. Previously, we would need to traverse the entire list of children to find a child, and then if it's not found, traverse the entire list again to properly locate it. Now we just find the exact location where the child is or should be, and then save that.
Test Plan: `PYTHONPATH=~/work/mercurial/facebook-hg-rpms/remotefilelog:~/work/mercurial/facebook-hg-rpms/fb-hgext/:~/work/mercurial/facebook-hg-rpms/remotenames/:~/work/mercurial/facebook-hg-rpms/lz4revlog/ /opt/local/bin/python2.7 ~/work/mercurial/facebook-hg-rpms/hg-crew/hg --config extensions.perftest=~/work/mercurial/facebook-hg-rpms/remotefilelog/tests/perftest.py testtree --kind flat,ctree,fast --test fulliter,diff,find --build "master~5::master"`
Reviewers: #fastmanifest, durham
Reviewed By: durham
Subscribers: mjpieters, durham, mitrandir
Differential Revision: https://phabricator.intern.facebook.com/D3772947
Signature: t1:3772947:1472758787:8e8597c30f2cf1512ac7195bfa44ed62c201deb2
2016-09-06 22:09:42 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
struct FindContext {
|
|
|
|
bool invalidate_checksums;
|
|
|
|
int32_t num_leaf_node_changes;
|
|
|
|
FindMode mode;
|
|
|
|
|
|
|
|
// reuse this space when fetching manifests.
|
|
|
|
std::string nodebuffer;
|
|
|
|
|
|
|
|
// any extra data the callback needs to complete the operation.
|
|
|
|
void *extras;
|
|
|
|
};
|
|
|
|
|
2016-08-23 01:45:06 +03:00
|
|
|
class PathIterator {
|
|
|
|
private:
|
|
|
|
std::string path;
|
|
|
|
size_t position;
|
|
|
|
public:
|
|
|
|
PathIterator(std::string path) {
|
|
|
|
this->path = path;
|
|
|
|
this->position = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool next(char const ** word, size_t *wordlen) {
|
|
|
|
if (this->isfinished()) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
*word = this->path.c_str() + this->position;
|
|
|
|
size_t slashoffset = this->path.find('/', this->position);
|
|
|
|
if (slashoffset == std::string::npos) {
|
|
|
|
*wordlen = this->path.length() - this->position;
|
|
|
|
} else {
|
|
|
|
*wordlen = slashoffset - this->position;
|
|
|
|
}
|
|
|
|
|
|
|
|
this->position += *wordlen + 1;
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool isfinished() {
|
|
|
|
return this->position >= this->path.length();
|
|
|
|
}
|
[ctree] replace _treemanifest_find with treemanifest_get
Summary: treemanifest_find is now a generic method to traverse a tree (like tree_path in cfastmanifest). It manages the tree by adding and removing intermediate nodes (if requested). Unlike _treemanifest_find, it uses lower-level APIs such as findChild and addChild, which uses std::list::iterator to avoid duplication of work. Previously, we would need to traverse the entire list of children to find a child, and then if it's not found, traverse the entire list again to properly locate it. Now we just find the exact location where the child is or should be, and then save that.
Test Plan: `PYTHONPATH=~/work/mercurial/facebook-hg-rpms/remotefilelog:~/work/mercurial/facebook-hg-rpms/fb-hgext/:~/work/mercurial/facebook-hg-rpms/remotenames/:~/work/mercurial/facebook-hg-rpms/lz4revlog/ /opt/local/bin/python2.7 ~/work/mercurial/facebook-hg-rpms/hg-crew/hg --config extensions.perftest=~/work/mercurial/facebook-hg-rpms/remotefilelog/tests/perftest.py testtree --kind flat,ctree,fast --test fulliter,diff,find --build "master~5::master"`
Reviewers: #fastmanifest, durham
Reviewed By: durham
Subscribers: mjpieters, durham, mitrandir
Differential Revision: https://phabricator.intern.facebook.com/D3772947
Signature: t1:3772947:1472758787:8e8597c30f2cf1512ac7195bfa44ed62c201deb2
2016-09-06 22:09:42 +03:00
|
|
|
|
|
|
|
void getPathToPosition(const char **word, size_t *wordlen) {
|
|
|
|
*word = path.c_str();
|
|
|
|
*wordlen = this->position;
|
|
|
|
}
|
2016-08-23 01:45:06 +03:00
|
|
|
};
|
|
|
|
|
2016-09-06 22:10:35 +03:00
|
|
|
/**
|
|
|
|
* A single instance of a treemanifest.
|
|
|
|
*/
|
|
|
|
struct treemanifest {
|
2016-09-06 22:38:15 +03:00
|
|
|
// Fetcher for the manifests.
|
|
|
|
ManifestFetcher fetcher;
|
2016-09-06 22:10:35 +03:00
|
|
|
|
2016-09-11 02:32:28 +03:00
|
|
|
ManifestEntry root;
|
2016-09-06 22:10:35 +03:00
|
|
|
|
2016-09-06 22:38:15 +03:00
|
|
|
treemanifest(PythonObj store, std::string rootNode) :
|
2016-09-11 02:32:28 +03:00
|
|
|
fetcher(store) {
|
|
|
|
std::string hexnode;
|
|
|
|
hexnode.reserve(HEX_NODE_SIZE);
|
|
|
|
|
|
|
|
hexfrombin(rootNode.c_str(), hexnode);
|
|
|
|
root.initialize(NULL, 0, hexnode.c_str(), MANIFEST_DIRECTORY_FLAG);
|
2016-09-18 23:38:01 +03:00
|
|
|
|
|
|
|
// ManifestEntry.initialize will create a blank manifest in .resolved.
|
|
|
|
// however, we actually want the resolution to happen through
|
|
|
|
// manifestfetcher. therefore, let's delete the field and clear it.
|
|
|
|
delete root.resolved;
|
|
|
|
root.resolved = NULL;
|
2016-09-06 22:38:15 +03:00
|
|
|
}
|
|
|
|
|
2016-09-12 21:44:47 +03:00
|
|
|
treemanifest(ManifestFetcher fetcher, ManifestEntry *otherRoot) :
|
|
|
|
fetcher(fetcher) {
|
|
|
|
root.initialize(otherRoot);
|
|
|
|
}
|
|
|
|
|
2016-09-06 22:40:32 +03:00
|
|
|
treemanifest(PythonObj store) :
|
2016-09-11 02:32:28 +03:00
|
|
|
fetcher(store) {
|
|
|
|
std::string hexnode;
|
|
|
|
hexnode.assign(HEX_NODE_SIZE, '\0');
|
2016-09-06 22:40:32 +03:00
|
|
|
|
2016-09-11 02:32:28 +03:00
|
|
|
root.initialize(NULL, 0, hexnode.c_str(), MANIFEST_DIRECTORY_FLAG);
|
|
|
|
}
|
2016-09-06 22:38:15 +03:00
|
|
|
|
2016-09-12 21:44:47 +03:00
|
|
|
treemanifest *copy();
|
|
|
|
|
2016-09-06 22:38:40 +03:00
|
|
|
void get(
|
2016-09-06 22:38:15 +03:00
|
|
|
const std::string &filename,
|
|
|
|
std::string *resultnode, char *resultflag);
|
2016-09-06 22:10:35 +03:00
|
|
|
|
2016-09-10 01:22:42 +03:00
|
|
|
SetResult set(
|
|
|
|
const std::string &filename,
|
|
|
|
const std::string &resultnode, char resultflag);
|
|
|
|
|
2016-09-10 01:24:31 +03:00
|
|
|
/**
|
|
|
|
* Removes a file from the treemanifest. Returns true iff the file was
|
|
|
|
* found and removed.
|
|
|
|
*/
|
|
|
|
bool remove(const std::string &filename);
|
|
|
|
|
2016-09-07 23:00:55 +03:00
|
|
|
Manifest *getRootManifest() {
|
2016-09-11 02:32:28 +03:00
|
|
|
if (this->root.resolved == NULL) {
|
|
|
|
std::string binnode;
|
|
|
|
binnode.reserve(BIN_NODE_SIZE);
|
|
|
|
|
|
|
|
appendbinfromhex(this->root.node, binnode);
|
|
|
|
this->root.resolved = this->fetcher.get("", 0, binnode);
|
2016-09-06 22:38:15 +03:00
|
|
|
}
|
2016-09-07 23:00:55 +03:00
|
|
|
|
2016-09-11 02:32:28 +03:00
|
|
|
return this->root.resolved;
|
2016-09-06 22:38:15 +03:00
|
|
|
}
|
2016-09-06 22:38:28 +03:00
|
|
|
|
2016-09-07 23:00:55 +03:00
|
|
|
private:
|
|
|
|
|
2016-09-06 22:38:28 +03:00
|
|
|
/**
|
|
|
|
* Basic mechanism to traverse a tree. Once the deepest directory in the
|
|
|
|
* path has been located, the supplied callback is executed. That callback
|
|
|
|
* is called with the manifest of the deepest directory and the leaf node's
|
|
|
|
* filename.
|
|
|
|
*
|
|
|
|
* For instance, if treemanifest_find is called on /abc/def/ghi, then the
|
|
|
|
* callback is executed with the manifest of /abc/def, and the filename
|
|
|
|
* passed in will be "ghi".
|
|
|
|
*/
|
2016-09-06 22:38:40 +03:00
|
|
|
FindResult find(
|
2016-09-11 02:32:28 +03:00
|
|
|
ManifestEntry *manifestentry,
|
2016-09-06 22:38:28 +03:00
|
|
|
PathIterator &path,
|
|
|
|
FindMode findMode,
|
|
|
|
FindContext *findContext,
|
|
|
|
FindResult (*callback)(
|
|
|
|
Manifest *manifest,
|
|
|
|
const char *filename, size_t filenamelen,
|
|
|
|
FindContext *findContext));
|
2016-09-06 22:10:35 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Represents a single stack frame in an iteration of the contents of the tree.
|
|
|
|
*/
|
|
|
|
struct stackframe {
|
|
|
|
Manifest *manifest;
|
|
|
|
ManifestIterator iterator;
|
|
|
|
|
|
|
|
stackframe(Manifest *manifest) :
|
|
|
|
manifest(manifest),
|
|
|
|
iterator(manifest->getIterator()) {
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* A helper struct representing the state of an iterator recursing over a tree.
|
|
|
|
*/
|
|
|
|
struct fileiter {
|
|
|
|
ManifestFetcher fetcher; // Instance to fetch tree content
|
|
|
|
std::vector<stackframe> frames;
|
|
|
|
std::string path; // The fullpath for the top entry in the stack.
|
|
|
|
|
|
|
|
// If provided, the given matcher filters the results by path
|
|
|
|
PythonObj matcher;
|
|
|
|
|
|
|
|
fileiter(treemanifest &tm) :
|
2016-09-06 22:11:11 +03:00
|
|
|
fetcher(tm.fetcher) {
|
2016-09-07 23:00:55 +03:00
|
|
|
this->frames.push_back(stackframe(tm.getRootManifest()));
|
2016-09-06 22:10:35 +03:00
|
|
|
this->path.reserve(1024);
|
|
|
|
}
|
|
|
|
|
|
|
|
fileiter(const fileiter &old) :
|
|
|
|
fetcher(old.fetcher),
|
|
|
|
frames(old.frames),
|
|
|
|
path(old.path) {
|
|
|
|
}
|
|
|
|
|
|
|
|
fileiter& operator=(const fileiter &other) {
|
|
|
|
this->fetcher = other.fetcher;
|
|
|
|
this->frames = other.frames;
|
|
|
|
this->path = other.path;
|
|
|
|
|
|
|
|
return *this;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2016-09-06 22:09:55 +03:00
|
|
|
extern void treemanifest_diffrecurse(
|
|
|
|
Manifest *selfmf,
|
|
|
|
Manifest *othermf,
|
|
|
|
std::string &path,
|
|
|
|
const PythonObj &diff,
|
|
|
|
const ManifestFetcher &fetcher);
|
|
|
|
|
2016-08-23 01:45:06 +03:00
|
|
|
#endif //REMOTEFILELOG_TREEMANIFEST_H
|