mirror of
https://github.com/facebook/sapling.git
synced 2024-09-11 18:47:10 +03:00
Remove HgImporter support for fetching trees
Summary: HgImporter is being removed - this change removes its support for fetching trees. Reviewed By: kmancini Differential Revision: D51872978 fbshipit-source-id: 5b373565c6b29ecc3dc7988695b6e4f62d9a04fb
This commit is contained in:
parent
62db053dc2
commit
0676b48d6a
@ -75,16 +75,6 @@ namespace {
|
||||
// Thread local HgImporter. This is only initialized on HgImporter threads.
|
||||
static folly::ThreadLocalPtr<Importer> threadLocalImporter;
|
||||
|
||||
/**
|
||||
* Checks that the thread local HgImporter is present and returns it.
|
||||
*/
|
||||
Importer& getThreadLocalImporter() {
|
||||
if (!threadLocalImporter) {
|
||||
EDEN_BUG() << "Attempting to get HgImporter from non-HgImporter thread";
|
||||
}
|
||||
return *threadLocalImporter;
|
||||
}
|
||||
|
||||
ObjectId hashFromRootId(const RootId& root) {
|
||||
return ObjectId::fromHex(root.value());
|
||||
}
|
||||
@ -313,7 +303,6 @@ folly::Future<TreePtr> HgBackingStore::fetchTreeFromImporter(
|
||||
edenTreeID,
|
||||
writeBatch,
|
||||
&liveImportTreeWatches = liveImportTreeWatches_] {
|
||||
Importer& importer = getThreadLocalImporter();
|
||||
folly::stop_watch<std::chrono::milliseconds> watch;
|
||||
RequestMetricsScope queueTracker{&liveImportTreeWatches};
|
||||
|
||||
@ -335,23 +324,8 @@ folly::Future<TreePtr> HgBackingStore::fetchTreeFromImporter(
|
||||
if (tree.hasValue()) {
|
||||
stats_->increment(&HgBackingStoreStats::fetchTreeRetrySuccess);
|
||||
result = tree.value();
|
||||
} else if (config_->getEdenConfig()
|
||||
->hgImporterFetchFallback.getValue()) {
|
||||
// Fall back to importer
|
||||
auto serializedTree = importer.fetchTree(path, manifestNode);
|
||||
if (serializedTree) {
|
||||
stats_->increment(&HgBackingStoreStats::importTreeSuccess);
|
||||
} else {
|
||||
stats_->increment(&HgBackingStoreStats::importTreeFailure);
|
||||
}
|
||||
result = processTree(
|
||||
std::move(serializedTree),
|
||||
manifestNode,
|
||||
edenTreeID,
|
||||
path,
|
||||
writeBatch.get());
|
||||
} else {
|
||||
// No fallback to importer, record miss and return error
|
||||
// Record miss and return error
|
||||
if (logger_) {
|
||||
logger_->logEvent(FetchMiss{
|
||||
datapackStore_.getRepoName(),
|
||||
@ -484,37 +458,6 @@ class Manifest {
|
||||
|
||||
} // namespace
|
||||
|
||||
TreePtr HgBackingStore::processTree(
|
||||
std::unique_ptr<IOBuf> content,
|
||||
const Hash20& manifestNode,
|
||||
const ObjectId& edenTreeID,
|
||||
RelativePathPiece path,
|
||||
LocalStore::WriteBatch* writeBatch) {
|
||||
auto manifest = Manifest(std::move(content));
|
||||
Tree::container entries{kPathMapDefaultCaseSensitive};
|
||||
auto hgObjectIdFormat = config_->getEdenConfig()->hgObjectIdFormat.getValue();
|
||||
const auto filteredPaths =
|
||||
config_->getEdenConfig()->hgFilteredPaths.getValue();
|
||||
|
||||
for (auto& entry : manifest) {
|
||||
XLOG(DBG9) << "tree: " << manifestNode << " " << entry.name
|
||||
<< " node: " << entry.node << " flag: " << entry.type;
|
||||
|
||||
auto relPath = path + entry.name;
|
||||
if (filteredPaths->count(relPath) == 0) {
|
||||
auto proxyHash =
|
||||
HgProxyHash::store(relPath, entry.node, hgObjectIdFormat);
|
||||
|
||||
entries.emplace(entry.name, proxyHash, entry.type);
|
||||
}
|
||||
}
|
||||
|
||||
writeBatch->flush();
|
||||
|
||||
return std::make_shared<TreePtr::element_type>(
|
||||
std::move(entries), edenTreeID);
|
||||
}
|
||||
|
||||
ImmediateFuture<folly::Unit> HgBackingStore::importTreeManifestForRoot(
|
||||
const RootId& rootId,
|
||||
const Hash20& manifestId,
|
||||
|
@ -161,12 +161,6 @@ class HgBackingStore {
|
||||
ObjectId edenTreeID,
|
||||
RelativePath path,
|
||||
std::shared_ptr<LocalStore::WriteBatch> writeBatch);
|
||||
TreePtr processTree(
|
||||
std::unique_ptr<folly::IOBuf> content,
|
||||
const Hash20& manifestNode,
|
||||
const ObjectId& edenTreeID,
|
||||
RelativePathPiece path,
|
||||
LocalStore::WriteBatch* writeBatch);
|
||||
|
||||
std::shared_ptr<LocalStore> localStore_;
|
||||
EdenStatsPtr stats_;
|
||||
|
@ -255,64 +255,6 @@ void HgImporter::stopHelperProcess() {
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<IOBuf> HgImporter::fetchTree(
|
||||
RelativePathPiece path,
|
||||
Hash20 pathManifestNode) {
|
||||
// Ask the hg_import_helper script to fetch data for this tree
|
||||
static constexpr auto getNumRequestsSinceLastLog =
|
||||
[](uint64_t& treeRequestsSinceLog) {
|
||||
uint64_t numRequests = 0;
|
||||
std::swap(numRequests, treeRequestsSinceLog);
|
||||
return numRequests;
|
||||
};
|
||||
XLOG_EVERY_MS(DBG1, 1000)
|
||||
<< "fetching data for tree \"" << path << "\" at manifest node "
|
||||
<< pathManifestNode << ". "
|
||||
<< getNumRequestsSinceLastLog(treeRequestsSinceLog_)
|
||||
<< " trees fetched since last log";
|
||||
treeRequestsSinceLog_++;
|
||||
|
||||
auto requestID = sendFetchTreeRequest(
|
||||
CMD_CAT_TREE, path, pathManifestNode, "CMD_CAT_TREE");
|
||||
|
||||
ChunkHeader header;
|
||||
header = readChunkHeader(requestID, "CMD_CAT_TREE");
|
||||
|
||||
auto buf = IOBuf::create(header.dataLength);
|
||||
|
||||
readFromHelper(
|
||||
buf->writableTail(), header.dataLength, "CMD_CAT_TREE response body");
|
||||
buf->append(header.dataLength);
|
||||
|
||||
// The last 8 bytes of the response are the body length.
|
||||
// Ensure that this looks correct, and advance the buffer past this data to
|
||||
// the start of the actual response body.
|
||||
//
|
||||
// This data doesn't really need to be present in the response. It is only
|
||||
// here so we can double-check that the response data appears valid.
|
||||
buf->trimEnd(sizeof(uint64_t));
|
||||
uint64_t bodyLength;
|
||||
memcpy(&bodyLength, buf->tail(), sizeof(uint64_t));
|
||||
bodyLength = Endian::big(bodyLength);
|
||||
if (bodyLength != header.dataLength - sizeof(uint64_t)) {
|
||||
auto msg = fmt::format(
|
||||
"inconsistent body length received when importing tree {} ({}, {}): "
|
||||
"bodyLength={} responseLength={}",
|
||||
pathManifestNode,
|
||||
path,
|
||||
pathManifestNode,
|
||||
bodyLength,
|
||||
header.dataLength);
|
||||
XLOG(ERR) << msg;
|
||||
throw std::runtime_error(std::move(msg));
|
||||
}
|
||||
|
||||
XLOG(DBG4) << "imported tree " << pathManifestNode << " (" << path << ", "
|
||||
<< pathManifestNode << "); length=" << bodyLength;
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
HgImporter::ChunkHeader HgImporter::readChunkHeader(
|
||||
TransactionID txnID,
|
||||
StringPiece cmdName) {
|
||||
@ -362,34 +304,6 @@ HgImporter::ChunkHeader HgImporter::readChunkHeader(
|
||||
throw HgImportPyError(errorType, message);
|
||||
}
|
||||
|
||||
HgImporter::TransactionID HgImporter::sendFetchTreeRequest(
|
||||
CommandType cmd,
|
||||
RelativePathPiece path,
|
||||
Hash20 pathManifestNode,
|
||||
StringPiece context) {
|
||||
stats_->increment(&HgImporterStats::fetchTree);
|
||||
|
||||
auto txnID = nextRequestID_++;
|
||||
ChunkHeader header;
|
||||
header.command = Endian::big<uint32_t>(cmd);
|
||||
header.requestID = Endian::big<uint32_t>(txnID);
|
||||
header.flags = 0;
|
||||
std::string_view pathStr = path.view();
|
||||
header.dataLength = Endian::big<uint32_t>(
|
||||
folly::to_narrow(Hash20::RAW_SIZE + pathStr.size()));
|
||||
|
||||
std::array<struct iovec, 3> iov;
|
||||
iov[0].iov_base = &header;
|
||||
iov[0].iov_len = sizeof(header);
|
||||
iov[1].iov_base = const_cast<uint8_t*>(pathManifestNode.getBytes().data());
|
||||
iov[1].iov_len = Hash20::RAW_SIZE;
|
||||
iov[2].iov_base = const_cast<char*>(pathStr.data());
|
||||
iov[2].iov_len = pathStr.size();
|
||||
writeToHelper(iov, context);
|
||||
|
||||
return txnID;
|
||||
}
|
||||
|
||||
void HgImporter::readFromHelper(void* buf, uint32_t size, StringPiece context) {
|
||||
size_t bytesRead;
|
||||
|
||||
@ -490,16 +404,6 @@ auto HgImporterManager::retryOnError(Fn&& fn, FetchMiss::MissType missType) {
|
||||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<IOBuf> HgImporterManager::fetchTree(
|
||||
RelativePathPiece path,
|
||||
Hash20 pathManifestNode) {
|
||||
return retryOnError(
|
||||
[&](HgImporter* importer) {
|
||||
return importer->fetchTree(path, pathManifestNode);
|
||||
},
|
||||
FetchMiss::Tree);
|
||||
}
|
||||
|
||||
HgImporter* HgImporterManager::getImporter() {
|
||||
if (!importer_) {
|
||||
importer_ =
|
||||
|
@ -57,16 +57,6 @@ struct ImporterOptions {
|
||||
class Importer {
|
||||
public:
|
||||
virtual ~Importer() = default;
|
||||
|
||||
/**
|
||||
* Import tree and store it in the datapack
|
||||
*
|
||||
* In the case where CMD_CAT_TREE is used, a valid IOBuf is return,
|
||||
* otherwise, nullptr is returned.
|
||||
*/
|
||||
virtual std::unique_ptr<folly::IOBuf> fetchTree(
|
||||
RelativePathPiece path,
|
||||
Hash20 pathManifestNode) = 0;
|
||||
};
|
||||
|
||||
/**
|
||||
@ -101,10 +91,6 @@ class HgImporter : public Importer {
|
||||
|
||||
ProcessStatus debugStopHelperProcess();
|
||||
|
||||
std::unique_ptr<folly::IOBuf> fetchTree(
|
||||
RelativePathPiece path,
|
||||
Hash20 pathManifestNode) override;
|
||||
|
||||
const ImporterOptions& getOptions() const;
|
||||
|
||||
private:
|
||||
@ -204,24 +190,9 @@ class HgImporter : public Importer {
|
||||
writeToHelper(iov.data(), iov.size(), context);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a request to the helper process asking it to prefetch data for trees
|
||||
* under the specified path, at the specified manifest node for the given
|
||||
* path.
|
||||
*/
|
||||
TransactionID sendFetchTreeRequest(
|
||||
CommandType cmd,
|
||||
RelativePathPiece path,
|
||||
Hash20 pathManifestNode,
|
||||
folly::StringPiece context);
|
||||
|
||||
SpawnedProcess helper_;
|
||||
EdenStatsPtr const stats_;
|
||||
ImporterOptions options_;
|
||||
uint32_t nextRequestID_{0};
|
||||
|
||||
// How many trees were fetched since the last time we logged?
|
||||
uint64_t treeRequestsSinceLog_{0};
|
||||
|
||||
/**
|
||||
* The input and output file descriptors to the helper subprocess.
|
||||
@ -256,10 +227,6 @@ class HgImporterManager : public Importer {
|
||||
std::shared_ptr<StructuredLogger> logger,
|
||||
std::optional<AbsolutePath> importHelperScript = std::nullopt);
|
||||
|
||||
std::unique_ptr<folly::IOBuf> fetchTree(
|
||||
RelativePathPiece path,
|
||||
Hash20 pathManifestNode) override;
|
||||
|
||||
private:
|
||||
template <typename Fn>
|
||||
auto retryOnError(Fn&& fn, FetchMiss::MissType missType);
|
||||
|
Loading…
Reference in New Issue
Block a user