/* * Copyright (c) Facebook, Inc. and its affiliates. * * This software may be used and distributed according to the terms of the * GNU General Public License version 2. */ #include "eden/fs/utils/FaultInjector.h" #include #include #include using folly::SemiFuture; using folly::StringPiece; using folly::Unit; namespace facebook { namespace eden { FaultInjector::Fault::Fault(StringPiece regex, FaultBehavior&& b, size_t count) : keyValueRegex(regex.begin(), regex.end()), countRemaining(count), behavior(std::move(b)) {} FaultInjector::FaultInjector(bool enabled) : enabled_{enabled} {} FaultInjector::~FaultInjector() { // If there are any blocked checks still pending on destruction // fail them all with an error. auto numUnblocked = unblockAllImpl(std::runtime_error("FaultInjector destroyed")); XLOG_IF(WARN, numUnblocked > 0) << "FaultInjector destroyed with " << numUnblocked << " blocked check calls still pending"; } SemiFuture FaultInjector::checkAsyncImpl( StringPiece keyClass, StringPiece keyValue) { auto behavior = findFault(keyClass, keyValue); return boost::apply_visitor( folly::overload( [&](const Unit&) { return folly::makeSemiFuture(); }, [&](const FaultInjector::Block&) { XLOG(DBG1) << "block fault hit: " << keyClass << ", " << keyValue; return addBlockedFault(keyClass, keyValue); }, [&](const FaultInjector::Delay& delay) -> SemiFuture { XLOG(DBG1) << "delay fault hit: " << keyClass << ", " << keyValue; if (delay.error.has_value()) { return folly::futures::sleepUnsafe(delay.duration) .thenValue([error = delay.error.value()](auto&&) { error.throw_exception(); }); } return folly::futures::sleepUnsafe(delay.duration); }, [&](const folly::exception_wrapper& error) { XLOG(DBG1) << "error fault hit: " << keyClass << ", " << keyValue; return folly::makeSemiFuture(error); }), behavior); } void FaultInjector::checkImpl(StringPiece keyClass, StringPiece keyValue) { auto behavior = findFault(keyClass, keyValue); boost::apply_visitor( folly::overload( [](const Unit&) {}, [&](const FaultInjector::Block&) { XLOG(DBG1) << "block fault hit: " << keyClass << ", " << keyValue; addBlockedFault(keyClass, keyValue).get(); }, [&](const FaultInjector::Delay& delay) { XLOG(DBG1) << "delay fault hit: " << keyClass << ", " << keyValue; /* sleep override */ std::this_thread::sleep_for(delay.duration); if (delay.error.has_value()) { delay.error.value().throw_exception(); } }, [&](const folly::exception_wrapper& error) { XLOG(DBG1) << "error fault hit: " << keyClass << ", " << keyValue; error.throw_exception(); }), behavior); } void FaultInjector::injectError( StringPiece keyClass, StringPiece keyValueRegex, folly::exception_wrapper error, size_t count) { XLOG(INFO) << "injectError(" << keyClass << ", " << keyValueRegex << ", count=" << count << ")"; injectFault(keyClass, keyValueRegex, error, count); } void FaultInjector::injectBlock( StringPiece keyClass, StringPiece keyValueRegex, size_t count) { XLOG(INFO) << "injectBlock(" << keyClass << ", " << keyValueRegex << ", count=" << count << ")"; injectFault(keyClass, keyValueRegex, Block{}, count); } void FaultInjector::injectDelay( StringPiece keyClass, StringPiece keyValueRegex, std::chrono::milliseconds duration, size_t count) { XLOG(INFO) << "injectDelay(" << keyClass << ", " << keyValueRegex << ", count=" << count << ")"; injectFault(keyClass, keyValueRegex, Delay{duration}, count); } void FaultInjector::injectDelayedError( StringPiece keyClass, StringPiece keyValueRegex, std::chrono::milliseconds duration, folly::exception_wrapper error, size_t count) { XLOG(INFO) << "injectDelayedError(" << keyClass << ", " << keyValueRegex << ", count=" << count << ")"; injectFault( keyClass, keyValueRegex, Delay{duration, std::move(error)}, count); } void FaultInjector::injectNoop( folly::StringPiece keyClass, folly::StringPiece keyValueRegex, size_t count) { XLOG(INFO) << "injectNoop(" << keyClass << ", " << keyValueRegex << ", count=" << count << ")"; injectFault(keyClass, keyValueRegex, folly::unit, count); } void FaultInjector::injectFault( StringPiece keyClass, StringPiece keyValueRegex, FaultBehavior&& behavior, size_t count) { if (!enabled_) { throw std::runtime_error("fault injection is disabled"); } auto state = state_.wlock(); state->faults[keyClass].emplace_back( keyValueRegex, std::move(behavior), count); } bool FaultInjector::removeFault( StringPiece keyClass, StringPiece keyValueRegex) { auto state = state_.wlock(); // Look for any faults matching this key class auto classIter = state->faults.find(keyClass); if (classIter == state->faults.end()) { XLOG(DBG2) << "removeFault(" << keyClass << ", " << keyValueRegex << ") --> no faults defined for class " << keyClass; return false; } // Scan all faults in this key class to find a matching regex auto& faultVector = classIter->second; for (auto iter = faultVector.begin(); iter != faultVector.end(); ++iter) { if (iter->keyValueRegex.str() == keyValueRegex) { XLOG(INFO) << "removeFault(" << keyClass << ", " << keyValueRegex << ")"; faultVector.erase(iter); if (faultVector.empty()) { state->faults.erase(classIter); } return true; } } XLOG(DBG2) << "removeFault(" << keyClass << ", " << keyValueRegex << ") --> no match"; return false; } size_t FaultInjector::unblock(StringPiece keyClass, StringPiece keyValueRegex) { XLOG(DBG1) << "unblock(" << keyClass << ", " << keyValueRegex << ")"; auto matches = extractBlockedChecks(keyClass, keyValueRegex); for (auto& match : matches) { match.promise.setValue(); } return matches.size(); } size_t FaultInjector::unblockWithError( StringPiece keyClass, StringPiece keyValueRegex, folly::exception_wrapper error) { XLOG(DBG1) << "unblockWithError(" << keyClass << ", " << keyValueRegex << ")"; auto matches = extractBlockedChecks(keyClass, keyValueRegex); for (auto& match : matches) { match.promise.setException(error); } return matches.size(); } size_t FaultInjector::unblockAll() { XLOG(DBG1) << "unblockAll()"; return unblockAllImpl(std::nullopt); } size_t FaultInjector::unblockAllWithError(folly::exception_wrapper error) { XLOG(DBG1) << "unblockAllWithError()"; return unblockAllImpl(std::move(error)); } FaultInjector::FaultBehavior FaultInjector::findFault( StringPiece keyClass, StringPiece keyValue) { XLOG(DBG4) << "findFault(" << keyClass << ", " << keyValue << ")"; auto state = state_.wlock(); // Look for any faults matching this key class auto classIter = state->faults.find(keyClass); if (classIter == state->faults.end()) { XLOG(DBG6) << "findFault(" << keyClass << ", " << keyValue << ") --> no faults for class " << keyClass; return folly::unit; } // Scan all faults in this key class to find a matching regex auto& faultVector = classIter->second; for (auto iter = faultVector.begin(); iter != faultVector.end(); ++iter) { if (!boost::regex_match( keyValue.begin(), keyValue.end(), iter->keyValueRegex)) { XLOG(DBG8) << "findFault(" << keyClass << ", " << keyValue << ") --> no match against /" << iter->keyValueRegex.str() << "/"; continue; } // Found a matching fault XLOG(DBG3) << "findFault(" << keyClass << ", " << keyValue << ") --> matched /" << iter->keyValueRegex.str() << "/"; auto behavior = iter->behavior; if (iter->countRemaining > 0) { --iter->countRemaining; if (iter->countRemaining == 0) { // This was the last match XLOG(DBG1) << "fault expired: " << keyClass << ", " << iter->keyValueRegex.str(); faultVector.erase(iter); } } return behavior; } XLOG(DBG6) << "findFault(" << keyClass << ", " << keyValue << ") --> no matches found"; return folly::unit; } SemiFuture FaultInjector::addBlockedFault( StringPiece keyClass, StringPiece keyValue) { auto state = state_.wlock(); auto [promise, future] = folly::makePromiseContract(); state->blockedChecks[keyClass].emplace_back(keyValue, std::move(promise)); return std::move(future); } std::vector FaultInjector::extractBlockedChecks( StringPiece keyClass, StringPiece keyValueRegex) { std::vector results; auto state = state_.wlock(); auto classIter = state->blockedChecks.find(keyClass); if (classIter == state->blockedChecks.end()) { return results; } auto& blockedChecks = classIter->second; // Walk through the list of blocked calls and extract out the ones that // match. We could use std::remove_if(), but that would still require an // extra move of all the matched values at the end. // // When we find a matching call we move it out to the results array. // Everything else we shift forwards in the blockedChecks array to fill in the // gaps from the matching calls that we extracted. auto writeIter = blockedChecks.begin(); auto end = blockedChecks.end(); boost::regex regex(keyValueRegex.begin(), keyValueRegex.end()); for (auto readIter = blockedChecks.begin(); readIter != end; ++readIter) { if (boost::regex_match(readIter->keyValue, regex)) { // Move this check out to the result list. results.emplace_back(std::move(*readIter)); } else { // This check doesn't match. Shift it forwards in the array to fill in // gaps left by extracted results. writeIter points to the location in // the array where we should shift forwards. // It will equal readIter if we haven't extracted any matches yet, so we // don't need to move the value anywhere in this case. if (readIter != writeIter) { *writeIter = std::move(*readIter); } ++writeIter; } } if (writeIter == blockedChecks.begin()) { // We extracted all blocked checks for this key class, // so just erase the key class from state->blockedChecks entirely. state->blockedChecks.erase(classIter); } else { // Chop off the end of the array that now has uninitialized values // that were moved forwards in the array. blockedChecks.erase(writeIter, blockedChecks.end()); } return results; } size_t FaultInjector::unblockAllImpl( std::optional error) { folly::StringKeyedUnorderedMap> blockedChecks; { auto state = state_.wlock(); state->blockedChecks.swap(blockedChecks); } size_t numUnblocked = 0; for (auto& classEntry : blockedChecks) { for (auto& check : classEntry.second) { if (error.has_value()) { check.promise.setException(error.value()); } else { check.promise.setValue(); } } numUnblocked += classEntry.second.size(); } return numUnblocked; } } // namespace eden } // namespace facebook