mirror of
https://github.com/facebook/sapling.git
synced 2024-10-06 23:07:18 +03:00
testlib: support Mononoke as a server
Summary: This ports the default Mononoke server test configuration from Bash to Python and allows the new test framework to run a Mononoke server in place of an EagerRepo server. Reviewed By: mitrandir77 Differential Revision: D35899361 fbshipit-source-id: a4f9b166b0e2b0b63b257d65dc7e22249c823290
This commit is contained in:
Durham Goode
Facebook GitHub Bot
parent
7f87b0f013
commit
e5aad65220
@ -11,7 +11,7 @@ from pathlib import Path
|
||||
from typing import Callable, TypeVar
|
||||
|
||||
from .repo import Repo
|
||||
from .server import LocalServer, Server
|
||||
from .server import LocalServer, MononokeServer, Server
|
||||
from .util import new_dir, test_globals
|
||||
from .workingcopy import WorkingCopy
|
||||
|
||||
@ -21,8 +21,9 @@ class BaseTest(unittest.TestCase):
|
||||
test_globals.setup()
|
||||
self.addCleanup(test_globals.cleanup)
|
||||
self.server = self.new_server()
|
||||
self.repo = self.server.clone()
|
||||
self.addCleanup(self.server.cleanup)
|
||||
self._add_production_configs(Path(test_globals.env["HGRCPATH"]))
|
||||
self.repo = self.server.clone()
|
||||
|
||||
def _add_production_configs(self, hgrc: Path) -> None:
|
||||
# Most production configs should be loaded via dynamicconfig. The ones
|
||||
@ -46,8 +47,35 @@ cachepath = {new_dir()}
|
||||
"""
|
||||
)
|
||||
|
||||
if os.environ.get("USE_MONONOKE", False):
|
||||
cert_dir = os.environ["HGTEST_CERTDIR"]
|
||||
f.write(
|
||||
f"""
|
||||
[auth]
|
||||
mononoke.cert={cert_dir}/localhost.crt
|
||||
mononoke.key={cert_dir}/localhost.key
|
||||
mononoke.cacerts={cert_dir}/root-ca.crt
|
||||
mononoke.prefix=mononoke://*
|
||||
mononoke.cn=localhost
|
||||
edenapi.cert={cert_dir}/localhost.crt
|
||||
edenapi.key={cert_dir}/localhost.key
|
||||
edenapi.prefix=localhost
|
||||
edenapi.schemes=https
|
||||
edenapi.cacerts={cert_dir}/root-ca.crt
|
||||
|
||||
[web]
|
||||
cacerts={cert_dir}/root-ca.crt
|
||||
|
||||
[edenapi]
|
||||
url=https://localhost:{self.server.port}/edenapi
|
||||
"""
|
||||
)
|
||||
|
||||
def new_server(self) -> Server:
|
||||
return LocalServer()
|
||||
if os.environ.get("USE_MONONOKE", False):
|
||||
return MononokeServer()
|
||||
else:
|
||||
return LocalServer()
|
||||
|
||||
|
||||
# Contravariance rules in pyre mean we can't specify a base type directly as an
|
||||
|
||||
@ -4,6 +4,15 @@
|
||||
# GNU General Public License version 2.
|
||||
|
||||
# pyre-strict
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
from .hg import hg
|
||||
from .repo import Repo
|
||||
@ -11,21 +20,342 @@ from .util import new_dir
|
||||
|
||||
|
||||
class Server:
|
||||
url: str
|
||||
|
||||
def __init__(self) -> None:
|
||||
# Satisfy pyre
|
||||
self.url = ""
|
||||
|
||||
def clone(self) -> Repo:
|
||||
def _clone(self, repoid: int, url: str) -> Repo:
|
||||
root = new_dir()
|
||||
hg(root).clone(self.url, root, noupdate=True)
|
||||
hg(root).clone(
|
||||
url, root, noupdate=True, config=f"remotefilelog.reponame=repo{repoid}"
|
||||
)
|
||||
with open(os.path.join(root, ".hg", "hgrc"), "a+") as f:
|
||||
f.write(
|
||||
f"""
|
||||
[remotefilelog]
|
||||
reponame=repo{repoid}
|
||||
"""
|
||||
)
|
||||
|
||||
return Repo(root)
|
||||
|
||||
def cleanup(self) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class LocalServer(Server):
|
||||
"""An EagerRepo backed EdenApi server."""
|
||||
|
||||
urls: Dict[int, str]
|
||||
|
||||
def __init__(self) -> None:
|
||||
dir = new_dir()
|
||||
self.url = f"eager://{dir}"
|
||||
self.urls = {}
|
||||
|
||||
def clone(self, repoid: int = 0) -> Repo:
|
||||
if repoid not in self.urls:
|
||||
self.urls[repoid] = f"eager://{new_dir()}"
|
||||
return self._clone(repoid, self.urls[repoid])
|
||||
|
||||
|
||||
class MononokeServer(Server):
|
||||
edenapi_url: str
|
||||
port: str
|
||||
process: subprocess.Popen[bytes]
|
||||
repo_count: int
|
||||
url_prefix: str
|
||||
|
||||
def __init__(self, repo_count=5) -> None:
|
||||
self.process, self.port = _start(repo_count)
|
||||
self.repo_count = repo_count
|
||||
|
||||
self.url_prefix = f"mononoke://localhost:{self.port}"
|
||||
self.edenapi_url = f"https://localhost:{self.port}/edenapi"
|
||||
|
||||
def clone(self, repoid: int = 0) -> Repo:
|
||||
if repoid >= self.repo_count:
|
||||
raise ValueError(
|
||||
"cannot request repo %s when there are only %s repos"
|
||||
% (repoid, self.repo_count)
|
||||
)
|
||||
return self._clone(repoid, self.url_prefix + f"/repo{repoid}")
|
||||
|
||||
def cleanup(self) -> None:
|
||||
self.process.kill()
|
||||
self.process.wait(timeout=5)
|
||||
|
||||
|
||||
def _start(repo_count) -> Tuple[subprocess.Popen[bytes], str, str]:
|
||||
executable = os.environ["HGTEST_MONONOKE_SERVER"]
|
||||
temp_dir = new_dir()
|
||||
cert_dir = os.environ["HGTEST_CERTDIR"]
|
||||
bind_addr = "[::1]:0" # Localhost
|
||||
configerator_path = str(new_dir())
|
||||
tunables_path = "mononoke_tunables.json"
|
||||
|
||||
def tjoin(path: str) -> str:
|
||||
return os.path.join(temp_dir, path)
|
||||
|
||||
def cjoin(path: str) -> str:
|
||||
return os.path.join(cert_dir, path)
|
||||
|
||||
addr_file = tjoin("mononoke_server_addr.txt")
|
||||
config_path = tjoin("mononoke-config")
|
||||
|
||||
_setup_mononoke_configs(config_path)
|
||||
_setup_configerator(configerator_path)
|
||||
for i in range(repo_count):
|
||||
_setup_repo(config_path, i)
|
||||
|
||||
process = subprocess.Popen(
|
||||
[
|
||||
executable,
|
||||
"--scribe-logging-directory",
|
||||
tjoin("scribe_logs"),
|
||||
"--ca-pem",
|
||||
cjoin("root-ca.crt"),
|
||||
"--private-key",
|
||||
cjoin("localhost.key"),
|
||||
"--cert",
|
||||
cjoin("localhost.crt"),
|
||||
"--ssl-ticket-seeds",
|
||||
cjoin("server.pem.seeds"),
|
||||
"--listening-host-port",
|
||||
bind_addr,
|
||||
"--bound-address-file",
|
||||
addr_file,
|
||||
"--mononoke-config-path",
|
||||
config_path,
|
||||
"--no-default-scuba-dataset",
|
||||
"--debug",
|
||||
"--skip-caching",
|
||||
"--mysql-master-only",
|
||||
"--tunables-config",
|
||||
tunables_path,
|
||||
"--local-configerator-path",
|
||||
configerator_path,
|
||||
"--log-exclude-tag",
|
||||
"futures_watchdog",
|
||||
"--with-test-megarepo-configs-client=true",
|
||||
],
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
close_fds=True,
|
||||
)
|
||||
|
||||
port = _wait(
|
||||
process,
|
||||
addr_file,
|
||||
cjoin("localhost.crt"),
|
||||
cjoin("localhost.key"),
|
||||
cjoin("root-ca.crt"),
|
||||
)
|
||||
|
||||
return process, port
|
||||
|
||||
|
||||
def _wait(
|
||||
process: subprocess.Popen, addr_file: str, cert: str, key: str, ca_cert: str
|
||||
) -> Tuple[str, str]:
|
||||
start = time.time()
|
||||
while not os.path.exists(addr_file) and (time.time() - start < 60):
|
||||
time.sleep(0.5)
|
||||
state = process.poll()
|
||||
if state is not None:
|
||||
raise Exception(
|
||||
"Mononoke server exited early (%s):\n%s\n%s"
|
||||
% (state, process.stdout.read(), process.stderr.read())
|
||||
)
|
||||
|
||||
if os.path.exists(addr_file):
|
||||
with open(addr_file) as f:
|
||||
content = f.read()
|
||||
split_idx = content.rfind(":")
|
||||
port = content[split_idx + 1 :].strip()
|
||||
else:
|
||||
raise Exception(
|
||||
"timed out waiting for Mononoke server %s" % (time.time() - start)
|
||||
)
|
||||
|
||||
response = requests.get(
|
||||
f"https://localhost:{port}/health_check", cert=(cert, key), verify=ca_cert
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
return port
|
||||
|
||||
|
||||
def _setup_mononoke_configs(config_dir: str) -> None:
|
||||
def write(path: str, content: str) -> None:
|
||||
path = os.path.join(config_dir, path)
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w+") as f:
|
||||
f.write(content)
|
||||
|
||||
db_path = new_dir()
|
||||
|
||||
repotype = "blob_sqlite"
|
||||
blobstorename = "blobstore"
|
||||
blobstorepath = os.path.join(config_dir, blobstorename)
|
||||
|
||||
write(
|
||||
"common/common.toml",
|
||||
f"""
|
||||
[redaction_config]
|
||||
blobstore = "{blobstorename}"
|
||||
darkstorm_blobstore = "{blobstorename}"
|
||||
redaction_sets_location = "scm/mononoke/redaction/redaction_sets"
|
||||
|
||||
[[whitelist_entry]]
|
||||
identity_type = "USER"
|
||||
identity_data = "myusername0"
|
||||
""",
|
||||
)
|
||||
write("common/commitsyncmap.toml", "")
|
||||
write(
|
||||
"common/storage.toml",
|
||||
f"""
|
||||
[{blobstorename}.metadata.local]
|
||||
local_db_path = "{db_path}"
|
||||
|
||||
[{blobstorename}.ephemeral_blobstore]
|
||||
initial_bubble_lifespan_secs = 1000
|
||||
bubble_expiration_grace_secs = 1000
|
||||
bubble_deletion_mode = 0
|
||||
blobstore = {{ blob_files = {{ path = "{blobstorepath}" }} }}
|
||||
|
||||
[{blobstorename}.ephemeral_blobstore.metadata.local]
|
||||
local_db_path = "{db_path}"
|
||||
|
||||
[{blobstorename}.blobstore]
|
||||
{repotype} = {{ path = "{blobstorepath}" }}
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
def _setup_repo(config_dir: str, repoid: int) -> None:
|
||||
reponame = f"repo{repoid}"
|
||||
|
||||
def write(path: str, content: str) -> None:
|
||||
path = os.path.join(config_dir, path)
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w+") as f:
|
||||
f.write(content)
|
||||
|
||||
write(
|
||||
f"repos/{reponame}/server.toml",
|
||||
"""
|
||||
hash_validation_percentage=100
|
||||
storage_config = "blobstore"
|
||||
|
||||
[pushrebase]
|
||||
forbid_p2_root_rebases = false
|
||||
rewritedates = false
|
||||
|
||||
[hook_manager_params]
|
||||
disable_acl_checker= true
|
||||
|
||||
[push]
|
||||
pure_push_allowed = true
|
||||
|
||||
[derived_data_config]
|
||||
enabled_config_name = "default"
|
||||
|
||||
[derived_data_config.available_configs.default]
|
||||
types=["blame", "changeset_info", "deleted_manifest", "fastlog", "filenodes", "fsnodes", "unodes", "hgchangesets", "skeleton_manifests"]
|
||||
""",
|
||||
)
|
||||
write(
|
||||
f"repo_definitions/{reponame}/server.toml",
|
||||
f"""
|
||||
repo_id={repoid}
|
||||
repo_name="{reponame}"
|
||||
repo_config="{reponame}"
|
||||
enabled=true
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
def _setup_configerator(cfgr_root: str) -> None:
|
||||
def write(path: str, content: str) -> None:
|
||||
path = os.path.join(cfgr_root, path)
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w+") as f:
|
||||
f.write(content)
|
||||
|
||||
write(
|
||||
"scm/mononoke/ratelimiting/ratelimits",
|
||||
"""
|
||||
{
|
||||
"rate_limits": [],
|
||||
"load_shed_limits": [],
|
||||
"datacenter_prefix_capacity": {},
|
||||
"commits_per_author": {
|
||||
"status": 0,
|
||||
"limit": 300,
|
||||
"window": 1800
|
||||
},
|
||||
"total_file_changes": {
|
||||
"status": 0,
|
||||
"limit": 80000,
|
||||
"window": 5
|
||||
}
|
||||
}
|
||||
""",
|
||||
)
|
||||
write(
|
||||
"scm/mononoke/pushredirect/enable",
|
||||
"""
|
||||
{
|
||||
"per_repo": {}
|
||||
}
|
||||
""",
|
||||
)
|
||||
write(
|
||||
"scm/mononoke/repos/commitsyncmaps/all",
|
||||
"""
|
||||
{}
|
||||
""",
|
||||
)
|
||||
write(
|
||||
"scm/mononoke/repos/commitsyncmaps/current",
|
||||
"""
|
||||
{}
|
||||
""",
|
||||
)
|
||||
write(
|
||||
"scm/mononoke/xdb_gc/default",
|
||||
"""
|
||||
{
|
||||
"put_generation": 2,
|
||||
"mark_generation": 1,
|
||||
"delete_generation": 0
|
||||
}
|
||||
""",
|
||||
)
|
||||
write(
|
||||
"scm/mononoke/observability/observability_config",
|
||||
"""
|
||||
{
|
||||
"slog_config": {
|
||||
"level": 4
|
||||
},
|
||||
"scuba_config": {
|
||||
"level": 1,
|
||||
"verbose_sessions": [],
|
||||
"verbose_unixnames": [],
|
||||
"verbose_source_hostnames": []
|
||||
}
|
||||
}
|
||||
""",
|
||||
)
|
||||
write(
|
||||
"scm/mononoke/redaction/redaction_sets",
|
||||
"""
|
||||
{
|
||||
"all_redactions": []
|
||||
}
|
||||
""",
|
||||
)
|
||||
write(
|
||||
"mononoke_tunables.json",
|
||||
"""
|
||||
{}
|
||||
""",
|
||||
)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user