mirror of
https://github.com/facebook/sapling.git
synced 2024-10-10 16:57:49 +03:00
mononoke/integration tests: separate out facebook-specific code for running integration tests
Summary: Not all facebook-specific code was moved out of integration_runner_real.py, but removing part of the code that is left would made the code less readable, the rest of it will be removed while the integration_runner_real.py is made usable for OSS Reviewed By: farnz Differential Revision: D22114948 fbshipit-source-id: d9c532a6a9ea653de2b12cffc92fbf45826dad37
This commit is contained in:
Lukas Piatkowski
Facebook GitHub Bot
parent
b02ad187cf
commit
6ebd409406
@ -1,44 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This software may be used and distributed according to the terms of the
|
||||
# GNU General Public License version 2.
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
from eden.mononoke.tests.integration.lib_buck import find_buck_out
|
||||
|
||||
|
||||
def map_name(k):
|
||||
v = os.environ[k]
|
||||
|
||||
# We want our test root to be the directory that actually contains the
|
||||
# files.
|
||||
if k == "TEST_ROOT_FACEBOOK":
|
||||
return os.path.join(v, "facebook")
|
||||
|
||||
return v
|
||||
|
||||
|
||||
def main():
|
||||
# We provide the output file and names as argument
|
||||
_, out, *names = sys.argv
|
||||
|
||||
# The INSTALL_DIR is provided by Buck's custom_rule.
|
||||
out = os.path.join(os.environ["INSTALL_DIR"], out)
|
||||
|
||||
# Locations are provided through the environment (using Buck location
|
||||
# macro). The paths we output must be relative to buck_out, since they might
|
||||
# have been built on a different host so we must avoid absolute paths.
|
||||
buck_out = find_buck_out(out)
|
||||
|
||||
manifest = {k: os.path.relpath(map_name(k), buck_out) for k in sorted(names)}
|
||||
|
||||
with open(out, "w") as f:
|
||||
json.dump(manifest, f, indent=2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This software may be used and distributed according to the terms of the
|
||||
# GNU General Public License version 2.
|
||||
|
||||
# This script is provided as a compatibility shim for those who want to run
|
||||
# individual tests, using Buck:
|
||||
#
|
||||
# buck run eden/mononoke/tests/integration:integration_runner -- TEST
|
||||
|
||||
function protip() {
|
||||
local real_runner real_manifest
|
||||
real_runner="$(readlink "$1")"
|
||||
real_manifest="$(readlink "$2")"
|
||||
shift
|
||||
shift
|
||||
|
||||
real_runner="$(readlink "$runner")"
|
||||
real_manifest="$(readlink "$manifest")"
|
||||
|
||||
echo >&2
|
||||
echo "======" >&2
|
||||
echo "|| Pro tip: save time on your incremental test runs!" >&2
|
||||
echo "|| Run this command instead:" >&2
|
||||
echo "||" >&2
|
||||
echo "|| $real_runner $real_manifest $*" >&2
|
||||
echo "||" >&2
|
||||
echo "|| Between test runs, manually rebuild only the binaries you care about using buck build." >&2
|
||||
echo "|| Check out the README.md for integration tests to learn more." >&2
|
||||
echo "======"
|
||||
echo >&2
|
||||
}
|
||||
|
||||
d="$BUCK_DEFAULT_RUNTIME_RESOURCES"
|
||||
runner="${d}/integration_runner_real"
|
||||
manifest="${d}/manifest.json"
|
||||
|
||||
protip "$runner" "$manifest" "$@"
|
||||
exec "$runner" "$manifest" "$@"
|
||||
@ -6,7 +6,6 @@
|
||||
|
||||
"""Runner for Mononoke/Mercurial integration tests."""
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
@ -18,11 +17,6 @@ import xml.etree.ElementTree as ET
|
||||
from typing import Any, Dict, List, NamedTuple, Set
|
||||
|
||||
import click
|
||||
from common.db.tests import DbDef
|
||||
from configerator.client import ConfigeratorClient
|
||||
from configerator.structs.mysql.table_schema.ttypes import DatabaseSchemas, TableSchema
|
||||
from eden.mononoke.tests.integration.lib_buck import find_buck_out
|
||||
from libfb.py.log import set_simple_logging
|
||||
|
||||
|
||||
ManifestEnv = Dict[str, str]
|
||||
@ -60,6 +54,15 @@ def is_mode_opt_buck_binary():
|
||||
return False
|
||||
|
||||
|
||||
def is_libfb_present():
|
||||
try:
|
||||
import libfb.py.log # noqa: F401
|
||||
|
||||
return True
|
||||
except ImportError:
|
||||
return False
|
||||
|
||||
|
||||
class TestFlags(NamedTuple):
|
||||
interactive: bool
|
||||
verbose: bool
|
||||
@ -106,52 +109,6 @@ class TestFlags(NamedTuple):
|
||||
return r
|
||||
|
||||
|
||||
def create_schema(mysql_schemas, conn):
|
||||
client = ConfigeratorClient()
|
||||
configs = []
|
||||
for schema_path in mysql_schemas:
|
||||
root = client.get_config_contents_as_thrift(
|
||||
"{}/.sql_schema_domains".format(schema_path), DatabaseSchemas
|
||||
)
|
||||
for table in root.tables.keys():
|
||||
configs.append("{}/{}.sql_table_schema".format(schema_path, table))
|
||||
for table_path in configs:
|
||||
table = client.get_config_contents_as_thrift(table_path, TableSchema)
|
||||
conn.query(table.sql)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def ephemeral_db_helper(shard_part, mysql_schemas):
|
||||
with DbDef.TestDatabaseManager(
|
||||
prefix=shard_part,
|
||||
oncall_shortname="source_control",
|
||||
force_ephemeral=True,
|
||||
ttl_minutes=15, # Some Mononoke tests last this long
|
||||
) as test_db_manager:
|
||||
if len(mysql_schemas) > 0:
|
||||
conn = test_db_manager.get_connection()
|
||||
create_schema(mysql_schemas, conn)
|
||||
yield {"DB_SHARD_NAME": test_db_manager.get_shard_name()}
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def devdb_db_helper(shard_part, mysql_schemas):
|
||||
from libfb.py import db_locator
|
||||
|
||||
shard_name = "devdb." + shard_part
|
||||
if len(mysql_schemas) > 0:
|
||||
locator = db_locator.Locator(tier_name=shard_name, role="scriptrw")
|
||||
locator.do_not_send_autocommit_query()
|
||||
conn = locator.create_connection()
|
||||
create_schema(mysql_schemas, conn)
|
||||
yield {"DB_SHARD_NAME": shard_name}
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def no_db_helper(*args, **kwargs):
|
||||
yield {}
|
||||
|
||||
|
||||
def public_test_root(manifest_env: ManifestEnv) -> str:
|
||||
return manifest_env["TEST_ROOT_PUBLIC"]
|
||||
|
||||
@ -160,15 +117,6 @@ def facebook_test_root(manifest_env: ManifestEnv) -> str:
|
||||
return manifest_env["TEST_ROOT_FACEBOOK"]
|
||||
|
||||
|
||||
def load_manifest_env(manifest_path: str) -> ManifestEnv:
|
||||
buck_out = find_buck_out(manifest_path)
|
||||
|
||||
with open(manifest_path) as f:
|
||||
manifest_env = json.load(f)
|
||||
|
||||
return {k: os.path.join(buck_out, v) for k, v in manifest_env.items()}
|
||||
|
||||
|
||||
def maybe_use_local_test_paths(manifest_env: ManifestEnv):
|
||||
# If we are running outside of Buck, then update the test paths to use the
|
||||
# actual files. This makes --interactive work, and allows for adding new
|
||||
@ -413,13 +361,20 @@ def run(
|
||||
mysql_schemas,
|
||||
devdb,
|
||||
):
|
||||
set_simple_logging(logging.INFO)
|
||||
if is_libfb_present():
|
||||
from libfb.py.log import set_simple_logging
|
||||
from eden.mononoke.tests.integration.facebook.lib_runner import (
|
||||
load_manifest_env,
|
||||
)
|
||||
|
||||
set_simple_logging(logging.INFO)
|
||||
manifest_env: ManifestEnv = load_manifest_env(manifest)
|
||||
else:
|
||||
with open(manifest) as f:
|
||||
manifest_env: ManifestEnv = json.load(f)
|
||||
|
||||
manifest_env = load_manifest_env(manifest)
|
||||
maybe_use_local_test_paths(manifest_env)
|
||||
|
||||
db_helper = no_db_helper
|
||||
|
||||
if dry_run:
|
||||
return run_discover_tests(ctx, manifest_env, output, mysql)
|
||||
|
||||
@ -448,29 +403,15 @@ def run(
|
||||
else:
|
||||
selected_tests.extend(tests)
|
||||
|
||||
shard_part = "monononoke_tests"
|
||||
if mysql:
|
||||
if dry_run:
|
||||
# We can dry-run for ephemeral tests
|
||||
pass
|
||||
elif len(selected_tests) <= 1:
|
||||
# One test is allowed to use an ephemeral db. This is OK, because
|
||||
# TestPilot actually asks us to run tests one by one.
|
||||
if devdb:
|
||||
shard_part = devdb
|
||||
db_helper = devdb_db_helper
|
||||
else:
|
||||
db_helper = ephemeral_db_helper
|
||||
else:
|
||||
# Too many tests! This won't work.
|
||||
raise click.BadParameter(
|
||||
"mysql tests allow at most 1 test at a time",
|
||||
ctx,
|
||||
param_hint="simple_test_selector",
|
||||
)
|
||||
if is_libfb_present():
|
||||
from eden.mononoke.tests.integration.facebook.lib_runner import fb_test_context
|
||||
|
||||
with db_helper(shard_part, mysql_schemas) as test_env:
|
||||
run_tests(ctx, manifest_env, output, selected_tests, test_flags, test_env)
|
||||
with fb_test_context(
|
||||
ctx, dry_run, mysql, mysql_schemas, devdb, selected_tests
|
||||
) as test_env:
|
||||
run_tests(ctx, manifest_env, output, selected_tests, test_flags, test_env)
|
||||
else:
|
||||
run_tests(ctx, manifest_env, output, selected_tests, test_flags, test_env={})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -1,17 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This software may be used and distributed according to the terms of the
|
||||
# GNU General Public License version 2.
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def find_buck_out(manifest_path):
|
||||
dir = manifest_path
|
||||
while dir:
|
||||
dir = os.path.dirname(dir)
|
||||
if os.path.exists(os.path.join(dir, "project_root")):
|
||||
return dir
|
||||
m = "%s does not appear to be in a buck-out directory" % manifest_path
|
||||
raise ValueError(m)
|
||||
Loading…
Reference in New Issue
Block a user