facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2025-01-07 14:10:42 +03:00
Code Issues Projects Releases Wiki Activity

clean up the find_executables module in the integration tests

Browse Source 
Summary:
Refactor the find_executables module to only look up executables when they are
needed, instead of performing all look-ups immediately even if the test in
question may not need all of the binaries.  Also add a _find_exe() helper
function to eliminate some code duplication.

Reviewed By: ryanmce

Differential Revision: D7512038

fbshipit-source-id: fdfb8ec70b3f6292603826b3fb22c01dbd1f0d72
This commit is contained in:
Adam Simpkins 2018-04-06 12:25:51 -07:00 committed by Facebook Github Bot
parent dc057b5b52
commit aa3009b2b8
5 changed files with 131 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
eden/integration/clone_test.py
View File

@ -17,7 +17,7 @@ import unittest
from eden.cli import util
from .lib import edenclient, testcase
from .lib.find_executables import EDEN_DAEMON
from .lib.find_executables import FindExe
# This is the name of the default repository created by EdenRepoTestBase.
@ -237,7 +237,7 @@ echo -n "$1" >> "{scratch_file}"
self.eden.run_cmd(
'clone',
'--daemon-binary',
EDEN_DAEMON,
FindExe.EDEN_DAEMON,
self.repo.path,
tmp,
'--daemon-args',

2
eden/integration/hg/lib/hg_extension_test_base.py
View File

@ -18,7 +18,7 @@ import subprocess
def _find_post_clone() -> str:
post_clone = (os.environ.get('EDENFS_POST_CLONE_PATH') or
os.path.join(find_executables.BUCK_OUT,
os.path.join(find_executables.FindExe.BUCK_OUT,
'gen/eden/hooks/hg/post-clone.par'))
if not os.access(post_clone, os.X_OK):
msg = ('unable to find post-clone script for integration testing: {!r}'

8
eden/integration/lib/edenclient.py
View File

@ -21,7 +21,7 @@ from eden.cli import util
import eden.thrift
from fb303.ttypes import fb_status
from .find_executables import EDEN_CLI, EDEN_DAEMON, EDEN_HG_IMPORT_HELPER
from .find_executables import FindExe
class EdenFS(object):
@ -143,7 +143,7 @@ class EdenFS(object):
A list of arguments to run Eden that can be used with
subprocess.Popen() or subprocess.check_call().
'''
cmd = [EDEN_CLI, '--config-dir', self._eden_dir]
cmd = [FindExe.EDEN_CLI, '--config-dir', self._eden_dir]
if self._etc_eden_dir:
cmd += ['--etc-eden-dir', self._etc_eden_dir]
if self._home_dir:
@ -183,7 +183,7 @@ class EdenFS(object):
# framework runs tests on each CPU core.
'--num_hg_import_threads', '2',
'--local_storage_engine_unsafe', self._storage_engine,
'--hgImportHelper', EDEN_HG_IMPORT_HELPER,
'--hgImportHelper', FindExe.EDEN_HG_IMPORT_HELPER,
# Disable falling back to importing mercurial data using
# flatmanifest when the repository supports treemanifest.
# If an error occurs importing treemanifest data in a test this is
@ -215,7 +215,7 @@ class EdenFS(object):
args = self._get_eden_args(
'daemon',
'--daemon-binary', EDEN_DAEMON,
'--daemon-binary', FindExe.EDEN_DAEMON,
'--foreground',
)

207
eden/integration/lib/find_executables.py
View File

@ -14,100 +14,131 @@ being run in buck-out.
import os
import sys
from typing import Callable, Dict, List, Optional, Type
def _find_directories():
'''Returns the paths to buck-out and the repo root.
class cached_property(object):
def __init__(self, find: Callable[['FindExeClass'], str]) -> None:
self.name: Optional[str] = None
self.find = find
Note that the path to buck-out may not be "buck-out" under the repo root
because Buck could have been run with `buck --config project.buck_out` and
sys.argv[0] could be the realpath rather than the symlink under buck-out.
def __get__(self, instance: 'FindExeClass',
owner: Type['FindExeClass']) -> str:
assert self.name is not None
result = instance._cache.get(self.name, None)
if result is None:
result = self.find(instance)
instance._cache[self.name] = result
return result
TODO: We will have to use a different heuristic for open source builds that
build with CMake. (Ultimately, we would prefer to build them with Buck.)
'''
executable = sys.argv[0]
path = os.path.dirname(os.path.abspath(executable))
while True:
parent = os.path.dirname(path)
parent_basename = os.path.basename(parent)
if parent_basename == 'buck-out':
repo_root = os.path.dirname(parent)
if os.path.basename(path) in ['bin', 'gen']:
buck_out = parent
else:
buck_out = path
return repo_root, buck_out
if parent == path:
raise Exception('Path to repo root not found from %s' % executable)
path = parent
def __set_name__(self, owner: Type['FindExeClass'], name: str) -> None:
self.name = name
REPO_ROOT, BUCK_OUT = _find_directories()
class FindExeClass(object):
def __init__(self) -> None:
self._cache: Dict[str, str] = {}
# The EDENFS_SUFFIX will be set to indicate if we should test with a
# particular variant of the edenfs daemon
EDENFS_SUFFIX = os.environ.get('EDENFS_SUFFIX', '')
@property
def BUCK_OUT(self) -> str:
if not hasattr(self, '_BUCK_OUT'):
self._find_repo_root_and_buck_out()
return self._BUCK_OUT
@property
def REPO_ROOT(self) -> str:
if not hasattr(self, '_REPO_ROOT'):
self._find_repo_root_and_buck_out()
return self._REPO_ROOT
@cached_property
def EDEN_CLI(self) -> str:
return self._find_exe(
'eden CLI',
env='EDENFS_CLI_PATH',
candidates=[os.path.join(self.BUCK_OUT, 'gen/eden/cli/cli.par')]
)
@cached_property
def EDEN_DAEMON(self) -> str:
edenfs_suffix = os.environ.get('EDENFS_SUFFIX', '')
edenfs = os.path.join(
self.BUCK_OUT, 'gen/eden/fs/service/edenfs%s' % edenfs_suffix
)
return self._find_exe(
'edenfs daemon',
env='EDENFS_SERVER_PATH',
candidates=[edenfs],
)
@cached_property
def EDEN_HG_IMPORT_HELPER(self) -> str:
return self._find_exe(
'hg_import_helper',
env='EDENFS_HG_IMPORT_HELPER',
candidates=[
os.path.join(
self.BUCK_OUT, 'gen/eden/fs/store/hg/hg_import_helper.par'
),
os.path.join(
self.REPO_ROOT, 'eden/fs/store/hg/hg_import_helper.py'
),
]
)
@cached_property
def FSATTR(self) -> str:
return self._find_exe(
'fsattr',
env='EDENFS_FSATTR_BIN',
candidates=[
os.path.join(self.BUCK_OUT, 'gen/eden/integration/fsattr')
]
)
def _find_exe(self, name, env, candidates):
if env is not None:
path = os.environ.get(env)
if path and not os.access(path, os.X_OK):
raise Exception(f'unable to find {name}: specified as {path!r} '
f'by ${env}, but not available there')
for path in candidates:
if os.access(path, os.X_OK):
return path
raise Exception(f'unable to find {name}')
def _find_repo_root_and_buck_out(self) -> None:
'''Finds the paths to buck-out and the repo root.
Note that the path to buck-out may not be "buck-out" under the repo
root because Buck could have been run with `buck --config
project.buck_out` and sys.argv[0] could be the realpath rather than the
symlink under buck-out.
TODO: We will have to use a different heuristic for open source builds
that build with CMake. (Ultimately, we would prefer to build them with
Buck.)
'''
executable = sys.argv[0]
path = os.path.dirname(os.path.abspath(executable))
while True:
parent = os.path.dirname(path)
parent_basename = os.path.basename(parent)
if parent_basename == 'buck-out':
self._REPO_ROOT = os.path.dirname(parent)
if os.path.basename(path) in ['bin', 'gen']:
self._BUCK_OUT = parent
else:
self._BUCK_OUT = path
return
if parent == path:
raise Exception(
'Path to repo root not found from %s' % executable
)
path = parent
def _find_cli():
cli = os.environ.get('EDENFS_CLI_PATH')
if not cli:
cli = os.path.join(BUCK_OUT, 'gen/eden/cli/cli.par')
if not os.access(cli, os.X_OK):
msg = 'unable to find eden CLI for integration testing: {!r}'.format(
cli)
raise Exception(msg)
return cli
EDEN_CLI = _find_cli()
def _find_daemon():
edenfs = os.environ.get('EDENFS_SERVER_PATH')
if not edenfs:
edenfs = os.path.join(BUCK_OUT,
'gen/eden/fs/service/edenfs%s' % EDENFS_SUFFIX)
if not os.access(edenfs, os.X_OK):
msg = 'unable to find eden daemon for integration testing: {!r}'.format(
edenfs)
raise Exception(msg)
return edenfs
EDEN_DAEMON = _find_daemon()
def _find_hg_import_helper():
import_helper = os.environ.get('EDENFS_HG_IMPORT_HELPER')
if import_helper and not os.access(import_helper, os.X_OK):
raise Exception(f'unable to find hg_import_helper script for '
'integration testing: {import_helper!r}')
candidates = (
os.path.join(BUCK_OUT, 'gen/eden/fs/store/hg/hg_import_helper.par'),
os.path.join(REPO_ROOT, 'eden/fs/store/hg/hg_import_helper.py'),
)
for path in candidates:
if os.access(path, os.X_OK):
return path
raise Exception(f'unable to find hg_import_helper script for integration '
'testing: checked paths {candidates!r}')
EDEN_HG_IMPORT_HELPER = _find_hg_import_helper()
def _find_fsattr():
fsattr = os.environ.get('EDENFS_FSATTR_BIN')
if not fsattr:
fsattr = os.path.join(BUCK_OUT, 'gen/eden/integration/fsattr')
if not os.access(fsattr, os.X_OK):
msg = 'unable to find fsattr for integration testing: {!r}'.format(
fsattr)
raise Exception(msg)
return fsattr
FSATTR = _find_fsattr()
# The main FindExe singleton
FindExe = FindExeClass()

8
eden/integration/lib/fs.py
View File

@ -12,12 +12,12 @@ import os
import subprocess
import typing
from .find_executables import FSATTR
from .find_executables import FindExe
def getxattr(abspath: str, attr: str) -> str:
raw_stdout = subprocess.check_output(
[FSATTR, '--attrName', attr, '--fileName', abspath]
[FindExe.FSATTR, '--attrName', attr, '--fileName', abspath]
)
result = json.loads(raw_stdout)
# fsattr should always return a string here. We just cast the result,
@ -26,7 +26,9 @@ def getxattr(abspath: str, attr: str) -> str:
def listxattr(abspath: str) -> typing.Dict[str, str]:
raw_stdout = subprocess.check_output([FSATTR, '--fileName', abspath])
raw_stdout = subprocess.check_output(
[FindExe.FSATTR, '--fileName', abspath]
)
result = json.loads(raw_stdout)
# fsattr should always return a dictionary here. We just cast the
# result, without actually validating it for now.