scripts: unify spwaning run-test logic

Summary:
Previously, lint.py and unit.py have different logic spawning
run-tests.py.

The logic in `unit.py` is more robust: it sets `PYTHONPATH` and
`cwd`. It sets `-j` according to CPU cores. It can find `run-tests.py`
even if `MERCURIALRUNTESTS` is not set. You can run the script
from any directory (not only reporoot).

Test Plan: Run `lint.py` and `unit.py`, from the `scripts` directory and reporoot.

Reviewers: #sourcecontrol, stash

Reviewed By: stash

Subscribers: stash, mjpieters

Differential Revision: https://phabricator.intern.facebook.com/D4095437

Signature: t1:4095437:1477663516:13a7ac4270435c272077132915f9d02cc98a5afb

scripts/lint.py

@@ -1,22 +1,16 @@
 #!/usr/bin/env python
+
+""" Convert mercurial check-code errors into a format
+that plays nicely with arc lint """
+
 import errno
 import os
 import subprocess
 import re
 import sys
 
-""" Convert mercurial check-code errors into a format
-that plays nicely with arc lint """
-
-runner = os.environ.get('MERCURIALRUNTEST', 'run-tests.py')
-
-if not os.path.exists(runner):
-    # If it looks like we're in facebook-hg-rpms, let's try
-    # running against the associated hg-crew tests
-    # otherwise, Popen will search for run-tests.py in the PATH
-    default_runner = os.path.relpath('../hg-crew/tests/run-tests.py')
-    if os.path.exists(default_runner):
-        runner = os.path.abspath(default_runner)
+sys.path.insert(0, os.path.dirname(__file__))
+import utils
 
 # Normalize the list of files that we should report on
 wanted = set()
@@ -27,26 +21,18 @@ for path in sys.argv[1:]:
 if wanted:
     os.environ['LINTFILES'] = '\n'.join(sorted(wanted))
 
+args = ['-l', 'test-check-code-hg.t', 'test-check-pyflakes-hg.t']
+
 try:
-    args = [runner, '-j2', '-l',
-            'test-check-code-hg.t',
-            'test-check-pyflakes-hg.t']
-
-    # Check lz4revlog requirement
-    reporoot = os.path.join(os.path.dirname(os.path.dirname(__file__)), '.hg')
-    with open(os.path.join(reporoot, 'requires'), 'r') as f:
-        if 'lz4revlog\n' in f:
-            args.append('--extra-config-opt=extensions.lz4revlog=')
-
-    proc = subprocess.Popen(args, stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE, cwd='tests')
+    proc = utils.spawnruntests(args, stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
 except OSError as ex:
     if ex.errno == errno.ENOENT:
         print 'lint.py:1: ERROR:ENVIRON: Please either set ' + \
               'MERCURIALRUNTEST var to the full path to run-tests.py, ' + \
               'or add the containing directory to your $PATH'
     else:
-        print 'lint.py:1: ERROR:OSError: %s: %s' % (runner, str(ex))
+        print 'lint.py:1: ERROR:OSError: %r' % ex
     sys.exit(0)
 
 output, error = proc.communicate()

scripts/unit.py

@@ -1,11 +1,4 @@
 #!/usr/bin/env python
-import json
-import multiprocessing
-import optparse
-import os
-import re
-import subprocess
-import sys
 
 """run a subset of tests that related to the current change
 
@@ -13,26 +6,22 @@ Optionally write result using JSON format. The JSON format can be parsed
 by MercurialTestEngine.php
 """
 
-reporoot = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
+import json
+import optparse
+import os
+import re
+import subprocess
+import sys
+
+sys.path.insert(0, os.path.dirname(__file__))
+import utils
+
+reporoot = utils.reporoot
 
 def info(message):
     """print message to stderr"""
     sys.stderr.write(message)
 
-def getrunner():
-    """return the path of run-tests.py. best-effort"""
-    runner = os.environ.get('MERCURIALRUNTEST', 'run-tests.py')
-    if os.path.exists(runner):
-        return runner
-    # Search some common places for run-tests.py
-    for prefix in ['..', os.path.expanduser('~')]:
-        for hgrepo in ['hg', 'hg-crew', 'hg-committed']:
-            path = os.path.abspath(os.path.join(prefix, hgrepo,
-                                                'tests', 'run-tests.py'))
-            if os.path.exists(path):
-                return path
-    return runner
-
 def checkoutput(*args, **kwds):
     """like subprocess.checked_output, but raise RuntimeError and return
     stderr as a second value.
@@ -94,13 +83,6 @@ def interestingtests(changed_files):
 
     return result
 
-def reporequires():
-    """return a list of string, which are the requirements of the hg repo"""
-    requirespath = os.path.join(reporoot, '.hg', 'requires')
-    if os.path.exists(requirespath):
-        return [s.rstrip() for s in open(requirespath, 'r')]
-    return []
-
 def runtests(tests=None):
     """run given tests
 
@@ -108,23 +90,9 @@ def runtests(tests=None):
     exitcode will be 0 on success, and non-zero on failure
     report is a dictionary of test results.
     """
-    cpucount = multiprocessing.cpu_count()
-    cmd = [getrunner(), '-j%d' % cpucount, '-l', '--json']
-    requires = reporequires()
-    if 'lz4revlog' in requires:
-        cmd += ['--extra-config-opt=extensions.lz4revlog=']
+    args = ['-l', '--json']
     if tests:
-        cmd += tests
-
-    # Include the repository root in PYTHONPATH so the unit tests will find
-    # the extensions from the local repository, rather than the versions
-    # already installed on the system.
-    env = os.environ.copy()
-    if 'PYTHONPATH' in env:
-        existing_pypath = [env['PYTHONPATH']]
-    else:
-        existing_pypath = []
-    env['PYTHONPATH'] = os.path.pathsep.join([reporoot] + existing_pypath)
+        args += tests
 
     # Run the tests.
     #
@@ -133,7 +101,7 @@ def runtests(tests=None):
     # should cause it to exit soon.  We want to wait for the test runner to
     # exit before we quit.  Otherwise may keep printing data even after we have
     # exited and returned control of the terminal to the user's shell.
-    proc = subprocess.Popen(cmd, cwd=os.path.join(reporoot, 'tests'), env=env)
+    proc = utils.spawnruntests(args)
     interruptcount = 0
     maxinterrupts = 3
     while True:

scripts/utils.py

@@ -0,0 +1,58 @@
+import multiprocessing
+import os
+import subprocess
+
+reporoot = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+def getrunner():
+    """return the path of run-tests.py. best-effort"""
+    runner = 'run-tests.py'
+    # Try MERCURIALRUNTEST
+    candidate = os.environ.get('MERCURIALRUNTEST')
+    if candidate and os.access(candidate, os.X_OK):
+        return candidate
+    # Search in PATH
+    for d in os.environ.get('PATH').split(os.path.pathsep):
+        candidate = os.path.abspath(os.path.join(d, runner))
+        if os.access(candidate, os.X_OK):
+            return candidate
+    # Search some common places for run-tests.py, as a nice default
+    # if we cannot find it otherwise.
+    for prefix in [os.path.dirname(reporoot), os.path.expanduser('~')]:
+        for hgrepo in ['hg', 'hg-crew', 'hg-committed']:
+            path = os.path.abspath(os.path.join(prefix, hgrepo,
+                                                'tests', runner))
+            if os.access(path, os.X_OK):
+                return path
+    return runner
+
+def reporequires():
+    """return a list of string, which are the requirements of the hg repo"""
+    requirespath = os.path.join(reporoot, '.hg', 'requires')
+    if os.path.exists(requirespath):
+        return [s.rstrip() for s in open(requirespath, 'r')]
+    return []
+
+def spawnruntests(args, **kwds):
+    cpucount = multiprocessing.cpu_count()
+    cmd = [getrunner(), '-j%d' % cpucount]
+
+    # Enable lz4revlog extension for lz4revlog repo.
+    if 'lz4revlog' in reporequires():
+        cmd += ['--extra-config-opt=extensions.lz4revlog=']
+
+    # Include the repository root in PYTHONPATH so the unit tests will find
+    # the extensions from the local repository, rather than the versions
+    # already installed on the system.
+    env = os.environ.copy()
+    if 'PYTHONPATH' in env:
+        existing_pypath = [env['PYTHONPATH']]
+    else:
+        existing_pypath = []
+    env['PYTHONPATH'] = os.path.pathsep.join([reporoot] + existing_pypath)
+
+    # Spawn the run-tests.py process.
+    cmd += args
+    cwd = os.path.join(reporoot, 'tests')
+    proc = subprocess.Popen(cmd, cwd=cwd, env=env, **kwds)
+    return proc