debugruntest: support stdlib doctest

Summary:
Python stdlib doctest is conceptually similar to what debugruntest
does - run code, check output. By adopting it in debugruntest,
we get the handy autofix feature for free, which is very handy
for files like `testing/t/transform.py`.

The Python doctest requires a Python module as the "starting point".
`doctest:<module name>` is added as a special syntax to indicate
this is a doctest.

A special case is added for edenscm/ source code. Python files will
automatically turn into doctest for convenience.

Reviewed By: DurhamG

Differential Revision: D34725126

fbshipit-source-id: 91b78505708ad930f7688cc3e51e53a94d9030e9

eden/scm/edenscm/testing/t/runner.py

@@ -4,8 +4,10 @@
 # GNU General Public License version 2.
 
 import collections
+import doctest
 import multiprocessing
 import os
+import re
 import sys
 import textwrap
 import threading
@@ -27,9 +29,44 @@ class TestId:
 
     @classmethod
     def frompath(cls, path: str):
+        if path.startswith("doctest:"):
+            name = path
+            modname = name[8:]
+            __import__(modname)
+            path = sys.modules[modname].__file__
+            return cls(name=name, path=path)
+
         path = os.path.abspath(path)
-        name = os.path.basename(path)
-        return cls(name=name, path=path)
+        if path.endswith(".py"):
+            # try to convert the .py path to doctest:module
+            modnames = sorted(n for n in sys.modules if "." not in n)
+            for name in modnames:
+                mod = sys.modules[name]
+                modpath = getattr(mod, "__file__", None)
+                if not modpath or os.path.basename(modpath) != "__init__.py":
+                    continue
+                prefix = os.path.join(os.path.dirname(modpath), "")
+                if not path.startswith(prefix):
+                    continue
+                relpath = path[len(prefix) :].replace("\\", "/")
+                for suffix in ["/__init__.py", ".py"]:
+                    if relpath.endswith(suffix):
+                        relpath = relpath[: -len(suffix)]
+                        break
+                modname = f"{mod.__name__}.{relpath.replace('/', '.')}"
+                return cls.frompath(f"doctest:{modname}")
+            raise RuntimeError(
+                f"cannot find Python module name for {path=} to run doctest"
+            )
+        else:
+            name = os.path.basename(path)
+            return cls(name=name, path=path)
+
+    @property
+    def modname(self) -> Optional[str]:
+        if self.name.startswith("doctest:"):
+            return self.name.split(":", 1)[1]
+        return None
 
 
 @dataclass
@@ -175,6 +212,69 @@ def runtest(testid: TestId, exts: List[str], mismatchcb: Callable[[Mismatch], No
     The generated Python code is written at __pycache__/ttest/<test>.py.
     Return output mismatches.
     """
+    if testid.modname:
+        return rundoctest(testid, mismatchcb)
+    else:
+        return runttest(testid, exts, mismatchcb)
+
+
+class doctestrunner(doctest.DocTestRunner):
+    """doctest runner that reports output mismatches as Mismatch"""
+
+    def __init__(self, testname: str, mismatchcb: Callable[[Mismatch], None]):
+        optionflags = doctest.IGNORE_EXCEPTION_DETAIL
+        super().__init__(verbose=False, optionflags=optionflags)
+        self.testname = testname
+        self.mismatchcb = mismatchcb
+
+    def report_failure(
+        self, out, test: doctest.DocTest, example: doctest.Example, got: str
+    ):
+        # see doctest.OutputChecker.output_difference
+        if not (self.optionflags & doctest.DONT_ACCEPT_BLANKLINE):
+            got = re.sub("(?m)^[ ]*(?=\n)", doctest.BLANKLINE_MARKER, got)
+
+        srcloc = test.lineno + example.lineno
+        outloc = srcloc + example.source.count("\n")
+        endloc = outloc + example.want.count("\n")
+        src = ">>> " + textwrap.indent(example.source, "... ")[4:]
+        mismatch = Mismatch(
+            actual=got,
+            expected=example.want,
+            src=src,
+            srcloc=srcloc,
+            outloc=outloc,
+            endloc=endloc,
+            indent=example.indent,
+            filename=test.filename,
+            testname=self.testname,
+        )
+        self.mismatchcb(mismatch)
+
+    def report_unexpected_exception(self, out, test, example, excinfo):
+        exctype, excvalue, exctb = excinfo
+        excmsg = str(excvalue)
+        exctypestr = exctype.__name__
+        if excmsg:
+            excstr = f"{exctypestr}: {excmsg}"
+        else:
+            excstr = exctypestr
+        got = f"Traceback (most recent call last):\n  ...\n{excstr}\n"
+        return self.report_failure(out, test, example, got)
+
+
+def rundoctest(testid: TestId, mismatchcb: Callable[[Mismatch], None]):
+    """run doctest for the given module, report Mismatch via mismatchcb"""
+    modname = testid.modname
+    __import__(modname)
+    mod = sys.modules[modname]
+    finder = doctest.DocTestFinder()
+    runner = doctestrunner(testid.name, mismatchcb)
+    for test in finder.find(mod):
+        runner.run(test)
+
+
+def runttest(testid: TestId, exts: List[str], mismatchcb: Callable[[Mismatch], None]):
     path = Path(testid.path)
     testdir = path.parent
 

eden/scm/tests/test-debugruntest.t

@@ -127,3 +127,91 @@ Autofix:
 
   $ hg debugruntest test-fail-sh.t
   # Ran 1 tests, 0 skipped, 0 failed.
+
+
+#if no-bash
+Doctest:
+
+  $ cat >> testmodule.py << 'EOF'
+  > """
+  > A module for doctest testing
+  >   >>> 1+1
+  >   3
+  > """
+  > def plus(a, b):
+  >     r"""a+b
+  >        >>> plus(10, 20)
+  >        31
+  >        32
+  >        >>> plus('a', 'b')
+  >        >>> plus('a', 3)
+  >     """
+  >     return a + b
+  > EOF
+
+  $ ls testmodule.py
+  testmodule.py
+
+  >>> import testmodule
+
+  $ hg debugruntest doctest:testmodule
+  doctest:testmodule -----------------------------------------------------------
+     3 >>> 1+1
+      -3
+      +2
+  
+     8 >>> plus(10, 20)
+      -31
+      -32
+      +30
+  
+    11 >>> plus('a', 'b')
+      +'ab'
+  
+    12 >>> plus('a', 3)
+      +Traceback (most recent call last):
+      +  ...
+      +TypeError: can only concatenate str (not "int") to str
+  
+  -----------------------------------------------------------------------------
+  Failed 1 test (output mismatch):
+    doctest:testmodule
+  
+  # Ran 1 tests, 0 skipped, 1 failed.
+  [1]
+
+Doctest can be auto fixed too:
+
+  $ hg debugruntest -q --fix doctest:testmodule
+  [1]
+  $ cat testmodule.py
+  """
+  A module for doctest testing
+    >>> 1+1
+    2
+  """
+  def plus(a, b):
+      r"""a+b
+         >>> plus(10, 20)
+         30
+         >>> plus('a', 'b')
+         'ab'
+         >>> plus('a', 3)
+         Traceback (most recent call last):
+           ...
+         TypeError: can only concatenate str (not "int") to str
+      """
+      return a + b
+
+Reload the cached module:
+
+    from importlib import reload
+    import testmodule
+    reload(testmodule)
+
+The doctest passes with the autofix changes:
+
+  $ hg debugruntest doctest:testmodule
+  # Ran 1 tests, 0 skipped, 0 failed.
+
+#endif