doctest: normalize b'', u'' and exception output on Python 3

The idea is described in the following page.

https://dirkjan.ochtman.nl/writing/2014/07/06/single-source-python-23-doctests.html

# no-check-commit

tests/test-doctest.py

@@ -4,6 +4,7 @@ from __future__ import absolute_import
 
 import doctest
 import os
+import re
 import sys
 
 ispy3 = (sys.version_info[0] >= 3)
@@ -11,6 +12,18 @@ ispy3 = (sys.version_info[0] >= 3)
 if 'TERM' in os.environ:
     del os.environ['TERM']
 
+class py3docchecker(doctest.OutputChecker):
+    def check_output(self, want, got, optionflags):
+        want2 = re.sub(r'''\bu(['"])(.*?)\1''', r'\1\2\1', want)  # py2: u''
+        got2 = re.sub(r'''\bb(['"])(.*?)\1''', r'\1\2\1', got)  # py3: b''
+        # py3: <exc.name>: b'<msg>' -> <name>: <msg>
+        #      <exc.name>: <others> -> <name>: <others>
+        got2 = re.sub(r'''^mercurial\.\w+\.(\w+): (['"])(.*?)\2''', r'\1: \3',
+                      got2, re.MULTILINE)
+        got2 = re.sub(r'^mercurial\.\w+\.(\w+): ', r'\1: ', got2, re.MULTILINE)
+        return any(doctest.OutputChecker.check_output(self, w, g, optionflags)
+                   for w, g in [(want, got), (want2, got2)])
+
 # TODO: migrate doctests to py3 and enable them on both versions
 def testmod(name, optionflags=0, testtarget=None, py2=True, py3=False):
     if not (not ispy3 and py2 or ispy3 and py3):
@@ -19,7 +32,16 @@ def testmod(name, optionflags=0, testtarget=None, py2=True, py3=False):
     mod = sys.modules[name]
     if testtarget is not None:
         mod = getattr(mod, testtarget)
-    doctest.testmod(mod, optionflags=optionflags)
+
+    # minimal copy of doctest.testmod()
+    finder = doctest.DocTestFinder()
+    checker = None
+    if ispy3:
+        checker = py3docchecker()
+    runner = doctest.DocTestRunner(checker=checker, optionflags=optionflags)
+    for test in finder.find(mod, name):
+        runner.run(test)
+    runner.summarize()
 
 testmod('mercurial.changegroup')
 testmod('mercurial.changelog')