py3: enable a number of hgsql related tests

Summary:
Now that we've upgraded mysql-connector-python, we can enable these
tests.

Reviewed By: quark-zju

Differential Revision: D21009185

fbshipit-source-id: e9ae62cadcc8d0a291381ab2cfb5c7bc04606d9e

eden/scm/edenscm/hgext/hgsql.py

@@ -70,7 +70,7 @@ from edenscm.mercurial import (
 )
 from edenscm.mercurial.i18n import _
 from edenscm.mercurial.node import bin, hex, nullid, nullrev
-from edenscm.mercurial.pycompat import encodeutf8, queue, range
+from edenscm.mercurial.pycompat import decodeutf8, encodeutf8, queue, range
 
 
 wrapcommand = extensions.wrapcommand
@@ -733,6 +733,8 @@ def wraprepo(repo):
 
             if reason is None:
                 reason = {MONONOKE_WRITE: MONONOKE_REASON}.get(state, DEFAULT_REASON)
+            else:
+                reason = decodeutf8(reason)
 
             return (readonly, reason)
 
@@ -763,12 +765,14 @@ def wraprepo(repo):
                 # SELECT GET_LOCK(...) will block. Break the lock attempt into
                 # smaller lock attempts
                 starttime = time.time()
+                locktimeout = float(self.locktimeout)
                 while True:
                     elapsed = time.time() - starttime
-                    if elapsed >= self.locktimeout:
+                    if elapsed >= locktimeout:
                         raise util.Abort(
                             "timed out waiting for mysql repo lock (%s)" % lockname
                         )
+
                     # Sync outside the SQL lock hoping that the repo is closer
                     # to the SQL repo when we got the lock.
                     self.pullfromdb(enforcepullfromdb=True)
@@ -922,7 +926,7 @@ def wraprepo(repo):
             sqlsynchash = sqlresults[0][0][0]
             if len(sqlsynchash) != 40:
                 raise RuntimeError("malicious SHA1 returned by MySQL: %r" % sqlsynchash)
-            return sqlsynchash
+            return decodeutf8(sqlsynchash)
 
         def needsync(self):
             """Returns True if the local repo is not in sync with the database.
@@ -939,11 +943,11 @@ def wraprepo(repo):
             sqlbookmarks = {}
             tip = -1
             for namespace, name, value in self.sqlcursor:
-                if namespace == "heads":
+                if namespace == b"heads":
                     sqlheads.add(bin(value))
-                elif namespace == "bookmarks":
-                    sqlbookmarks[name] = bin(value)
-                elif namespace == "tip":
+                elif namespace == b"bookmarks":
+                    sqlbookmarks[decodeutf8(name)] = bin(value)
+                elif namespace == b"tip":
                     tip = int(value)
 
             # Since we don't have the lock right now, and since this is the
@@ -1156,7 +1160,10 @@ def wraprepo(repo):
                         WHERE namespace = 'bookmarks' AND repo = %s""",
                         (self.sqlreponame,),
                     )
-                    fetchedbookmarks = self.sqlcursor.fetchall()
+                    fetchedbookmarks = [
+                        (decodeutf8(name), node)
+                        for name, node in self.sqlcursor.fetchall()
+                    ]
 
                     changes = []
                     for name, node in fetchedbookmarks:
@@ -1184,7 +1191,7 @@ def wraprepo(repo):
             # that don't exist in the loaded changelog. So let's force loading
             # bookmarks now.
             bm = self._bookmarks
-            self.sharedvfs.write("lastsqlsync", str(int(time.time())))
+            self.sharedvfs.write("lastsqlsync", encodeutf8(str(int(time.time()))))
 
         def fetchthread(self, queue, abort, fetchstart, fetchend):
             """Fetches every revision from fetchstart to fetchend (inclusive)
@@ -1209,6 +1216,8 @@ def wraprepo(repo):
                     # Put split chunks back together into a single revision
                     groupedrevdata = {}
                     for revdata in self.sqlcursor:
+                        revdata = (decodeutf8(revdata[0]),) + revdata[1:]
+
                         name = revdata[0]
                         chunk = revdata[1]
                         linkrev = revdata[3]
@@ -1286,7 +1295,7 @@ def wraprepo(repo):
             )
             headsindb = cursor.fetchall()
             for head in headsindb:
-                head = head[0]
+                head = decodeutf8(head[0])
                 if head in newheads:
                     newheads.discard(head)
                 else:
@@ -1316,6 +1325,8 @@ def wraprepo(repo):
             )
             bookmarksindb = cursor.fetchall()
             for k, v in bookmarksindb:
+                k = decodeutf8(k)
+                v = decodeutf8(v)
                 if newbookmarks.get(k) == v:
                     del newbookmarks[k]
                 else:
@@ -1596,6 +1607,8 @@ def wraprepo(repo):
                 )
 
                 for path, rev, node in cursor:
+                    path = decodeutf8(path)
+                    node = decodeutf8(node)
                     rev = int(rev)
                     checkrevs.remove((path, rev))
                     rl = None
@@ -1665,16 +1678,19 @@ def wraprepo(repo):
         (versus the default byte arrays)."""
 
         def _STRING_to_python(self, value, dsc=None):
-            return str(value)
+            return bytes(value)
 
         def _VAR_STRING_to_python(self, value, dsc=None):
-            return str(value)
+            return bytes(value)
 
         def _BLOB_to_python(self, value, dsc=None):
-            return str(value)
+            return bytes(value)
 
         def _bytearray_to_mysql(self, value, dsc=None):
-            return str(value)
+            return bytes(value)
+
+        def _memoryview_to_mysql(self, value, dsc=None):
+            return value.tobytes()
 
 
 class bufferedopener(object):
@@ -1700,7 +1716,7 @@ class bufferedopener(object):
 
         if buffer:
             fp = self.opener(self.path, self.mode)
-            fp.write("".join(buffer))
+            fp.write(b"".join(buffer))
             fp.close()
 
     def close(self):

eden/scm/edenscm/hgext/infinitepush/sqlindex.py

@@ -436,14 +436,14 @@ class CustomConverter(mysql.connector.conversion.MySQLConverter):
     (versus the default byte arrays)."""
 
     def _STRING_to_python(self, value, dsc=None):
-        return str(value)
+        return bytes(value)
 
     def _VAR_STRING_to_python(self, value, dsc=None):
-        return str(value)
+        return bytes(value)
 
     def _BLOB_to_python(self, value, dsc=None):
-        return str(value)
+        return bytes(value)
 
     # localstr is Mercurial-specific. See encoding.py
     def _localstr_to_mysql(self, value):
-        return str(value)
+        return bytes(value)

eden/scm/edenscm/mercurial/pycompat.py

@@ -185,16 +185,16 @@ else:
     rawinput = raw_input  # noqa
 
     def encodeutf8(s):
-        # type: (bytes) -> bytes
+        # type: (str) -> bytes
         if istest():
-            assert isinstance(s, bytes)
+            assert isinstance(s, str), "expected str, actual %s" % s.__class__
 
         return s
 
     def decodeutf8(s, errors="strict"):
         # type: (bytes, str) -> bytes
         if istest():
-            assert isinstance(s, bytes)
+            assert isinstance(s, bytes), "expected bytes, actual %s" % s.__class__
         return s
 
     def iteritems(s):

eden/scm/edenscm/mercurial/statprof.py

@@ -709,14 +709,16 @@ def display_hotpath(data, fp, limit=0.05, **kwargs):
     def _write(node, depth, multiple_siblings):
         site = node.site
         visiblechildren = [
-            c for c in node.children.itervalues() if c.count >= (limit * root.count)
+            c
+            for c in pycompat.itervalues(node.children)
+            if c.count >= (limit * root.count)
         ]
         if site:
             indent = depth * 2 - 1
             filename = ""
             function = ""
             if len(node.children) > 0:
-                childsite = list(node.children.itervalues())[0].site
+                childsite = list(node.children.values())[0].site
                 filename = (childsite.filename() + ":").ljust(15)
                 function = childsite.function
 
@@ -735,7 +737,7 @@ def display_hotpath(data, fp, limit=0.05, **kwargs):
             codestring = codepattern % ("line", site.lineno, site.getsource(30))
 
             finalstring = liststring + codestring
-            childrensamples = sum([c.count for c in node.children.itervalues()])
+            childrensamples = sum([c.count for c in pycompat.itervalues(node.children)])
             # Make frames that performed more than 10% of the operation red
             if node.count - childrensamples > (0.1 * root.count):
                 finalstring = redformat % finalstring

eden/scm/tests/test-commitcloud-backup-sql1.t

@@ -156,14 +156,14 @@ hg cloud listbackups should also go in MRU order
   $ hg cloud listbackups --json
   {
       "mydevhost": [
-          "$TESTTMP/client5", 
+          "$TESTTMP/client5",* (glob)
           "$TESTTMP/client1"
-      ], 
+      ],* (glob)
       "ourdevhost": [
           "$TESTTMP/client4"
-      ], 
+      ],* (glob)
       "devhost": [
-          "$TESTTMP/client3", 
+          "$TESTTMP/client3",* (glob)
           "$TESTTMP/client2\\dev"
       ]
   }

eden/scm/tests/test-fb-hgext-pushrebase-hgsql-mfs.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
 TODO: Make this test work with obsstore

eden/scm/tests/test-hgsql-bundle2.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
   $ . "$TESTDIR/hgsql/library.sh"

eden/scm/tests/test-hgsql-corruption.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
   $ . "$TESTDIR/hgsql/library.sh"
@@ -30,8 +29,8 @@
 - The huge number in the output below is because we're trying to apply rev 0
 (which contains the generaldelta bit in the offset int) to a non-rev 0
 location (so the generaldelta bit isn't stripped before the comparison)
-  $ hg log -l 1 2>&1 | egrep 'Corruption'
-  CorruptionException: revision offset doesn't match prior length (12884967424 offset vs 3 length): data/z.i
+  $ hg log -l 1 2>&1 | egrep 'Corruption.*offset'
+  *CorruptionException: revision offset doesn't match prior length (12884967424 offset vs 3 length): data/z.i (glob)
 
 # Recover middle commit, but on top, then try syncing (succeeds)
 
@@ -51,8 +50,8 @@ location (so the generaldelta bit isn't stripped before the comparison)
   $ echo a > a
   $ hg commit -qAm a
   $ cd ../master
-  $ hg pull ../client 2>&1 | egrep 'Corruption'
-  CorruptionException: expected node d34c38483be9d08f205eaae60c380a29b48e0189 at rev 1 of 00changelog.i but found bc3a71defa4a8fb6e8e5c192c02a26d94853d281
+  $ hg pull ../client 2>&1 | egrep 'Corruption.*node'
+  *CorruptionException: expected node d34c38483be9d08f205eaae60c380a29b48e0189 at rev 1 of 00changelog.i but found bc3a71defa4a8fb6e8e5c192c02a26d94853d281 (glob)
 
 # Fix ordering, can pull now
 

eden/scm/tests/test-hgsql-hggit.t

@@ -1,4 +1,3 @@
-#require py2
 
   $ . "$RUNTESTDIR/hgsql/library.sh"
   $ . "$RUNTESTDIR/hggit/testutil"

eden/scm/tests/test-hgsql-lfs-sync.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
 Inital setup

eden/scm/tests/test-hgsql-profiler.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
   $ . "$TESTDIR/hgsql/library.sh"

eden/scm/tests/test-hgsql-pushpull.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
   $ . "$TESTDIR/hgsql/library.sh"

eden/scm/tests/test-hgsql-pushrebase.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
   $ . "$TESTDIR/hgsql/library.sh"

eden/scm/tests/test-hgsql-pushrebase2.t

@@ -1,4 +1,3 @@
-#require py2
 #require no-windows
 
 Test:
@@ -57,10 +56,11 @@ Another prepushrebase hook just to warm up in-memory repo states (changelog and
 manifest).
 
   $ cat > $TESTTMP/prepushrebase.py <<EOF
+  > from mercurial.pycompat import decodeutf8
   > def warmup(ui, repo, *args, **kwds):
   >     # Just have some side-effect loading the changelog and manifest
   >     data = repo['tip']['A'].data()
-  >     ui.write_err('prepushrebase hook called. A = %r\n' % data)
+  >     ui.write_err('prepushrebase hook called. A = %r\n' % decodeutf8(data))
   > EOF
 
 Setup prepushrebase hooks.

eden/scm/tests/test-hgsql-race-conditions.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
 no-check-code

eden/scm/tests/test-hgsql-repolock.t

@@ -1,4 +1,3 @@
-#require py2
   $ . "$TESTDIR/hgsql/library.sh"
   $ initserver master masterrepo
   $ cd master

eden/scm/tests/test-hgsql-requires.t

@@ -1,4 +1,3 @@
-#require py2
   $ . "$TESTDIR/hgsql/library.sh"
   $ disable treemanifest
 

eden/scm/tests/test-hgsql-sqlrefill.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
   $ . "$TESTDIR/hgsql/library.sh"
@@ -29,7 +28,7 @@ Configure master as a server backed by sql.
   $ configureserver master masterrepo
   $ cd master
   $ hg log -GT '{files}' 2>&1 | grep "CorruptionException:"
-  CorruptionException: heads don't match after sync
+  *CorruptionException: heads don't match after sync (glob)
 
 
 Fix the server using sqlrefill.

eden/scm/tests/test-hgsql-sqlverify.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
   $ . "$TESTDIR/hgsql/library.sh"
@@ -34,8 +33,8 @@ Run with incorrect local revlogs
   $ hg log -r tip --forcesync -T '{desc}\n'
   add b
   $ hg sqlverify >$TESTTMP/sqlverify.out 2>&1 || true
-  $ grep Corruption $TESTTMP/sqlverify.out || cat $TESTTMP/sqlverify.out
-  CorruptionException: * with linkrev *, disk does not match mysql (glob)
+  $ grep "Corruption.*linkrev" $TESTTMP/sqlverify.out || cat $TESTTMP/sqlverify.out
+  *CorruptionException: * with linkrev *, disk does not match mysql (glob)
 
   $ hg debugstrip -q -r 1: --config hgsql.bypass=True --no-backup
   $ hg log -r tip --forcesync -T '\n'

eden/scm/tests/test-hgsql-strip.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
   $ . "$TESTDIR/hgsql/library.sh"
@@ -115,7 +114,7 @@ Verify master is broken
 
   $ cd ../master
   $ hg log 2>&1 | egrep 'CorruptionException:'
-  CorruptionException: heads don't match after sync
+  *CorruptionException: heads don't match after sync (glob)
 
 Run sqlstrip on master as well
 

eden/scm/tests/test-hgsql-sync-interval.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
 Test hgsql tries to sync before entering the critical section

eden/scm/tests/test-hgsql-sync.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
 #testcases case-innodb case-rocksdb
@@ -155,14 +154,16 @@
   > from edenscm.mercurial import demandimport, extensions
   > with demandimport.deactivated():
   >     import mysql.connector
-  > watchstrings = os.environ.get("INSPECTSQL")
-  > if watchstrings:
-  >     watchstrings = watchstrings.split(',')
-  > def printsql(orig, *args, **kwargs):
-  >     if not watchstrings or any(s for s in watchstrings if s in args[1]):
-  >         print(args[1] % args[2], file=sys.stderr)
-  >     return orig(*args, **kwargs)
-  > extensions.wrapfunction(mysql.connector.cursor.MySQLCursor, "execute", printsql)
+  > def uisetup(ui):
+  >     watchstrings = os.environ.get("INSPECTSQL")
+  >     if watchstrings:
+  >         watchstrings = watchstrings.split(',')
+  >     def printsql(orig, *args, **kwargs):
+  >         if not watchstrings or any(s for s in watchstrings if s in args[1]):
+  >             ui.warn(args[1] % args[2], file=sys.stderr)
+  >             ui.warn("\n")
+  >         return orig(*args, **kwargs)
+  >     extensions.wrapfunction(mysql.connector.cursor.MySQLCursor, "execute", printsql)
   > EOF
   $ cat >> $HGRCPATH <<EOF
   > [extensions]

eden/scm/tests/test-hgsql-treemanifest.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible