copytrace: modifying buildstate behavior

Summary: Using it on 10000 commits makes SQL crashs because the queries are too big. Modifies buildstate to check if the oldest and most recent commits on the public branch, rebased to, are in the database, if yes: Assumes all of the nodes in between are. If not: goes through all the nodes by 1000 to check/retrieve them from the server. To avoid double checking if the hash is present in the database, during a retrievepkg, missing hashes are manually added and never asked to the server (this should mostly never happem)

Test Plan: The former tests still pass and buildstate doesn't raise an exception anymore when rebasing over 10000 commits

Reviewers: #sourcecontrol, rmcelroy

Differential Revision: https://phabricator.fb.com/D2718840

Tasks: 8660367

copytrace/copytrace.py

@@ -370,9 +370,23 @@ def buildstate(orig, repo, dest, rebaseset, collapsef, obsoletenotrebased):
             rev = rebaseset.first()
             rebased = repo[rev]
             ca = rebased.ancestor(dest)
-            ctxlist = list(repo.set("only(%r, %r)" % (dest.rev(), ca.rev())))
-            if ctxlist:
-                dbutil.checkpresence(repo, [ctx.hex() for ctx in ctxlist])
+
+            # Checking if the first and last revs are in the database
+            notin = dbutil.checkpresence(repo, [dest.hex(), ca.hex()],
+                                         True, False)
+
+            # If one of them is missing go through all
+            # Else assume that the ones in between should all be in
+            if notin:
+                ctxlist = list(repo.set("only(%r, %r)" %
+                               (dest.rev(), ca.rev())))
+                if ctxlist:
+                    maxi = int(repo.ui.config('copytrace', 'maxquery', '500'))
+                    length = len(ctxlist)
+                    for i in range(0, length, maxi):
+                        subctx = ctxlist[i:min(i+maxi, length)]
+                        dbutil.checkpresence(repo,
+                             [ctx.hex() for ctx in subctx], True, False)
 
     except Exception as e:
         error.logfailure(repo, e, "buildstate")

copytrace/dbutil.py

@@ -216,13 +216,13 @@ def insertrawdata(repo, dic, mapping={}):
     _close(conn, cursor)
 
 
-def retrievedatapkg(repo, ctxlist, move=False, askserver=True, addmissing=True):
+def retrievedatapkg(repo, ctxlist, move=False, askserver=False, addmissing=True):
     """
     retrieves {ctxhash: {dst: src}} for ctxhash in ctxlist for moves or copies
     """
     # Checks if the database has the data, else, asks it to the server, or adds
     # it manually
-    checkpresence(repo, ctxlist, askserver)
+    checkpresence(repo, ctxlist, askserver, addmissing)
 
     dbname, conn, cursor = _connect(repo)
 
@@ -317,20 +317,23 @@ def removectx(repo, ctx):
     _close(conn, cursor)
 
 
-def checkpresence(repo, ctxhashs, askserver=True):
+def checkpresence(repo, ctxhashs, askserver=False, addmissing=False):
     """
     checks if the ctx in ctxhashs are in the local database or requests for it
     """
+    missing = []
+
     if askserver:
         missing = _processmissing(repo, ctxhashs, mutable=False)
         # Requests the missing data to the server
         if missing and not repo.copytraceremote:
             _requestdata(repo, missing)
 
-    missing = _processmissing(repo, ctxhashs)
-    # Manually adds the still missing data
-    if missing:
-        _addmissingmoves(repo, missing)
+    if addmissing:
+        missing = _processmissing(repo, ctxhashs)
+        # Manually adds the still missing data
+        if missing:
+            _addmissingmoves(repo, missing)
 
     return missing
 

copytrace/filldb.py

@@ -128,7 +128,8 @@ def _fillctx(repo, ctxlist):
     check the presence of the ctx move data or adds it returning its parents
     """
     try:
-        added = dbutil.checkpresence(repo, ctxlist, askserver=False)
+        added = dbutil.checkpresence(repo, ctxlist, askserver=False,
+                                     addmissing=True)
         # The ctx was already processed, we don't check its parents
         if not added:
             return []
@@ -142,4 +143,4 @@ def _fillctx(repo, ctxlist):
                 parents.append(ctx.p2())
             return parents
     except:
-        repo.ui.warn("%s failed\n" % curctx.hex())
+        repo.ui.warn("%s failed\n" % ''.join(ctxlist))

tests/test-bundle2.t

@@ -90,7 +90,7 @@ REQUESTS MISSING MOVES DURING REBASE
   9c11d01510faa13840e36ea2d8acdd0b126cca67|||0
   $ hg rebase -s 9c11d0 -d 274c7e
   pulling move data from ssh://user@dummy/serverrepo
-  moves for 1 changesets retrieved
+  moves for 2 changesets retrieved
   rebasing 3:9c11d01510fa "mv a c" (tip)
   note: possible conflict - a was renamed multiple times to:
    b
@@ -103,6 +103,8 @@ REQUESTS MISSING MOVES DURING REBASE
   274c7e2c58b0256e17dc0f128380c8600bb0ee43|||0
   9c11d01510faa13840e36ea2d8acdd0b126cca67|a|c|1
   9c11d01510faa13840e36ea2d8acdd0b126cca67|||0
+  ac82d8b1f7c418c61a493ed229ffaa981bda8e90|||0
+  ac82d8b1f7c418c61a493ed229ffaa981bda8e90|||1
   $ cd ..
 
 
@@ -125,8 +127,12 @@ REBASING ON ANOTHER DRAFT BRANCH -- SERVER HAS NO MOVE DATA -- LOCAL DATA ERASED
   756b298ed880909df1cec4e7c763b22cc22064ff|||0
   9c11d01510faa13840e36ea2d8acdd0b126cca67|a|c|1
   9c11d01510faa13840e36ea2d8acdd0b126cca67|||0
+  ac82d8b1f7c418c61a493ed229ffaa981bda8e90|||0
+  ac82d8b1f7c418c61a493ed229ffaa981bda8e90|||1
   $ rm .hg/moves.db
   $ hg rebase -s 165d58 -d 756b29
+  pulling move data from ssh://user@dummy/serverrepo
+  moves for 1 changesets retrieved
   rebasing 5:165d58c1c606 "mv c e" (tip)
   note: possible conflict - c was renamed multiple times to:
    d
@@ -137,5 +143,7 @@ REBASING ON ANOTHER DRAFT BRANCH -- SERVER HAS NO MOVE DATA -- LOCAL DATA ERASED
   165d58c1c606c35cdad6f4fe1939d578513e6806|||0
   756b298ed880909df1cec4e7c763b22cc22064ff|c|d|1
   756b298ed880909df1cec4e7c763b22cc22064ff|||0
+  89c7ee4b298e2371d470910ff5a4ecce28ee49d9|b|c|1
+  89c7ee4b298e2371d470910ff5a4ecce28ee49d9|||0
   fa511326cccaa2c9933c752bd0009407f7cfcd2d|||0
   fa511326cccaa2c9933c752bd0009407f7cfcd2d|||1

tests/test-copytrace.t

@@ -71,9 +71,9 @@ Setup repo
 Rebase
   $ hg rebase -s 90e435664a -d 8762e63a42
   rebasing 1:90e435664a9d "mv a c, mod b"
+  merging c and cc to cc
   note: possible conflict - a was deleted and renamed to:
    aaa
-  merging c and cc to cc
   saved backup bundle to $TESTTMP/repo/.hg/strip-backup/90e435664a9d-d4904a7d-backup.hg (glob)
 
   $ hg log -G -T 'changeset: {node}\n desc: {desc}\n'