copytrace: requesting missing move data to the server when info is missing

Summary: If some contexts are not present in the local database during a retrieve, a request is send to the server for those moves one time through a 'pull(revs)' call. When doing a rebase, a check is performed beforehand to see if the local database contains the contexts of the rebased-to branch.

Test Plan: Rebasing in the middle of a branch for which the moves are not known

Reviewers: #sourcecontrol, rmcelroy

Differential Revision: https://phabricator.fb.com/D2634365

Tasks: 8660367

copytrace/__init__.py

@@ -5,6 +5,7 @@ import filldb
 import copytrace
 import bundle2
 
+
 def extsetup(ui):
     wrapfunction(cmdutil, 'commit', filldb.commit)
     wrapfunction(cmdutil, 'amend', filldb.amend)
@@ -12,6 +13,7 @@ def extsetup(ui):
 
     wrapfunction(copies, 'mergecopies', copytrace.mergecopieswithdb)
     wrapfunction(copies, 'pathcopies', copytrace.pathcopieswithdb)
+    wrapfunction(rebase, 'buildstate', copytrace.buildstate)
 
     wrapfunction(exchange, '_pullbundle2extraprepare',
                 bundle2._pullbundle2extraprepare)

copytrace/bundle2.py

@@ -3,15 +3,56 @@
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
-from mercurial import bundle2, util, exchange
+from mercurial import bundle2, util, exchange, hg, error
+from mercurial.i18n import _
 import dbutil
 
 
-# Temporarily used to force to load the module, will be completed later
+# Temporarily used to force to load the module
 def _pullbundle2extraprepare(orig, pullop, kwargs):
     return orig(pullop, kwargs)
 
 
+def pullmoves(repo, nodelist, source="default"):
+    """
+    Fetch move data from the server
+    """
+    source, branches = hg.parseurl(repo.ui.expandpath(source))
+    # No default server defined
+    try:
+        remote = hg.peer(repo, {}, source)
+    except:
+        return
+    repo.ui.status(_('pulling move data from %s\n') % util.hidepassword(source))
+    pullop = exchange.pulloperation(repo, remote, nodelist, False)
+    lock = pullop.repo.lock()
+    try:
+        pullop.trmanager = exchange.transactionmanager(repo, 'pull',
+                                                       remote.url())
+        _pullmovesbundle2(pullop)
+        pullop.trmanager.close()
+    finally:
+        pullop.trmanager.release()
+        lock.release()
+
+
+def _pullmovesbundle2(pullop):
+    """
+    Pull move data
+    """
+    kwargs = {}
+    kwargs['bundlecaps'] = exchange.caps20to10(pullop.repo)
+    kwargs['movedatareq'] = pullop.heads
+    kwargs['common'] = pullop.heads
+    kwargs['heads'] = pullop.heads
+    kwargs['cg'] = False
+    bundle = pullop.remote.getbundle('pull', **kwargs)
+    try:
+        op = bundle2.processbundle(pullop.repo, bundle, pullop.gettransaction)
+    except error.BundleValueError as exc:
+        raise error.Abort('missing support for %s' % exc)
+
+
 @exchange.getbundle2partsgenerator('pull:movedata')
 def _getbundlemovedata(bundler, repo, source, bundlecaps=None, heads=None,
                        common=None,  b2caps=None, **kwargs):

copytrace/copytrace.py

@@ -266,6 +266,7 @@ def _processrenames(repo, ctx, renamed, move=False):
         if not src in m:
             del renamed[src]
 
+
 def _forwardrenameswithdb(a, b, match=None, move=False):
     """
     find {dst@b: src@a} renames mapping where a is an ancestor of b
@@ -329,3 +330,18 @@ def pathcopieswithdb(orig, x, y, match=None):
         return _backwardrenameswithdb(x, y)
     return _chain(x, y, _backwardrenameswithdb(x, a),
                    _forwardrenameswithdb(a, y, match=match))
+
+
+def buildstate(orig, repo, dest, rebaseset, collapsef, obsoletenotrebased):
+    """
+    wraps the command to get the set of revs that will be involved in the
+    rebase and checks if they are in the database
+    """
+    if rebaseset:
+        rev = rebaseset.first()
+        rebased = repo[rev]
+        ca = rebased.ancestor(dest)
+        ctxlist = list(repo.set("only(%r, %r)" % (dest.rev(), ca.rev())))
+        if ctxlist:
+            dbutil.checkpresence(repo, ctxlist)
+    return orig(repo, dest, rebaseset, collapsef, obsoletenotrebased)

copytrace/dbutil.py

@@ -8,7 +8,8 @@
 # GNU General Public License version 2 or any later version.
 
 
-from mercurial import scmutil, util
+from mercurial import scmutil, util, commands
+import bundle2
 import sqlite3
 
 
@@ -79,7 +80,6 @@ def insertdata(repo, ctx, mvdict, cpdict, remote=False):
         ctxhash = '0'
     else:
         ctxhash = str(ctx.hex())
-
     try:
         insertitem(c, ctxhash, mvdict, True)
         insertitem(c, ctxhash, cpdict, False)
@@ -90,7 +90,7 @@ def insertdata(repo, ctx, mvdict, cpdict, remote=False):
     _close(conn)
 
 
-def retrievedata(repo, ctx, move=False, remote=False):
+def retrievedata(repo, ctx, move=False, remote=False, askserver=True):
     """
     returns the {dst:src} dictonary for moves if move = True or of copies if
     move = False for ctx
@@ -113,6 +113,14 @@ def retrievedata(repo, ctx, move=False, remote=False):
     all_rows = c.fetchall()
     _close(conn)
     ret = {}
+
+    # The local database doesn't have the data for this ctx and hasn't tried
+    # to retrieve it yet (askserver)
+    if askserver and not remote and not all_rows:
+        _requestdata(repo, [ctx])
+        return retrievedata(repo, ctx, move=move, remote=remote,
+                            askserver=False)
+
     for src, dst in all_rows:
         # this ctx is registered but has no move data
         if not dst:
@@ -142,7 +150,7 @@ def insertrawdata(repo, dic, remote=False):
     _close(conn)
 
 
-def retrieverawdata(repo, ctxlist, remote=False):
+def retrieverawdata(repo, ctxlist, remote=False, askserver=True):
     """
     retrieves {ctxhash: {dst: src}} for ctxhash in ctxlist for moves or copies
     """
@@ -167,6 +175,17 @@ def retrieverawdata(repo, ctxlist, remote=False):
         ret.setdefault(ctxhash.encode('utf8'), []).append((src.encode('utf8'),
              dst.encode('utf8'), mv.encode('utf8')))
 
+    processed = ret.keys()
+    missing = [f for f in ctxlist if f not in processed]
+
+    # The local database doesn't have the data for this ctx and hasn't tried
+    # to retrieve it yet (askserver)
+    if askserver and not remote and missing:
+        _requestdata(repo, missing)
+        add = retrieverawdata(repo, missing, move=move, remote=remote,
+                              askserver=False)
+        ret.update(add)
+
     return ret
 
 
@@ -187,3 +206,33 @@ def removectx(repo, ctx, remote=False):
         raise util.Abort('could not delete ctx from the %s database' % dbname)
 
     _close(conn)
+
+
+def checkpresence(repo, ctxlist):
+    """
+    checks if the ctx in ctxlist are present in the database or requests for it
+    """
+    ctxhashs = [ctx.hex() for ctx in ctxlist]
+    dbname, conn, c = _connect(repo, False)
+    try:
+        c.execute('SELECT DISTINCT hash FROM Moves WHERE hash IN (%s)'
+                  % (','.join('?' * len(ctxhashs))), ctxhashs)
+    except:
+        raise util.Abort('could not check ctx presence in the %s database'
+                         % dbname)
+    processed = c.fetchall()
+    _close(conn)
+    processed = [ctx[0].encode('utf8') for ctx in processed]
+    missing = [repo[f].node() for f in ctxlist if f not in processed]
+    if missing:
+        _requestdata(repo, missing)
+
+
+def _requestdata(repo, nodelist):
+    """
+    Requests missing ctx data to a server
+    """
+    try:
+        bundle2.pullmoves(repo, nodelist)
+    except:
+        pass

tests/test-bundle2.t

@@ -102,3 +102,86 @@ PULLS FROM SERVER
   ec660297011163dd7658d444365b6590c0dd67b3|b|d|1
   ec660297011163dd7658d444365b6590c0dd67b3|||0
   $ cd ..
+  $ rm -rf serverrepo
+  $ rm -rf clientrepo
+
+
+
+!! TEST 2: pulling missing move data from repo when rebasing !!
+
+SETUP SERVER REPO
+
+  $ hg init serverrepo
+  $ cd serverrepo
+  $ touch a b
+  $ hg add a b
+  $ hg commit -m "add a b"
+  $ hg mv a c
+  $ hg commit -m "mv a c"
+  $ hg mv c d
+  $ hg commit -m "mv c d"
+  $ hg mv d e
+  $ hg commit -m "mv d e"
+  $ cd ..
+
+SETUP CLIENT REPO
+
+  $ hg clone serverrepo clientrepo
+  updating to branch default
+  2 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  $ cd clientrepo
+  $ hg log -G -T 'changeset: {node}\n desc: {desc}\n'
+  @  changeset: 2a998f0bae7cad015586b9a9e5e8a05b4b7d281f
+  |   desc: mv d e
+  o  changeset: d4670020b03d62be270c7f8c22d1bf620c4c8f4a
+  |   desc: mv c d
+  o  changeset: a003d50a0eea20c381b92e9200e323f3c945c473
+  |   desc: mv a c
+  o  changeset: 2f1222a290f07a1758cc927c57cc22805d6696ed
+      desc: add a b
+  $ hg update -q 2f1222
+  $ hg mv b z
+  $ hg commit -q -m "mv b z"
+  $ hg log -G -T 'changeset: {node}\n desc: {desc}\n'
+  @  changeset: d9e9933769659048c7efa24b53b2e38a1d8205b2
+  |   desc: mv b z
+  | o  changeset: 2a998f0bae7cad015586b9a9e5e8a05b4b7d281f
+  | |   desc: mv d e
+  | o  changeset: d4670020b03d62be270c7f8c22d1bf620c4c8f4a
+  | |   desc: mv c d
+  | o  changeset: a003d50a0eea20c381b92e9200e323f3c945c473
+  |/    desc: mv a c
+  o  changeset: 2f1222a290f07a1758cc927c57cc22805d6696ed
+      desc: add a b
+  $ sqlite3 .hg/moves.db "SELECT * FROM Moves" | sort
+  d9e9933769659048c7efa24b53b2e38a1d8205b2|b|z|1
+  d9e9933769659048c7efa24b53b2e38a1d8205b2|||0
+  $ hg rebase -s d9e993 -d d46700
+  pulling move data from $TESTTMP/serverrepo
+  moves for 2 changesets retrieved
+  rebasing 4:d9e993376965 "mv b z" (tip)
+  saved backup bundle to $TESTTMP/clientrepo/.hg/strip-backup/d9e993376965-0332a78c-backup.hg (glob)
+  $ hg log -G -T 'changeset: {node}\n desc: {desc}\n'
+  @  changeset: daf6369e3e011c90ecd56144609c0e8fd823e83b
+  |   desc: mv b z
+  | o  changeset: 2a998f0bae7cad015586b9a9e5e8a05b4b7d281f
+  |/    desc: mv d e
+  o  changeset: d4670020b03d62be270c7f8c22d1bf620c4c8f4a
+  |   desc: mv c d
+  o  changeset: a003d50a0eea20c381b92e9200e323f3c945c473
+  |   desc: mv a c
+  o  changeset: 2f1222a290f07a1758cc927c57cc22805d6696ed
+      desc: add a b
+  $ sqlite3 .hg/moves.db "SELECT * FROM Moves" | sort
+  0|a|d|0
+  0|b|z|0
+  0|||1
+  a003d50a0eea20c381b92e9200e323f3c945c473|a|c|1
+  a003d50a0eea20c381b92e9200e323f3c945c473|||0
+  d4670020b03d62be270c7f8c22d1bf620c4c8f4a|c|d|1
+  d4670020b03d62be270c7f8c22d1bf620c4c8f4a|||0
+  d9e9933769659048c7efa24b53b2e38a1d8205b2|b|z|1
+  d9e9933769659048c7efa24b53b2e38a1d8205b2|||0
+  daf6369e3e011c90ecd56144609c0e8fd823e83b|b|z|1
+  daf6369e3e011c90ecd56144609c0e8fd823e83b|||0
+