mirror of
https://github.com/facebook/sapling.git
synced 2024-10-10 08:47:12 +03:00
copytrace: modifying buildstate behavior
Summary: Using it on 10000 commits makes SQL crashs because the queries are too big. Modifies buildstate to check if the oldest and most recent commits on the public branch, rebased to, are in the database, if yes: Assumes all of the nodes in between are. If not: goes through all the nodes by 1000 to check/retrieve them from the server. To avoid double checking if the hash is present in the database, during a retrievepkg, missing hashes are manually added and never asked to the server (this should mostly never happem) Test Plan: The former tests still pass and buildstate doesn't raise an exception anymore when rebasing over 10000 commits Reviewers: #sourcecontrol, rmcelroy Differential Revision: https://phabricator.fb.com/D2718840 Tasks: 8660367
This commit is contained in:
parent
75943ea342
commit
6064bfa18a
@ -370,9 +370,23 @@ def buildstate(orig, repo, dest, rebaseset, collapsef, obsoletenotrebased):
|
||||
rev = rebaseset.first()
|
||||
rebased = repo[rev]
|
||||
ca = rebased.ancestor(dest)
|
||||
ctxlist = list(repo.set("only(%r, %r)" % (dest.rev(), ca.rev())))
|
||||
if ctxlist:
|
||||
dbutil.checkpresence(repo, [ctx.hex() for ctx in ctxlist])
|
||||
|
||||
# Checking if the first and last revs are in the database
|
||||
notin = dbutil.checkpresence(repo, [dest.hex(), ca.hex()],
|
||||
True, False)
|
||||
|
||||
# If one of them is missing go through all
|
||||
# Else assume that the ones in between should all be in
|
||||
if notin:
|
||||
ctxlist = list(repo.set("only(%r, %r)" %
|
||||
(dest.rev(), ca.rev())))
|
||||
if ctxlist:
|
||||
maxi = int(repo.ui.config('copytrace', 'maxquery', '500'))
|
||||
length = len(ctxlist)
|
||||
for i in range(0, length, maxi):
|
||||
subctx = ctxlist[i:min(i+maxi, length)]
|
||||
dbutil.checkpresence(repo,
|
||||
[ctx.hex() for ctx in subctx], True, False)
|
||||
|
||||
except Exception as e:
|
||||
error.logfailure(repo, e, "buildstate")
|
||||
|
@ -216,13 +216,13 @@ def insertrawdata(repo, dic, mapping={}):
|
||||
_close(conn, cursor)
|
||||
|
||||
|
||||
def retrievedatapkg(repo, ctxlist, move=False, askserver=True, addmissing=True):
|
||||
def retrievedatapkg(repo, ctxlist, move=False, askserver=False, addmissing=True):
|
||||
"""
|
||||
retrieves {ctxhash: {dst: src}} for ctxhash in ctxlist for moves or copies
|
||||
"""
|
||||
# Checks if the database has the data, else, asks it to the server, or adds
|
||||
# it manually
|
||||
checkpresence(repo, ctxlist, askserver)
|
||||
checkpresence(repo, ctxlist, askserver, addmissing)
|
||||
|
||||
dbname, conn, cursor = _connect(repo)
|
||||
|
||||
@ -317,20 +317,23 @@ def removectx(repo, ctx):
|
||||
_close(conn, cursor)
|
||||
|
||||
|
||||
def checkpresence(repo, ctxhashs, askserver=True):
|
||||
def checkpresence(repo, ctxhashs, askserver=False, addmissing=False):
|
||||
"""
|
||||
checks if the ctx in ctxhashs are in the local database or requests for it
|
||||
"""
|
||||
missing = []
|
||||
|
||||
if askserver:
|
||||
missing = _processmissing(repo, ctxhashs, mutable=False)
|
||||
# Requests the missing data to the server
|
||||
if missing and not repo.copytraceremote:
|
||||
_requestdata(repo, missing)
|
||||
|
||||
missing = _processmissing(repo, ctxhashs)
|
||||
# Manually adds the still missing data
|
||||
if missing:
|
||||
_addmissingmoves(repo, missing)
|
||||
if addmissing:
|
||||
missing = _processmissing(repo, ctxhashs)
|
||||
# Manually adds the still missing data
|
||||
if missing:
|
||||
_addmissingmoves(repo, missing)
|
||||
|
||||
return missing
|
||||
|
||||
|
@ -128,7 +128,8 @@ def _fillctx(repo, ctxlist):
|
||||
check the presence of the ctx move data or adds it returning its parents
|
||||
"""
|
||||
try:
|
||||
added = dbutil.checkpresence(repo, ctxlist, askserver=False)
|
||||
added = dbutil.checkpresence(repo, ctxlist, askserver=False,
|
||||
addmissing=True)
|
||||
# The ctx was already processed, we don't check its parents
|
||||
if not added:
|
||||
return []
|
||||
@ -142,4 +143,4 @@ def _fillctx(repo, ctxlist):
|
||||
parents.append(ctx.p2())
|
||||
return parents
|
||||
except:
|
||||
repo.ui.warn("%s failed\n" % curctx.hex())
|
||||
repo.ui.warn("%s failed\n" % ''.join(ctxlist))
|
||||
|
@ -90,7 +90,7 @@ REQUESTS MISSING MOVES DURING REBASE
|
||||
9c11d01510faa13840e36ea2d8acdd0b126cca67|||0
|
||||
$ hg rebase -s 9c11d0 -d 274c7e
|
||||
pulling move data from ssh://user@dummy/serverrepo
|
||||
moves for 1 changesets retrieved
|
||||
moves for 2 changesets retrieved
|
||||
rebasing 3:9c11d01510fa "mv a c" (tip)
|
||||
note: possible conflict - a was renamed multiple times to:
|
||||
b
|
||||
@ -103,6 +103,8 @@ REQUESTS MISSING MOVES DURING REBASE
|
||||
274c7e2c58b0256e17dc0f128380c8600bb0ee43|||0
|
||||
9c11d01510faa13840e36ea2d8acdd0b126cca67|a|c|1
|
||||
9c11d01510faa13840e36ea2d8acdd0b126cca67|||0
|
||||
ac82d8b1f7c418c61a493ed229ffaa981bda8e90|||0
|
||||
ac82d8b1f7c418c61a493ed229ffaa981bda8e90|||1
|
||||
$ cd ..
|
||||
|
||||
|
||||
@ -125,8 +127,12 @@ REBASING ON ANOTHER DRAFT BRANCH -- SERVER HAS NO MOVE DATA -- LOCAL DATA ERASED
|
||||
756b298ed880909df1cec4e7c763b22cc22064ff|||0
|
||||
9c11d01510faa13840e36ea2d8acdd0b126cca67|a|c|1
|
||||
9c11d01510faa13840e36ea2d8acdd0b126cca67|||0
|
||||
ac82d8b1f7c418c61a493ed229ffaa981bda8e90|||0
|
||||
ac82d8b1f7c418c61a493ed229ffaa981bda8e90|||1
|
||||
$ rm .hg/moves.db
|
||||
$ hg rebase -s 165d58 -d 756b29
|
||||
pulling move data from ssh://user@dummy/serverrepo
|
||||
moves for 1 changesets retrieved
|
||||
rebasing 5:165d58c1c606 "mv c e" (tip)
|
||||
note: possible conflict - c was renamed multiple times to:
|
||||
d
|
||||
@ -137,5 +143,7 @@ REBASING ON ANOTHER DRAFT BRANCH -- SERVER HAS NO MOVE DATA -- LOCAL DATA ERASED
|
||||
165d58c1c606c35cdad6f4fe1939d578513e6806|||0
|
||||
756b298ed880909df1cec4e7c763b22cc22064ff|c|d|1
|
||||
756b298ed880909df1cec4e7c763b22cc22064ff|||0
|
||||
89c7ee4b298e2371d470910ff5a4ecce28ee49d9|b|c|1
|
||||
89c7ee4b298e2371d470910ff5a4ecce28ee49d9|||0
|
||||
fa511326cccaa2c9933c752bd0009407f7cfcd2d|||0
|
||||
fa511326cccaa2c9933c752bd0009407f7cfcd2d|||1
|
||||
|
@ -71,9 +71,9 @@ Setup repo
|
||||
Rebase
|
||||
$ hg rebase -s 90e435664a -d 8762e63a42
|
||||
rebasing 1:90e435664a9d "mv a c, mod b"
|
||||
merging c and cc to cc
|
||||
note: possible conflict - a was deleted and renamed to:
|
||||
aaa
|
||||
merging c and cc to cc
|
||||
saved backup bundle to $TESTTMP/repo/.hg/strip-backup/90e435664a9d-d4904a7d-backup.hg (glob)
|
||||
|
||||
$ hg log -G -T 'changeset: {node}\n desc: {desc}\n'
|
||||
|
Loading…
Reference in New Issue
Block a user