mirror of
https://github.com/facebook/sapling.git
synced 2024-10-07 07:17:55 +03:00
p4fastimport: fix sync commit - complete workflow
Differential Revision: https://phabricator.intern.facebook.com/D5966213
This commit is contained in:
Zhihui Huang
parent
e4f962c4d2
commit
a6d5ff8f57
@ -346,37 +346,22 @@ def p4syncimport(ui, repo, client, **opts):
|
||||
scmutil.checknewlabel(repo, opts['bookmark'], 'bookmark')
|
||||
|
||||
if len(repo) == 0:
|
||||
raise error.Abort(_('p4 blob commit does not support empty repo yet.'))
|
||||
raise error.Abort(_('p4 sync commit does not support empty repo yet.'))
|
||||
|
||||
p1ctx, startcl, __ = startfrom(ui, repo, opts)
|
||||
p1ctx, __, __ = startfrom(ui, repo, opts)
|
||||
|
||||
# Fail if the specified client does not exist
|
||||
if not p4.exists_client(client):
|
||||
raise error.Abort(_('p4 client %s does not exist.') % client)
|
||||
|
||||
# Return all the changelists touching files in a given client view.
|
||||
ui.note(_('loading changelist numbers.\n'))
|
||||
changelists = sorted(p4.parse_changes(
|
||||
client, startcl=startcl, endcl=startcl))
|
||||
ui.note(_('%d changelists to import.\n') % len(changelists))
|
||||
if len(changelists) == 0:
|
||||
return
|
||||
# Get a list of files that we will have to import
|
||||
basepath = opts.get('path')
|
||||
startcl, endcl = startcl, startcl
|
||||
ui.note(_('loading list of files.\n'))
|
||||
filelist = getfilelist(ui, p4.parse_filelist(client, startcl, endcl))
|
||||
ui.note(_('%d files to import.\n') % len(filelist))
|
||||
latestcl = p4.get_latest_cl(client)
|
||||
if latestcl is None:
|
||||
raise error.Abort(_('Cannot find latest p4 changelist number.'))
|
||||
|
||||
importset = importer.ImportSet(repo, client, changelists,
|
||||
filelist, basepath)
|
||||
p4filelogs = []
|
||||
for i, f in enumerate(importset.filelogs()):
|
||||
ui.debug('reading filelog %s\n' % f.depotfile)
|
||||
ui.progress(_('reading filelog'), i, unit=_('filelogs'),
|
||||
total=len(filelist))
|
||||
p4filelogs.append(f)
|
||||
ui.progress(_('reading filelog'), None)
|
||||
ui.note(_('Latest change list number %s\n') % latestcl)
|
||||
p4filelogs = p4.get_filelogs_at_cl(client, latestcl)
|
||||
p4filelogs = sorted(p4filelogs)
|
||||
|
||||
# sync import.
|
||||
with repo.wlock(), repo.lock():
|
||||
@ -384,12 +369,12 @@ def p4syncimport(ui, repo, client, **opts):
|
||||
count = 0
|
||||
fileinfo = {}
|
||||
largefileslist = []
|
||||
p4filelogs = sorted(p4filelogs)
|
||||
tr = repo.transaction('syncimport')
|
||||
try:
|
||||
for p4fl in p4filelogs:
|
||||
bfi = importer.BlobFileImporter(ui, repo, importset, p4fl)
|
||||
# Create filelog.
|
||||
bfi = importer.SyncFileImporter(
|
||||
ui, repo, client, latestcl, p4fl)
|
||||
# Create hg filelog
|
||||
fileflags, largefiles, oldtiprev, newtiprev = bfi.create(tr)
|
||||
fileinfo[p4fl.depotfile] = {
|
||||
'flags': fileflags,
|
||||
@ -399,8 +384,8 @@ def p4syncimport(ui, repo, client, **opts):
|
||||
largefileslist.extend(largefiles)
|
||||
count += 1
|
||||
# Generate manifest and changelog
|
||||
clog = importer.BlobChangeManifestImporter(ui, repo, importset,
|
||||
p1ctx=p1ctx)
|
||||
clog = importer.SyncChangeManifestImporter(
|
||||
ui, repo, client, latestcl, p1ctx=p1ctx)
|
||||
revisions = []
|
||||
for cl, hgnode in clog.creategen(tr, fileinfo):
|
||||
revisions.append((cl, hex(hgnode)))
|
||||
@ -411,12 +396,11 @@ def p4syncimport(ui, repo, client, **opts):
|
||||
|
||||
if ui.config('p4fastimport', 'lfsmetadata', None) is not None:
|
||||
ui.note(_('writing lfs metadata to sqlite\n'))
|
||||
writelfsmetadata(largefiles, revisions,
|
||||
writelfsmetadata(largefileslist, revisions,
|
||||
ui.config('p4fastimport', 'lfsmetadata', None))
|
||||
|
||||
tr.close()
|
||||
ui.note(_('%d revision(s), %d file(s) imported.\n') % (
|
||||
len(changelists), count))
|
||||
ui.note(_('1 revision, %d file(s) imported.\n') % count)
|
||||
finally:
|
||||
tr.release()
|
||||
|
||||
|
||||
@ -22,6 +22,9 @@ KEYWORD_REGEX = "\$(Id|Header|DateTime|" + \
|
||||
"Date|Change|File|" + \
|
||||
"Revision|Author).*?\$"
|
||||
|
||||
#TODO: make p4 user configurable
|
||||
P4_ADMIN_USER = 'p4admin'
|
||||
|
||||
def relpath(client, depotfile):
|
||||
where = p4.parse_where(client, depotfile)
|
||||
filename = where['clientFile'].replace('//%s/' % client, '')
|
||||
@ -147,7 +150,48 @@ class ChangeManifestImporter(object):
|
||||
user=username, date=date,
|
||||
extra={'p4changelist': cl})
|
||||
|
||||
class BlobChangeManifestImporter(ChangeManifestImporter):
|
||||
class SyncChangeManifestImporter(ChangeManifestImporter):
|
||||
def __init__(self, ui, repo, client, cl, p1ctx):
|
||||
self._ui = ui
|
||||
self._repo = repo
|
||||
self._client = client
|
||||
self._cl = cl
|
||||
self._p1ctx = p1ctx
|
||||
|
||||
def creategen(self, tr, fileinfo):
|
||||
mrevlog = self._repo.manifestlog._revlog
|
||||
clog = self._repo.changelog
|
||||
cp1 = self._p1ctx.node()
|
||||
cp2 = nullid
|
||||
p1 = self._repo[cp1]
|
||||
mp1 = p1.manifestnode()
|
||||
mp2 = nullid
|
||||
mf = manifest.manifestdict()
|
||||
changed = []
|
||||
|
||||
for info in fileinfo.values():
|
||||
localname = info['localname']
|
||||
baserev = info['baserev']
|
||||
mf[localname] = self._repo.file(localname).node(baserev)
|
||||
changed.append(localname)
|
||||
linkrev = len(self._repo)
|
||||
oldmp1 = mp1
|
||||
mp1 = mrevlog.addrevision(mf.text(mrevlog._usemanifestv2), tr,
|
||||
linkrev, mp1, mp2)
|
||||
|
||||
self._ui.debug('changelist %d: writing manifest. '
|
||||
'node: %s p1: %s p2: %s linkrev: %d\n' % (
|
||||
self._cl, short(mp1), short(oldmp1), short(mp2), linkrev))
|
||||
|
||||
desc = 'p4fastimport synchronizing client view'
|
||||
username = P4_ADMIN_USER
|
||||
self._ui.debug('changelist %d: writing changelog: %s\n' % (
|
||||
self._cl, desc))
|
||||
cp1 = self.writechangelog(
|
||||
clog, mp1, changed, desc, tr, cp1, cp2,
|
||||
username, None, self._cl)
|
||||
yield self._cl, cp1
|
||||
|
||||
def writechangelog(
|
||||
self, clog, mp1, changed, desc, tr, cp1, cp2, username, date, cl):
|
||||
return clog.add(
|
||||
@ -378,61 +422,65 @@ class FileImporter(object):
|
||||
newlen = len(hgfilelog)
|
||||
return fileflags, largefiles, origlen, newlen
|
||||
|
||||
class BlobFileImporter(FileImporter):
|
||||
class SyncFileImporter(FileImporter):
|
||||
def __init__(self, ui, repo, client, cl, p4filelog):
|
||||
self._ui = ui
|
||||
self._repo = repo
|
||||
self._client = client
|
||||
self._cl = cl
|
||||
self._p4filelog = p4filelog
|
||||
|
||||
@util.propertycache
|
||||
def relpath(self):
|
||||
return relpath(self._client, self._p4filelog.depotfile)
|
||||
|
||||
def create(self, tr):
|
||||
assert tr is not None
|
||||
p4fi = P4FileImporter(self._p4filelog)
|
||||
revs = set()
|
||||
for c in self._importset.changelists:
|
||||
if c.cl in p4fi.revisions and not self._p4filelog.isdeleted(c.cl):
|
||||
revs.add(c)
|
||||
|
||||
fileflags = collections.defaultdict(dict)
|
||||
lastlinkrev = 0
|
||||
|
||||
hgfilelog = self.hgfilelog()
|
||||
origlen = len(hgfilelog)
|
||||
largefiles = []
|
||||
lfsext = self.findlfs()
|
||||
|
||||
for c in sorted(revs):
|
||||
linkrev = self._importset.linkrev(c.cl)
|
||||
fparent1, fparent2 = nullid, nullid
|
||||
linkrev = len(self._repo)
|
||||
fparent1, fparent2 = nullid, nullid
|
||||
localfile = self.relpath
|
||||
hgfilelog = self._repo.file(localfile)
|
||||
if len(hgfilelog) > 0:
|
||||
fparent1 = hgfilelog.tip()
|
||||
|
||||
assert linkrev >= lastlinkrev
|
||||
lastlinkrev = linkrev
|
||||
# Only read files from p4 for a sync commit
|
||||
text, src = p4.get_file(self._p4filelog.depotfile, clnum=self._cl), 'p4'
|
||||
if text is None:
|
||||
raise error.Abort('error generating file content %d %s' % (
|
||||
self._cl, localfile))
|
||||
|
||||
if len(hgfilelog) > 0:
|
||||
fparent1 = hgfilelog.tip()
|
||||
meta = {}
|
||||
if self._p4filelog.isexec(self._cl):
|
||||
fileflags[self._cl] = 'x'
|
||||
else:
|
||||
fileflags[self._cl] = ''
|
||||
|
||||
text = None
|
||||
# Only read files from p4 for a blob commit
|
||||
text, src = p4fi.content(c.cl), 'p4'
|
||||
if text is None:
|
||||
raise error.Abort('error generating file content %d %s' % (
|
||||
c.cl, self.relpath))
|
||||
if self._p4filelog.issymlink(self._cl):
|
||||
fileflags[self._cl] = 'l'
|
||||
else:
|
||||
fileflags[self._cl] = ''
|
||||
|
||||
meta = {}
|
||||
fileflags[c.cl] = ' '
|
||||
if self._p4filelog.isexec(c.cl):
|
||||
fileflags[c.cl] = 'x'
|
||||
if self._p4filelog.issymlink(c.cl):
|
||||
fileflags[c.cl] = 'l'
|
||||
if self._p4filelog.iskeyworded(c.cl):
|
||||
text = re.sub(KEYWORD_REGEX, r'$\1$', text)
|
||||
if self._p4filelog.iskeyworded(self._cl):
|
||||
text = re.sub(KEYWORD_REGEX, r'$\1$', text)
|
||||
|
||||
node = hgfilelog.add(text, meta, tr, linkrev, fparent1, fparent2)
|
||||
self._ui.debug(
|
||||
'writing filelog: %s, p1 %s, linkrev %d, %d bytes, src: %s, '
|
||||
'path: %s\n' % (short(node), short(fparent1), linkrev,
|
||||
len(text), src, self.relpath))
|
||||
origlen = len(hgfilelog)
|
||||
node = hgfilelog.add(text, meta, tr, linkrev, fparent1, fparent2)
|
||||
self._ui.debug(
|
||||
'writing filelog: %s, p1 %s, linkrev %d, %d bytes, src: %s, '
|
||||
'path: %s\n' % (short(node), short(fparent1), linkrev,
|
||||
len(text), src, self.relpath))
|
||||
|
||||
if lfsext and lfsext.wrapper._islfs(hgfilelog, node):
|
||||
lfspointer = lfsext.pointer.deserialize(
|
||||
hgfilelog.revision(node, raw=True))
|
||||
oid = lfspointer.oid()
|
||||
largefiles.append((c.cl, self.depotfile, oid))
|
||||
self._ui.debug('largefile: %s, oid: %s\n' % (self.relpath, oid))
|
||||
largefiles = []
|
||||
if lfsext and lfsext.wrapper._islfs(hgfilelog, node):
|
||||
lfspointer = lfsext.pointer.deserialize(
|
||||
hgfilelog.revision(node, raw=True))
|
||||
oid = lfspointer.oid()
|
||||
largefiles.append((self._cl, self._p4filelog.depotfile, oid))
|
||||
self._ui.debug('largefile: %s, oid: %s\n' % (self.relpath, oid))
|
||||
|
||||
newlen = len(hgfilelog)
|
||||
return fileflags, largefiles, origlen, newlen
|
||||
|
||||
@ -37,7 +37,7 @@ populate the depot
|
||||
edit //depot/Main/largefile#2
|
||||
Change 2 submitted.
|
||||
|
||||
Blob Commit
|
||||
Sync Commit
|
||||
|
||||
$ cd $hgwd
|
||||
$ hg init --config 'format.usefncache=False'
|
||||
@ -75,19 +75,18 @@ Blob Commit
|
||||
$ cd $hgwd
|
||||
$ hg p4syncimport --debug -P $P4ROOT hg-p4-import
|
||||
incremental import from changelist: 3, node: * (glob)
|
||||
loading changelist numbers.
|
||||
1 changelists to import.
|
||||
loading list of files.
|
||||
1 files to import.
|
||||
reading filelog * (glob)
|
||||
Latest change list number 3
|
||||
running a sync import.
|
||||
writing filelog: fc4d9d827df8, p1 a80d06849b33, linkrev 2, 4 bytes, src: *, path: Main/a (glob)
|
||||
writing filelog: cf38a89d2b54, p1 000000000000, linkrev 2, 23 bytes, src: *, path: Main/anotherlargefile (glob)
|
||||
largefile: Main/anotherlargefile, oid: 9703972eff7a4df07317eda436ab7ef827ed16ea28c62abdcd7de269745c610c
|
||||
changelist 3: writing manifest. node: edf0e5bc6eac p1: c14352bb3510 p2: 000000000000 linkrev: 2
|
||||
changelist 3: writing changelog: third
|
||||
writing filelog: fa4df9fc788c, p1 9f14f96519e1, linkrev 2, 88 bytes, src: *, path: Main/largefile (glob)
|
||||
largefile: Main/largefile, oid: b0d5c1968efbabbff9d94160f284cd7b52686ca3c46cfffdd351de07384fce9c
|
||||
changelist 3: writing manifest. node: a6a6dbeefd0a p1: c14352bb3510 p2: 000000000000 linkrev: 2
|
||||
changelist 3: writing changelog: p4fastimport synchronizing client view
|
||||
writing lfs metadata to sqlite
|
||||
updating the branch cache
|
||||
1 revision(s), 1 file(s) imported.
|
||||
1 revision, 3 file(s) imported.
|
||||
|
||||
Verify
|
||||
(waiting for https://patchwork.mercurial-scm.org/patch/20582/)
|
||||
@ -98,7 +97,7 @@ Verify
|
||||
checking manifests
|
||||
crosschecking files in changesets and manifests
|
||||
checking files
|
||||
3 files, 3 changesets, 5 total revisions
|
||||
3 files, 3 changesets, 7 total revisions
|
||||
|
||||
$ test -d .hg/store/lfs/objects
|
||||
[1]
|
||||
@ -106,6 +105,7 @@ Verify
|
||||
1|1|*|37a7b43abd9e105a0e6b22088b140735a02f288767fe7a6f4f436cb46b064ca9|//depot/Main/largefile (glob)
|
||||
2|2|*|b0d5c1968efbabbff9d94160f284cd7b52686ca3c46cfffdd351de07384fce9c|//depot/Main/largefile (glob)
|
||||
3|3|*|9703972eff7a4df07317eda436ab7ef827ed16ea28c62abdcd7de269745c610c|//depot/Main/anotherlargefile (glob)
|
||||
4|3|*|b0d5c1968efbabbff9d94160f284cd7b52686ca3c46cfffdd351de07384fce9c|//depot/Main/largefile (glob)
|
||||
|
||||
End Test
|
||||
|
||||
|
||||
@ -25,7 +25,7 @@ Populate the Depot
|
||||
edit //depot/Main/a#2
|
||||
Change 2 submitted.
|
||||
|
||||
Sync import
|
||||
Fast Import
|
||||
|
||||
$ cd $hgwd
|
||||
$ hg init --config 'format.usefncache=False'
|
||||
@ -60,35 +60,54 @@ Sync import
|
||||
//depot/Main/c#1 - opened for add
|
||||
//depot/Main/d/e#1 - opened for add
|
||||
//depot/Main/d/f/g#1 - opened for add
|
||||
$ p4 delete Main/a
|
||||
//depot/Main/a#2 - opened for delete
|
||||
$ p4 submit -d third
|
||||
Submitting change 3.
|
||||
Locking 4 files ...
|
||||
Locking 5 files ...
|
||||
delete //depot/Main/a#3
|
||||
edit //depot/Main/b#2
|
||||
add //depot/Main/c#1
|
||||
add //depot/Main/d/e#1
|
||||
add //depot/Main/d/f/g#1
|
||||
Change 3 submitted.
|
||||
|
||||
Sync Import
|
||||
|
||||
$ cd $hgwd
|
||||
$ cd $hgwd
|
||||
$ hg p4syncimport --bookmark master --debug -P $P4ROOT hg-p4-import
|
||||
incremental import from changelist: 3, node: * (glob)
|
||||
loading changelist numbers.
|
||||
1 changelists to import.
|
||||
loading list of files.
|
||||
4 files to import.
|
||||
reading filelog * (glob)
|
||||
reading filelog * (glob)
|
||||
reading filelog * (glob)
|
||||
reading filelog * (glob)
|
||||
Latest change list number 3
|
||||
running a sync import.
|
||||
writing filelog: 861f64b39056, p1 1e88685f5dde, linkrev 2, 4 bytes, src: *, path: Main/b (glob)
|
||||
writing filelog: 149da44f2a4e, p1 000000000000, linkrev 2, 2 bytes, src: *, path: Main/c (glob)
|
||||
writing filelog: 6b67ccefd5ce, p1 000000000000, linkrev 2, 2 bytes, src: *, path: Main/d/e (glob)
|
||||
writing filelog: 0973eb1b2ecc, p1 000000000000, linkrev 2, 2 bytes, *: p4, path: Main/d/f/g (glob)
|
||||
changelist 3: writing manifest. node: 2f8836be616d p1: 5c8695bebd8f p2: 000000000000 linkrev: 2
|
||||
changelist 3: writing changelog: third
|
||||
writing filelog: 0973eb1b2ecc, p1 000000000000, linkrev 2, 2 bytes, src: *, path: Main/d/f/g (glob)
|
||||
changelist 3: writing manifest. node: f0ca72fbd536 p1: 5c8695bebd8f p2: 000000000000 linkrev: 2
|
||||
changelist 3: writing changelog: p4fastimport synchronizing client view
|
||||
writing bookmark
|
||||
updating the branch cache
|
||||
1 revision(s), 4 file(s) imported.
|
||||
1 revision, 4 file(s) imported.
|
||||
|
||||
Sync Import without New Changes
|
||||
|
||||
$ hg p4syncimport --bookmark master --debug -P $P4ROOT hg-p4-import
|
||||
incremental import from changelist: 4, node: * (glob)
|
||||
Latest change list number 3
|
||||
running a sync import.
|
||||
writing filelog: 0a738da2fcb2, p1 861f64b39056, linkrev 3, 4 bytes, src: *, path: Main/b (glob)
|
||||
writing filelog: 825765709d02, p1 149da44f2a4e, linkrev 3, 2 bytes, src: *, path: Main/c (glob)
|
||||
writing filelog: c3bd1155d122, p1 6b67ccefd5ce, linkrev 3, 2 bytes, src: *, path: Main/d/e (glob)
|
||||
writing filelog: 7fd4bac0fb3a, p1 0973eb1b2ecc, linkrev 3, 2 bytes, src: *, path: Main/d/f/g (glob)
|
||||
changelist 3: writing manifest. node: c11fdd72fcbf p1: f0ca72fbd536 p2: 000000000000 linkrev: 3
|
||||
changelist 3: writing changelog: p4fastimport synchronizing client view
|
||||
writing bookmark
|
||||
updating the branch cache
|
||||
1 revision, 4 file(s) imported.
|
||||
|
||||
Fast Import after Sync Import
|
||||
|
||||
$ hg p4fastimport --bookmark master --debug -P $P4ROOT hg-p4-import
|
||||
incremental import from changelist: 4, node: * (glob)
|
||||
loading changelist numbers.
|
||||
@ -101,14 +120,13 @@ Verify
|
||||
checking manifests
|
||||
crosschecking files in changesets and manifests
|
||||
checking files
|
||||
5 files, 3 changesets, 7 total revisions
|
||||
5 files, 4 changesets, 11 total revisions
|
||||
|
||||
$ hg update master
|
||||
5 files updated, 0 files merged, 0 files removed, 0 files unresolved
|
||||
4 files updated, 0 files merged, 0 files removed, 0 files unresolved
|
||||
(activating bookmark master)
|
||||
|
||||
$ hg manifest -r master
|
||||
Main/a
|
||||
Main/b
|
||||
Main/c
|
||||
Main/d/e
|
||||
|
||||
Loading…
Reference in New Issue
Block a user