2009-03-04 00:32:23 +03:00
|
|
|
# Perforce source for convert extension.
|
|
|
|
#
|
|
|
|
# Copyright 2009, Frank Kingswood <frank@kingswood-consulting.co.uk>
|
|
|
|
#
|
2009-04-26 03:08:54 +04:00
|
|
|
# This software may be used and distributed according to the terms of the
|
2010-01-20 07:20:08 +03:00
|
|
|
# GNU General Public License version 2 or any later version.
|
2016-03-02 18:31:15 +03:00
|
|
|
from __future__ import absolute_import
|
2009-03-04 00:32:23 +03:00
|
|
|
|
|
|
|
import marshal
|
2009-06-18 13:39:04 +04:00
|
|
|
import re
|
2009-03-04 00:32:23 +03:00
|
|
|
|
2016-05-14 08:03:12 +03:00
|
|
|
from mercurial.i18n import _
|
2016-03-02 18:31:15 +03:00
|
|
|
from mercurial import (
|
|
|
|
error,
|
|
|
|
util,
|
|
|
|
)
|
|
|
|
|
|
|
|
from . import common
|
|
|
|
|
2009-03-04 00:32:23 +03:00
|
|
|
def loaditer(f):
|
|
|
|
"Yield the dictionary objects generated by p4"
|
|
|
|
try:
|
|
|
|
while True:
|
|
|
|
d = marshal.load(f)
|
|
|
|
if not d:
|
|
|
|
break
|
|
|
|
yield d
|
|
|
|
except EOFError:
|
|
|
|
pass
|
|
|
|
|
2015-07-14 18:23:57 +03:00
|
|
|
def decodefilename(filename):
|
|
|
|
"""Perforce escapes special characters @, #, *, or %
|
|
|
|
with %40, %23, %2A, or %25 respectively
|
|
|
|
|
|
|
|
>>> decodefilename('portable-net45%252Bnetcore45%252Bwp8%252BMonoAndroid')
|
|
|
|
'portable-net45%2Bnetcore45%2Bwp8%2BMonoAndroid'
|
|
|
|
>>> decodefilename('//Depot/Directory/%2525/%2523/%23%40.%2A')
|
|
|
|
'//Depot/Directory/%25/%23/#@.*'
|
|
|
|
"""
|
|
|
|
replacements = [('%2A', '*'), ('%23', '#'), ('%40', '@'), ('%25', '%')]
|
|
|
|
for k, v in replacements:
|
|
|
|
filename = filename.replace(k, v)
|
|
|
|
return filename
|
|
|
|
|
2016-03-02 18:31:15 +03:00
|
|
|
class p4_source(common.converter_source):
|
2015-07-08 20:27:43 +03:00
|
|
|
def __init__(self, ui, path, revs=None):
|
2015-07-22 18:57:11 +03:00
|
|
|
# avoid import cycle
|
2016-03-02 18:31:15 +03:00
|
|
|
from . import convcmd
|
2015-07-22 18:57:11 +03:00
|
|
|
|
2015-07-08 20:27:43 +03:00
|
|
|
super(p4_source, self).__init__(ui, path, revs=revs)
|
2009-03-04 00:32:23 +03:00
|
|
|
|
2009-06-18 13:39:04 +04:00
|
|
|
if "/" in path and not path.startswith('//'):
|
2016-03-02 18:31:15 +03:00
|
|
|
raise common.NoRepo(_('%s does not look like a P4 repository') %
|
|
|
|
path)
|
2009-04-04 21:28:13 +04:00
|
|
|
|
2016-03-02 18:31:15 +03:00
|
|
|
common.checktool('p4', abort=False)
|
2009-03-04 00:32:23 +03:00
|
|
|
|
2016-12-14 12:45:57 +03:00
|
|
|
self.revmap = {}
|
2009-03-04 00:32:23 +03:00
|
|
|
self.p4changes = {}
|
2016-12-14 12:43:47 +03:00
|
|
|
self.heads = []
|
2009-03-04 00:32:23 +03:00
|
|
|
self.changeset = {}
|
|
|
|
self.files = {}
|
2015-07-08 20:05:27 +03:00
|
|
|
self.copies = {}
|
2015-07-22 18:57:11 +03:00
|
|
|
self.encoding = self.ui.config('convert', 'p4.encoding',
|
|
|
|
default=convcmd.orig_encoding)
|
2009-03-04 00:32:23 +03:00
|
|
|
self.depotname = {} # mapping from local name to depot name
|
2015-07-08 20:05:27 +03:00
|
|
|
self.localname = {} # mapping from depot name to local name
|
2010-01-25 09:05:27 +03:00
|
|
|
self.re_type = re.compile(
|
|
|
|
"([a-z]+)?(text|binary|symlink|apple|resource|unicode|utf\d+)"
|
|
|
|
"(\+\w+)?$")
|
|
|
|
self.re_keywords = re.compile(
|
|
|
|
r"\$(Id|Header|Date|DateTime|Change|File|Revision|Author)"
|
|
|
|
r":[^$\n]*\$")
|
2009-06-18 13:39:04 +04:00
|
|
|
self.re_keywords_old = re.compile("\$(Id|Header):[^$\n]*\$")
|
2009-03-04 00:32:23 +03:00
|
|
|
|
2015-07-08 20:27:43 +03:00
|
|
|
if revs and len(revs) > 1:
|
2015-10-08 22:55:45 +03:00
|
|
|
raise error.Abort(_("p4 source does not support specifying "
|
2015-07-08 20:27:43 +03:00
|
|
|
"multiple revisions"))
|
2009-03-04 00:32:23 +03:00
|
|
|
self._parse(ui, path)
|
|
|
|
|
2016-12-14 12:45:57 +03:00
|
|
|
def setrevmap(self, revmap):
|
|
|
|
"""Sets the parsed revmap dictionary.
|
|
|
|
|
|
|
|
Revmap stores mappings from a source revision to a target revision.
|
|
|
|
It is set in convertcmd.convert and provided by the user as a file
|
|
|
|
on the commandline.
|
|
|
|
|
|
|
|
Revisions in the map are considered beeing present in the
|
|
|
|
repository and ignored during _parse(). This allows for incremental
|
|
|
|
imports if a revmap is provided.
|
|
|
|
"""
|
|
|
|
self.revmap = revmap
|
|
|
|
|
2009-03-04 00:32:23 +03:00
|
|
|
def _parse_view(self, path):
|
|
|
|
"Read changes affecting the path"
|
2010-06-14 19:38:36 +04:00
|
|
|
cmd = 'p4 -G changes -s submitted %s' % util.shellquote(path)
|
2009-09-23 17:36:51 +04:00
|
|
|
stdout = util.popen(cmd, mode='rb')
|
2009-03-04 00:32:23 +03:00
|
|
|
for d in loaditer(stdout):
|
|
|
|
c = d.get("change", None)
|
|
|
|
if c:
|
|
|
|
self.p4changes[c] = True
|
|
|
|
|
|
|
|
def _parse(self, ui, path):
|
|
|
|
"Prepare list of P4 filenames and revisions to import"
|
|
|
|
ui.status(_('reading p4 views\n'))
|
|
|
|
|
|
|
|
# read client spec or view
|
|
|
|
if "/" in path:
|
|
|
|
self._parse_view(path)
|
|
|
|
if path.startswith("//") and path.endswith("/..."):
|
|
|
|
views = {path[:-3]:""}
|
|
|
|
else:
|
|
|
|
views = {"//": ""}
|
|
|
|
else:
|
2010-06-14 19:38:36 +04:00
|
|
|
cmd = 'p4 -G client -o %s' % util.shellquote(path)
|
2009-09-23 17:36:51 +04:00
|
|
|
clientspec = marshal.load(util.popen(cmd, mode='rb'))
|
2009-03-23 15:11:11 +03:00
|
|
|
|
2009-03-04 00:32:23 +03:00
|
|
|
views = {}
|
|
|
|
for client in clientspec:
|
|
|
|
if client.startswith("View"):
|
|
|
|
sview, cview = clientspec[client].split()
|
|
|
|
self._parse_view(sview)
|
|
|
|
if sview.endswith("...") and cview.endswith("..."):
|
|
|
|
sview = sview[:-3]
|
|
|
|
cview = cview[:-3]
|
|
|
|
cview = cview[2:]
|
|
|
|
cview = cview[cview.find("/") + 1:]
|
|
|
|
views[sview] = cview
|
|
|
|
|
|
|
|
# list of changes that affect our source files
|
|
|
|
self.p4changes = self.p4changes.keys()
|
|
|
|
self.p4changes.sort(key=int)
|
|
|
|
|
|
|
|
# list with depot pathnames, longest first
|
|
|
|
vieworder = views.keys()
|
2009-07-05 14:50:14 +04:00
|
|
|
vieworder.sort(key=len, reverse=True)
|
2009-03-04 00:32:23 +03:00
|
|
|
|
|
|
|
# handle revision limiting
|
|
|
|
startrev = self.ui.config('convert', 'p4.startrev', default=0)
|
|
|
|
|
|
|
|
# now read the full changelists to get the list of file revisions
|
|
|
|
ui.status(_('collecting p4 changelists\n'))
|
|
|
|
lastid = None
|
|
|
|
for change in self.p4changes:
|
2016-12-14 08:49:58 +03:00
|
|
|
if startrev and int(change) < int(startrev):
|
|
|
|
continue
|
|
|
|
if self.revs and int(change) > int(self.revs[0]):
|
|
|
|
continue
|
2016-12-14 12:45:57 +03:00
|
|
|
if change in self.revmap:
|
|
|
|
# Ignore already present revisions, but set the parent pointer.
|
|
|
|
lastid = change
|
|
|
|
continue
|
2016-12-14 08:49:58 +03:00
|
|
|
|
2009-03-04 00:32:23 +03:00
|
|
|
if lastid:
|
|
|
|
parents = [lastid]
|
|
|
|
else:
|
|
|
|
parents = []
|
2009-03-23 15:11:11 +03:00
|
|
|
|
2016-12-14 08:49:58 +03:00
|
|
|
d = self._fetch_revision(change)
|
|
|
|
c = self._construct_commit(d, parents)
|
|
|
|
|
|
|
|
shortdesc = c.desc.splitlines(True)[0].rstrip('\r\n')
|
|
|
|
t = '%s %s' % (c.rev, repr(shortdesc)[1:-1])
|
|
|
|
ui.status(util.ellipsis(t, 80) + '\n')
|
2009-03-04 00:32:23 +03:00
|
|
|
|
|
|
|
files = []
|
2015-07-08 20:05:27 +03:00
|
|
|
copies = {}
|
|
|
|
copiedfiles = []
|
2009-03-04 00:32:23 +03:00
|
|
|
i = 0
|
|
|
|
while ("depotFile%d" % i) in d and ("rev%d" % i) in d:
|
|
|
|
oldname = d["depotFile%d" % i]
|
|
|
|
filename = None
|
|
|
|
for v in vieworder:
|
2015-07-13 17:05:03 +03:00
|
|
|
if oldname.lower().startswith(v.lower()):
|
2015-07-14 18:23:57 +03:00
|
|
|
filename = decodefilename(views[v] + oldname[len(v):])
|
2009-03-04 00:32:23 +03:00
|
|
|
break
|
|
|
|
if filename:
|
|
|
|
files.append((filename, d["rev%d" % i]))
|
|
|
|
self.depotname[filename] = oldname
|
2015-07-08 20:05:27 +03:00
|
|
|
if (d.get("action%d" % i) == "move/add"):
|
|
|
|
copiedfiles.append(filename)
|
|
|
|
self.localname[oldname] = filename
|
2009-03-04 00:32:23 +03:00
|
|
|
i += 1
|
2015-07-08 20:05:27 +03:00
|
|
|
|
|
|
|
# Collect information about copied files
|
|
|
|
for filename in copiedfiles:
|
|
|
|
oldname = self.depotname[filename]
|
|
|
|
|
|
|
|
flcmd = 'p4 -G filelog %s' \
|
|
|
|
% util.shellquote(oldname)
|
|
|
|
flstdout = util.popen(flcmd, mode='rb')
|
|
|
|
|
|
|
|
copiedfilename = None
|
|
|
|
for d in loaditer(flstdout):
|
|
|
|
copiedoldname = None
|
|
|
|
|
|
|
|
i = 0
|
|
|
|
while ("change%d" % i) in d:
|
|
|
|
if (d["change%d" % i] == change and
|
|
|
|
d["action%d" % i] == "move/add"):
|
|
|
|
j = 0
|
|
|
|
while ("file%d,%d" % (i, j)) in d:
|
|
|
|
if d["how%d,%d" % (i, j)] == "moved from":
|
|
|
|
copiedoldname = d["file%d,%d" % (i, j)]
|
|
|
|
break
|
|
|
|
j += 1
|
|
|
|
i += 1
|
|
|
|
|
|
|
|
if copiedoldname and copiedoldname in self.localname:
|
|
|
|
copiedfilename = self.localname[copiedoldname]
|
|
|
|
break
|
|
|
|
|
|
|
|
if copiedfilename:
|
|
|
|
copies[filename] = copiedfilename
|
|
|
|
else:
|
|
|
|
ui.warn(_("cannot find source for copied file: %s@%s\n")
|
|
|
|
% (filename, change))
|
|
|
|
|
2009-03-04 00:32:23 +03:00
|
|
|
self.changeset[change] = c
|
|
|
|
self.files[change] = files
|
2015-07-08 20:05:27 +03:00
|
|
|
self.copies[change] = copies
|
2009-03-04 00:32:23 +03:00
|
|
|
lastid = change
|
2009-03-23 15:11:11 +03:00
|
|
|
|
2016-12-14 08:49:58 +03:00
|
|
|
if lastid and len(self.changeset) > 0:
|
2009-03-04 00:32:23 +03:00
|
|
|
self.heads = [lastid]
|
|
|
|
|
|
|
|
def getheads(self):
|
|
|
|
return self.heads
|
|
|
|
|
|
|
|
def getfile(self, name, rev):
|
2010-06-15 13:05:17 +04:00
|
|
|
cmd = 'p4 -G print %s' \
|
|
|
|
% util.shellquote("%s#%s" % (self.depotname[name], rev))
|
2009-03-04 00:32:23 +03:00
|
|
|
|
2015-07-08 20:11:40 +03:00
|
|
|
lasterror = None
|
|
|
|
while True:
|
|
|
|
stdout = util.popen(cmd, mode='rb')
|
2009-03-04 00:32:23 +03:00
|
|
|
|
2015-07-08 20:11:40 +03:00
|
|
|
mode = None
|
2015-07-30 02:58:05 +03:00
|
|
|
contents = []
|
2015-07-08 20:11:40 +03:00
|
|
|
keywords = None
|
|
|
|
|
|
|
|
for d in loaditer(stdout):
|
|
|
|
code = d["code"]
|
|
|
|
data = d.get("data")
|
|
|
|
|
|
|
|
if code == "error":
|
|
|
|
# if this is the first time error happened
|
|
|
|
# re-attempt getting the file
|
|
|
|
if not lasterror:
|
|
|
|
lasterror = IOError(d["generic"], data)
|
|
|
|
# this will exit inner-most for-loop
|
|
|
|
break
|
|
|
|
else:
|
|
|
|
raise lasterror
|
|
|
|
|
|
|
|
elif code == "stat":
|
|
|
|
action = d.get("action")
|
|
|
|
if action in ["purge", "delete", "move/delete"]:
|
|
|
|
return None, None
|
|
|
|
p4type = self.re_type.match(d["type"])
|
|
|
|
if p4type:
|
|
|
|
mode = ""
|
|
|
|
flags = ((p4type.group(1) or "")
|
|
|
|
+ (p4type.group(3) or ""))
|
|
|
|
if "x" in flags:
|
|
|
|
mode = "x"
|
|
|
|
if p4type.group(2) == "symlink":
|
|
|
|
mode = "l"
|
|
|
|
if "ko" in flags:
|
|
|
|
keywords = self.re_keywords_old
|
|
|
|
elif "k" in flags:
|
|
|
|
keywords = self.re_keywords
|
|
|
|
|
|
|
|
elif code == "text" or code == "binary":
|
2015-07-30 02:58:05 +03:00
|
|
|
contents.append(data)
|
2015-07-08 20:11:40 +03:00
|
|
|
|
|
|
|
lasterror = None
|
|
|
|
|
|
|
|
if not lasterror:
|
|
|
|
break
|
2009-03-04 00:32:23 +03:00
|
|
|
|
|
|
|
if mode is None:
|
2014-08-27 00:03:32 +04:00
|
|
|
return None, None
|
2009-03-04 00:32:23 +03:00
|
|
|
|
2015-07-30 02:58:05 +03:00
|
|
|
contents = ''.join(contents)
|
|
|
|
|
2009-06-18 13:39:04 +04:00
|
|
|
if keywords:
|
|
|
|
contents = keywords.sub("$\\1$", contents)
|
|
|
|
if mode == "l" and contents.endswith("\n"):
|
|
|
|
contents = contents[:-1]
|
|
|
|
|
2010-05-09 23:52:34 +04:00
|
|
|
return contents, mode
|
2009-03-04 00:32:23 +03:00
|
|
|
|
convert: introduce --full for converting all files
Convert will normally only process files that were changed in a source
revision, apply the filemap, and record it has a change in the target
repository. (If it ends up not really changing anything, nothing changes.)
That means that _if_ the filemap is changed before continuing an incremental
convert, the change will only kick in when the files it affects are modified in
a source revision and thus processed.
With --full, convert will make a full conversion every time and process
all files in the source repo and remove target repo files that shouldn't be
there. Filemap changes will thus kick in on the first converted revision, no
matter what is changed.
This flag should in most cases not make any difference but will make convert
significantly slower.
Other names has been considered for this feature, such as "resync", "sync",
"checkunmodified", "all" or "allfiles", but I found that they were less obvious
and required more explanation than "full" and were harder to describe
consistently.
2014-08-27 00:03:32 +04:00
|
|
|
def getchanges(self, rev, full):
|
|
|
|
if full:
|
2015-10-14 09:06:54 +03:00
|
|
|
raise error.Abort(_("convert from p4 does not support --full"))
|
2015-07-08 20:05:27 +03:00
|
|
|
return self.files[rev], self.copies[rev], set()
|
2009-03-04 00:32:23 +03:00
|
|
|
|
2016-12-14 08:49:58 +03:00
|
|
|
def _construct_commit(self, obj, parents=None):
|
|
|
|
"""
|
|
|
|
Constructs a common.commit object from an unmarshalled
|
|
|
|
`p4 describe` output
|
|
|
|
"""
|
|
|
|
desc = self.recode(obj.get("desc", ""))
|
|
|
|
date = (int(obj["time"]), 0) # timezone not set
|
|
|
|
if parents is None:
|
|
|
|
parents = []
|
|
|
|
|
|
|
|
return common.commit(author=self.recode(obj["user"]),
|
|
|
|
date=util.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'),
|
|
|
|
parents=parents, desc=desc, branch=None, rev=obj['change'],
|
|
|
|
extra={"p4": obj['change'], "convert_revision": obj['change']})
|
|
|
|
|
|
|
|
def _fetch_revision(self, rev):
|
|
|
|
"""Return an output of `p4 describe` including author, commit date as
|
|
|
|
a dictionary."""
|
|
|
|
cmd = "p4 -G describe -s %s" % rev
|
|
|
|
stdout = util.popen(cmd, mode='rb')
|
|
|
|
return marshal.load(stdout)
|
|
|
|
|
2009-03-04 00:32:23 +03:00
|
|
|
def getcommit(self, rev):
|
2016-12-14 23:07:23 +03:00
|
|
|
if rev in self.changeset:
|
|
|
|
return self.changeset[rev]
|
|
|
|
elif rev in self.revmap:
|
|
|
|
d = self._fetch_revision(rev)
|
|
|
|
return self._construct_commit(d, parents=None)
|
|
|
|
raise error.Abort(
|
|
|
|
_("cannot find %s in the revmap or parsed changesets") % rev)
|
2009-03-04 00:32:23 +03:00
|
|
|
|
|
|
|
def gettags(self):
|
2016-12-14 12:45:17 +03:00
|
|
|
return {}
|
2009-03-04 00:32:23 +03:00
|
|
|
|
|
|
|
def getchangedfiles(self, rev, i):
|
2009-04-27 01:50:44 +04:00
|
|
|
return sorted([x[0] for x in self.files[rev]])
|