mirror of
https://github.com/facebook/sapling.git
synced 2024-10-11 17:27:53 +03:00
7088442e8d
Summary: StringIO.pos exists, but cStringIO.pos doesn't. Let's use a method that exists instead! Reviewed By: xavierd Differential Revision: D17790697 fbshipit-source-id: 9132a8ca9c0bfc35c0acc573b92399d5fc3f420d
1865 lines
69 KiB
Python
1865 lines
69 KiB
Python
import collections
|
|
import itertools
|
|
import os
|
|
import re
|
|
import shutil
|
|
|
|
import _ssh
|
|
import compat
|
|
import git2hg
|
|
import hg2git
|
|
import util
|
|
from bindings import nodemap
|
|
from dulwich import client, config as dul_config, diff_tree
|
|
from dulwich.errors import GitProtocolError, HangupException
|
|
from dulwich.objects import Blob, Commit, Tag, Tree, parse_timezone
|
|
from dulwich.pack import apply_delta, create_delta
|
|
from dulwich.repo import Repo, check_ref_format
|
|
from edenscm.mercurial import (
|
|
bookmarks,
|
|
commands,
|
|
context,
|
|
encoding,
|
|
error,
|
|
phases,
|
|
progress,
|
|
pycompat,
|
|
util as hgutil,
|
|
vfs as vfsmod,
|
|
)
|
|
from edenscm.mercurial.i18n import _
|
|
from edenscm.mercurial.node import bin, hex, nullid, nullrev
|
|
from overlay import overlayrepo
|
|
|
|
|
|
RE_GIT_AUTHOR = re.compile("^(.*?) ?\<(.*?)(?:\>(.*))?$")
|
|
|
|
RE_GIT_SANITIZE_AUTHOR = re.compile("[<>\n]")
|
|
|
|
RE_GIT_AUTHOR_EXTRA = re.compile("^(.*?)\ ext:\((.*)\) <(.*)\>$")
|
|
|
|
RE_GIT_EXTRA_KEY = re.compile("GIT([0-9]*)-(.*)")
|
|
|
|
# Test for git:// and git+ssh:// URI.
|
|
# Support several URL forms, including separating the
|
|
# host and path with either a / or : (sepr)
|
|
RE_GIT_URI = re.compile(
|
|
r"^(?P<scheme>git([+]ssh)?://)(?P<host>.*?)(:(?P<port>\d+))?"
|
|
r"(?P<sepr>[:/])(?P<path>.*)$"
|
|
)
|
|
|
|
RE_NEWLINES = re.compile("[\r\n]")
|
|
RE_GIT_PROGRESS = re.compile("\((\d+)/(\d+)\)")
|
|
|
|
RE_AUTHOR_FILE = re.compile("\s*=\s*")
|
|
|
|
CALLBACK_BUFFER = ""
|
|
|
|
|
|
class GitProgress(object):
|
|
"""convert git server progress strings into mercurial progress"""
|
|
|
|
def __init__(self, ui, prog):
|
|
self.ui = ui
|
|
self.prog = prog
|
|
|
|
self.lasttopic = None
|
|
self.msgbuf = ""
|
|
|
|
def progress(self, msg):
|
|
# 'Counting objects: 33640, done.\n'
|
|
# 'Compressing objects: 0% (1/9955) \r
|
|
msgs = RE_NEWLINES.split(self.msgbuf + msg)
|
|
self.msgbuf = msgs.pop()
|
|
|
|
for msg in msgs:
|
|
td = msg.split(":", 1)
|
|
data = td.pop()
|
|
if not td:
|
|
self.ui.note(data + "\n")
|
|
continue
|
|
topic = td[0]
|
|
|
|
m = RE_GIT_PROGRESS.search(data)
|
|
if m:
|
|
pos, total = map(int, m.group(1, 2))
|
|
if topic != self.lasttopic:
|
|
self.prog.reset(topic, total=total)
|
|
self.lasttopic = topic
|
|
self.prog.value = pos
|
|
else:
|
|
self.ui.note(msg + "\n")
|
|
|
|
|
|
class GitHandler(object):
|
|
map_file = "git-mapfile"
|
|
remote_refs_file = "git-remote-refs"
|
|
tags_file = "git-tags"
|
|
|
|
def __init__(self, dest_repo, ui):
|
|
self.repo = dest_repo
|
|
self.ui = ui
|
|
self.vfs = self.repo.sharedvfs
|
|
|
|
if compat.config(ui, "bool", "git", "intree"):
|
|
self.gitdir = self.repo.wvfs.join(".git")
|
|
else:
|
|
self.gitdir = self.vfs.join("git")
|
|
|
|
self.init_author_file()
|
|
|
|
self.paths = ui.configitems("paths")
|
|
|
|
self.branch_bookmark_suffix = compat.config(
|
|
ui, "string", "git", "branch_bookmark_suffix"
|
|
)
|
|
|
|
self._map_real = None
|
|
self._map_hg_modifications = set()
|
|
self.load_tags()
|
|
self._remote_refs = None
|
|
|
|
@property
|
|
def _map(self):
|
|
if self._map_real is None:
|
|
self.load_map()
|
|
return self._map_real
|
|
|
|
@property
|
|
def remote_refs(self):
|
|
if self._remote_refs is None:
|
|
self.load_remote_refs()
|
|
return self._remote_refs
|
|
|
|
@hgutil.propertycache
|
|
def git(self):
|
|
# Dulwich is going to try and join unicode ref names against
|
|
# the repository path to try and read unpacked refs. This
|
|
# doesn't match hg's bytes-only view of filesystems, we just
|
|
# have to cope with that. As a workaround, try decoding our
|
|
# (bytes) path to the repo in hg's active encoding and hope
|
|
# for the best.
|
|
gitpath = self.gitdir.decode(encoding.encoding, encoding.encodingmode)
|
|
# make the git data directory
|
|
if os.path.exists(self.gitdir):
|
|
return Repo(gitpath)
|
|
else:
|
|
# Set disallowinitbare to prevent hggit from creating a .hg/git
|
|
# directory. This is useful when the .hg/git directory should be
|
|
# managed externally.
|
|
if self.ui.configbool("hggit", "disallowinitbare"):
|
|
raise error.Abort(_("missing .hg/git repo"))
|
|
os.mkdir(self.gitdir)
|
|
return Repo.init_bare(gitpath)
|
|
|
|
def init_author_file(self):
|
|
self.author_map = {}
|
|
authors_path = compat.config(self.ui, "string", "git", "authors")
|
|
if authors_path:
|
|
with open(self.repo.wvfs.join(authors_path)) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or line.startswith("#"):
|
|
continue
|
|
from_, to = RE_AUTHOR_FILE.split(line, 2)
|
|
self.author_map[from_] = to
|
|
|
|
# FILE LOAD AND SAVE METHODS
|
|
|
|
def map_set(self, gitsha, hgsha):
|
|
hgnode = bin(hgsha)
|
|
self._map.add(bin(gitsha), hgnode)
|
|
self._map_hg_modifications.add(hgnode)
|
|
|
|
def map_hg_get(self, gitsha):
|
|
node = self._map.lookupbyfirst(bin(gitsha))
|
|
if node is not None:
|
|
node = hex(node)
|
|
return node
|
|
|
|
def map_git_get(self, hgsha):
|
|
node = self._map.lookupbysecond(bin(hgsha))
|
|
if node is not None:
|
|
node = hex(node)
|
|
return node
|
|
|
|
def load_map(self):
|
|
if self.ui.configbool("hggit", "indexedlognodemap", False):
|
|
dir = self.vfs.join(self.map_file + "-log")
|
|
|
|
# Check for log existance before we instantiate it.
|
|
logexists = self.vfs.exists(dir)
|
|
|
|
self._map_real = nodemap.nodemap(dir)
|
|
|
|
# If the indexedlog map doesn't exist, populate it.
|
|
if not logexists and self.vfs.exists(self.map_file):
|
|
# Don't construct
|
|
oldmap = GitMap(self.vfs(self.map_file))
|
|
for gitsha, hgsha in oldmap.items():
|
|
self._map_real.add(gitsha, hgsha)
|
|
self._map_real.flush()
|
|
else:
|
|
content = []
|
|
if os.path.exists(self.vfs.join(self.map_file)):
|
|
content = self.vfs(self.map_file)
|
|
|
|
self._map_real = GitMap(content)
|
|
|
|
def save_map(self, map_file):
|
|
wlock = self.repo.wlock()
|
|
try:
|
|
map = self._map
|
|
|
|
if self.ui.configbool("hggit", "indexedlognodemap", False):
|
|
# If using index log, also write the flat map, so we can roll
|
|
# back easily.
|
|
self._map_real.flush()
|
|
|
|
file = self.vfs(map_file, "a+", atomictemp=True)
|
|
buf = hgutil.stringio()
|
|
bwrite = buf.write
|
|
# Append new entries to the end of the file so we can search
|
|
# backwards from the end for recently added entries.
|
|
for hgnode in self._map_hg_modifications:
|
|
gitnode = map.lookupbysecond(hgnode)
|
|
if gitnode is None:
|
|
raise KeyError(hex(hgnode))
|
|
bwrite("%s %s\n" % (hex(gitnode), hex(hgnode)))
|
|
self._map_hg_modifications.clear()
|
|
file.write(buf.getvalue())
|
|
buf.close()
|
|
# If this complains, atomictempfile no longer has close
|
|
file.close()
|
|
finally:
|
|
wlock.release()
|
|
|
|
def load_tags(self):
|
|
self.tags = {}
|
|
if os.path.exists(self.vfs.join(self.tags_file)):
|
|
for line in self.vfs(self.tags_file):
|
|
sha, name = line.strip().split(" ", 1)
|
|
self.tags[name] = sha
|
|
|
|
def save_tags(self):
|
|
file = self.vfs(self.tags_file, "w+", atomictemp=True)
|
|
for name, sha in sorted(self.tags.iteritems()):
|
|
if not self.repo.tagtype(name) == "global":
|
|
file.write("%s %s\n" % (sha, name))
|
|
# If this complains, atomictempfile no longer has close
|
|
file.close()
|
|
|
|
def load_remote_refs(self):
|
|
self._remote_refs = {}
|
|
refdir = os.path.join(self.git.path, "refs", "remotes")
|
|
|
|
paths = self.paths
|
|
# if paths are set, we should still check 'default'
|
|
if not paths:
|
|
paths = [("default", None)]
|
|
|
|
# we avoid using dulwich's refs method because it is incredibly slow;
|
|
# on a repo with a few hundred branches and a few thousand tags,
|
|
# dulwich took about 200ms
|
|
for p in paths:
|
|
remotedir = os.path.join(refdir, p[0])
|
|
for root, dirs, files in os.walk(remotedir):
|
|
for f in files:
|
|
try:
|
|
ref = root.replace(refdir + pycompat.ossep, "") + "/"
|
|
node = open(os.path.join(root, f)).read().strip()
|
|
hgsha = self._map.lookupbyfirst(bin(node))
|
|
if hgsha is None:
|
|
raise KeyError(hex(node))
|
|
self._remote_refs[ref + f] = hgsha
|
|
except (KeyError, IOError):
|
|
pass
|
|
|
|
# END FILE LOAD AND SAVE METHODS
|
|
|
|
# COMMANDS METHODS
|
|
|
|
def import_commits(self, remote_name):
|
|
refs = self.git.refs.as_dict()
|
|
filteredrefs = self.filter_min_date(refs)
|
|
self.import_git_objects(remote_name, filteredrefs)
|
|
self.update_hg_bookmarks(refs)
|
|
self.save_map(self.map_file)
|
|
|
|
def fetch(self, remote, heads):
|
|
refs = self.fetch_pack(remote, heads)
|
|
remote_name = self.remote_name(remote)
|
|
|
|
# if remote returns a symref for HEAD, then let's store that
|
|
rhead = None
|
|
rnode = None
|
|
oldheads = self.repo.changelog.heads()
|
|
imported = 0
|
|
if refs:
|
|
filteredrefs = self.filter_min_date(self.filter_refs(refs, heads))
|
|
imported = self.import_git_objects(remote_name, filteredrefs)
|
|
self.import_tags(refs)
|
|
self.update_hg_bookmarks(refs)
|
|
|
|
try:
|
|
symref = refs["HEAD"]
|
|
if symref.startswith("refs/heads"):
|
|
rhead = symref.replace("refs/heads/", "")
|
|
|
|
rnode = refs["refs/heads/%s" % rhead]
|
|
hgrnode = self._map.lookupbyfirst(bin(rnode))
|
|
if hgrnode is None:
|
|
raise KeyError(rnode)
|
|
rnode = self.repo[hgrnode].node()
|
|
except KeyError:
|
|
# if there is any error make sure to clear the variables
|
|
rhead = None
|
|
rnode = None
|
|
|
|
if remote_name:
|
|
self.update_remote_branches(remote_name, refs)
|
|
elif not self.paths:
|
|
# intial cloning
|
|
self.update_remote_branches("default", refs)
|
|
|
|
# "Activate" a tipmost bookmark.
|
|
bms = self.repo["tip"].bookmarks()
|
|
|
|
# override the 'tipmost' behavior if we know the remote HEAD
|
|
if rnode:
|
|
# make sure the bookmark exists; at the point the remote
|
|
# branches has already been set up
|
|
suffix = self.branch_bookmark_suffix or ""
|
|
changes = [(rhead + suffix, rnode)]
|
|
util.updatebookmarks(self.repo, changes)
|
|
bms = [rhead + suffix]
|
|
|
|
if bms:
|
|
try:
|
|
bookmarks.activate(self.repo, bms[0])
|
|
except AttributeError:
|
|
# hg < 3.5
|
|
bookmarks.setcurrent(self.repo, bms[0])
|
|
|
|
self.save_map(self.map_file)
|
|
|
|
# also mark public any branches the user specified
|
|
blist = [
|
|
self.repo[branch].node() for branch in self.ui.configlist("git", "public")
|
|
]
|
|
if rnode and self.ui.configbool("hggit", "usephases"):
|
|
blist.append(rnode)
|
|
|
|
if blist:
|
|
lock = self.repo.lock()
|
|
try:
|
|
tr = self.repo.transaction("phase")
|
|
try:
|
|
phases.advanceboundary(self.repo, tr, phases.public, blist)
|
|
except TypeError:
|
|
# hg < 3.2
|
|
phases.advanceboundary(self.repo, phases.public, blist)
|
|
tr.close()
|
|
finally:
|
|
if tr is not None:
|
|
tr.release()
|
|
lock.release()
|
|
|
|
if imported == 0:
|
|
return 0
|
|
|
|
# code taken from localrepo.py:addchangegroup
|
|
dh = 0
|
|
if oldheads:
|
|
heads = self.repo.changelog.heads()
|
|
dh = len(heads) - len(oldheads)
|
|
for h in heads:
|
|
if h not in oldheads and self.repo[h].closesbranch():
|
|
dh -= 1
|
|
|
|
if dh < 0:
|
|
return dh - 1
|
|
else:
|
|
return dh + 1
|
|
|
|
def export_commits(self):
|
|
try:
|
|
self.export_git_objects()
|
|
self.export_hg_tags()
|
|
self.update_references()
|
|
finally:
|
|
self.save_map(self.map_file)
|
|
|
|
def get_refs(self, remote):
|
|
self.export_commits()
|
|
client, path = self.get_transport_and_path(remote)
|
|
old_refs = {}
|
|
new_refs = {}
|
|
|
|
def changed(refs):
|
|
old_refs.update(refs)
|
|
exportable = self.get_exportable()
|
|
new_refs.update(self.get_changed_refs(refs, exportable, True))
|
|
return refs # always return the same refs to make the send a no-op
|
|
|
|
try:
|
|
client.send_pack(path, changed, lambda have, want: [])
|
|
|
|
changed_refs = [
|
|
ref for ref, sha in new_refs.iteritems() if sha != old_refs.get(ref)
|
|
]
|
|
new = [bin(self.map_hg_get(new_refs[ref])) for ref in changed_refs]
|
|
old = {}
|
|
for r in old_refs:
|
|
old_ref = self.map_hg_get(old_refs[r])
|
|
if old_ref:
|
|
old[bin(old_ref)] = 1
|
|
|
|
return old, new
|
|
except (HangupException, GitProtocolError) as e:
|
|
raise error.Abort(_("git remote error: ") + str(e))
|
|
|
|
def push(self, remote, revs, force):
|
|
self.export_commits()
|
|
old_refs, new_refs = self.upload_pack(remote, revs, force)
|
|
remote_name = self.remote_name(remote)
|
|
|
|
if remote_name and new_refs:
|
|
for ref, new_sha in sorted(new_refs.iteritems()):
|
|
old_sha = old_refs.get(ref)
|
|
if old_sha is None:
|
|
if self.ui.verbose:
|
|
self.ui.note(
|
|
_("adding reference %s::%s => GIT:%s\n")
|
|
% (remote_name, ref, new_sha[0:8])
|
|
)
|
|
else:
|
|
self.ui.status(_("adding reference %s\n") % ref)
|
|
elif new_sha != old_sha:
|
|
if self.ui.verbose:
|
|
self.ui.note(
|
|
_("updating reference %s::%s => GIT:%s\n")
|
|
% (remote_name, ref, new_sha[0:8])
|
|
)
|
|
else:
|
|
self.ui.status(_("updating reference %s\n") % ref)
|
|
else:
|
|
self.ui.debug(
|
|
"unchanged reference %s::%s => GIT:%s\n"
|
|
% (remote_name, ref, new_sha[0:8])
|
|
)
|
|
|
|
self.update_remote_branches(remote_name, new_refs)
|
|
if old_refs == new_refs:
|
|
self.ui.status(_("no changes found\n"))
|
|
ret = None
|
|
elif len(new_refs) > len(old_refs):
|
|
ret = 1 + (len(new_refs) - len(old_refs))
|
|
elif len(old_refs) > len(new_refs):
|
|
ret = -1 - (len(new_refs) - len(old_refs))
|
|
else:
|
|
ret = 1
|
|
return ret
|
|
|
|
def clear(self):
|
|
mapfile = self.vfs.join(self.map_file)
|
|
if self.ui.configbool("hggit", "indexedlognodemap", False):
|
|
shutil.rmtree(mapfile + "-log")
|
|
|
|
if os.path.exists(self.gitdir):
|
|
for root, dirs, files in os.walk(self.gitdir, topdown=False):
|
|
for name in files:
|
|
os.remove(os.path.join(root, name))
|
|
for name in dirs:
|
|
os.rmdir(os.path.join(root, name))
|
|
os.rmdir(self.gitdir)
|
|
if os.path.exists(mapfile):
|
|
os.remove(mapfile)
|
|
|
|
# incoming support
|
|
def getremotechanges(self, remote, revs):
|
|
self.export_commits()
|
|
refs = self.fetch_pack(remote.path, revs)
|
|
|
|
# refs contains all remote refs. Prune to only those requested.
|
|
if revs:
|
|
reqrefs = {}
|
|
for rev in revs:
|
|
for n in ("refs/heads/" + rev, "refs/tags/" + rev):
|
|
if n in refs:
|
|
reqrefs[n] = refs[n]
|
|
else:
|
|
reqrefs = refs
|
|
|
|
commits = [bin(c) for c in self.get_git_incoming(reqrefs).commits]
|
|
|
|
b = overlayrepo(self, commits, refs)
|
|
|
|
return (b, commits, lambda: None)
|
|
|
|
# CHANGESET CONVERSION METHODS
|
|
|
|
def export_git_objects(self):
|
|
self.ui.note(_("finding hg commits to export\n"))
|
|
repo = self.repo
|
|
clnode = repo.changelog.node
|
|
clparents = repo.changelog.parentrevs
|
|
|
|
# Starting at the heads, walk back and find any commits that aren't in
|
|
# the git/hg mapping.
|
|
pending = []
|
|
if len(repo) > 0:
|
|
# repo.heads() returns the nullrev in an empty repo
|
|
pending = list(repo.changelog.headrevs())
|
|
exportrevs = set()
|
|
while pending:
|
|
rev = pending.pop()
|
|
node = clnode(rev)
|
|
if self._map.lookupbysecond(node) is None:
|
|
exportrevs.add(rev)
|
|
for parentrev in clparents(rev):
|
|
if parentrev != nullrev and parentrev not in exportrevs:
|
|
pending.append(parentrev)
|
|
# Sorting here is important, because the below code expects to process
|
|
# these in topological order.
|
|
to_export = list(repo[r] for r in sorted(exportrevs))
|
|
|
|
todo_total = len(exportrevs)
|
|
pos = 0
|
|
export = []
|
|
with progress.bar(
|
|
repo.ui, "find commits to export", "commits", todo_total
|
|
) as prog:
|
|
for ctx in to_export:
|
|
item = hex(ctx.node())
|
|
pos += 1
|
|
prog.value = (pos, item)
|
|
if ctx.extra().get("hg-git", None) != "octopus":
|
|
export.append(ctx)
|
|
|
|
total = len(export)
|
|
if not total:
|
|
return
|
|
|
|
self.ui.note(_("exporting hg objects to git\n"))
|
|
|
|
# By only exporting deltas, the assertion is that all previous objects
|
|
# for all other changesets are already present in the Git repository.
|
|
# This assertion is necessary to prevent redundant work. Here, nodes,
|
|
# and therefore export, is in topological order. By definition,
|
|
# export[0]'s parents must be present in Git, so we start the
|
|
# incremental exporter from there.
|
|
pctx = export[0].p1()
|
|
pnode = pctx.node()
|
|
if pnode == nullid:
|
|
gitcommit = None
|
|
else:
|
|
gitnode = self._map.lookupbysecond(pnode)
|
|
if gitnode is None:
|
|
raise KeyError(hex(pnode))
|
|
gitsha = hex(gitnode)
|
|
try:
|
|
gitcommit = self.git[gitsha]
|
|
except KeyError:
|
|
raise error.Abort(
|
|
_("Parent SHA-1 not present in Git " "repo: %s") % gitsha
|
|
)
|
|
|
|
exporter = hg2git.IncrementalChangesetExporter(
|
|
self.repo, pctx, self.git.object_store, gitcommit
|
|
)
|
|
|
|
mapsavefreq = compat.config(self.ui, "int", "hggit", "mapsavefrequency")
|
|
with progress.bar(self.ui, _("exporting"), total=total) as prog:
|
|
for i, ctx in enumerate(export):
|
|
prog.value = i
|
|
self.export_hg_commit(ctx.node(), exporter)
|
|
if mapsavefreq and i % mapsavefreq == 0:
|
|
self.ui.debug("saving mapfile\n")
|
|
self.save_map(self.map_file)
|
|
|
|
def set_commiter_from_author(self, commit):
|
|
commit.committer = commit.author
|
|
commit.commit_time = commit.author_time
|
|
commit.commit_timezone = commit.author_timezone
|
|
|
|
# convert this commit into git objects
|
|
# go through the manifest, convert all blobs/trees we don't have
|
|
# write the commit object (with metadata info)
|
|
def export_hg_commit(self, rev, exporter):
|
|
self.ui.note(_("converting revision %s\n") % hex(rev))
|
|
|
|
oldenc = self.swap_out_encoding()
|
|
|
|
ctx = self.repo.changectx(rev)
|
|
extra = ctx.extra()
|
|
|
|
commit = Commit()
|
|
|
|
(time, timezone) = ctx.date()
|
|
# work around to bad timezone offets - dulwich does not handle
|
|
# sub minute based timezones. In the one known case, it was a
|
|
# manual edit that led to the unusual value. Based on that,
|
|
# there is no reason to round one way or the other, so do the
|
|
# simplest and round down.
|
|
timezone -= timezone % 60
|
|
commit.author = self.get_git_author(ctx)
|
|
commit.author_time = int(time)
|
|
commit.author_timezone = -timezone
|
|
|
|
if "committer" in extra:
|
|
try:
|
|
# fixup timezone
|
|
(name, timestamp, timezone) = extra["committer"].rsplit(" ", 2)
|
|
commit.committer = name
|
|
commit.commit_time = timestamp
|
|
|
|
# work around a timezone format change
|
|
if int(timezone) % 60 != 0: # pragma: no cover
|
|
timezone = parse_timezone(timezone)
|
|
# Newer versions of Dulwich return a tuple here
|
|
if isinstance(timezone, tuple):
|
|
timezone, neg_utc = timezone
|
|
commit._commit_timezone_neg_utc = neg_utc
|
|
else:
|
|
timezone = -int(timezone)
|
|
commit.commit_timezone = timezone
|
|
except Exception: # extra is essentially user-supplied; be careful
|
|
self.set_commiter_from_author(commit)
|
|
else:
|
|
self.set_commiter_from_author(commit)
|
|
|
|
commit.parents = []
|
|
for parent in self.get_git_parents(ctx):
|
|
hgsha = hex(parent.node())
|
|
git_sha = self.map_git_get(hgsha)
|
|
if git_sha:
|
|
if git_sha not in self.git.object_store:
|
|
raise error.Abort(
|
|
_("Parent SHA-1 not present in Git " "repo: %s") % git_sha
|
|
)
|
|
|
|
commit.parents.append(git_sha)
|
|
|
|
commit.message, extra = self.get_git_message_and_extra(ctx)
|
|
commit.extra.extend(extra)
|
|
|
|
if "encoding" in extra:
|
|
commit.encoding = extra["encoding"]
|
|
|
|
for obj, nodeid in exporter.update_changeset(ctx):
|
|
# In theory we should check if the object exists before adding it,
|
|
# but in practice it's unlikely to exist, and scanning all the packs
|
|
# to determine that is expensive.
|
|
self.git.object_store.add_object(obj)
|
|
|
|
tree_sha = exporter.root_tree_sha
|
|
|
|
if tree_sha not in self.git.object_store:
|
|
raise error.Abort(_("Tree SHA-1 not present in Git repo: %s") % tree_sha)
|
|
|
|
commit.tree = tree_sha
|
|
|
|
if commit.id not in self.git.object_store:
|
|
self.git.object_store.add_object(commit)
|
|
self.map_set(commit.id, ctx.hex())
|
|
|
|
self.swap_out_encoding(oldenc)
|
|
return commit.id
|
|
|
|
def get_valid_git_username_email(self, name):
|
|
r"""Sanitize usernames and emails to fit git's restrictions.
|
|
|
|
The following is taken from the man page of git's fast-import
|
|
command:
|
|
|
|
[...] Likewise LF means one (and only one) linefeed [...]
|
|
|
|
committer
|
|
The committer command indicates who made this commit,
|
|
and when they made it.
|
|
|
|
Here <name> is the person's display name (for example
|
|
"Com M Itter") and <email> is the person's email address
|
|
("cm@example.com[1]"). LT and GT are the literal
|
|
less-than (\x3c) and greater-than (\x3e) symbols. These
|
|
are required to delimit the email address from the other
|
|
fields in the line. Note that <name> and <email> are
|
|
free-form and may contain any sequence of bytes, except
|
|
LT, GT and LF. <name> is typically UTF-8 encoded.
|
|
|
|
Accordingly, this function makes sure that there are none of the
|
|
characters <, >, or \n in any string which will be used for
|
|
a git username or email. Before this, it first removes left
|
|
angle brackets and spaces from the beginning, and right angle
|
|
brackets and spaces from the end, of this string, to convert
|
|
such things as " <john@doe.com> " to "john@doe.com" for
|
|
convenience.
|
|
|
|
TESTS:
|
|
|
|
>>> from collections import namedtuple
|
|
>>> from edenscm.mercurial.ui import ui
|
|
>>> mockrepo = namedtuple('localrepo', ['sharedvfs'])
|
|
>>> mockrepo.sharedvfs = ''
|
|
>>> g = GitHandler(mockrepo, ui()).get_valid_git_username_email
|
|
>>> g('John Doe')
|
|
'John Doe'
|
|
>>> g('john@doe.com')
|
|
'john@doe.com'
|
|
>>> g(' <john@doe.com> ')
|
|
'john@doe.com'
|
|
>>> g(' <random<\n<garbage\n> > > ')
|
|
'random???garbage?'
|
|
>>> g('Typo in hgrc >but.hg-git@handles.it.gracefully>')
|
|
'Typo in hgrc ?but.hg-git@handles.it.gracefully'
|
|
"""
|
|
return RE_GIT_SANITIZE_AUTHOR.sub("?", name.lstrip("< ").rstrip("> "))
|
|
|
|
def get_git_author(self, ctx):
|
|
# hg authors might not have emails
|
|
author = ctx.user()
|
|
|
|
# see if a translation exists
|
|
author = self.author_map.get(author, author)
|
|
|
|
# check for git author pattern compliance
|
|
a = RE_GIT_AUTHOR.match(author)
|
|
|
|
if a:
|
|
name = self.get_valid_git_username_email(a.group(1))
|
|
email = self.get_valid_git_username_email(a.group(2))
|
|
if a.group(3) is not None and len(a.group(3)) != 0:
|
|
name += " ext:(" + hgutil.urlreq.quote(a.group(3)) + ")"
|
|
author = "%s <%s>" % (
|
|
self.get_valid_git_username_email(name),
|
|
self.get_valid_git_username_email(email),
|
|
)
|
|
elif "@" in author:
|
|
author = "%s <%s>" % (
|
|
self.get_valid_git_username_email(author),
|
|
self.get_valid_git_username_email(author),
|
|
)
|
|
else:
|
|
author = self.get_valid_git_username_email(author) + " <none@none>"
|
|
|
|
if "author" in ctx.extra():
|
|
author = "".join(apply_delta(author, ctx.extra()["author"]))
|
|
|
|
return author
|
|
|
|
def get_git_parents(self, ctx):
|
|
def is_octopus_part(ctx):
|
|
olist = ("octopus", "octopus-done")
|
|
return ctx.extra().get("hg-git", None) in olist
|
|
|
|
parents = []
|
|
if ctx.extra().get("hg-git", None) == "octopus-done":
|
|
# implode octopus parents
|
|
part = ctx
|
|
while is_octopus_part(part):
|
|
(p1, p2) = part.parents()
|
|
assert ctx.extra().get("hg-git", None) != "octopus"
|
|
parents.append(p1)
|
|
part = p2
|
|
parents.append(p2)
|
|
else:
|
|
parents = ctx.parents()
|
|
|
|
return parents
|
|
|
|
def get_git_message_and_extra(self, ctx):
|
|
extra = ctx.extra()
|
|
|
|
message = ctx.description() + "\n"
|
|
if "message" in extra:
|
|
message = "".join(apply_delta(message, extra["message"]))
|
|
|
|
# HG EXTRA INFORMATION
|
|
|
|
# test only -- do not document this!
|
|
extra_in_message = compat.config(self.ui, "bool", "git", "debugextrainmessage")
|
|
extra_message = ""
|
|
git_extra = []
|
|
if ctx.branch() != "default":
|
|
# we always store the branch in the extra message
|
|
extra_message += "branch : " + ctx.branch() + "\n"
|
|
|
|
# Git native extra items always come first, followed by hg renames,
|
|
# followed by hg extra keys
|
|
git_extraitems = []
|
|
for key, value in extra.items():
|
|
m = RE_GIT_EXTRA_KEY.match(key)
|
|
if m is not None:
|
|
git_extraitems.append((int(m.group(1)), m.group(2), value))
|
|
del extra[key]
|
|
|
|
git_extraitems.sort()
|
|
for i, field, value in git_extraitems:
|
|
git_extra.append(
|
|
(hgutil.urlreq.unquote(field), hgutil.urlreq.unquote(value))
|
|
)
|
|
|
|
if extra.get("hg-git-rename-source", None) != "git":
|
|
renames = []
|
|
for f in ctx.files():
|
|
if f not in ctx.manifest():
|
|
continue
|
|
rename = ctx.filectx(f).renamed()
|
|
if rename:
|
|
renames.append((rename[0], f))
|
|
|
|
if renames:
|
|
for oldfile, newfile in renames:
|
|
if extra_in_message:
|
|
extra_message += "rename : " + oldfile + " => " + newfile + "\n"
|
|
else:
|
|
spec = "%s:%s" % (
|
|
hgutil.urlreq.quote(oldfile),
|
|
hgutil.urlreq.quote(newfile),
|
|
)
|
|
git_extra.append(("HG:rename", spec))
|
|
|
|
# hg extra items always go at the end
|
|
extraitems = extra.items()
|
|
extraitems.sort()
|
|
for key, value in extraitems:
|
|
if key in (
|
|
"author",
|
|
"committer",
|
|
"encoding",
|
|
"message",
|
|
"branch",
|
|
"hg-git",
|
|
"hg-git-rename-source",
|
|
):
|
|
continue
|
|
else:
|
|
if extra_in_message:
|
|
extra_message += (
|
|
"extra : " + key + " : " + hgutil.urlreq.quote(value) + "\n"
|
|
)
|
|
else:
|
|
spec = "%s:%s" % (
|
|
hgutil.urlreq.quote(key),
|
|
hgutil.urlreq.quote(value),
|
|
)
|
|
git_extra.append(("HG:extra", spec))
|
|
|
|
if extra_message:
|
|
message += "\n--HG--\n" + extra_message
|
|
|
|
if (
|
|
extra.get("hg-git-rename-source", None) != "git"
|
|
and not extra_in_message
|
|
and not git_extra
|
|
and extra_message == ""
|
|
):
|
|
# We need to store this if no other metadata is stored. This
|
|
# indicates that when reimporting the commit into Mercurial we'll
|
|
# know not to detect renames.
|
|
git_extra.append(("HG:rename-source", "hg"))
|
|
|
|
return message, git_extra
|
|
|
|
def get_git_incoming(self, refs):
|
|
return git2hg.find_incoming(self.git.object_store, self._map, refs)
|
|
|
|
def import_git_objects(self, remote_name, refs):
|
|
result = self.get_git_incoming(refs)
|
|
commits = result.commits
|
|
commit_cache = result.commit_cache
|
|
# import each of the commits, oldest first
|
|
total = len(commits)
|
|
if total:
|
|
self.ui.status(_("importing git objects into hg\n"))
|
|
else:
|
|
self.ui.status(_("no changes found\n"))
|
|
|
|
mapsavefreq = compat.config(self.ui, "int", "hggit", "mapsavefrequency")
|
|
with progress.bar(self.ui, _("importing"), "commits", total=total) as prog:
|
|
icommits = enumerate(commits)
|
|
while True:
|
|
isubcommits = list(itertools.islice(icommits, mapsavefreq or 1))
|
|
if not isubcommits:
|
|
break
|
|
|
|
with self.repo.transaction("git import"):
|
|
for i, csha in isubcommits:
|
|
prog.value = i
|
|
commit = commit_cache[csha]
|
|
self.import_git_commit(commit)
|
|
self.ui.debug("committing transaction\n")
|
|
|
|
if mapsavefreq:
|
|
self.ui.debug("saving mapfile\n")
|
|
self.save_map(self.map_file)
|
|
|
|
# TODO if the tags cache is used, remove any dangling tag references
|
|
return total
|
|
|
|
def import_git_commit(self, commit):
|
|
self.ui.debug("importing: %s\n" % commit.id)
|
|
|
|
detect_renames = False
|
|
(strip_message, hg_renames, hg_branch, extra) = git2hg.extract_hg_metadata(
|
|
commit.message, commit.extra
|
|
)
|
|
|
|
# Store the original git commit hash in the Mercurial extras. This
|
|
# breaks bidirectionality, but makes it possible for a Mercurial client
|
|
# to compute the git/hg mapping without having the entire git repo.
|
|
# "convert_revision" was chosen to match the hgsubversion and hg convert
|
|
# extra field.
|
|
extra["convert_revision"] = commit.id
|
|
|
|
if hg_renames is None:
|
|
detect_renames = True
|
|
# We have to store this unconditionally, even if there are no
|
|
# renames detected from Git. This is because we export an extra
|
|
# 'HG:rename-source' Git parameter when this isn't set, which will
|
|
# break bidirectionality.
|
|
extra["hg-git-rename-source"] = "git"
|
|
else:
|
|
renames = hg_renames
|
|
|
|
gparents = map(self.map_hg_get, commit.parents)
|
|
|
|
for parent in gparents:
|
|
if parent not in self.repo:
|
|
raise error.Abort(
|
|
_("you appear to have run strip - " "please run hg git-cleanup")
|
|
)
|
|
|
|
# get a list of the changed, added, removed files and gitlinks
|
|
files, gitlinks, git_renames = self.get_files_changed(commit, detect_renames)
|
|
if detect_renames:
|
|
renames = git_renames
|
|
|
|
date = (commit.author_time, -commit.author_timezone)
|
|
text = strip_message
|
|
|
|
origtext = text
|
|
try:
|
|
text.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
text = self.decode_guess(text, commit.encoding)
|
|
|
|
text = "\n".join([l.rstrip() for l in text.splitlines()]).strip("\n")
|
|
if text + "\n" != origtext:
|
|
extra["message"] = create_delta(text + "\n", origtext)
|
|
|
|
author = commit.author
|
|
|
|
# convert extra data back to the end
|
|
if " ext:" in commit.author:
|
|
m = RE_GIT_AUTHOR_EXTRA.match(commit.author)
|
|
if m:
|
|
name = m.group(1)
|
|
ex = hgutil.urlreq.unquote(m.group(2))
|
|
email = m.group(3)
|
|
author = name + " <" + email + ">" + ex
|
|
|
|
if " <none@none>" in commit.author:
|
|
author = commit.author[:-12]
|
|
|
|
try:
|
|
author.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
origauthor = author
|
|
author = self.decode_guess(author, commit.encoding)
|
|
extra["author"] = create_delta(author, origauthor)
|
|
|
|
oldenc = self.swap_out_encoding()
|
|
|
|
def findconvergedfiles(p1, p2):
|
|
# If any files have the same contents in both parents of a merge
|
|
# (and are therefore not reported as changed by Git) but are at
|
|
# different file revisions in Mercurial (because they arrived at
|
|
# those contents in different ways), we need to include them in
|
|
# the list of changed files so that Mercurial can join up their
|
|
# filelog histories (same as if the merge was done in Mercurial to
|
|
# begin with).
|
|
if p2 == nullid:
|
|
return []
|
|
manifest1 = self.repo.changectx(p1).manifest()
|
|
manifest2 = self.repo.changectx(p2).manifest()
|
|
return [
|
|
path
|
|
for path, node1 in manifest1.iteritems()
|
|
if path not in files and manifest2.get(path, node1) != node1
|
|
]
|
|
|
|
def getfilectx(repo, memctx, f):
|
|
info = files.get(f)
|
|
if info is not None:
|
|
# it's a file reported as modified from Git
|
|
delete, mode, sha = info
|
|
if delete:
|
|
if getattr(memctx, "_returnnoneformissingfiles", False):
|
|
return None
|
|
else: # Mercurial < 3.2
|
|
raise IOError
|
|
|
|
if not sha: # indicates there's no git counterpart
|
|
e = ""
|
|
copied_path = None
|
|
else:
|
|
data = self.git[sha].data
|
|
copied_path = renames.get(f)
|
|
e = self.convert_git_int_mode(mode)
|
|
else:
|
|
# it's a converged file
|
|
fc = context.filectx(self.repo, f, changeid=memctx.p1().rev())
|
|
data = fc.data()
|
|
e = fc.flags()
|
|
copied_path = None
|
|
copied = fc.renamed()
|
|
if copied:
|
|
copied_path = copied[0]
|
|
|
|
try:
|
|
return context.memfilectx(
|
|
self.repo,
|
|
memctx,
|
|
f,
|
|
data,
|
|
islink="l" in e,
|
|
isexec="x" in e,
|
|
copied=copied_path,
|
|
)
|
|
except TypeError:
|
|
try:
|
|
return context.memfilectx(
|
|
self.repo,
|
|
f,
|
|
data,
|
|
islink="l" in e,
|
|
isexec="x" in e,
|
|
copied=copied_path,
|
|
)
|
|
except TypeError:
|
|
return context.memfilectx(
|
|
f, data, islink="l" in e, isexec="x" in e, copied=copied_path
|
|
)
|
|
|
|
p1, p2 = (nullid, nullid)
|
|
octopus = False
|
|
|
|
if len(gparents) > 1:
|
|
# merge, possibly octopus
|
|
def commit_octopus(p1, p2):
|
|
ctx = context.memctx(
|
|
self.repo,
|
|
(p1, p2),
|
|
text,
|
|
list(files) + findconvergedfiles(p1, p2),
|
|
getfilectx,
|
|
author,
|
|
date,
|
|
{"hg-git": "octopus"},
|
|
)
|
|
return hex(self.repo.commitctx(ctx))
|
|
|
|
octopus = len(gparents) > 2
|
|
p2 = gparents.pop()
|
|
p1 = gparents.pop()
|
|
while len(gparents) > 0:
|
|
p2 = commit_octopus(p1, p2)
|
|
p1 = gparents.pop()
|
|
else:
|
|
if gparents:
|
|
p1 = gparents.pop()
|
|
|
|
# if named branch, add to extra
|
|
if hg_branch:
|
|
extra["branch"] = hg_branch
|
|
else:
|
|
extra["branch"] = "default"
|
|
|
|
# if committer is different than author, add it to extra
|
|
if (
|
|
commit.author != commit.committer
|
|
or commit.author_time != commit.commit_time
|
|
or commit.author_timezone != commit.commit_timezone
|
|
):
|
|
extra["committer"] = "%s %d %d" % (
|
|
commit.committer,
|
|
commit.commit_time,
|
|
-commit.commit_timezone,
|
|
)
|
|
|
|
if commit.encoding:
|
|
extra["encoding"] = commit.encoding
|
|
|
|
if octopus:
|
|
extra["hg-git"] = "octopus-done"
|
|
|
|
ctx = context.memctx(
|
|
self.repo,
|
|
(p1, p2),
|
|
text,
|
|
list(files) + findconvergedfiles(p1, p2),
|
|
getfilectx,
|
|
author,
|
|
date,
|
|
extra,
|
|
)
|
|
node = self.repo.commitctx(ctx)
|
|
|
|
self.swap_out_encoding(oldenc)
|
|
|
|
# save changeset to mapping file
|
|
cs = hex(node)
|
|
self.map_set(commit.id, cs)
|
|
|
|
# PACK UPLOADING AND FETCHING
|
|
|
|
def upload_pack(self, remote, revs, force):
|
|
client, path = self.get_transport_and_path(remote)
|
|
old_refs = {}
|
|
change_totals = {}
|
|
|
|
def changed(refs):
|
|
self.ui.status(_("searching for changes\n"))
|
|
old_refs.update(refs)
|
|
all_exportable = self.get_exportable()
|
|
if revs is None:
|
|
exportable = all_exportable
|
|
else:
|
|
exportable = {}
|
|
for rev in (hex(r) for r in revs):
|
|
if rev not in all_exportable:
|
|
raise error.Abort(
|
|
"revision %s cannot be pushed since"
|
|
" it doesn't have a bookmark" % self.repo[rev]
|
|
)
|
|
exportable[rev] = all_exportable[rev]
|
|
return self.get_changed_refs(refs, exportable, force)
|
|
|
|
def genpack(have, want):
|
|
commits = []
|
|
for mo in self.git.object_store.find_missing_objects(have, want):
|
|
(sha, name) = mo
|
|
o = self.git.object_store[sha]
|
|
t = type(o)
|
|
change_totals[t] = change_totals.get(t, 0) + 1
|
|
if isinstance(o, Commit):
|
|
commits.append(sha)
|
|
commit_count = len(commits)
|
|
self.ui.note(_("%d commits found\n") % commit_count)
|
|
if commit_count > 0:
|
|
self.ui.debug("list of commits:\n")
|
|
for commit in commits:
|
|
self.ui.debug("%s\n" % commit)
|
|
self.ui.status(_("adding objects\n"))
|
|
return self.git.object_store.generate_pack_contents(have, want)
|
|
|
|
def callback(remote_info):
|
|
# dulwich (perhaps git?) wraps remote output at a fixed width but
|
|
# signifies the end of transmission with a double new line
|
|
global CALLBACK_BUFFER
|
|
if remote_info and not remote_info.endswith("\n\n"):
|
|
CALLBACK_BUFFER += remote_info
|
|
return
|
|
|
|
remote_info = CALLBACK_BUFFER + remote_info
|
|
CALLBACK_BUFFER = ""
|
|
if not remote_info:
|
|
remote_info = "\n"
|
|
|
|
for line in remote_info[:-1].split("\n"):
|
|
self.ui.status(_("remote: %s\n") % line)
|
|
|
|
try:
|
|
new_refs = client.send_pack(path, changed, genpack, progress=callback)
|
|
if len(change_totals) > 0:
|
|
self.ui.status(
|
|
_("added %d commits with %d trees" " and %d blobs\n")
|
|
% (
|
|
change_totals.get(Commit, 0),
|
|
change_totals.get(Tree, 0),
|
|
change_totals.get(Blob, 0),
|
|
)
|
|
)
|
|
return old_refs, new_refs
|
|
except (HangupException, GitProtocolError) as e:
|
|
raise error.Abort(_("git remote error: ") + str(e))
|
|
|
|
def get_changed_refs(self, refs, exportable, force):
|
|
new_refs = refs.copy()
|
|
|
|
# The remote repo is empty and the local one doesn't have
|
|
# bookmarks/tags
|
|
#
|
|
# (older dulwich versions return the proto-level
|
|
# capabilities^{} key when the dict should have been
|
|
# empty. That check can probably be removed at some point in
|
|
# the future.)
|
|
if not refs or refs.keys()[0] == "capabilities^{}":
|
|
if not exportable:
|
|
tip = self.repo.lookup("tip")
|
|
if tip != nullid:
|
|
if "capabilities^{}" in new_refs:
|
|
del new_refs["capabilities^{}"]
|
|
tip = hex(tip)
|
|
try:
|
|
commands.bookmark(
|
|
self.ui, self.repo, "master", rev=tip, force=True
|
|
)
|
|
except NameError:
|
|
bookmarks.bookmark(
|
|
self.ui, self.repo, "master", rev=tip, force=True
|
|
)
|
|
try:
|
|
bookmarks.activate(self.repo, "master")
|
|
except AttributeError:
|
|
# hg < 3.5
|
|
bookmarks.setcurrent(self.repo, "master")
|
|
new_refs["refs/heads/master"] = self.map_git_get(tip)
|
|
|
|
# mapped nodes might be hidden
|
|
unfiltered = self.repo.unfiltered()
|
|
for rev, rev_refs in exportable.iteritems():
|
|
ctx = self.repo[rev]
|
|
if not rev_refs:
|
|
raise error.Abort(
|
|
"revision %s cannot be pushed since"
|
|
" it doesn't have a bookmark" % ctx
|
|
)
|
|
|
|
# Check if the tags the server is advertising are annotated tags,
|
|
# by attempting to retrieve it from the our git repo, and building
|
|
# a list of these tags.
|
|
#
|
|
# This is possible, even though (currently) annotated tags are
|
|
# dereferenced and stored as lightweight ones, as the annotated tag
|
|
# is still stored in the git repo.
|
|
uptodate_annotated_tags = []
|
|
for ref in rev_refs.tags:
|
|
# Check tag.
|
|
if ref not in refs:
|
|
continue
|
|
try:
|
|
# We're not using Repo.tag(), as it's deprecated.
|
|
tag = self.git.get_object(refs[ref])
|
|
if not isinstance(tag, Tag):
|
|
continue
|
|
except KeyError:
|
|
continue
|
|
|
|
# If we've reached here, the tag's good.
|
|
uptodate_annotated_tags.append(ref)
|
|
|
|
for ref in rev_refs:
|
|
if ref not in refs:
|
|
new_refs[ref] = self.map_git_get(ctx.hex())
|
|
elif self._map.lookupbyfirst(bin(new_refs[ref])) is not None:
|
|
rctx = unfiltered[self.map_hg_get(new_refs[ref])]
|
|
if rctx.ancestor(ctx) == rctx or force:
|
|
new_refs[ref] = self.map_git_get(ctx.hex())
|
|
else:
|
|
raise error.Abort("pushing %s overwrites %s" % (ref, ctx))
|
|
elif ref in uptodate_annotated_tags:
|
|
# we already have the annotated tag.
|
|
pass
|
|
else:
|
|
raise error.Abort(
|
|
"branch '%s' changed on the server, "
|
|
"please pull and merge before pushing" % ref
|
|
)
|
|
|
|
return new_refs
|
|
|
|
def fetch_pack(self, remote_name, heads=None):
|
|
localclient, path = self.get_transport_and_path(remote_name)
|
|
|
|
# The dulwich default walk only checks refs/heads/. We also want to
|
|
# consider remotes when doing discovery, so we build our own list. We
|
|
# can't just do 'refs/' here because the tag class doesn't have a
|
|
# parents function for walking, and older versions of dulwich don't like
|
|
# that.
|
|
haveheads = self.git.refs.as_dict("refs/remotes/").values()
|
|
haveheads.extend(self.git.refs.as_dict("refs/heads/").values())
|
|
graphwalker = self.git.get_graph_walker(heads=haveheads)
|
|
|
|
def determine_wants(refs):
|
|
if refs is None:
|
|
return None
|
|
filteredrefs = self.filter_refs(refs, heads)
|
|
return [x for x in filteredrefs.itervalues() if x not in self.git]
|
|
|
|
try:
|
|
with progress.bar(self.ui, "") as prog:
|
|
gitprogress = GitProgress(self.ui, prog)
|
|
f = pycompat.stringio()
|
|
|
|
# monkey patch dulwich's read_pkt_refs so that we can determine
|
|
# on clone which bookmark to activate
|
|
client.read_pkt_refs = compat.read_pkt_refs
|
|
ret = localclient.fetch_pack(
|
|
path, determine_wants, graphwalker, f.write, gitprogress.progress
|
|
)
|
|
if f.tell() != 0:
|
|
f.seek(0)
|
|
self.git.object_store.add_thin_pack(f.read, None)
|
|
|
|
# For empty repos dulwich gives us None, but since later
|
|
# we want to iterate over this, we really want an empty
|
|
# iterable
|
|
return ret if ret else {}
|
|
except (HangupException, GitProtocolError) as e:
|
|
raise error.Abort(_("git remote error: ") + str(e))
|
|
|
|
# REFERENCES HANDLING
|
|
|
|
def filter_refs(self, refs, heads):
|
|
"""For a dictionary of refs: shas, if heads is None then return refs
|
|
that match the heads. Otherwise, return refs that are heads or tags.
|
|
|
|
"""
|
|
filteredrefs = []
|
|
if heads is not None:
|
|
# contains pairs of ('refs/(heads|tags|...)/foo', 'foo')
|
|
# if ref is just '<foo>', then we get ('foo', 'foo')
|
|
stripped_refs = [(r, r[r.find("/", r.find("/") + 1) + 1 :]) for r in refs]
|
|
for h in heads:
|
|
if h.endswith("/*"):
|
|
prefix = h[:-1] # include the / but not the *
|
|
r = [
|
|
pair[0] for pair in stripped_refs if pair[1].startswith(prefix)
|
|
]
|
|
r.sort()
|
|
filteredrefs.extend(r)
|
|
else:
|
|
r = [pair[0] for pair in stripped_refs if pair[1] == h]
|
|
if not r:
|
|
raise error.Abort("ref %s not found on remote server" % h)
|
|
elif len(r) == 1:
|
|
filteredrefs.append(r[0])
|
|
else:
|
|
raise error.Abort("ambiguous reference %s: %r" % (h, r))
|
|
else:
|
|
for ref, sha in refs.iteritems():
|
|
if not ref.endswith("^{}") and (
|
|
ref.startswith("refs/heads/") or ref.startswith("refs/tags/")
|
|
):
|
|
filteredrefs.append(ref)
|
|
filteredrefs.sort()
|
|
|
|
# the choice of OrderedDict vs plain dict has no impact on stock
|
|
# hg-git, but allows extensions to customize the order in which refs
|
|
# are returned
|
|
return util.OrderedDict((r, refs[r]) for r in filteredrefs)
|
|
|
|
def filter_min_date(self, refs):
|
|
"""filter refs by minimum date
|
|
|
|
This only works for refs that are available locally."""
|
|
min_date = compat.config(self.ui, "string", "git", "mindate")
|
|
if min_date is None:
|
|
return refs
|
|
|
|
# filter refs older than min_timestamp
|
|
min_timestamp, min_offset = hgutil.parsedate(min_date)
|
|
|
|
def check_min_time(obj):
|
|
if isinstance(obj, Tag):
|
|
return obj.tag_time >= min_timestamp
|
|
else:
|
|
return obj.commit_time >= min_timestamp
|
|
|
|
return util.OrderedDict(
|
|
(ref, sha) for ref, sha in refs.iteritems() if check_min_time(self.git[sha])
|
|
)
|
|
|
|
def update_references(self):
|
|
exportable = self.get_exportable()
|
|
|
|
# Create a local Git branch name for each
|
|
# Mercurial bookmark.
|
|
for hg_sha, refs in exportable.iteritems():
|
|
for git_ref in refs.heads:
|
|
git_sha = self.map_git_get(hg_sha)
|
|
if git_sha:
|
|
self.git.refs[git_ref] = git_sha
|
|
|
|
def export_hg_tags(self):
|
|
for tag, sha in self.repo.tags().iteritems():
|
|
if self.repo.tagtype(tag) in ("global", "git"):
|
|
tag = tag.replace(" ", "_")
|
|
target = self.map_git_get(hex(sha))
|
|
if target is not None:
|
|
tag_refname = "refs/tags/" + tag
|
|
if check_ref_format(tag_refname):
|
|
self.git.refs[tag_refname] = target
|
|
self.tags[tag] = hex(sha)
|
|
else:
|
|
self.repo.ui.warn(
|
|
_(
|
|
"Skipping export of tag %s because "
|
|
"it has invalid name as a git "
|
|
"refname.\n"
|
|
)
|
|
% tag
|
|
)
|
|
else:
|
|
self.repo.ui.warn(
|
|
_(
|
|
"Skipping export of tag %s because it "
|
|
"has no matching git revision.\n"
|
|
)
|
|
% tag
|
|
)
|
|
|
|
def _filter_for_bookmarks(self, bms):
|
|
if not self.branch_bookmark_suffix:
|
|
return [(bm, bm) for bm in bms]
|
|
else:
|
|
|
|
def _filter_bm(bm):
|
|
if bm.endswith(self.branch_bookmark_suffix):
|
|
return bm[0 : -(len(self.branch_bookmark_suffix))]
|
|
else:
|
|
return bm
|
|
|
|
return [(_filter_bm(bm), bm) for bm in bms]
|
|
|
|
def get_exportable(self):
|
|
class heads_tags(object):
|
|
def __init__(self):
|
|
self.heads = set()
|
|
self.tags = set()
|
|
|
|
def __iter__(self):
|
|
return itertools.chain(self.heads, self.tags)
|
|
|
|
def __nonzero__(self):
|
|
return bool(self.heads) or bool(self.tags)
|
|
|
|
res = collections.defaultdict(heads_tags)
|
|
|
|
bms = self.repo._bookmarks
|
|
for filtered_bm, bm in self._filter_for_bookmarks(bms):
|
|
res[hex(bms[bm])].heads.add("refs/heads/" + filtered_bm)
|
|
for tag, sha in self.tags.iteritems():
|
|
res[sha].tags.add("refs/tags/" + tag)
|
|
return res
|
|
|
|
def import_tags(self, refs):
|
|
keys = refs.keys()
|
|
if not keys:
|
|
return
|
|
repotags = self.repo.tags()
|
|
for k in keys[:]:
|
|
ref_name = k
|
|
parts = k.split("/")
|
|
if parts[0] == "refs" and parts[1] == "tags":
|
|
ref_name = "/".join([v for v in parts[2:]])
|
|
# refs contains all the refs in the server, not just
|
|
# the ones we are pulling
|
|
if refs[k] not in self.git.object_store:
|
|
continue
|
|
if ref_name[-3:] == "^{}":
|
|
ref_name = ref_name[:-3]
|
|
if ref_name not in repotags:
|
|
obj = self.git.get_object(refs[k])
|
|
sha = None
|
|
if isinstance(obj, Commit): # lightweight
|
|
sha = self.map_hg_get(refs[k])
|
|
if sha is not None:
|
|
self.tags[ref_name] = sha
|
|
elif isinstance(obj, Tag): # annotated
|
|
(obj_type, obj_sha) = obj.object
|
|
obj = self.git.get_object(obj_sha)
|
|
if isinstance(obj, Commit):
|
|
sha = self.map_hg_get(obj_sha)
|
|
# TODO: better handling for annotated tags
|
|
if sha is not None:
|
|
self.tags[ref_name] = sha
|
|
self.save_tags()
|
|
|
|
def update_hg_bookmarks(self, refs):
|
|
try:
|
|
bms = self.repo._bookmarks
|
|
|
|
heads = dict(
|
|
[(ref[11:], refs[ref]) for ref in refs if ref.startswith("refs/heads/")]
|
|
)
|
|
|
|
suffix = self.branch_bookmark_suffix or ""
|
|
changes = []
|
|
for head, sha in heads.iteritems():
|
|
# refs contains all the refs in the server, not just
|
|
# the ones we are pulling
|
|
hgsha = self.map_hg_get(sha)
|
|
if hgsha is None:
|
|
continue
|
|
hgsha = bin(hgsha)
|
|
if head not in bms:
|
|
# new branch
|
|
changes.append((head + suffix, hgsha))
|
|
else:
|
|
bm = self.repo[bms[head]]
|
|
if bm.ancestor(self.repo[hgsha]) == bm:
|
|
# fast forward
|
|
changes.append((head + suffix, hgsha))
|
|
|
|
if heads:
|
|
util.updatebookmarks(self.repo, changes)
|
|
|
|
except AttributeError:
|
|
self.ui.warn(
|
|
_("creating bookmarks failed, do you have" " bookmarks enabled?\n")
|
|
)
|
|
|
|
def update_remote_branches(self, remote_name, refs):
|
|
remote_refs = self.remote_refs
|
|
# since we re-write all refs for this remote each time, prune
|
|
# all entries matching this remote from our refs list now so
|
|
# that we avoid any stale refs hanging around forever
|
|
for t in list(remote_refs):
|
|
if t.startswith(remote_name + "/"):
|
|
del remote_refs[t]
|
|
for ref_name, sha in refs.iteritems():
|
|
if ref_name.startswith("refs/heads"):
|
|
hgsha = self.map_hg_get(sha)
|
|
if hgsha is None or hgsha not in self.repo:
|
|
continue
|
|
head = ref_name[11:]
|
|
remote_refs["/".join((remote_name, head))] = bin(hgsha)
|
|
# TODO(durin42): what is this doing?
|
|
new_ref = "refs/remotes/%s/%s" % (remote_name, head)
|
|
self.git.refs[new_ref] = sha
|
|
elif ref_name.startswith("refs/tags") and not ref_name.endswith("^{}"):
|
|
self.git.refs[ref_name] = sha
|
|
|
|
# UTILITY FUNCTIONS
|
|
|
|
def convert_git_int_mode(self, mode):
|
|
# TODO: make these into constants
|
|
convert = {0o100644: "", 0o100755: "x", 0o120000: "l"}
|
|
if mode in convert:
|
|
return convert[mode]
|
|
return ""
|
|
|
|
def get_file(self, commit, f):
|
|
otree = self.git.tree(commit.tree)
|
|
parts = f.split("/")
|
|
for part in parts:
|
|
(mode, sha) = otree[part]
|
|
obj = self.git.get_object(sha)
|
|
if isinstance(obj, Blob):
|
|
return (mode, sha, obj._text)
|
|
elif isinstance(obj, Tree):
|
|
otree = obj
|
|
|
|
def get_files_changed(self, commit, detect_renames):
|
|
tree = commit.tree
|
|
btree = None
|
|
|
|
if commit.parents:
|
|
btree = self.git[commit.parents[0]].tree
|
|
|
|
files = {}
|
|
gitlinks = {}
|
|
renames = None
|
|
rename_detector = None
|
|
if detect_renames:
|
|
renames = {}
|
|
rename_detector = self._rename_detector
|
|
|
|
# this set is unused if rename detection isn't enabled -- that makes
|
|
# the code below simpler
|
|
renamed_out = set()
|
|
|
|
changes = diff_tree.tree_changes(
|
|
self.git.object_store, btree, tree, rename_detector=rename_detector
|
|
)
|
|
|
|
for change in changes:
|
|
oldfile, oldmode, oldsha = change.old
|
|
newfile, newmode, newsha = change.new
|
|
# actions are described by the following table ('no' means 'does
|
|
# not exist'):
|
|
# old new | action
|
|
# no file | record file
|
|
# no gitlink | record gitlink
|
|
# file no | delete file
|
|
# file file | record file
|
|
# file gitlink | delete file and record gitlink
|
|
# gitlink no | delete gitlink
|
|
# gitlink file | delete gitlink and record file
|
|
# gitlink gitlink | record gitlink
|
|
#
|
|
# There's an edge case here -- symlink <-> regular file transitions
|
|
# are returned by dulwich as separate deletes and adds, not
|
|
# modifications. The order of those results is unspecified and
|
|
# could be either way round. Handle both cases: delete first, then
|
|
# add -- delete stored in 'old = file' case, then overwritten by
|
|
# 'new = file' case. add first, then delete -- record stored in
|
|
# 'new = file' case, then membership check fails in 'old = file'
|
|
# case so is not overwritten there. This is not an issue for
|
|
# gitlink <-> {symlink, regular file} transitions because they
|
|
# write to separate dictionaries.
|
|
#
|
|
# There's a similar edge case when rename detection is enabled: if
|
|
# a file is renamed and then replaced by a symlink (typically to
|
|
# the new location), it is returned by dulwich as an add and a
|
|
# rename. The order of those results is unspecified. Handle both
|
|
# cases: rename first, then add -- delete stored in 'new = file'
|
|
# case with renamed_out, then renamed_out check passes in 'old =
|
|
# file' case so is overwritten. add first, then rename -- add
|
|
# stored in 'old = file' case, then membership check fails in 'new
|
|
# = file' case so is overwritten.
|
|
if newmode == 0o160000:
|
|
# new = gitlink
|
|
gitlinks[newfile] = newsha
|
|
if change.type == diff_tree.CHANGE_RENAME:
|
|
# don't record the rename because only file -> file renames
|
|
# make sense in Mercurial
|
|
gitlinks[oldfile] = None
|
|
if oldmode is not None and oldmode != 0o160000:
|
|
# file -> gitlink
|
|
files[oldfile] = True, None, None
|
|
continue
|
|
if oldmode == 0o160000 and newmode != 0o160000:
|
|
# gitlink -> no/file (gitlink -> gitlink is covered above)
|
|
gitlinks[oldfile] = None
|
|
continue
|
|
if newfile is not None:
|
|
self.audit_hg_path(newfile)
|
|
# new = file
|
|
files[newfile] = False, newmode, newsha
|
|
if renames is not None and newfile != oldfile:
|
|
renames[newfile] = oldfile
|
|
renamed_out.add(oldfile)
|
|
# the membership check is explained in a comment above
|
|
if change.type == diff_tree.CHANGE_RENAME and oldfile not in files:
|
|
files[oldfile] = True, None, None
|
|
else:
|
|
# old = file
|
|
# files renamed_out | action
|
|
# no * | write
|
|
# yes no | ignore
|
|
# yes yes | write
|
|
if oldfile not in files or oldfile in renamed_out:
|
|
files[oldfile] = True, None, None
|
|
|
|
return files, gitlinks, renames
|
|
|
|
@hgutil.propertycache
|
|
def _rename_detector(self):
|
|
# disabled by default to avoid surprises
|
|
similarity = compat.config(self.ui, "int", "git", "similarity")
|
|
if similarity < 0 or similarity > 100:
|
|
raise error.Abort(_("git.similarity must be between 0 and 100"))
|
|
if similarity == 0:
|
|
return None
|
|
|
|
# default is borrowed from Git
|
|
max_files = compat.config(self.ui, "int", "git", "renamelimit")
|
|
if max_files < 0:
|
|
raise error.Abort(_("git.renamelimit must be non-negative"))
|
|
if max_files == 0:
|
|
max_files = None
|
|
|
|
find_copies_harder = compat.config(self.ui, "bool", "git", "findcopiesharder")
|
|
return diff_tree.RenameDetector(
|
|
self.git.object_store,
|
|
rename_threshold=similarity,
|
|
max_files=max_files,
|
|
find_copies_harder=find_copies_harder,
|
|
)
|
|
|
|
def parse_gitmodules(self, tree_obj):
|
|
"""Parse .gitmodules from a git tree specified by tree_obj
|
|
|
|
:return: list of tuples (submodule path, url, name),
|
|
where name is quoted part of the section's name, or
|
|
empty list if nothing found
|
|
"""
|
|
rv = []
|
|
try:
|
|
unused_mode, gitmodules_sha = tree_obj[".gitmodules"]
|
|
except KeyError:
|
|
return rv
|
|
gitmodules_content = self.git[gitmodules_sha].data
|
|
fo = pycompat.stringio(gitmodules_content)
|
|
tt = dul_config.ConfigFile.from_file(fo)
|
|
for section in tt.keys():
|
|
section_kind, section_name = section
|
|
if section_kind == "submodule":
|
|
sm_path = tt.get(section, "path")
|
|
sm_url = tt.get(section, "url")
|
|
rv.append((sm_path, sm_url, section_name))
|
|
return rv
|
|
|
|
def git_file_readlines(self, tree_obj, fname):
|
|
"""Read content of a named entry from the git commit tree
|
|
|
|
:return: list of lines
|
|
"""
|
|
if fname in tree_obj:
|
|
unused_mode, sha = tree_obj[fname]
|
|
content = self.git[sha].data
|
|
return content.splitlines()
|
|
return []
|
|
|
|
def remote_name(self, remote):
|
|
names = [name for name, path in self.paths if path == remote]
|
|
if names:
|
|
return names[0]
|
|
|
|
def audit_hg_path(self, path):
|
|
if ".hg" in path.split(os.path.sep):
|
|
if compat.config(self.ui, "bool", "git", "blockdothg"):
|
|
raise error.Abort(
|
|
("Refusing to import problematic path %r" % path),
|
|
hint=(
|
|
"Mercurial cannot check out paths inside nested "
|
|
+ "repositories; if you need to continue, then set "
|
|
+ "'[git] blockdothg = false' in your hgrc."
|
|
),
|
|
)
|
|
self.ui.warn(
|
|
(
|
|
"warning: path %r is within a nested repository, "
|
|
+ "which Mercurial cannot check out.\n"
|
|
)
|
|
% path
|
|
)
|
|
|
|
# Stolen from hgsubversion
|
|
def swap_out_encoding(self, new_encoding="UTF-8"):
|
|
try:
|
|
from edenscm.mercurial import encoding
|
|
|
|
old = encoding.encoding
|
|
encoding.encoding = new_encoding
|
|
except (AttributeError, ImportError):
|
|
old = hgutil._encoding
|
|
hgutil._encoding = new_encoding
|
|
return old
|
|
|
|
def decode_guess(self, string, encoding):
|
|
# text is not valid utf-8, try to make sense of it
|
|
if encoding:
|
|
try:
|
|
return string.decode(encoding).encode("utf-8")
|
|
except UnicodeDecodeError:
|
|
pass
|
|
|
|
try:
|
|
return string.decode("latin-1").encode("utf-8")
|
|
except UnicodeDecodeError:
|
|
return string.decode("ascii", "replace").encode("utf-8")
|
|
|
|
def get_transport_and_path(self, uri):
|
|
"""Method that sets up the transport (either ssh or http(s))
|
|
|
|
Tests:
|
|
|
|
>>> from collections import namedtuple
|
|
>>> from dulwich.client import HttpGitClient, SSHGitClient
|
|
>>> from edenscm.mercurial.ui import ui
|
|
>>> mockrepo = namedtuple('localrepo', ['sharedvfs'])
|
|
>>> mockrepo.sharedvfs = ''
|
|
>>> g = GitHandler(mockrepo, ui())
|
|
>>> client, url = g.get_transport_and_path('http://fqdn.com/test.git')
|
|
>>> print isinstance(client, HttpGitClient)
|
|
True
|
|
>>> print url
|
|
http://fqdn.com/test.git
|
|
>>> client, url = g.get_transport_and_path('git@fqdn.com:user/repo.git')
|
|
>>> print isinstance(client, SSHGitClient)
|
|
True
|
|
>>> print url
|
|
user/repo.git
|
|
>>> print client.host
|
|
git@fqdn.com
|
|
"""
|
|
# pass hg's ui.ssh config to dulwich
|
|
if not issubclass(client.get_ssh_vendor, _ssh.SSHVendor):
|
|
client.get_ssh_vendor = _ssh.generate_ssh_vendor(self.ui)
|
|
|
|
# test for raw git ssh uri here so that we can reuse the logic below
|
|
if util.isgitsshuri(uri):
|
|
uri = "git+ssh://" + uri
|
|
|
|
git_match = RE_GIT_URI.match(uri)
|
|
if git_match:
|
|
res = git_match.groupdict()
|
|
host, port, sepr = res["host"], res["port"], res["sepr"]
|
|
transport = client.TCPGitClient
|
|
if "ssh" in res["scheme"]:
|
|
util.checksafessh(host)
|
|
transport = client.SSHGitClient
|
|
path = res["path"]
|
|
if sepr == "/" and not path.startswith("~"):
|
|
path = "/" + path
|
|
# strip trailing slash for heroku-style URLs
|
|
# ssh+git://git@heroku.com:project.git/
|
|
if sepr == ":" and path.endswith(".git/"):
|
|
path = path.rstrip("/")
|
|
if port:
|
|
client.port = port
|
|
|
|
return transport(host, port=port), path
|
|
|
|
if uri.startswith("git+http://") or uri.startswith("git+https://"):
|
|
uri = uri[4:]
|
|
|
|
if uri.startswith("http://") or uri.startswith("https://"):
|
|
pmgr = compat.passwordmgr(self.ui)
|
|
auth = hgutil.urlreq.HTTPBasicAuthHandler(pmgr)
|
|
|
|
opener = hgutil.urlreq.build_opener(auth)
|
|
ua = "git/20x6 (hg-git ; uses dulwich and hg ; like git-core)"
|
|
opener.addheaders = [("User-Agent", ua)]
|
|
try:
|
|
return client.HttpGitClient(uri, opener=opener), uri
|
|
except TypeError as e:
|
|
if e.message.find("unexpected keyword argument 'opener'") >= 0:
|
|
# Dulwich 0.9.4, which is the latest version that ships
|
|
# with Ubuntu 14.04, doesn't support the 'opener' keyword.
|
|
# Try without authentication.
|
|
return client.HttpGitClient(uri), uri
|
|
else:
|
|
raise
|
|
|
|
# if its not git or git+ssh, try a local url..
|
|
return client.SubprocessGitClient(), uri
|
|
|
|
|
|
class GitMap(object):
|
|
def __init__(self, content):
|
|
mapgit = {}
|
|
maphg = {}
|
|
for line in content:
|
|
# format is <40 hex digits> <40 hex digits>\n
|
|
if len(line) != 82:
|
|
raise ValueError(
|
|
_("corrupt mapfile: incorrect line length %d %s")
|
|
% (len(line), content)
|
|
)
|
|
gitnode = bin(line[:40])
|
|
hgnode = bin(line[41:81])
|
|
mapgit[gitnode] = hgnode
|
|
maphg[hgnode] = gitnode
|
|
self._mapgit = mapgit
|
|
self._maphg = maphg
|
|
|
|
def lookupbyfirst(self, gitnode):
|
|
return self._mapgit.get(gitnode)
|
|
|
|
def lookupbysecond(self, hgnode):
|
|
return self._maphg.get(hgnode)
|
|
|
|
def add(self, gitnode, hgnode):
|
|
self._mapgit[gitnode] = hgnode
|
|
self._maphg[hgnode] = gitnode
|
|
|
|
def items(self):
|
|
return self._mapgit.items()
|