2015-10-31 01:37:17 +03:00
|
|
|
# dirsync.py - keep two directories synchronized at commit time
|
|
|
|
#
|
|
|
|
# Copyright 2015 Facebook, Inc.
|
|
|
|
#
|
|
|
|
# This software may be used and distributed according to the terms of the
|
|
|
|
# GNU General Public License version 2 or any later version.
|
|
|
|
"""
|
|
|
|
dirsync is an extension for keeping directories in a repo synchronized.
|
|
|
|
|
|
|
|
Configure it by adding the following config options to your .hg/hgrc.
|
|
|
|
|
|
|
|
[dirsync]
|
|
|
|
projectX.dir1 = path/to/dir1
|
|
|
|
projectX.dir2 = path/dir2
|
|
|
|
|
|
|
|
The configs are of the form "group.name = path-to-dir". Every config entry with
|
|
|
|
the same `group` will be mirrored amongst each other. The `name` is just used to
|
|
|
|
separate them and is not used anywhere. The `path` is the path to the directory
|
|
|
|
from the repo root. It must be a directory, but it doesn't matter if you specify
|
|
|
|
the trailing '/' or not.
|
|
|
|
|
|
|
|
Multiple mirror groups can be specified at once, and you can mirror between an
|
|
|
|
arbitrary number of directories. Ex:
|
|
|
|
|
|
|
|
[dirsync]
|
|
|
|
projectX.dir1 = path/to/dir1
|
|
|
|
projectX.dir2 = path/dir2
|
|
|
|
projectY.dir1 = otherpath/dir1
|
|
|
|
projectY.dir2 = foo/bar
|
|
|
|
projectY.dir3 = foo/goo/hoo
|
|
|
|
"""
|
|
|
|
|
2017-03-18 02:35:08 +03:00
|
|
|
from __future__ import absolute_import
|
|
|
|
|
2015-10-31 01:37:17 +03:00
|
|
|
from collections import defaultdict
|
dirsync: Behave better with already-inconsistent mirror
Summary:
If a mirror of a file is missing, dirsync now behaves better:
- When deleting the source, it allows the commit and does no mirroring since the two paths are already in sync (previously, it raised an exception when trying to delete the mirror)
- When updating the source, it creates a file in the mirrored location to match
- When updating (creating) the mirrored file, the source (which already exists) is updated too
Fixes https://www.facebook.com/groups/sourcecontrol/permalink/929877003728587/.
Test Plan: `../../hg/tests/run-tests.py test-dirsync.t`
Reviewers: #sourcecontrol, durham, lcharignon, ttung
Reviewed By: durham
Subscribers: poshannessy, mjpieters, durham
Differential Revision: https://phabricator.fb.com/D2794928
Signature: t1:2794928:1456446576:c18d06a42eb89b5168c25d048ba4fa1fa3f0c6d3
2016-02-26 03:50:43 +03:00
|
|
|
import errno
|
2017-03-18 02:35:08 +03:00
|
|
|
from mercurial import (
|
2017-09-08 20:01:05 +03:00
|
|
|
cmdutil,
|
2017-03-18 02:35:08 +03:00
|
|
|
error,
|
|
|
|
extensions,
|
|
|
|
localrepo,
|
|
|
|
match as matchmod,
|
2017-09-08 20:01:05 +03:00
|
|
|
scmutil,
|
2017-03-18 02:35:08 +03:00
|
|
|
util,
|
|
|
|
)
|
2016-03-05 03:39:48 +03:00
|
|
|
from mercurial.i18n import _
|
2015-10-31 01:37:17 +03:00
|
|
|
|
2016-11-29 16:24:07 +03:00
|
|
|
testedwith = 'ships-with-fb-hgext'
|
2015-10-31 01:37:17 +03:00
|
|
|
|
2017-03-18 05:42:50 +03:00
|
|
|
_disabled = [False]
|
|
|
|
|
2015-10-31 01:37:17 +03:00
|
|
|
def extsetup(ui):
|
2017-09-08 20:01:05 +03:00
|
|
|
extensions.wrapfunction(cmdutil, 'amend', _amend)
|
2015-10-31 01:37:17 +03:00
|
|
|
extensions.wrapfunction(localrepo.localrepository, 'commit', _commit)
|
2017-09-08 20:01:05 +03:00
|
|
|
|
dirsync: disable dirsync during shelve/unshelve
Summary:
Shelve and unshelve use commits under the hood to bundle up data. dirsync sees
these commits happening and performs mirrors, which then shows up at unbundle
time, making it impossible to mirror the change again. Example: if I change X,
and it gets mirrored to Y, if I go back and change X later after a unshelve, hg
commit now fails because Y has pending changes and differs from X.
Test Plan: Added a test
Reviewers: #mercurial, ttung, quark
Reviewed By: quark
Subscribers: quark, wez, mjpieters, frantic, wluh
Differential Revision: https://phabricator.intern.facebook.com/D3266602
Signature: t1:3266602:1462484017:738f67c0ab4b5af999819d3855c1f4ba6b2ea338
2016-05-06 00:51:59 +03:00
|
|
|
def wrapshelve(loaded=False):
|
|
|
|
try:
|
|
|
|
shelvemod = extensions.find('shelve')
|
|
|
|
extensions.wrapcommand(shelvemod.cmdtable, 'shelve',
|
|
|
|
_bypassdirsync)
|
|
|
|
extensions.wrapcommand(shelvemod.cmdtable, 'unshelve',
|
|
|
|
_bypassdirsync)
|
|
|
|
except KeyError:
|
|
|
|
pass
|
|
|
|
extensions.afterloaded('shelve', wrapshelve)
|
|
|
|
|
|
|
|
def _bypassdirsync(orig, ui, repo, *args, **kwargs):
|
2017-03-18 05:42:50 +03:00
|
|
|
_disabled[0] = True
|
dirsync: disable dirsync during shelve/unshelve
Summary:
Shelve and unshelve use commits under the hood to bundle up data. dirsync sees
these commits happening and performs mirrors, which then shows up at unbundle
time, making it impossible to mirror the change again. Example: if I change X,
and it gets mirrored to Y, if I go back and change X later after a unshelve, hg
commit now fails because Y has pending changes and differs from X.
Test Plan: Added a test
Reviewers: #mercurial, ttung, quark
Reviewed By: quark
Subscribers: quark, wez, mjpieters, frantic, wluh
Differential Revision: https://phabricator.intern.facebook.com/D3266602
Signature: t1:3266602:1462484017:738f67c0ab4b5af999819d3855c1f4ba6b2ea338
2016-05-06 00:51:59 +03:00
|
|
|
try:
|
|
|
|
return orig(ui, repo, *args, **kwargs)
|
|
|
|
finally:
|
2017-03-18 05:42:50 +03:00
|
|
|
_disabled[0] = False
|
2015-10-31 01:37:17 +03:00
|
|
|
|
2017-03-18 02:35:08 +03:00
|
|
|
def getconfigs(repo):
|
|
|
|
# bypass "repoui.copy = baseui.copy # prevent copying repo configuration"
|
|
|
|
ui = repo.ui.__class__.copy(repo.ui)
|
|
|
|
|
2017-03-20 20:55:43 +03:00
|
|
|
# also read from wvfs/.hgdirsync
|
|
|
|
filename = '.hgdirsync'
|
|
|
|
content = repo.wvfs.tryread(filename)
|
2017-03-18 02:35:08 +03:00
|
|
|
if content:
|
2017-03-20 20:55:43 +03:00
|
|
|
ui._tcfg.parse(filename, '[dirsync]\n%s' % content, ['dirsync'])
|
2017-03-18 02:35:08 +03:00
|
|
|
|
2015-10-31 01:37:17 +03:00
|
|
|
maps = defaultdict(list)
|
|
|
|
for key, value in ui.configitems('dirsync'):
|
|
|
|
if '.' not in key:
|
|
|
|
continue
|
|
|
|
name, disambig = key.split('.', 1)
|
|
|
|
# Normalize paths to have / at the end. For easy concatenation later.
|
|
|
|
if value[-1] != '/':
|
|
|
|
value = value + '/'
|
|
|
|
maps[name].append(value)
|
|
|
|
return maps
|
|
|
|
|
|
|
|
def getmirrors(maps, filename):
|
|
|
|
for key, mirrordirs in maps.iteritems():
|
|
|
|
for subdir in mirrordirs:
|
|
|
|
if filename.startswith(subdir):
|
|
|
|
return mirrordirs
|
|
|
|
|
|
|
|
return []
|
|
|
|
|
2017-09-08 20:01:05 +03:00
|
|
|
def _updateworkingcopy(repo, matcher):
|
|
|
|
maps = getconfigs(repo)
|
|
|
|
mirroredfiles = set()
|
|
|
|
if maps:
|
|
|
|
status = repo.status()
|
|
|
|
|
|
|
|
for added in status.added:
|
|
|
|
mirrors = getmirrors(maps, added)
|
|
|
|
if mirrors and matcher(added):
|
|
|
|
mirroredfiles.update(applytomirrors(repo, status, added,
|
|
|
|
mirrors, 'a'))
|
|
|
|
|
|
|
|
for modified in status.modified:
|
|
|
|
mirrors = getmirrors(maps, modified)
|
|
|
|
if mirrors and matcher(modified):
|
|
|
|
mirroredfiles.update(applytomirrors(repo, status, modified,
|
|
|
|
mirrors, 'm'))
|
|
|
|
|
|
|
|
for removed in status.removed:
|
|
|
|
mirrors = getmirrors(maps, removed)
|
|
|
|
if mirrors and matcher(removed):
|
|
|
|
mirroredfiles.update(applytomirrors(repo, status, removed,
|
|
|
|
mirrors, 'r'))
|
|
|
|
|
|
|
|
return mirroredfiles
|
|
|
|
|
|
|
|
def _amend(orig, ui, repo, commitfunc, old, extra, pats, opts):
|
|
|
|
# Only wrap if not disabled and repo is instance of
|
|
|
|
# localrepo.localrepository
|
|
|
|
if _disabled[0] or not isinstance(repo, localrepo.localrepository):
|
|
|
|
return orig(ui, repo, old, extra, pats, opts)
|
|
|
|
|
|
|
|
with repo.wlock(), repo.lock(), repo.transaction('dirsyncamend'):
|
|
|
|
wctx = repo[None]
|
|
|
|
matcher = scmutil.match(wctx, pats, opts)
|
|
|
|
if (opts.get('addremove')
|
|
|
|
and scmutil.addremove(repo, matcher, "", opts)):
|
|
|
|
raise error.Abort(
|
|
|
|
_("failed to mark all new/missing files as added/removed"))
|
|
|
|
|
|
|
|
mirroredfiles = _updateworkingcopy(repo, matcher)
|
|
|
|
if mirroredfiles and not matcher.always():
|
|
|
|
# Ensure that all the files to be amended (original + synced) are
|
|
|
|
# under consideration during the amend operation. We do so by
|
|
|
|
# setting the value against 'include' key in opts as the only source
|
|
|
|
# of truth.
|
|
|
|
pats = ()
|
|
|
|
opts['include'] = [
|
|
|
|
f for f in wctx.files() if matcher(f)] + list(mirroredfiles)
|
|
|
|
|
|
|
|
return orig(ui, repo, commitfunc, old, extra, pats, opts)
|
|
|
|
|
2015-10-31 01:37:17 +03:00
|
|
|
def _commit(orig, self, *args, **kwargs):
|
2017-03-18 05:42:50 +03:00
|
|
|
if _disabled[0]:
|
dirsync: disable dirsync during shelve/unshelve
Summary:
Shelve and unshelve use commits under the hood to bundle up data. dirsync sees
these commits happening and performs mirrors, which then shows up at unbundle
time, making it impossible to mirror the change again. Example: if I change X,
and it gets mirrored to Y, if I go back and change X later after a unshelve, hg
commit now fails because Y has pending changes and differs from X.
Test Plan: Added a test
Reviewers: #mercurial, ttung, quark
Reviewed By: quark
Subscribers: quark, wez, mjpieters, frantic, wluh
Differential Revision: https://phabricator.intern.facebook.com/D3266602
Signature: t1:3266602:1462484017:738f67c0ab4b5af999819d3855c1f4ba6b2ea338
2016-05-06 00:51:59 +03:00
|
|
|
return orig(self, *args, **kwargs)
|
|
|
|
|
2017-09-08 20:01:05 +03:00
|
|
|
with self.wlock(), self.lock(), self.transaction('dirsynccommit'):
|
|
|
|
matcher = args[3] if len(args) >= 4 else kwargs.get('match')
|
|
|
|
matcher = matcher or matchmod.always(self.root, '')
|
|
|
|
|
|
|
|
mirroredfiles = _updateworkingcopy(self, matcher)
|
|
|
|
if mirroredfiles and not matcher.always():
|
|
|
|
origmatch = matcher.matchfn
|
2015-12-12 01:46:29 +03:00
|
|
|
def extramatches(path):
|
2015-11-25 22:21:47 +03:00
|
|
|
return path in mirroredfiles or origmatch(path)
|
2017-09-08 20:01:05 +03:00
|
|
|
matcher.matchfn = extramatches
|
|
|
|
matcher._files.extend(mirroredfiles)
|
|
|
|
matcher._fileset.update(mirroredfiles)
|
|
|
|
|
2015-10-31 01:37:17 +03:00
|
|
|
return orig(self, *args, **kwargs)
|
|
|
|
|
|
|
|
def applytomirrors(repo, status, sourcepath, mirrors, action):
|
|
|
|
"""Applies the changes that are in the sourcepath to all the mirrors."""
|
2015-11-25 22:21:47 +03:00
|
|
|
mirroredfiles = set()
|
|
|
|
|
2015-10-31 01:37:17 +03:00
|
|
|
# Detect which mirror this file comes from
|
|
|
|
sourcemirror = None
|
|
|
|
for mirror in mirrors:
|
|
|
|
if sourcepath.startswith(mirror):
|
|
|
|
sourcemirror = mirror
|
|
|
|
break
|
|
|
|
if not sourcemirror:
|
2016-03-05 03:39:48 +03:00
|
|
|
raise error.Abort(_("unable to detect source mirror of '%s'") %
|
2016-03-07 13:50:04 +03:00
|
|
|
(sourcepath,))
|
2015-10-31 01:37:17 +03:00
|
|
|
|
|
|
|
relpath = sourcepath[len(sourcemirror):]
|
|
|
|
|
|
|
|
# Apply the change to each mirror one by one
|
|
|
|
allchanges = set(status.modified + status.removed + status.added)
|
|
|
|
for mirror in mirrors:
|
|
|
|
if mirror == sourcemirror:
|
|
|
|
continue
|
|
|
|
|
|
|
|
mirrorpath = mirror + relpath
|
2015-11-25 22:21:47 +03:00
|
|
|
mirroredfiles.add(mirrorpath)
|
2015-10-31 01:37:17 +03:00
|
|
|
if mirrorpath in allchanges:
|
|
|
|
wctx = repo[None]
|
|
|
|
if (sourcepath not in wctx and mirrorpath not in wctx and
|
|
|
|
sourcepath in status.removed and mirrorpath in status.removed):
|
2016-01-06 00:52:13 +03:00
|
|
|
if repo.ui.verbose:
|
2016-03-05 03:39:48 +03:00
|
|
|
repo.ui.status(_("not mirroring remove of '%s' to '%s';"
|
|
|
|
" it is already removed\n")
|
2016-03-07 13:50:04 +03:00
|
|
|
% (sourcepath, mirrorpath))
|
2015-10-31 01:37:17 +03:00
|
|
|
continue
|
|
|
|
|
|
|
|
if wctx[sourcepath].data() == wctx[mirrorpath].data():
|
2016-01-06 00:52:13 +03:00
|
|
|
if repo.ui.verbose:
|
2016-03-05 03:39:48 +03:00
|
|
|
repo.ui.status(_("not mirroring '%s' to '%s'; it already "
|
|
|
|
"matches\n") % (sourcepath, mirrorpath))
|
2015-10-31 01:37:17 +03:00
|
|
|
continue
|
2016-03-07 13:50:04 +03:00
|
|
|
raise error.Abort(_("path '%s' needs to be mirrored to '%s', but "
|
|
|
|
"the target already has pending changes") %
|
|
|
|
(sourcepath, mirrorpath))
|
2015-10-31 01:37:17 +03:00
|
|
|
|
|
|
|
fullsource = repo.wjoin(sourcepath)
|
|
|
|
fulltarget = repo.wjoin(mirrorpath)
|
|
|
|
|
|
|
|
dirstate = repo.dirstate
|
|
|
|
if action == 'm' or action == 'a':
|
|
|
|
mirrorpathdir, unused = util.split(mirrorpath)
|
|
|
|
util.makedirs(repo.wjoin(mirrorpathdir))
|
|
|
|
|
|
|
|
util.copyfile(fullsource, fulltarget)
|
dirsync: Behave better with already-inconsistent mirror
Summary:
If a mirror of a file is missing, dirsync now behaves better:
- When deleting the source, it allows the commit and does no mirroring since the two paths are already in sync (previously, it raised an exception when trying to delete the mirror)
- When updating the source, it creates a file in the mirrored location to match
- When updating (creating) the mirrored file, the source (which already exists) is updated too
Fixes https://www.facebook.com/groups/sourcecontrol/permalink/929877003728587/.
Test Plan: `../../hg/tests/run-tests.py test-dirsync.t`
Reviewers: #sourcecontrol, durham, lcharignon, ttung
Reviewed By: durham
Subscribers: poshannessy, mjpieters, durham
Differential Revision: https://phabricator.fb.com/D2794928
Signature: t1:2794928:1456446576:c18d06a42eb89b5168c25d048ba4fa1fa3f0c6d3
2016-02-26 03:50:43 +03:00
|
|
|
if dirstate[mirrorpath] in '?r':
|
|
|
|
dirstate.add(mirrorpath)
|
2015-10-31 01:37:17 +03:00
|
|
|
|
2016-01-11 21:19:22 +03:00
|
|
|
|
2015-10-31 01:37:17 +03:00
|
|
|
if action == 'a':
|
|
|
|
# For adds, detect copy data as well
|
|
|
|
copysource = dirstate.copied(sourcepath)
|
|
|
|
if copysource and copysource.startswith(sourcemirror):
|
|
|
|
mirrorcopysource = mirror + copysource[len(sourcemirror):]
|
|
|
|
dirstate.copy(mirrorcopysource, mirrorpath)
|
2016-03-05 03:39:48 +03:00
|
|
|
repo.ui.status(_("mirrored copy '%s -> %s' to '%s -> %s'\n")
|
2016-03-07 13:50:04 +03:00
|
|
|
% (copysource, sourcepath,
|
|
|
|
mirrorcopysource, mirrorpath))
|
2015-10-31 01:37:17 +03:00
|
|
|
else:
|
2016-03-05 03:39:48 +03:00
|
|
|
repo.ui.status(_("mirrored adding '%s' to '%s'\n") %
|
2015-10-31 01:37:17 +03:00
|
|
|
(sourcepath, mirrorpath))
|
|
|
|
else:
|
2016-03-05 03:39:48 +03:00
|
|
|
repo.ui.status(_("mirrored changes in '%s' to '%s'\n") %
|
2015-10-31 01:37:17 +03:00
|
|
|
(sourcepath, mirrorpath))
|
|
|
|
elif action == 'r':
|
dirsync: Behave better with already-inconsistent mirror
Summary:
If a mirror of a file is missing, dirsync now behaves better:
- When deleting the source, it allows the commit and does no mirroring since the two paths are already in sync (previously, it raised an exception when trying to delete the mirror)
- When updating the source, it creates a file in the mirrored location to match
- When updating (creating) the mirrored file, the source (which already exists) is updated too
Fixes https://www.facebook.com/groups/sourcecontrol/permalink/929877003728587/.
Test Plan: `../../hg/tests/run-tests.py test-dirsync.t`
Reviewers: #sourcecontrol, durham, lcharignon, ttung
Reviewed By: durham
Subscribers: poshannessy, mjpieters, durham
Differential Revision: https://phabricator.fb.com/D2794928
Signature: t1:2794928:1456446576:c18d06a42eb89b5168c25d048ba4fa1fa3f0c6d3
2016-02-26 03:50:43 +03:00
|
|
|
try:
|
|
|
|
util.unlink(fulltarget)
|
|
|
|
except OSError as e:
|
|
|
|
if e.errno == errno.ENOENT:
|
2016-03-07 13:50:04 +03:00
|
|
|
repo.ui.status(_("not mirroring remove of '%s' to '%s'; it "
|
|
|
|
"is already removed\n") %
|
dirsync: Behave better with already-inconsistent mirror
Summary:
If a mirror of a file is missing, dirsync now behaves better:
- When deleting the source, it allows the commit and does no mirroring since the two paths are already in sync (previously, it raised an exception when trying to delete the mirror)
- When updating the source, it creates a file in the mirrored location to match
- When updating (creating) the mirrored file, the source (which already exists) is updated too
Fixes https://www.facebook.com/groups/sourcecontrol/permalink/929877003728587/.
Test Plan: `../../hg/tests/run-tests.py test-dirsync.t`
Reviewers: #sourcecontrol, durham, lcharignon, ttung
Reviewed By: durham
Subscribers: poshannessy, mjpieters, durham
Differential Revision: https://phabricator.fb.com/D2794928
Signature: t1:2794928:1456446576:c18d06a42eb89b5168c25d048ba4fa1fa3f0c6d3
2016-02-26 03:50:43 +03:00
|
|
|
(sourcepath, mirrorpath))
|
|
|
|
else:
|
|
|
|
raise
|
|
|
|
else:
|
|
|
|
dirstate.remove(mirrorpath)
|
2016-03-05 03:39:48 +03:00
|
|
|
repo.ui.status(_("mirrored remove of '%s' to '%s'\n") %
|
dirsync: Behave better with already-inconsistent mirror
Summary:
If a mirror of a file is missing, dirsync now behaves better:
- When deleting the source, it allows the commit and does no mirroring since the two paths are already in sync (previously, it raised an exception when trying to delete the mirror)
- When updating the source, it creates a file in the mirrored location to match
- When updating (creating) the mirrored file, the source (which already exists) is updated too
Fixes https://www.facebook.com/groups/sourcecontrol/permalink/929877003728587/.
Test Plan: `../../hg/tests/run-tests.py test-dirsync.t`
Reviewers: #sourcecontrol, durham, lcharignon, ttung
Reviewed By: durham
Subscribers: poshannessy, mjpieters, durham
Differential Revision: https://phabricator.fb.com/D2794928
Signature: t1:2794928:1456446576:c18d06a42eb89b5168c25d048ba4fa1fa3f0c6d3
2016-02-26 03:50:43 +03:00
|
|
|
(sourcepath, mirrorpath))
|
2015-11-25 22:21:47 +03:00
|
|
|
|
|
|
|
return mirroredfiles
|