mirror of
https://github.com/facebook/sapling.git
synced 2024-10-06 06:47:41 +03:00
sparse: allow changing sparse profiles via configs
Summary: If we mess up a sparse profile, it can be expensive to wait for all users to rebase forward in order for it to get fixed. Let's support adding includes/excludes via configuration instead. This will also help us incrementally roll out exclusions, to limit the extent of breakages. Like by only rolling out to sandcastle staging. Differential Revision: D31705827 fbshipit-source-id: 444c690deaf57f0bc15c1cc5a9c2e6b7d73e6ea6
This commit is contained in:
Durham Goode
Facebook GitHub Bot
parent
f211e4943f
commit
7505e3434c
@ -158,6 +158,7 @@ from edenscm.mercurial import (
|
||||
extensions,
|
||||
hg,
|
||||
hintutil,
|
||||
json,
|
||||
localrepo,
|
||||
match as matchmod,
|
||||
merge as mergemod,
|
||||
@ -208,6 +209,8 @@ configitem(
|
||||
configitem("sparse", "warnfullcheckout", default=None)
|
||||
configitem("sparse", "bypassfullcheckoutwarn", default=False)
|
||||
|
||||
profilecachefile = "sparseprofileconfigs"
|
||||
|
||||
|
||||
def uisetup(ui):
|
||||
_setupupdates(ui)
|
||||
@ -309,6 +312,10 @@ def _setupupdates(ui):
|
||||
oldrevs = [pctx.rev() for pctx in wctx.parents()]
|
||||
oldsparsematch = repo.sparsematch(*oldrevs)
|
||||
|
||||
repo._clearpendingprofileconfig(all=True)
|
||||
oldprofileconfigs = repo._getcachedprofileconfigs()
|
||||
newprofileconfigs = repo._creatependingprofileconfigs()
|
||||
|
||||
if branchmerge:
|
||||
# If we're merging, use the wctx filter, since we're merging into
|
||||
# the wctx.
|
||||
@ -364,7 +371,7 @@ def _setupupdates(ui):
|
||||
dirstate.normal(file)
|
||||
|
||||
profiles = repo.getactiveprofiles()
|
||||
changedprofiles = profiles & files
|
||||
changedprofiles = (profiles & files) or (oldprofileconfigs != newprofileconfigs)
|
||||
# If an active profile changed during the update, refresh the checkout.
|
||||
# Don't do this during a branch merge, since all incoming changes should
|
||||
# have been handled by the temporary includes above.
|
||||
@ -407,12 +414,17 @@ def _setupupdates(ui):
|
||||
extensions.wrapfunction(mergemod, "calculateupdates", _calculateupdates)
|
||||
|
||||
def _update(orig, repo, node, branchmerge, *args, **kwargs):
|
||||
results = orig(repo, node, branchmerge, *args, **kwargs)
|
||||
try:
|
||||
results = orig(repo, node, branchmerge, *args, **kwargs)
|
||||
except Exception:
|
||||
repo._clearpendingprofileconfig()
|
||||
raise
|
||||
|
||||
# If we're updating to a location, clean up any stale temporary includes
|
||||
# (ex: this happens during hg rebase --abort).
|
||||
if not branchmerge and util.safehasattr(repo, "sparsematch"):
|
||||
repo.prunetemporaryincludes()
|
||||
|
||||
return results
|
||||
|
||||
extensions.wrapfunction(mergemod, "update", _update)
|
||||
@ -993,6 +1005,94 @@ def _wraprepo(ui, repo):
|
||||
}
|
||||
return RawSparseConfig(filename, lines, profiles, metadata)
|
||||
|
||||
def _getlatestprofileconfigs(self):
|
||||
includes = collections.defaultdict(list)
|
||||
excludes = collections.defaultdict(list)
|
||||
for key, value in self.ui.configitems("sparseprofile"):
|
||||
# "exclude:" is the same length as "include.", so no need for
|
||||
# handling both.
|
||||
section = key[: len("include.")]
|
||||
name = key[len("include.") :]
|
||||
if section == "include.":
|
||||
for include in self.ui.configlist("sparseprofile", key):
|
||||
includes[name].append(include)
|
||||
elif section == "exclude.":
|
||||
for exclude in self.ui.configlist("sparseprofile", key):
|
||||
excludes[name].append(exclude)
|
||||
|
||||
results = {}
|
||||
keys = set(includes.keys())
|
||||
keys.update(excludes.keys())
|
||||
for key in keys:
|
||||
config = ""
|
||||
if key in includes:
|
||||
config += "[include]\n"
|
||||
for include in includes[key]:
|
||||
config += include + "\n"
|
||||
if key in excludes:
|
||||
config += "[exclude]\n"
|
||||
for exclude in excludes[key]:
|
||||
config += exclude + "\n"
|
||||
results[key] = config
|
||||
|
||||
return results
|
||||
|
||||
def _pendingprofileconfigname(self):
|
||||
return "%s.%s" % (profilecachefile, os.getpid())
|
||||
|
||||
def _getcachedprofileconfigs(self):
|
||||
"""gets the currently cached sparse profile config value
|
||||
|
||||
This may be a process-local value, if this process is in the middle
|
||||
of a checkout.
|
||||
"""
|
||||
# First check for a process-local profilecache. This let's an
|
||||
# ongoing checkout see the new sparse profile before we persist it
|
||||
# for other processes to see.
|
||||
pendingfile = self._pendingprofileconfigname()
|
||||
for name in [pendingfile, profilecachefile]:
|
||||
if self.localvfs.exists(name):
|
||||
serialized = self.localvfs.readutf8(name)
|
||||
return json.loads(serialized)
|
||||
return {}
|
||||
|
||||
def _creatependingprofileconfigs(self):
|
||||
"""creates a new process-local sparse profile config value
|
||||
|
||||
This will be read for future sparse matchers in this process.
|
||||
"""
|
||||
pendingfile = self._pendingprofileconfigname()
|
||||
latest = self._getlatestprofileconfigs()
|
||||
serialized = json.dumps(latest)
|
||||
self.localvfs.writeutf8(pendingfile, serialized)
|
||||
return latest
|
||||
|
||||
def _clearpendingprofileconfig(self, all=False):
|
||||
"""deletes all pending sparse profile config files
|
||||
|
||||
all=True causes it delete all the pending profile configs, for all
|
||||
processes. This should only be used while holding the wlock, so you don't
|
||||
accidentally delete a pending file out from under another process.
|
||||
"""
|
||||
prefix = "%s." % profilecachefile
|
||||
pid = str(os.getpid())
|
||||
for name in self.localvfs.listdir():
|
||||
if name.startswith(prefix):
|
||||
suffix = name[len(prefix) :]
|
||||
if all or suffix == pid:
|
||||
self.localvfs.unlink(name)
|
||||
|
||||
def _persistprofileconfigs(self):
|
||||
"""upgrades the current process-local sparse profile config value to
|
||||
be the global value
|
||||
"""
|
||||
# The pending file should exist in all cases when this code path is
|
||||
# hit. But if it does't this will throw an exception. That's
|
||||
# probably fine though, since that indicates something went very
|
||||
# wrong.
|
||||
pendingfile = self._pendingprofileconfigname()
|
||||
self.localvfs.rename(pendingfile, profilecachefile)
|
||||
|
||||
def getsparsepatterns(self, rev, rawconfig=None, debugversion=None):
|
||||
"""Produce the full sparse config for a revision as a SparseConfig
|
||||
|
||||
@ -1020,6 +1120,8 @@ def _wraprepo(ui, repo):
|
||||
"be a RawSparseConfig, not: %s" % rawconfig
|
||||
)
|
||||
|
||||
profileconfigs = self._getcachedprofileconfigs()
|
||||
|
||||
includes = set()
|
||||
excludes = set()
|
||||
rules = ["glob:.hg*"]
|
||||
@ -1027,7 +1129,7 @@ def _wraprepo(ui, repo):
|
||||
onlyv1 = True
|
||||
for kind, value in rawconfig.lines:
|
||||
if kind == "profile":
|
||||
profile = self.readsparseprofile(rev, value)
|
||||
profile = self.readsparseprofile(rev, value, profileconfigs)
|
||||
if profile is not None:
|
||||
profiles.append(profile)
|
||||
# v1 config's put all includes before all excludes, so
|
||||
@ -1072,7 +1174,7 @@ def _wraprepo(ui, repo):
|
||||
rawconfig.metadata,
|
||||
)
|
||||
|
||||
def readsparseprofile(self, rev, name):
|
||||
def readsparseprofile(self, rev, name, profileconfigs):
|
||||
ctx = self[rev]
|
||||
try:
|
||||
raw = self.getrawprofile(name, ctx.hex())
|
||||
@ -1095,7 +1197,7 @@ def _wraprepo(ui, repo):
|
||||
for kind, value in rawconfig.lines:
|
||||
if kind == "profile":
|
||||
profiles.add(value)
|
||||
profile = self.readsparseprofile(rev, value)
|
||||
profile = self.readsparseprofile(rev, value, profileconfigs)
|
||||
if profile is not None:
|
||||
for rule in profile.rules:
|
||||
rules.append(rule)
|
||||
@ -1106,6 +1208,18 @@ def _wraprepo(ui, repo):
|
||||
elif kind == "exclude":
|
||||
rules.append("!" + value)
|
||||
|
||||
if profileconfigs:
|
||||
raw = profileconfigs.get(name)
|
||||
if raw:
|
||||
rawprofileconfig = self.readsparseconfig(
|
||||
raw, filename=name + "-hgrc.dynamic"
|
||||
)
|
||||
for kind, value in rawprofileconfig.lines:
|
||||
if kind == "include":
|
||||
rules.append(value)
|
||||
elif kind == "exclude":
|
||||
rules.append("!" + value)
|
||||
|
||||
return SparseProfile(name, rules, profiles, rawconfig.metadata)
|
||||
|
||||
def _warnfullcheckout(self):
|
||||
|
||||
@ -2587,6 +2587,16 @@ def update(
|
||||
# update completed, clear state
|
||||
util.unlink(repo.localvfs.join("updatestate"))
|
||||
|
||||
# After recordupdates has finished, the checkout is considered
|
||||
# finished and we should persist the sparse profile config
|
||||
# changes.
|
||||
#
|
||||
# Ideally this would be part of some wider transaction framework
|
||||
# that ensures these things all happen atomically, but that
|
||||
# doesn't exist for the dirstate right now.
|
||||
if util.safehasattr(repo, "_persistprofileconfigs"):
|
||||
repo._persistprofileconfigs()
|
||||
|
||||
if not branchmerge:
|
||||
repo.dirstate.setbranch(p2.branch())
|
||||
|
||||
@ -2606,6 +2616,16 @@ def getsparsematchers(repo, fp1, fp2, matcher=None):
|
||||
if sparsematch is not None:
|
||||
revs = {fp1, fp2}
|
||||
revs -= {nullid}
|
||||
|
||||
repo._clearpendingprofileconfig(all=True)
|
||||
oldpatterns = repo.getsparsepatterns(fp1)
|
||||
oldmatcher = sparsematch(fp1)
|
||||
|
||||
repo._creatependingprofileconfigs()
|
||||
|
||||
newpatterns = repo.getsparsepatterns(fp2)
|
||||
newmatcher = sparsematch(fp2)
|
||||
|
||||
sparsematcher = sparsematch(*list(revs))
|
||||
# Ignore files that are not in either source or target sparse match
|
||||
# This is not enough if sparse profile changes, but works for checkout within same sparse profile
|
||||
@ -2615,8 +2635,8 @@ def getsparsematchers(repo, fp1, fp2, matcher=None):
|
||||
# This signals to nativecheckout that there isn't a sparse
|
||||
# profile transition.
|
||||
oldnewmatchers = None
|
||||
if not repo.getsparsepatterns(fp1).equivalent(repo.getsparsepatterns(fp2)):
|
||||
oldnewmatchers = (sparsematch(fp1), sparsematch(fp2))
|
||||
if not oldpatterns.equivalent(newpatterns):
|
||||
oldnewmatchers = (oldmatcher, newmatcher)
|
||||
|
||||
# This can be optimized - if matchers are same, we can set sparsematchers = None
|
||||
# sparse.py does not do it, so we are not making things worse
|
||||
@ -2727,6 +2747,16 @@ def donativecheckout(repo, p1, p2, xp1, xp2, matcher, force, partial, wc, prerec
|
||||
repo.localvfs.unlink("updatestate")
|
||||
repo.localvfs.unlink("updateprogress")
|
||||
|
||||
# After recordupdates has finished, the checkout is considered
|
||||
# finished and we should persist the sparse profile config
|
||||
# changes.
|
||||
#
|
||||
# Ideally this would be part of some wider transaction framework
|
||||
# that ensures these things all happen atomically, but that
|
||||
# doesn't exist for the dirstate right now.
|
||||
if util.safehasattr(repo, "_persistprofileconfigs"):
|
||||
repo._persistprofileconfigs()
|
||||
|
||||
if not partial:
|
||||
repo.hook("update", parent1=xp1, parent2=xp2, error=stats[3])
|
||||
postrecrawls = querywatchmanrecrawls(repo)
|
||||
|
||||
186
eden/scm/tests/test-sparse-hgrc-profile.t
Normal file
186
eden/scm/tests/test-sparse-hgrc-profile.t
Normal file
@ -0,0 +1,186 @@
|
||||
#testcases normalcheckout nativecheckout
|
||||
$ configure modernclient
|
||||
$ enable sparse
|
||||
|
||||
#if nativecheckout
|
||||
$ setconfig experimental.nativecheckout=True
|
||||
#endif
|
||||
|
||||
$ newclientrepo
|
||||
$ touch init
|
||||
$ hg commit -Aqm initial
|
||||
$ touch alwaysincluded
|
||||
$ touch excludedbyconfig
|
||||
$ touch includedbyconfig
|
||||
$ cat >> sparseprofile <<EOF
|
||||
> [include]
|
||||
> alwaysincluded
|
||||
> excludedbyconfig
|
||||
> EOF
|
||||
$ hg commit -Aqm 'add files'
|
||||
$ echo >> alwaysincluded
|
||||
$ hg commit -Aqm 'modify'
|
||||
$ hg sparse enable sparseprofile
|
||||
$ ls
|
||||
alwaysincluded
|
||||
excludedbyconfig
|
||||
|
||||
Test hg update reads new hgrc profile config
|
||||
$ cp .hg/hgrc .hg/hgrc.bak
|
||||
$ cat >> .hg/hgrc <<EOF
|
||||
> [sparseprofile]
|
||||
> include.sparseprofile=includedbyconfig
|
||||
> exclude.sparseprofile=excludedbyconfig
|
||||
> EOF
|
||||
# Run a no-op command to verify it does not refresh the sparse profile with the
|
||||
# new config.
|
||||
$ hg log -r . -T '{desc}\n'
|
||||
modify
|
||||
|
||||
$ ls
|
||||
alwaysincluded
|
||||
excludedbyconfig
|
||||
|
||||
# hg up should update to use the new config
|
||||
$ hg up -q .^
|
||||
$ ls
|
||||
alwaysincluded
|
||||
includedbyconfig
|
||||
|
||||
$ cat .hg/sparseprofileconfigs
|
||||
{"sparseprofile": "[include]\nincludedbyconfig\n[exclude]\nexcludedbyconfig\n"} (no-eol)
|
||||
$ ls .hg/sparseprofileconfigs*
|
||||
.hg/sparseprofileconfigs
|
||||
|
||||
Test hg updating back to original location keeps the new hgrc profile config
|
||||
$ hg up -q tip
|
||||
$ ls
|
||||
alwaysincluded
|
||||
includedbyconfig
|
||||
|
||||
$ cat .hg/sparseprofileconfigs
|
||||
{"sparseprofile": "[include]\nincludedbyconfig\n[exclude]\nexcludedbyconfig\n"} (no-eol)
|
||||
$ ls .hg/sparseprofileconfigs*
|
||||
.hg/sparseprofileconfigs
|
||||
|
||||
Reset to remove hgrc profile config
|
||||
$ cp .hg/hgrc.bak .hg/hgrc
|
||||
$ hg up -q .^
|
||||
$ hg up -q .~-1
|
||||
$ ls
|
||||
alwaysincluded
|
||||
excludedbyconfig
|
||||
|
||||
$ cat .hg/sparseprofileconfigs
|
||||
{} (no-eol)
|
||||
$ ls .hg/sparseprofileconfigs*
|
||||
.hg/sparseprofileconfigs
|
||||
|
||||
Test hg commit does not read new hgrc profile config
|
||||
$ cat >> .hg/hgrc <<EOF
|
||||
> [sparseprofile]
|
||||
> include.sparseprofile=includedbyconfig
|
||||
> exclude.sparseprofile=excludedbyconfig
|
||||
> EOF
|
||||
$ echo >> alwaysincluded
|
||||
$ hg commit -m 'modify alwaysincluded'
|
||||
$ ls
|
||||
alwaysincluded
|
||||
excludedbyconfig
|
||||
|
||||
$ cat .hg/sparseprofileconfigs
|
||||
{} (no-eol)
|
||||
$ ls .hg/sparseprofileconfigs*
|
||||
.hg/sparseprofileconfigs
|
||||
|
||||
Update to get latest config
|
||||
$ hg up -q .^
|
||||
$ hg up -q .~-1
|
||||
$ cat .hg/sparseprofileconfigs
|
||||
{"sparseprofile": "[include]\nincludedbyconfig\n[exclude]\nexcludedbyconfig\n"} (no-eol)
|
||||
$ ls .hg/sparseprofileconfigs*
|
||||
.hg/sparseprofileconfigs
|
||||
|
||||
Reset
|
||||
$ cp .hg/hgrc.bak .hg/hgrc
|
||||
$ hg up -q .^
|
||||
$ hg up -q .~-1
|
||||
|
||||
Cleanly crash an update and verify the new config was not applied
|
||||
$ cat > ../killer.py << EOF
|
||||
> from edenscm.mercurial import error, extensions, localrepo
|
||||
> def setparents(orig, repo, *args, **kwargs):
|
||||
> raise error.Abort("bad thing happened")
|
||||
>
|
||||
> def extsetup(ui):
|
||||
> extensions.wrapfunction(localrepo.localrepository, "setparents",
|
||||
> setparents)
|
||||
> EOF
|
||||
|
||||
$ cat >> .hg/hgrc <<EOF
|
||||
> [sparseprofile]
|
||||
> include.sparseprofile=includedbyconfig
|
||||
> exclude.sparseprofile=excludedbyconfig
|
||||
> EOF
|
||||
$ hg up .^ --config extensions.killer=$TESTTMP/killer.py
|
||||
abort: bad thing happened
|
||||
[255]
|
||||
$ cat .hg/sparseprofileconfigs
|
||||
{} (no-eol)
|
||||
$ ls .hg/sparseprofileconfigs*
|
||||
.hg/sparseprofileconfigs
|
||||
|
||||
But a successful update does get the new config
|
||||
$ hg up -q .^
|
||||
$ cat .hg/sparseprofileconfigs
|
||||
{"sparseprofile": "[include]\nincludedbyconfig\n[exclude]\nexcludedbyconfig\n"} (no-eol)
|
||||
$ ls .hg/sparseprofileconfigs*
|
||||
.hg/sparseprofileconfigs
|
||||
|
||||
Reset
|
||||
$ cp .hg/hgrc.bak .hg/hgrc
|
||||
$ hg up -q .^
|
||||
$ hg up -q .~-1
|
||||
|
||||
Hard killing the process leaves the pending config file around
|
||||
$ cat >> .hg/hgrc <<EOF
|
||||
> [sparseprofile]
|
||||
> include.sparseprofile=includedbyconfig
|
||||
> exclude.sparseprofile=excludedbyconfig
|
||||
> EOF
|
||||
|
||||
$ cat > ../killer.py << EOF
|
||||
> import os
|
||||
> from edenscm.mercurial import extensions, localrepo
|
||||
> def setparents(orig, repo, *args, **kwargs):
|
||||
> # os._exit skips all cleanup
|
||||
> os._exit(100)
|
||||
>
|
||||
> def extsetup(ui):
|
||||
> extensions.wrapfunction(localrepo.localrepository, "setparents",
|
||||
> setparents)
|
||||
> EOF
|
||||
$ hg up .^ --config extensions.killer=$TESTTMP/killer.py
|
||||
[100]
|
||||
$ cat .hg/sparseprofileconfigs
|
||||
{} (no-eol)
|
||||
$ ls .hg/sparseprofileconfigs*
|
||||
.hg/sparseprofileconfigs
|
||||
.hg/sparseprofileconfigs.* (glob)
|
||||
|
||||
But it is not consumed (alwaysincluded should show up in the list)
|
||||
$ hg files
|
||||
alwaysincluded
|
||||
excludedbyconfig
|
||||
|
||||
And is cleaned up on the next update
|
||||
$ hg up -q .^
|
||||
$ ls .hg/sparseprofileconfigs*
|
||||
.hg/sparseprofileconfigs
|
||||
$ hg files
|
||||
alwaysincluded
|
||||
includedbyconfig
|
||||
|
||||
Reset
|
||||
$ cp .hg/hgrc.bak .hg/hgrc
|
||||
$ hg up -q .~-1
|
||||
@ -881,7 +881,6 @@ File count and size data for hg explain is cached in the simplecache extension:
|
||||
sparseprofile:profiles__bar__eggs:*:v2 (glob)
|
||||
sparseprofile:profiles__bar__ham:*:v2 (glob)
|
||||
|
||||
|
||||
Test non-existing profiles are properly reported
|
||||
$ newrepo sparsenoprofile
|
||||
$ cat > .hg/hgrc <<EOF
|
||||
|
||||
Loading…
Reference in New Issue
Block a user