2016-09-08 18:08:12 +03:00
|
|
|
# Copyright 2016-present Facebook. All Rights Reserved.
|
|
|
|
#
|
|
|
|
# context: context needed to annotate a file
|
|
|
|
#
|
|
|
|
# This software may be used and distributed according to the terms of the
|
|
|
|
# GNU General Public License version 2 or any later version.
|
|
|
|
|
|
|
|
from __future__ import absolute_import
|
|
|
|
|
2016-10-19 18:05:22 +03:00
|
|
|
import collections
|
2016-09-08 18:46:27 +03:00
|
|
|
import contextlib
|
2016-10-08 17:58:21 +03:00
|
|
|
import hashlib
|
2016-09-08 18:46:27 +03:00
|
|
|
import os
|
|
|
|
|
2016-09-08 18:08:12 +03:00
|
|
|
from mercurial import (
|
2016-09-28 17:56:31 +03:00
|
|
|
error,
|
2016-09-08 18:46:27 +03:00
|
|
|
lock as lockmod,
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
mdiff,
|
|
|
|
node,
|
|
|
|
scmutil,
|
2016-09-08 18:08:12 +03:00
|
|
|
util,
|
|
|
|
)
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
from mercurial.i18n import _
|
2016-09-08 18:08:12 +03:00
|
|
|
|
2016-10-19 18:05:22 +03:00
|
|
|
from . import (
|
|
|
|
error as faerror,
|
|
|
|
revmap as revmapmod,
|
|
|
|
)
|
|
|
|
|
2016-09-08 18:46:27 +03:00
|
|
|
import linelog as linelogmod
|
|
|
|
|
2016-09-28 17:56:31 +03:00
|
|
|
# given path, get filelog, cached
|
2016-09-08 18:08:12 +03:00
|
|
|
@util.lrucachefunc
|
2016-09-28 17:56:31 +03:00
|
|
|
def _getflog(repo, path):
|
|
|
|
return repo.file(path)
|
2016-09-08 18:08:12 +03:00
|
|
|
|
|
|
|
# extracted from mercurial.context.basefilectx.annotate
|
|
|
|
def _parents(f, follow=True):
|
|
|
|
# Cut _descendantrev here to mitigate the penalty of lazy linkrev
|
|
|
|
# adjustment. Otherwise, p._adjustlinkrev() would walk changelog
|
|
|
|
# from the topmost introrev (= srcrev) down to p.linkrev() if it
|
|
|
|
# isn't an ancestor of the srcrev.
|
|
|
|
f._changeid
|
|
|
|
pl = f.parents()
|
|
|
|
|
|
|
|
# Don't return renamed parents if we aren't following.
|
|
|
|
if not follow:
|
|
|
|
pl = [p for p in pl if p.path() == f.path()]
|
|
|
|
|
|
|
|
# renamed filectx won't have a filelog yet, so set it
|
|
|
|
# from the cache to save time
|
|
|
|
for p in pl:
|
|
|
|
if not '_filelog' in p.__dict__:
|
2016-09-28 17:56:31 +03:00
|
|
|
p._filelog = _getflog(f._repo, p.path())
|
2016-09-08 18:08:12 +03:00
|
|
|
|
|
|
|
return pl
|
|
|
|
|
|
|
|
# extracted from mercurial.context.basefilectx.annotate. slightly modified
|
|
|
|
# so it takes a fctx instead of a pair of text and fctx.
|
|
|
|
def _decorate(fctx):
|
|
|
|
text = fctx.data()
|
|
|
|
linecount = text.count('\n')
|
fastannotate: handle empty file correctly
Summary:
`_decorate` backported from upstream does not handle empty file correctly.
It would raise an assertion error when annotating an empty file:
File "fastannotate/commands.py", line 138, in fastannotate
not showdeleted))
File "fastannotate/context.py", line 331, in annotate
return self._refineannotateresult(result, revfctx, showpath, showlines)
File "fastannotate/context.py", line 503, in _refineannotateresult
if len(lines) != len(result):
AssertionError
This patch fixes it.
Test Plan: Run the modified test.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3944132
Signature: t1:3944132:1475166894:e2610c6364806b77c8533315a1a0a08b6c158fe5
2016-09-29 18:48:40 +03:00
|
|
|
if text and not text.endswith('\n'):
|
2016-09-08 18:08:12 +03:00
|
|
|
linecount += 1
|
|
|
|
return ([(fctx, i) for i in xrange(linecount)], text)
|
|
|
|
|
|
|
|
# extracted from mercurial.context.basefilectx.annotate. slightly modified
|
|
|
|
# so it takes an extra "blocks" parameter calculated elsewhere, instead of
|
|
|
|
# calculating diff here.
|
|
|
|
def _pair(parent, child, blocks):
|
|
|
|
for (a1, a2, b1, b2), t in blocks:
|
|
|
|
# Changed blocks ('!') or blocks made only of blank lines ('~')
|
|
|
|
# belong to the child.
|
|
|
|
if t == '=':
|
|
|
|
child[0][b1:b2] = parent[0][a1:a2]
|
|
|
|
return child
|
2016-09-08 18:09:48 +03:00
|
|
|
|
2016-09-28 17:56:31 +03:00
|
|
|
# like scmutil.revsingle, but with lru cache, so their states (like manifests)
|
|
|
|
# could be reused
|
|
|
|
_revsingle = util.lrucachefunc(scmutil.revsingle)
|
|
|
|
|
|
|
|
def resolvefctx(repo, rev, path, resolverev=False, adjustctx=None):
|
|
|
|
"""(repo, str, str) -> fctx
|
|
|
|
|
|
|
|
get the filectx object from repo, rev, path, in an efficient way.
|
|
|
|
|
|
|
|
if resolverev is True, "rev" is a revision specified by the revset
|
|
|
|
language, otherwise "rev" is a nodeid, or a revision number that can
|
|
|
|
be consumed by repo.__getitem__.
|
|
|
|
|
|
|
|
if adjustctx is not None, the returned fctx will point to a changeset
|
2016-09-28 17:56:47 +03:00
|
|
|
that introduces the change (last modified the file). if adjustctx
|
|
|
|
is 'linkrev', trust the linkrev and do not adjust it. this is noticeably
|
|
|
|
faster for big repos but is incorrect for some cases.
|
2016-09-28 17:56:31 +03:00
|
|
|
"""
|
fastannotate: support replacing fctx.annotate directly
Summary:
Previously, I have performance concern about the signature of
`fctx.annotate`: it returns fctxs, not ideal for performance since my
initial goal is to get rid of reading revlogs in the best case. And letting
the low-level `fctx.annotate` have some side effects writing to the disk is
not that pretty.
Because of that, fastannotate had re-invent part of the formatter (also
optimized for performance somehow), and still cannot support all the
features the original annotate supports, namely, templates.
Now it makes sense to just replace `fctx.annotate` with a sub-optimal
implementation, just for the flexibility of the template support of the
original annotate command. We can use a "fake" or "lazy" fctx object to
minimal the performance impact when people only need to print changeset
nodes and line numbers - in which case we don't read revlog.
Actually, the hgweb support already did a similar thing - converting
fastannotate output to annotate output. So we can reuse some code.
The next planned steps are:
- Make the original "annotate" command aware of fastannotate protocol, do
the pre-download stuff (downloading cache files at `fctx.annotate` is
possible but inefficient because of not being batched).
- "fastannotate" command remains a separated command optimized for perf
and provides extra features like "--deleted".
Because of the plan, the "commands" option does not make much sense -
"fastannotate" command will not replace "annotate" directly thus dropped
from the config interface. A new "modes" config option is added to control
how fastannotate works.
Test Plan: Modified existing tests
Reviewers: #sourcecontrol, simonfar
Reviewed By: simonfar
Subscribers: mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4238998
Signature: t1:4238998:1480447679:c48e0e565663c086293265e104d9cf414d913aa7
2016-11-29 14:51:08 +03:00
|
|
|
if resolverev and not isinstance(rev, int) and rev is not None:
|
2016-09-28 17:56:31 +03:00
|
|
|
ctx = _revsingle(repo, rev)
|
|
|
|
else:
|
|
|
|
ctx = repo[rev]
|
2016-11-03 23:55:16 +03:00
|
|
|
fctx = ctx[path]
|
2016-09-28 17:56:31 +03:00
|
|
|
if adjustctx is not None:
|
2016-09-28 17:56:47 +03:00
|
|
|
if adjustctx == 'linkrev':
|
|
|
|
introrev = fctx.linkrev()
|
|
|
|
else:
|
|
|
|
introrev = fctx.introrev()
|
2016-09-28 17:56:31 +03:00
|
|
|
if introrev != ctx.rev():
|
|
|
|
fctx._changeid = introrev
|
|
|
|
fctx._changectx = repo[introrev]
|
|
|
|
return fctx
|
|
|
|
|
2016-09-29 18:33:46 +03:00
|
|
|
# like mercurial.store.encodedir, but use linelog suffixes: .m, .l, .lock
|
|
|
|
def encodedir(path):
|
|
|
|
return (path
|
|
|
|
.replace('.hg/', '.hg.hg/')
|
|
|
|
.replace('.l/', '.l.hg/')
|
|
|
|
.replace('.m/', '.m.hg/')
|
|
|
|
.replace('.lock/', '.lock.hg/'))
|
|
|
|
|
2016-10-08 17:58:21 +03:00
|
|
|
def hashdiffopts(diffopts):
|
2016-10-27 02:19:05 +03:00
|
|
|
diffoptstr = str(sorted(
|
|
|
|
(k, getattr(diffopts, k))
|
|
|
|
for k in mdiff.diffopts.defaults.iterkeys()
|
|
|
|
))
|
2016-10-08 17:58:21 +03:00
|
|
|
return hashlib.sha1(diffoptstr).hexdigest()[:6]
|
|
|
|
|
|
|
|
_defaultdiffopthash = hashdiffopts(mdiff.defaultopts)
|
|
|
|
|
2016-09-08 18:09:48 +03:00
|
|
|
class annotateopts(object):
|
|
|
|
"""like mercurial.mdiff.diffopts, but is for annotate
|
|
|
|
|
|
|
|
followrename: follow renames, like "hg annotate -f"
|
|
|
|
followmerge: follow p2 of a merge changeset, otherwise p2 is ignored
|
|
|
|
"""
|
|
|
|
|
|
|
|
defaults = {
|
2016-10-08 17:58:21 +03:00
|
|
|
'diffopts': None,
|
2016-09-08 18:09:48 +03:00
|
|
|
'followrename': True,
|
|
|
|
'followmerge': True,
|
|
|
|
}
|
|
|
|
|
|
|
|
def __init__(self, **opts):
|
|
|
|
for k, v in self.defaults.iteritems():
|
|
|
|
setattr(self, k, opts.get(k, v))
|
|
|
|
|
2016-10-08 17:58:21 +03:00
|
|
|
@util.propertycache
|
2016-09-08 18:09:48 +03:00
|
|
|
def shortstr(self):
|
|
|
|
"""represent opts in a short string, suitable for a directory name"""
|
|
|
|
result = ''
|
|
|
|
if not self.followrename:
|
|
|
|
result += 'r0'
|
|
|
|
if not self.followmerge:
|
|
|
|
result += 'm0'
|
2016-10-08 17:58:21 +03:00
|
|
|
if self.diffopts is not None:
|
|
|
|
assert isinstance(self.diffopts, mdiff.diffopts)
|
|
|
|
diffopthash = hashdiffopts(self.diffopts)
|
|
|
|
if diffopthash != _defaultdiffopthash:
|
|
|
|
result += 'i' + diffopthash
|
2016-09-08 18:09:48 +03:00
|
|
|
return result or 'default'
|
|
|
|
|
|
|
|
defaultopts = annotateopts()
|
2016-09-08 18:46:27 +03:00
|
|
|
|
|
|
|
class _annotatecontext(object):
|
|
|
|
"""do not use this class directly as it does not use lock to protect
|
|
|
|
writes. use "with annotatecontext(...)" instead.
|
|
|
|
"""
|
|
|
|
|
2016-10-29 21:45:16 +03:00
|
|
|
def __init__(self, repo, path, linelogpath, revmappath, opts):
|
2016-09-08 18:46:27 +03:00
|
|
|
self.repo = repo
|
|
|
|
self.ui = repo.ui
|
|
|
|
self.path = path
|
|
|
|
self.opts = opts
|
2016-10-29 21:45:16 +03:00
|
|
|
self.linelogpath = linelogpath
|
|
|
|
self.revmappath = revmappath
|
|
|
|
self._linelog = None
|
|
|
|
self._revmap = None
|
fastannotate: handle "draft" paths, and renames correctly
Summary:
Previously we can only answer the "path" information when the revision is in
the linelog revmap, and the code would crash if a revision is in a side
branch, and the user requests path information. This diff fixes it.
Besides, this diff improves rename handling. For example, given the following
chart:
```
o---o -o file name: a
/
o---o- file name: b
^ ^ ^
1 2 3 revisions
```
Depending on the position of the `main branch` reference, fastannotate may
or may not use linelog:
- main branch is at rev 2, annotate a -r 3 will not take advantage of linelog
(fallback to slow annotate)
- main branch is at rev 3, annotate a -r 2 will not take advantage of linelog
This is not ideal, but seems to be the best we can do for now.
Test Plan:
Added a new test, updated existing relevant tests. Some debug messages are
changed to reflect internals more precisely.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4010964
Signature: t1:4010964:1476458201:79875d96399d023d0000d0c4bb8b8d40ea43eef0
2016-10-09 21:47:01 +03:00
|
|
|
self._node2path = {} # {str: str}
|
2016-09-08 18:46:27 +03:00
|
|
|
|
2016-10-29 21:45:16 +03:00
|
|
|
@property
|
|
|
|
def linelog(self):
|
|
|
|
if self._linelog is None:
|
|
|
|
self._linelog = linelogmod.linelog(self.linelogpath)
|
|
|
|
return self._linelog
|
|
|
|
|
|
|
|
@property
|
|
|
|
def revmap(self):
|
|
|
|
if self._revmap is None:
|
|
|
|
self._revmap = revmapmod.revmap(self.revmappath)
|
|
|
|
return self._revmap
|
|
|
|
|
|
|
|
def close(self):
|
|
|
|
if self._revmap is not None:
|
|
|
|
self._revmap.flush()
|
|
|
|
self._revmap = None
|
|
|
|
if self._linelog is not None:
|
|
|
|
self._linelog.close()
|
|
|
|
self._linelog = None
|
|
|
|
|
|
|
|
__del__ = close
|
|
|
|
|
|
|
|
def rebuild(self):
|
|
|
|
"""delete linelog and revmap, useful for rebuilding"""
|
|
|
|
self.close()
|
|
|
|
self._node2path.clear()
|
|
|
|
_unlinkpaths([self.revmappath, self.linelogpath])
|
|
|
|
|
2016-10-29 22:24:48 +03:00
|
|
|
@util.propertycache
|
|
|
|
def lastnode(self):
|
|
|
|
"""return last node in revmap, or None if revmap is empty"""
|
|
|
|
if self._revmap is None:
|
|
|
|
# fast path, read revmap without loading its full content
|
|
|
|
return revmapmod.getlastnode(self.revmappath)
|
|
|
|
else:
|
|
|
|
return self._revmap.rev2hsh(self._revmap.maxrev)
|
|
|
|
|
2016-11-26 01:06:18 +03:00
|
|
|
def isuptodate(self, master, strict=True):
|
2016-10-29 22:24:48 +03:00
|
|
|
"""return True if the revmap / linelog is up-to-date, or the file
|
|
|
|
does not exist in the master revision. False otherwise.
|
|
|
|
|
2016-11-26 01:06:18 +03:00
|
|
|
it tries to be fast and could return false negatives, because of the
|
|
|
|
use of linkrev instead of introrev.
|
2016-10-29 22:24:48 +03:00
|
|
|
|
|
|
|
useful for both server and client to decide whether to update
|
|
|
|
fastannotate cache or not.
|
2016-11-26 01:06:18 +03:00
|
|
|
|
|
|
|
if strict is True, even if fctx exists in the revmap, but is not the
|
|
|
|
last node, isuptodate will return False. it's good for performance - no
|
|
|
|
expensive check was done.
|
|
|
|
|
|
|
|
if strict is False, if fctx exists in the revmap, this function may
|
|
|
|
return True. this is useful for the client to skip downloading the
|
|
|
|
cache if the client's master is behind the server's.
|
2016-10-29 22:24:48 +03:00
|
|
|
"""
|
|
|
|
lastnode = self.lastnode
|
|
|
|
try:
|
|
|
|
f = self._resolvefctx(master, resolverev=True)
|
2016-11-26 01:06:18 +03:00
|
|
|
# choose linkrev instead of introrev as the check is meant to be
|
|
|
|
# *fast*.
|
|
|
|
linknode = self.repo.changelog.rev(f.linkrev())[-1]
|
|
|
|
if not strict and lastnode:
|
|
|
|
# perform the mtime check first, it's faster than loading the
|
|
|
|
# revamp.
|
|
|
|
try:
|
|
|
|
mtime = os.stat(self.revmappath).st_mtime
|
|
|
|
except OSError: # not fatal
|
|
|
|
pass
|
|
|
|
else:
|
|
|
|
if f.date()[0] < mtime:
|
|
|
|
# if mtime check passes, check if f.node() is in the
|
|
|
|
# revmap. note: this loads the revmap and can be slow.
|
|
|
|
return self.revmap.hsh2rev(linknode) is not None
|
2016-10-29 22:24:48 +03:00
|
|
|
# avoid resolving old manifest, or slow adjustlinkrev to be fast,
|
|
|
|
# false negatives are acceptable in this case.
|
2016-11-26 01:06:18 +03:00
|
|
|
return linknode == lastnode
|
2016-10-29 22:24:48 +03:00
|
|
|
except LookupError:
|
|
|
|
# master does not have the file, or the revmap is ahead
|
|
|
|
return True
|
|
|
|
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
def annotate(self, rev, master=None, showpath=False, showlines=False):
|
|
|
|
"""incrementally update the cache so it includes revisions in the main
|
|
|
|
branch till 'master'. and run annotate on 'rev', which may or may not be
|
|
|
|
included in the main branch.
|
|
|
|
|
2016-09-28 17:56:31 +03:00
|
|
|
if master is None, do not update linelog.
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
|
|
|
|
the first value returned is the annotate result, it is [(node, linenum)]
|
|
|
|
by default. [(node, linenum, path)] if showpath is True.
|
|
|
|
|
|
|
|
if showlines is True, a second value will be returned, it is a list of
|
|
|
|
corresponding line contents.
|
|
|
|
"""
|
|
|
|
|
2016-12-07 03:09:04 +03:00
|
|
|
# the fast path test requires commit hash, convert rev number to hash,
|
|
|
|
# so it may hit the fast path. note: in the "fctx" mode, the "annotate"
|
|
|
|
# command could give us a revision number even if the user passes a
|
|
|
|
# commit hash.
|
|
|
|
if isinstance(rev, int):
|
|
|
|
rev = self.repo.changelog.node(rev)
|
|
|
|
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
# fast path: if rev is in the main branch already
|
|
|
|
directly, revfctx = self.canannotatedirectly(rev)
|
|
|
|
if directly:
|
|
|
|
if self.ui.debugflag:
|
fastannotate: handle "draft" paths, and renames correctly
Summary:
Previously we can only answer the "path" information when the revision is in
the linelog revmap, and the code would crash if a revision is in a side
branch, and the user requests path information. This diff fixes it.
Besides, this diff improves rename handling. For example, given the following
chart:
```
o---o -o file name: a
/
o---o- file name: b
^ ^ ^
1 2 3 revisions
```
Depending on the position of the `main branch` reference, fastannotate may
or may not use linelog:
- main branch is at rev 2, annotate a -r 3 will not take advantage of linelog
(fallback to slow annotate)
- main branch is at rev 3, annotate a -r 2 will not take advantage of linelog
This is not ideal, but seems to be the best we can do for now.
Test Plan:
Added a new test, updated existing relevant tests. Some debug messages are
changed to reflect internals more precisely.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4010964
Signature: t1:4010964:1476458201:79875d96399d023d0000d0c4bb8b8d40ea43eef0
2016-10-09 21:47:01 +03:00
|
|
|
self.ui.debug('fastannotate: %s: using fast path '
|
|
|
|
'(resolved fctx: %s)\n'
|
|
|
|
% (self.path, util.safehasattr(revfctx, 'node')))
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
return self.annotatedirectly(revfctx, showpath, showlines)
|
|
|
|
|
|
|
|
# resolve master
|
|
|
|
masterfctx = None
|
|
|
|
if master:
|
fastannotate: handle "draft" paths, and renames correctly
Summary:
Previously we can only answer the "path" information when the revision is in
the linelog revmap, and the code would crash if a revision is in a side
branch, and the user requests path information. This diff fixes it.
Besides, this diff improves rename handling. For example, given the following
chart:
```
o---o -o file name: a
/
o---o- file name: b
^ ^ ^
1 2 3 revisions
```
Depending on the position of the `main branch` reference, fastannotate may
or may not use linelog:
- main branch is at rev 2, annotate a -r 3 will not take advantage of linelog
(fallback to slow annotate)
- main branch is at rev 3, annotate a -r 2 will not take advantage of linelog
This is not ideal, but seems to be the best we can do for now.
Test Plan:
Added a new test, updated existing relevant tests. Some debug messages are
changed to reflect internals more precisely.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4010964
Signature: t1:4010964:1476458201:79875d96399d023d0000d0c4bb8b8d40ea43eef0
2016-10-09 21:47:01 +03:00
|
|
|
try:
|
|
|
|
masterfctx = self._resolvefctx(master, resolverev=True,
|
|
|
|
adjustctx=True)
|
|
|
|
except LookupError: # master does not have the file
|
|
|
|
pass
|
|
|
|
else:
|
|
|
|
if masterfctx in self.revmap: # no need to update linelog
|
|
|
|
masterfctx = None
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
|
|
|
|
# ... - @ <- rev (can be an arbitrary changeset,
|
|
|
|
# / not necessarily a descendant
|
|
|
|
# master -> o of master)
|
|
|
|
# |
|
|
|
|
# a merge -> o 'o': new changesets in the main branch
|
|
|
|
# |\ '#': revisions in the main branch that
|
|
|
|
# o * exist in linelog / revmap
|
|
|
|
# | . '*': changesets in side branches, or
|
|
|
|
# last master -> # . descendants of master
|
|
|
|
# | .
|
|
|
|
# # * joint: '#', and is a parent of a '*'
|
|
|
|
# |/
|
|
|
|
# a joint -> # ^^^^ --- side branches
|
|
|
|
# |
|
|
|
|
# ^ --- main branch (in linelog)
|
|
|
|
|
|
|
|
# these DFSes are similar to the traditional annotate algorithm.
|
|
|
|
# we cannot really reuse the code for perf reason.
|
|
|
|
|
|
|
|
# 1st DFS calculates merges, joint points, and needed.
|
|
|
|
# "needed" is a simple reference counting dict to free items in
|
|
|
|
# "hist", reducing its memory usage otherwise could be huge.
|
|
|
|
initvisit = [revfctx]
|
|
|
|
if masterfctx:
|
2016-10-21 22:50:37 +03:00
|
|
|
if masterfctx.rev() is None:
|
|
|
|
raise error.Abort(_('cannot update linelog to wdir()'),
|
|
|
|
hint=_('set fastannotate.mainbranch'))
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
initvisit.append(masterfctx)
|
|
|
|
visit = initvisit[:]
|
|
|
|
pcache = {}
|
|
|
|
needed = {revfctx: 1}
|
|
|
|
hist = {} # {fctx: ([(llrev or fctx, linenum)], text)}
|
|
|
|
while visit:
|
|
|
|
f = visit.pop()
|
|
|
|
if f in pcache or f in hist:
|
|
|
|
continue
|
|
|
|
if f in self.revmap: # in the old main branch, it's a joint
|
|
|
|
llrev = self.revmap.hsh2rev(f.node())
|
|
|
|
self.linelog.annotate(llrev)
|
|
|
|
result = self.linelog.annotateresult
|
|
|
|
hist[f] = (result, f.data())
|
|
|
|
continue
|
|
|
|
pl = self._parentfunc(f)
|
|
|
|
pcache[f] = pl
|
|
|
|
for p in pl:
|
|
|
|
needed[p] = needed.get(p, 0) + 1
|
|
|
|
if p not in pcache:
|
|
|
|
visit.append(p)
|
|
|
|
|
|
|
|
# 2nd (simple) DFS calculates new changesets in the main branch
|
|
|
|
# ('o' nodes in # the above graph), so we know when to update linelog.
|
|
|
|
newmainbranch = set()
|
|
|
|
f = masterfctx
|
|
|
|
while f and f not in self.revmap:
|
|
|
|
newmainbranch.add(f)
|
|
|
|
pl = pcache[f]
|
|
|
|
if pl:
|
|
|
|
f = pl[0]
|
|
|
|
else:
|
|
|
|
f = None
|
|
|
|
break
|
|
|
|
|
|
|
|
# f, if present, is the position where the last build stopped at, and
|
|
|
|
# should be the "master" last time. check to see if we can continue
|
|
|
|
# building the linelog incrementally. (we cannot if diverged)
|
|
|
|
if masterfctx is not None:
|
|
|
|
self._checklastmasterhead(f)
|
|
|
|
|
|
|
|
if self.ui.debugflag:
|
fastannotate: handle "draft" paths, and renames correctly
Summary:
Previously we can only answer the "path" information when the revision is in
the linelog revmap, and the code would crash if a revision is in a side
branch, and the user requests path information. This diff fixes it.
Besides, this diff improves rename handling. For example, given the following
chart:
```
o---o -o file name: a
/
o---o- file name: b
^ ^ ^
1 2 3 revisions
```
Depending on the position of the `main branch` reference, fastannotate may
or may not use linelog:
- main branch is at rev 2, annotate a -r 3 will not take advantage of linelog
(fallback to slow annotate)
- main branch is at rev 3, annotate a -r 2 will not take advantage of linelog
This is not ideal, but seems to be the best we can do for now.
Test Plan:
Added a new test, updated existing relevant tests. Some debug messages are
changed to reflect internals more precisely.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4010964
Signature: t1:4010964:1476458201:79875d96399d023d0000d0c4bb8b8d40ea43eef0
2016-10-09 21:47:01 +03:00
|
|
|
if newmainbranch:
|
|
|
|
self.ui.debug('fastannotate: %s: %d new changesets in the main'
|
|
|
|
' branch\n' % (self.path, len(newmainbranch)))
|
|
|
|
elif not hist: # no joints, no updates
|
|
|
|
self.ui.debug('fastannotate: %s: linelog cannot help in '
|
|
|
|
'annotating this revision\n' % self.path)
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
|
|
|
|
# prepare annotateresult so we can update linelog incrementally
|
|
|
|
self.linelog.annotate(self.linelog.maxrev)
|
|
|
|
|
|
|
|
# 3rd DFS does the actual annotate
|
|
|
|
visit = initvisit[:]
|
|
|
|
progress = 0
|
|
|
|
while visit:
|
|
|
|
f = visit[-1]
|
fastannotate: handle "draft" paths, and renames correctly
Summary:
Previously we can only answer the "path" information when the revision is in
the linelog revmap, and the code would crash if a revision is in a side
branch, and the user requests path information. This diff fixes it.
Besides, this diff improves rename handling. For example, given the following
chart:
```
o---o -o file name: a
/
o---o- file name: b
^ ^ ^
1 2 3 revisions
```
Depending on the position of the `main branch` reference, fastannotate may
or may not use linelog:
- main branch is at rev 2, annotate a -r 3 will not take advantage of linelog
(fallback to slow annotate)
- main branch is at rev 3, annotate a -r 2 will not take advantage of linelog
This is not ideal, but seems to be the best we can do for now.
Test Plan:
Added a new test, updated existing relevant tests. Some debug messages are
changed to reflect internals more precisely.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4010964
Signature: t1:4010964:1476458201:79875d96399d023d0000d0c4bb8b8d40ea43eef0
2016-10-09 21:47:01 +03:00
|
|
|
if f in hist:
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
visit.pop()
|
|
|
|
continue
|
|
|
|
|
|
|
|
ready = True
|
|
|
|
pl = pcache[f]
|
|
|
|
for p in pl:
|
|
|
|
if p not in hist:
|
|
|
|
ready = False
|
|
|
|
visit.append(p)
|
|
|
|
if not ready:
|
|
|
|
continue
|
|
|
|
|
|
|
|
visit.pop()
|
|
|
|
blocks = None # mdiff blocks, used for appending linelog
|
|
|
|
ismainbranch = (f in newmainbranch)
|
|
|
|
# curr is the same as the traditional annotate algorithm,
|
|
|
|
# if we only care about linear history (do not follow merge),
|
|
|
|
# then curr is not actually used.
|
|
|
|
assert f not in hist
|
|
|
|
curr = _decorate(f)
|
|
|
|
for i, p in enumerate(pl):
|
2016-10-08 17:58:21 +03:00
|
|
|
bs = list(self._diffblocks(hist[p][1], curr[1]))
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
if i == 0 and ismainbranch:
|
|
|
|
blocks = bs
|
|
|
|
curr = _pair(hist[p], curr, bs)
|
|
|
|
if needed[p] == 1:
|
|
|
|
del hist[p]
|
|
|
|
del needed[p]
|
|
|
|
else:
|
|
|
|
needed[p] -= 1
|
|
|
|
|
|
|
|
hist[f] = curr
|
|
|
|
del pcache[f]
|
|
|
|
|
|
|
|
if ismainbranch: # need to write to linelog
|
|
|
|
if not self.ui.quiet:
|
|
|
|
progress += 1
|
|
|
|
self.ui.progress(_('building cache'), progress,
|
|
|
|
total=len(newmainbranch))
|
|
|
|
bannotated = None
|
|
|
|
if len(pl) == 2 and self.opts.followmerge: # merge
|
|
|
|
bannotated = curr[0]
|
|
|
|
if blocks is None: # no parents, add an empty one
|
2016-10-08 17:58:21 +03:00
|
|
|
blocks = list(self._diffblocks('', curr[1]))
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
self._appendrev(f, blocks, bannotated)
|
fastannotate: handle "draft" paths, and renames correctly
Summary:
Previously we can only answer the "path" information when the revision is in
the linelog revmap, and the code would crash if a revision is in a side
branch, and the user requests path information. This diff fixes it.
Besides, this diff improves rename handling. For example, given the following
chart:
```
o---o -o file name: a
/
o---o- file name: b
^ ^ ^
1 2 3 revisions
```
Depending on the position of the `main branch` reference, fastannotate may
or may not use linelog:
- main branch is at rev 2, annotate a -r 3 will not take advantage of linelog
(fallback to slow annotate)
- main branch is at rev 3, annotate a -r 2 will not take advantage of linelog
This is not ideal, but seems to be the best we can do for now.
Test Plan:
Added a new test, updated existing relevant tests. Some debug messages are
changed to reflect internals more precisely.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4010964
Signature: t1:4010964:1476458201:79875d96399d023d0000d0c4bb8b8d40ea43eef0
2016-10-09 21:47:01 +03:00
|
|
|
elif showpath: # not append linelog, but we need to record path
|
|
|
|
self._node2path[f.node()] = f.path()
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
|
|
|
|
if progress: # clean progress bar
|
|
|
|
self.ui.write()
|
|
|
|
|
|
|
|
result = [
|
2016-11-17 01:19:34 +03:00
|
|
|
((self.revmap.rev2hsh(fr) if isinstance(fr, int) else fr.node()), l)
|
|
|
|
for fr, l in hist[revfctx][0]] # [(node, linenumber)]
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
return self._refineannotateresult(result, revfctx, showpath, showlines)
|
|
|
|
|
|
|
|
def canannotatedirectly(self, rev):
|
|
|
|
"""(str) -> bool, fctx or node.
|
|
|
|
return (True, f) if we can annotate without updating the linelog, pass
|
|
|
|
f to annotatedirectly.
|
|
|
|
return (False, f) if we need extra calculation. f is the fctx resolved
|
|
|
|
from rev.
|
|
|
|
"""
|
|
|
|
result = True
|
|
|
|
f = None
|
fastannotate: support replacing fctx.annotate directly
Summary:
Previously, I have performance concern about the signature of
`fctx.annotate`: it returns fctxs, not ideal for performance since my
initial goal is to get rid of reading revlogs in the best case. And letting
the low-level `fctx.annotate` have some side effects writing to the disk is
not that pretty.
Because of that, fastannotate had re-invent part of the formatter (also
optimized for performance somehow), and still cannot support all the
features the original annotate supports, namely, templates.
Now it makes sense to just replace `fctx.annotate` with a sub-optimal
implementation, just for the flexibility of the template support of the
original annotate command. We can use a "fake" or "lazy" fctx object to
minimal the performance impact when people only need to print changeset
nodes and line numbers - in which case we don't read revlog.
Actually, the hgweb support already did a similar thing - converting
fastannotate output to annotate output. So we can reuse some code.
The next planned steps are:
- Make the original "annotate" command aware of fastannotate protocol, do
the pre-download stuff (downloading cache files at `fctx.annotate` is
possible but inefficient because of not being batched).
- "fastannotate" command remains a separated command optimized for perf
and provides extra features like "--deleted".
Because of the plan, the "commands" option does not make much sense -
"fastannotate" command will not replace "annotate" directly thus dropped
from the config interface. A new "modes" config option is added to control
how fastannotate works.
Test Plan: Modified existing tests
Reviewers: #sourcecontrol, simonfar
Reviewed By: simonfar
Subscribers: mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4238998
Signature: t1:4238998:1480447679:c48e0e565663c086293265e104d9cf414d913aa7
2016-11-29 14:51:08 +03:00
|
|
|
if not isinstance(rev, int) and rev is not None:
|
fastannotate: handle "draft" paths, and renames correctly
Summary:
Previously we can only answer the "path" information when the revision is in
the linelog revmap, and the code would crash if a revision is in a side
branch, and the user requests path information. This diff fixes it.
Besides, this diff improves rename handling. For example, given the following
chart:
```
o---o -o file name: a
/
o---o- file name: b
^ ^ ^
1 2 3 revisions
```
Depending on the position of the `main branch` reference, fastannotate may
or may not use linelog:
- main branch is at rev 2, annotate a -r 3 will not take advantage of linelog
(fallback to slow annotate)
- main branch is at rev 3, annotate a -r 2 will not take advantage of linelog
This is not ideal, but seems to be the best we can do for now.
Test Plan:
Added a new test, updated existing relevant tests. Some debug messages are
changed to reflect internals more precisely.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4010964
Signature: t1:4010964:1476458201:79875d96399d023d0000d0c4bb8b8d40ea43eef0
2016-10-09 21:47:01 +03:00
|
|
|
hsh = {20: bytes, 40: node.bin}.get(len(rev), lambda x: None)(rev)
|
|
|
|
if hsh is not None and (hsh, self.path) in self.revmap:
|
|
|
|
f = hsh
|
2016-10-09 17:19:18 +03:00
|
|
|
if f is None:
|
2016-09-28 17:56:47 +03:00
|
|
|
adjustctx = 'linkrev' if self._perfhack else True
|
2016-09-28 17:56:31 +03:00
|
|
|
f = self._resolvefctx(rev, adjustctx=adjustctx, resolverev=True)
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
result = f in self.revmap
|
2016-09-28 17:56:47 +03:00
|
|
|
if not result and self._perfhack:
|
|
|
|
# redo the resolution without perfhack - as we are going to
|
|
|
|
# do write operations, we need a correct fctx.
|
|
|
|
f = self._resolvefctx(rev, adjustctx=True, resolverev=True)
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
return result, f
|
|
|
|
|
2016-09-12 00:52:29 +03:00
|
|
|
def annotatealllines(self, rev, showpath=False, showlines=False):
|
|
|
|
"""(rev : str) -> [(node : str, linenum : int, path : str)]
|
|
|
|
|
|
|
|
the result has the same format with annotate, but include all (including
|
|
|
|
deleted) lines up to rev. call this after calling annotate(rev, ...) for
|
|
|
|
better performance and accuracy.
|
|
|
|
"""
|
2016-09-28 17:56:31 +03:00
|
|
|
revfctx = self._resolvefctx(rev, resolverev=True, adjustctx=True)
|
2016-09-12 00:52:29 +03:00
|
|
|
|
|
|
|
# find a chain from rev to anything in the mainbranch
|
|
|
|
if revfctx not in self.revmap:
|
|
|
|
chain = [revfctx]
|
|
|
|
a = ''
|
|
|
|
while True:
|
|
|
|
f = chain[-1]
|
|
|
|
pl = self._parentfunc(f)
|
|
|
|
if not pl:
|
|
|
|
break
|
|
|
|
if pl[0] in self.revmap:
|
|
|
|
a = pl[0].data()
|
|
|
|
break
|
|
|
|
chain.append(pl[0])
|
|
|
|
|
|
|
|
# both self.linelog and self.revmap is backed by filesystem. now
|
|
|
|
# we want to modify them but do not want to write changes back to
|
|
|
|
# files. so we create in-memory objects and copy them. it's like
|
|
|
|
# a "fork".
|
|
|
|
linelog = linelogmod.linelog()
|
|
|
|
linelog.copyfrom(self.linelog)
|
2016-10-08 03:39:17 +03:00
|
|
|
linelog.annotate(linelog.maxrev)
|
2016-09-12 00:52:29 +03:00
|
|
|
revmap = revmapmod.revmap()
|
|
|
|
revmap.copyfrom(self.revmap)
|
|
|
|
|
|
|
|
for f in reversed(chain):
|
|
|
|
b = f.data()
|
2016-10-08 17:58:21 +03:00
|
|
|
blocks = list(self._diffblocks(a, b))
|
2016-09-12 00:52:29 +03:00
|
|
|
self._doappendrev(linelog, revmap, f, blocks)
|
|
|
|
a = b
|
|
|
|
else:
|
|
|
|
# fastpath: use existing linelog, revmap as we don't write to them
|
|
|
|
linelog = self.linelog
|
|
|
|
revmap = self.revmap
|
|
|
|
|
|
|
|
lines = linelog.getalllines()
|
|
|
|
hsh = revfctx.node()
|
|
|
|
llrev = revmap.hsh2rev(hsh)
|
|
|
|
result = [(revmap.rev2hsh(r), l) for r, l in lines if r <= llrev]
|
|
|
|
# cannot use _refineannotateresult since we need custom logic for
|
|
|
|
# resolving line contents
|
|
|
|
if showpath:
|
|
|
|
result = self._addpathtoresult(result, revmap)
|
|
|
|
if showlines:
|
|
|
|
linecontents = self._resolvelines(result, revmap, linelog)
|
|
|
|
result = (result, linecontents)
|
|
|
|
return result
|
|
|
|
|
|
|
|
def _resolvelines(self, annotateresult, revmap, linelog):
|
|
|
|
"""(annotateresult) -> [line]. designed for annotatealllines.
|
|
|
|
this is probably the most inefficient code in the whole fastannotate
|
|
|
|
directory. but we have made a decision that the linelog does not
|
|
|
|
store line contents. so getting them requires random accesses to
|
|
|
|
the revlog data, since they can be many, it can be very slow.
|
|
|
|
"""
|
|
|
|
# [llrev]
|
|
|
|
revs = [revmap.hsh2rev(l[0]) for l in annotateresult]
|
|
|
|
result = [None] * len(annotateresult)
|
|
|
|
# {(rev, linenum): [lineindex]}
|
2016-10-19 18:05:22 +03:00
|
|
|
key2idxs = collections.defaultdict(list)
|
2016-09-12 00:52:29 +03:00
|
|
|
for i in xrange(len(result)):
|
|
|
|
key2idxs[(revs[i], annotateresult[i][1])].append(i)
|
|
|
|
while key2idxs:
|
|
|
|
# find an unresolved line and its linelog rev to annotate
|
|
|
|
hsh = None
|
|
|
|
try:
|
|
|
|
for (rev, _linenum), idxs in key2idxs.iteritems():
|
|
|
|
if revmap.rev2flag(rev) & revmapmod.sidebranchflag:
|
|
|
|
continue
|
|
|
|
hsh = annotateresult[idxs[0]][0]
|
|
|
|
break
|
|
|
|
except StopIteration: # no more unresolved lines
|
|
|
|
return result
|
|
|
|
if hsh is None:
|
|
|
|
# the remaining key2idxs are not in main branch, resolving them
|
|
|
|
# using the hard way...
|
|
|
|
revlines = {}
|
|
|
|
for (rev, linenum), idxs in key2idxs.iteritems():
|
|
|
|
if rev not in revlines:
|
|
|
|
hsh = annotateresult[idxs[0]][0]
|
|
|
|
if self.ui.debugflag:
|
|
|
|
self.ui.debug('fastannotate: reading %s line #%d '
|
|
|
|
'to resolve lines %r\n'
|
|
|
|
% (node.short(hsh), linenum, idxs))
|
2016-09-28 17:56:31 +03:00
|
|
|
fctx = self._resolvefctx(hsh, revmap.rev2path(rev))
|
2016-09-12 00:52:29 +03:00
|
|
|
lines = mdiff.splitnewlines(fctx.data())
|
|
|
|
revlines[rev] = lines
|
|
|
|
for idx in idxs:
|
|
|
|
result[idx] = revlines[rev][linenum]
|
|
|
|
assert all(x is not None for x in result)
|
|
|
|
return result
|
|
|
|
|
|
|
|
# run the annotate and the lines should match to the file content
|
|
|
|
self.ui.debug('fastannotate: annotate %s to resolve lines\n'
|
|
|
|
% node.short(hsh))
|
|
|
|
linelog.annotate(rev)
|
2016-09-28 17:56:31 +03:00
|
|
|
fctx = self._resolvefctx(hsh, revmap.rev2path(rev))
|
2016-09-12 00:52:29 +03:00
|
|
|
annotated = linelog.annotateresult
|
|
|
|
lines = mdiff.splitnewlines(fctx.data())
|
|
|
|
assert len(lines) == len(annotated)
|
|
|
|
# resolve lines from the annotate result
|
|
|
|
for i, line in enumerate(lines):
|
|
|
|
k = annotated[i]
|
|
|
|
if k in key2idxs:
|
|
|
|
for idx in key2idxs[k]:
|
|
|
|
result[idx] = line
|
|
|
|
del key2idxs[k]
|
|
|
|
return result
|
|
|
|
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
def annotatedirectly(self, f, showpath, showlines):
|
|
|
|
"""like annotate, but when we know that f is in linelog.
|
|
|
|
f can be either a 20-char str (node) or a fctx. this is for perf - in
|
|
|
|
the best case, the user provides a node and we don't need to read the
|
|
|
|
filelog or construct any filecontext.
|
|
|
|
"""
|
|
|
|
if isinstance(f, str):
|
|
|
|
hsh = f
|
|
|
|
else:
|
|
|
|
hsh = f.node()
|
|
|
|
llrev = self.revmap.hsh2rev(hsh)
|
|
|
|
assert llrev
|
|
|
|
assert (self.revmap.rev2flag(llrev) & revmapmod.sidebranchflag) == 0
|
|
|
|
self.linelog.annotate(llrev)
|
|
|
|
result = [(self.revmap.rev2hsh(r), l)
|
|
|
|
for r, l in self.linelog.annotateresult]
|
|
|
|
return self._refineannotateresult(result, f, showpath, showlines)
|
|
|
|
|
|
|
|
def _refineannotateresult(self, result, f, showpath, showlines):
|
|
|
|
"""add the missing path or line contents, they can be expensive.
|
|
|
|
f could be either node or fctx.
|
|
|
|
"""
|
|
|
|
if showpath:
|
|
|
|
result = self._addpathtoresult(result)
|
|
|
|
if showlines:
|
|
|
|
if isinstance(f, str): # f: node or fctx
|
|
|
|
llrev = self.revmap.hsh2rev(f)
|
2016-09-28 17:56:31 +03:00
|
|
|
fctx = self._resolvefctx(f, self.revmap.rev2path(llrev))
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
else:
|
|
|
|
fctx = f
|
|
|
|
lines = mdiff.splitnewlines(fctx.data())
|
2016-11-25 22:49:13 +03:00
|
|
|
if len(lines) != len(result): # linelog is probably corrupted
|
|
|
|
raise faerror.CorruptedFileError()
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
result = (result, lines)
|
|
|
|
return result
|
|
|
|
|
|
|
|
def _appendrev(self, fctx, blocks, bannotated=None):
|
|
|
|
self._doappendrev(self.linelog, self.revmap, fctx, blocks, bannotated)
|
|
|
|
|
2016-10-08 17:58:21 +03:00
|
|
|
def _diffblocks(self, a, b):
|
|
|
|
return mdiff.allblocks(a, b, self.opts.diffopts)
|
|
|
|
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
@staticmethod
|
|
|
|
def _doappendrev(linelog, revmap, fctx, blocks, bannotated=None):
|
|
|
|
"""append a revision to linelog and revmap"""
|
|
|
|
|
|
|
|
def getllrev(f):
|
|
|
|
"""(fctx) -> int"""
|
|
|
|
# f should not be a linelog revision
|
|
|
|
assert not isinstance(f, int)
|
|
|
|
# f is a fctx, allocate linelog rev on demand
|
|
|
|
hsh = f.node()
|
|
|
|
rev = revmap.hsh2rev(hsh)
|
|
|
|
if rev is None:
|
|
|
|
rev = revmap.append(hsh, sidebranch=True, path=f.path())
|
|
|
|
return rev
|
|
|
|
|
|
|
|
# append sidebranch revisions to revmap
|
|
|
|
siderevs = []
|
|
|
|
siderevmap = {} # node: int
|
|
|
|
if bannotated is not None:
|
|
|
|
for (a1, a2, b1, b2), op in blocks:
|
|
|
|
if op != '=':
|
|
|
|
# f could be either linelong rev, or fctx.
|
|
|
|
siderevs += [f for f, l in bannotated[b1:b2]
|
|
|
|
if not isinstance(f, int)]
|
|
|
|
siderevs = set(siderevs)
|
|
|
|
if fctx in siderevs: # mainnode must be appended seperately
|
|
|
|
siderevs.remove(fctx)
|
|
|
|
for f in siderevs:
|
|
|
|
siderevmap[f] = getllrev(f)
|
|
|
|
|
|
|
|
# the changeset in the main branch, could be a merge
|
|
|
|
llrev = revmap.append(fctx.node(), path=fctx.path())
|
|
|
|
siderevmap[fctx] = llrev
|
|
|
|
|
|
|
|
for (a1, a2, b1, b2), op in reversed(blocks):
|
|
|
|
if op == '=':
|
|
|
|
continue
|
|
|
|
if bannotated is None:
|
|
|
|
linelog.replacelines(llrev, a1, a2, b1, b2)
|
|
|
|
else:
|
|
|
|
blines = [((r if isinstance(r, int) else siderevmap[r]), l)
|
|
|
|
for r, l in bannotated[b1:b2]]
|
|
|
|
linelog.replacelines_vec(llrev, a1, a2, blines)
|
|
|
|
|
|
|
|
def _addpathtoresult(self, annotateresult, revmap=None):
|
|
|
|
"""(revmap, [(node, linenum)]) -> [(node, linenum, path)]"""
|
|
|
|
if revmap is None:
|
|
|
|
revmap = self.revmap
|
fastannotate: handle "draft" paths, and renames correctly
Summary:
Previously we can only answer the "path" information when the revision is in
the linelog revmap, and the code would crash if a revision is in a side
branch, and the user requests path information. This diff fixes it.
Besides, this diff improves rename handling. For example, given the following
chart:
```
o---o -o file name: a
/
o---o- file name: b
^ ^ ^
1 2 3 revisions
```
Depending on the position of the `main branch` reference, fastannotate may
or may not use linelog:
- main branch is at rev 2, annotate a -r 3 will not take advantage of linelog
(fallback to slow annotate)
- main branch is at rev 3, annotate a -r 2 will not take advantage of linelog
This is not ideal, but seems to be the best we can do for now.
Test Plan:
Added a new test, updated existing relevant tests. Some debug messages are
changed to reflect internals more precisely.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4010964
Signature: t1:4010964:1476458201:79875d96399d023d0000d0c4bb8b8d40ea43eef0
2016-10-09 21:47:01 +03:00
|
|
|
|
|
|
|
def _getpath(nodeid):
|
|
|
|
path = self._node2path.get(nodeid)
|
|
|
|
if path is None:
|
|
|
|
path = revmap.rev2path(revmap.hsh2rev(nodeid))
|
|
|
|
self._node2path[nodeid] = path
|
|
|
|
return path
|
|
|
|
|
|
|
|
return [(n, l, _getpath(n)) for n, l in annotateresult]
|
fastannotate: implement the annotate algorithm
Summary:
This diff implements the `annotate` algorithm. Unlike the vanilla one, the
annotate method takes 2 revisions: the revision specified for annotating,
and the head of the main branch. The algorithm will do a "hybrid" annotate:
incrementally update the linelog (the cache) so it can answer queries of
any revision in the main branch. And use the traditional algorithm to deal
with revisions not in the main branch: like a side branch of a merge commit,
or the revision the user specified not in the main branch.
The main branch is supposed to be something like `master` or `@`, and their
p1s.
Building up linelog with merge handled reasonably for the main branch, and
the non-linelog part that produces final result share a lot internal states
and logic so they are deeply coupled. Splitting them will probably reduce
performance, or have difficulty (no clean way) to share internal states.
If the caller only wants to build linelog without annotate things, just pass
`rev = master`.
While some attempts are made to support "merge" changeset, the result can
still be different from the vanilla one sometimes. In those cases, both
results make sense. It's really hard, if not impossible, to make the new
implementation 100% same with the vanilla one because of the linear history
restriction of linelog so I guess currently it's good enough. The differences
will be covered by a `.t` test later.
Test Plan: Code Review. A `.t` file will be added.
Reviewers: #sourcecontrol, stash
Reviewed By: stash
Subscribers: stash, mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D3836438
Signature: t1:3836438:1473778829:27978479a01920833fa146f427178292ea1f5306
2016-09-08 18:54:21 +03:00
|
|
|
|
|
|
|
def _checklastmasterhead(self, fctx):
|
|
|
|
"""check if fctx is the master's head last time, raise if not"""
|
|
|
|
if fctx is None:
|
|
|
|
llrev = 0
|
|
|
|
else:
|
|
|
|
llrev = self.revmap.hsh2rev(fctx.node())
|
|
|
|
assert llrev
|
|
|
|
if self.linelog.maxrev != llrev:
|
|
|
|
raise faerror.CannotReuseError()
|
|
|
|
|
|
|
|
@util.propertycache
|
|
|
|
def _parentfunc(self):
|
|
|
|
"""-> (fctx) -> [fctx]"""
|
|
|
|
followrename = self.opts.followrename
|
|
|
|
followmerge = self.opts.followmerge
|
|
|
|
def parents(f):
|
|
|
|
pl = _parents(f, follow=followrename)
|
|
|
|
if not followmerge:
|
|
|
|
pl = pl[:1]
|
|
|
|
return pl
|
|
|
|
return parents
|
|
|
|
|
2016-09-28 17:56:47 +03:00
|
|
|
@util.propertycache
|
|
|
|
def _perfhack(self):
|
|
|
|
return self.ui.configbool('fastannotate', 'perfhack')
|
|
|
|
|
2016-09-28 17:56:31 +03:00
|
|
|
def _resolvefctx(self, rev, path=None, **kwds):
|
|
|
|
return resolvefctx(self.repo, rev, (path or self.path), **kwds)
|
|
|
|
|
2016-09-08 18:46:27 +03:00
|
|
|
def _unlinkpaths(paths):
|
|
|
|
"""silent, best-effort unlink"""
|
|
|
|
for path in paths:
|
|
|
|
try:
|
|
|
|
util.unlink(path)
|
|
|
|
except OSError:
|
|
|
|
pass
|
|
|
|
|
2016-10-22 03:37:08 +03:00
|
|
|
class pathhelper(object):
|
|
|
|
"""helper for getting paths for lockfile, linelog and revmap"""
|
|
|
|
|
fastannotate: implement a simple wireprotocol transferring the annotate cache
Summary:
This diff adds a simple wireprotocol that the client can download fastannotate
cache files from the server.
Design decisions:
- Transfer full files. The linelog is not append-only and rsync-like
incremental updates is not trivial. Since the files look small (usually <
1MB), send full copies for now. If file size becomes an issue, we can
improve it later.
- The server can send arbitrary number of files with paths defined. This
provides the flexibility to send extra files without changing the protocol.
For example, if we want to speed up `--deleted`, we may want to download
`linemap` files.
- No compression. The revmap is basically not compress-able because it
contains the hashes. The linelog could be compressed though, but they are
small enough, and we may rely on compression on other layers, like SSH.
- `lastnode` check - if the last node of revmap the client reports matches
what the server has, the server will assume it's up-to-date and do nothing.
Note that this diff is only about the protocol and methods on peer, localrepo
objects. The client-side command is not aware of the feature yet.
Test Plan:
- Clone from a local repo A to B, via the ssh protocol.
- Config both repos so B is the client, A is the server.
- Run `hg debugshell --debug` from B, make sure the client can get contents
correctly (check the file contents from both the client and the server),
also make sure the non-existed files will be ignored:
In [1]: repo.prefetchfastannotate(['README', 'Makefile', 'NOT-EXIStED'])
running ssh localhost ...
sending hello command
sending between command
remote: 383
remote: capabilities: .... getannotate
remote: 1
fastannotate: requesting 2 files
sending batch command
fastannotate: server returned
fastannotate: writing 4976 bytes to fastannotate/default/README.l
fastannotate: writing 813 bytes to fastannotate/default/README.m
fastannotate: writing 9024 bytes to fastannotate/default/Makefile.l
fastannotate: writing 2453 bytes to fastannotate/default/Makefile.m
- Make sure the second time fetching these files, they will be skipped because
of the uptodate check:
In [2]: repo.prefetchfastannotate(['README', 'Makefile', 'NOT-EXIStED'])
- Manually create a diverge case where the client and the server disagrees
about what "master" (or "mainbranch") is. Make sure that although the
request will be send by the client, the server can return nothing because of
the lastnode check.
Reviewers: #mercurial, durham, stash
Reviewed By: stash
Subscribers: mjpieters
Differential Revision: https://phabricator.intern.facebook.com/D4063249
Signature: t1:4063249:1477920081:9bfab3031aa3118060c433f6e27c5b32aa25040c
2016-10-30 00:45:20 +03:00
|
|
|
def __init__(self, repo, path, opts=defaultopts):
|
2016-10-22 03:37:08 +03:00
|
|
|
# different options use different directories
|
|
|
|
self._vfspath = os.path.join('fastannotate',
|
|
|
|
opts.shortstr, encodedir(path))
|
|
|
|
self._repo = repo
|
|
|
|
|
|
|
|
@property
|
|
|
|
def dirname(self):
|
|
|
|
return os.path.dirname(self._repo.vfs.join(self._vfspath))
|
|
|
|
|
|
|
|
@property
|
|
|
|
def linelogpath(self):
|
|
|
|
return self._repo.vfs.join(self._vfspath + '.l')
|
|
|
|
|
|
|
|
def lock(self):
|
|
|
|
return lockmod.lock(self._repo.vfs, self._vfspath + '.lock')
|
|
|
|
|
2016-10-30 05:13:07 +03:00
|
|
|
@contextlib.contextmanager
|
|
|
|
def _lockflock(self):
|
|
|
|
"""the same as 'lock' but use flock instead of lockmod.lock, to avoid
|
|
|
|
creating temporary symlinks."""
|
|
|
|
import fcntl
|
|
|
|
lockpath = self.linelogpath
|
|
|
|
util.makedirs(os.path.dirname(lockpath))
|
|
|
|
lockfd = os.open(lockpath, os.O_RDONLY | os.O_CREAT, 0o664)
|
|
|
|
fcntl.flock(lockfd, fcntl.LOCK_EX)
|
|
|
|
try:
|
|
|
|
yield
|
|
|
|
finally:
|
|
|
|
fcntl.flock(lockfd, fcntl.LOCK_UN)
|
|
|
|
os.close(lockfd)
|
|
|
|
|
2016-10-22 03:37:08 +03:00
|
|
|
@property
|
|
|
|
def revmappath(self):
|
|
|
|
return self._repo.vfs.join(self._vfspath + '.m')
|
|
|
|
|
2016-09-08 18:46:27 +03:00
|
|
|
@contextlib.contextmanager
|
|
|
|
def annotatecontext(repo, path, opts=defaultopts, rebuild=False):
|
|
|
|
"""context needed to perform (fast) annotate on a file
|
|
|
|
|
|
|
|
an annotatecontext of a single file consists of two structures: the
|
|
|
|
linelog and the revmap. this function takes care of locking. only 1
|
|
|
|
process is allowed to write that file's linelog and revmap at a time.
|
|
|
|
|
|
|
|
when something goes wrong, this function will assume the linelog and the
|
|
|
|
revmap are in a bad state, and remove them from disk.
|
|
|
|
|
|
|
|
use this function in the following way:
|
|
|
|
|
|
|
|
with annotatecontext(...) as actx:
|
|
|
|
actx. ....
|
|
|
|
"""
|
2016-10-22 03:37:08 +03:00
|
|
|
helper = pathhelper(repo, path, opts)
|
|
|
|
util.makedirs(helper.dirname)
|
|
|
|
revmappath = helper.revmappath
|
|
|
|
linelogpath = helper.linelogpath
|
2016-10-29 21:45:16 +03:00
|
|
|
actx = None
|
2016-09-08 18:46:27 +03:00
|
|
|
try:
|
2016-10-22 03:37:08 +03:00
|
|
|
with helper.lock():
|
2016-10-29 21:45:16 +03:00
|
|
|
actx = _annotatecontext(repo, path, linelogpath, revmappath, opts)
|
2016-09-08 18:46:27 +03:00
|
|
|
if rebuild:
|
2016-10-29 21:45:16 +03:00
|
|
|
actx.rebuild()
|
|
|
|
yield actx
|
2016-09-08 18:46:27 +03:00
|
|
|
except Exception:
|
2016-10-29 21:45:16 +03:00
|
|
|
if actx is not None:
|
|
|
|
actx.rebuild()
|
2016-09-08 18:46:27 +03:00
|
|
|
repo.ui.debug('fastannotate: %s: cache broken and deleted\n' % path)
|
|
|
|
raise
|
|
|
|
finally:
|
2016-10-29 21:45:16 +03:00
|
|
|
if actx is not None:
|
|
|
|
actx.close()
|
2016-12-01 05:03:44 +03:00
|
|
|
|
|
|
|
def fctxannotatecontext(fctx, follow=True, diffopts=None, rebuild=False):
|
|
|
|
"""like annotatecontext but get the context from a fctx. convenient when
|
|
|
|
used in fctx.annotate
|
|
|
|
"""
|
|
|
|
repo = fctx._repo
|
|
|
|
path = fctx._path
|
|
|
|
if repo.ui.configbool('fastannotate', 'forcefollow', True):
|
|
|
|
follow = True
|
|
|
|
aopts = annotateopts(diffopts=diffopts, followrename=follow)
|
|
|
|
return annotatecontext(repo, path, aopts, rebuild)
|