2010-04-03 04:58:16 +04:00
|
|
|
# similar.py - mechanisms for finding similar files
|
|
|
|
#
|
|
|
|
# Copyright 2005-2007 Matt Mackall <mpm@selenic.com>
|
|
|
|
#
|
|
|
|
# This software may be used and distributed according to the terms of the
|
|
|
|
# GNU General Public License version 2 or any later version.
|
|
|
|
|
2015-12-13 10:17:22 +03:00
|
|
|
from __future__ import absolute_import
|
|
|
|
|
|
|
|
from .i18n import _
|
|
|
|
from . import (
|
|
|
|
bdiff,
|
|
|
|
mdiff,
|
|
|
|
)
|
2010-04-03 04:58:16 +04:00
|
|
|
|
2010-04-03 04:58:16 +04:00
|
|
|
def _findexactmatches(repo, added, removed):
|
|
|
|
'''find renamed files that have no changes
|
|
|
|
|
|
|
|
Takes a list of new filectxs and a list of removed filectxs, and yields
|
|
|
|
(before, after) tuples of exact matches.
|
|
|
|
'''
|
|
|
|
numfiles = len(added) + len(removed)
|
|
|
|
|
2017-03-23 14:57:27 +03:00
|
|
|
# Build table of removed files: {hash(fctx.data()): [fctx, ...]}.
|
|
|
|
# We use hash() to discard fctx.data() from memory.
|
2010-04-03 04:58:16 +04:00
|
|
|
hashes = {}
|
2017-03-23 14:57:27 +03:00
|
|
|
for i, fctx in enumerate(removed):
|
2016-03-11 17:29:20 +03:00
|
|
|
repo.ui.progress(_('searching for exact renames'), i, total=numfiles,
|
|
|
|
unit=_('files'))
|
2017-03-23 14:57:27 +03:00
|
|
|
h = hash(fctx.data())
|
|
|
|
if h not in hashes:
|
|
|
|
hashes[h] = [fctx]
|
|
|
|
else:
|
|
|
|
hashes[h].append(fctx)
|
2010-04-03 04:58:16 +04:00
|
|
|
|
|
|
|
# For each added file, see if it corresponds to a removed file.
|
|
|
|
for i, fctx in enumerate(added):
|
|
|
|
repo.ui.progress(_('searching for exact renames'), i + len(removed),
|
2016-03-11 17:29:20 +03:00
|
|
|
total=numfiles, unit=_('files'))
|
similar: compare between actual file contents for exact identity
Before this patch, similarity detection logic (for addremove and
automv) depends entirely on SHA-1 digesting. But this causes incorrect
rename detection, if:
- removing file A and adding file B occur at same committing, and
- SHA-1 hash values of file A and B are same
This may prevent security experts from managing sample files for
SHAttered issue in Mercurial repository, for example.
https://security.googleblog.com/2017/02/announcing-first-sha1-collision.html
https://shattered.it/
Hash collision itself isn't so serious for core repository
functionality of Mercurial, described by mpm as below, though.
https://www.mercurial-scm.org/wiki/mpm/SHA1
This patch compares between actual file contents after hash comparison
for exact identity.
Even after this patch, SHA-1 is still used, because it is reasonable
enough to quickly detect existence of "(almost) same" file.
- replacing SHA-1 causes decreasing performance, and
- replacement of it has ambiguity, yet
Getting content of removed file (= rfctx.data()) at each exact
comparison should be cheap enough, even though getting content of
added one costs much.
======= ============== =====================
file fctx data() reads from
======= ============== =====================
removed filectx in-memory revlog data
added workingfilectx storage
======= ============== =====================
2017-03-02 20:57:06 +03:00
|
|
|
adata = fctx.data()
|
2017-03-23 14:57:27 +03:00
|
|
|
h = hash(adata)
|
|
|
|
for rfctx in hashes.get(h, []):
|
similar: compare between actual file contents for exact identity
Before this patch, similarity detection logic (for addremove and
automv) depends entirely on SHA-1 digesting. But this causes incorrect
rename detection, if:
- removing file A and adding file B occur at same committing, and
- SHA-1 hash values of file A and B are same
This may prevent security experts from managing sample files for
SHAttered issue in Mercurial repository, for example.
https://security.googleblog.com/2017/02/announcing-first-sha1-collision.html
https://shattered.it/
Hash collision itself isn't so serious for core repository
functionality of Mercurial, described by mpm as below, though.
https://www.mercurial-scm.org/wiki/mpm/SHA1
This patch compares between actual file contents after hash comparison
for exact identity.
Even after this patch, SHA-1 is still used, because it is reasonable
enough to quickly detect existence of "(almost) same" file.
- replacing SHA-1 causes decreasing performance, and
- replacement of it has ambiguity, yet
Getting content of removed file (= rfctx.data()) at each exact
comparison should be cheap enough, even though getting content of
added one costs much.
======= ============== =====================
file fctx data() reads from
======= ============== =====================
removed filectx in-memory revlog data
added workingfilectx storage
======= ============== =====================
2017-03-02 20:57:06 +03:00
|
|
|
# compare between actual file contents for exact identity
|
|
|
|
if adata == rfctx.data():
|
|
|
|
yield (rfctx, fctx)
|
2017-03-23 14:57:27 +03:00
|
|
|
break
|
2010-04-03 04:58:16 +04:00
|
|
|
|
|
|
|
# Done
|
|
|
|
repo.ui.progress(_('searching for exact renames'), None)
|
|
|
|
|
2017-01-08 07:47:57 +03:00
|
|
|
def _ctxdata(fctx):
|
|
|
|
# lazily load text
|
|
|
|
orig = fctx.data()
|
|
|
|
return orig, mdiff.splitnewlines(orig)
|
|
|
|
|
2017-01-13 22:42:36 +03:00
|
|
|
def _score(fctx, otherdata):
|
|
|
|
orig, lines = otherdata
|
|
|
|
text = fctx.data()
|
2017-01-08 07:47:57 +03:00
|
|
|
# bdiff.blocks() returns blocks of matching lines
|
|
|
|
# count the number of bytes in each
|
|
|
|
equal = 0
|
|
|
|
matches = bdiff.blocks(text, orig)
|
|
|
|
for x1, x2, y1, y2 in matches:
|
|
|
|
for line in lines[y1:y2]:
|
|
|
|
equal += len(line)
|
|
|
|
|
|
|
|
lengths = len(text) + len(orig)
|
|
|
|
return equal * 2.0 / lengths
|
|
|
|
|
2017-01-13 22:42:36 +03:00
|
|
|
def score(fctx1, fctx2):
|
|
|
|
return _score(fctx1, _ctxdata(fctx2))
|
|
|
|
|
2010-04-03 04:58:16 +04:00
|
|
|
def _findsimilarmatches(repo, added, removed, threshold):
|
|
|
|
'''find potentially renamed files based on similar file content
|
|
|
|
|
|
|
|
Takes a list of new filectxs and a list of removed filectxs, and yields
|
|
|
|
(before, after, score) tuples of partial matches.
|
|
|
|
'''
|
2010-04-03 04:58:16 +04:00
|
|
|
copies = {}
|
|
|
|
for i, r in enumerate(removed):
|
2012-05-12 17:54:54 +04:00
|
|
|
repo.ui.progress(_('searching for similar files'), i,
|
2016-03-11 17:29:20 +03:00
|
|
|
total=len(removed), unit=_('files'))
|
2010-04-03 04:58:16 +04:00
|
|
|
|
2017-01-13 22:42:36 +03:00
|
|
|
data = None
|
2010-04-03 04:58:16 +04:00
|
|
|
for a in added:
|
|
|
|
bestscore = copies.get(a, (None, threshold))[1]
|
2017-01-13 22:42:36 +03:00
|
|
|
if data is None:
|
|
|
|
data = _ctxdata(r)
|
|
|
|
myscore = _score(a, data)
|
2017-03-23 14:52:41 +03:00
|
|
|
if myscore > bestscore:
|
2010-04-03 04:58:16 +04:00
|
|
|
copies[a] = (r, myscore)
|
|
|
|
repo.ui.progress(_('searching'), None)
|
|
|
|
|
|
|
|
for dest, v in copies.iteritems():
|
2017-01-08 07:43:49 +03:00
|
|
|
source, bscore = v
|
|
|
|
yield source, dest, bscore
|
2010-04-03 04:58:16 +04:00
|
|
|
|
2017-03-23 15:17:08 +03:00
|
|
|
def _dropempty(fctxs):
|
|
|
|
return [x for x in fctxs if x.size() > 0]
|
|
|
|
|
2010-04-03 04:58:16 +04:00
|
|
|
def findrenames(repo, added, removed, threshold):
|
|
|
|
'''find renamed files -- yields (before, after, score) tuples'''
|
2017-03-23 15:10:45 +03:00
|
|
|
wctx = repo[None]
|
|
|
|
pctx = wctx.p1()
|
2010-04-03 04:58:16 +04:00
|
|
|
|
|
|
|
# Zero length files will be frequently unrelated to each other, and
|
|
|
|
# tracking the deletion/addition of such a file will probably cause more
|
|
|
|
# harm than good. We strip them out here to avoid matching them later on.
|
2017-03-23 15:17:08 +03:00
|
|
|
addedfiles = _dropempty(wctx[fp] for fp in sorted(added))
|
|
|
|
removedfiles = _dropempty(pctx[fp] for fp in sorted(removed) if fp in pctx)
|
2010-04-03 04:58:16 +04:00
|
|
|
|
|
|
|
# Find exact matches.
|
2017-03-23 14:50:33 +03:00
|
|
|
matchedfiles = set()
|
|
|
|
for (a, b) in _findexactmatches(repo, addedfiles, removedfiles):
|
|
|
|
matchedfiles.add(b)
|
2010-04-03 04:58:16 +04:00
|
|
|
yield (a.path(), b.path(), 1.0)
|
|
|
|
|
|
|
|
# If the user requested similar files to be matched, search for them also.
|
|
|
|
if threshold < 1.0:
|
2017-03-23 14:50:33 +03:00
|
|
|
addedfiles = [x for x in addedfiles if x not in matchedfiles]
|
2015-03-15 12:58:56 +03:00
|
|
|
for (a, b, score) in _findsimilarmatches(repo, addedfiles,
|
|
|
|
removedfiles, threshold):
|
2010-04-03 04:58:16 +04:00
|
|
|
yield (a.path(), b.path(), score)
|