mirror of
https://github.com/facebook/sapling.git
synced 2024-10-07 07:17:55 +03:00
introduce store classes
move store walking from streamclone.py into store.py
This commit is contained in:
parent
34afc460b4
commit
904617b7d8
@ -60,30 +60,13 @@ class localrepository(repo.repository):
|
||||
if r not in self.supported:
|
||||
raise repo.RepoError(_("requirement '%s' not supported") % r)
|
||||
|
||||
# setup store
|
||||
if "store" in requirements:
|
||||
self.encodefn = store.encodefilename
|
||||
self.decodefn = store.decodefilename
|
||||
self.spath = os.path.join(self.path, "store")
|
||||
else:
|
||||
self.encodefn = lambda x: x
|
||||
self.decodefn = lambda x: x
|
||||
self.spath = self.path
|
||||
self.store = store.store(requirements, self.path)
|
||||
|
||||
try:
|
||||
# files in .hg/ will be created using this mode
|
||||
mode = os.stat(self.spath).st_mode
|
||||
# avoid some useless chmods
|
||||
if (0777 & ~util._umask) == (0777 & mode):
|
||||
mode = None
|
||||
except OSError:
|
||||
mode = None
|
||||
|
||||
self._createmode = mode
|
||||
self.opener.createmode = mode
|
||||
sopener = util.opener(self.spath)
|
||||
sopener.createmode = mode
|
||||
self.sopener = store.encodedopener(sopener, self.encodefn)
|
||||
self.spath = self.store.path
|
||||
self.sopener = self.store.opener
|
||||
self.sjoin = self.store.join
|
||||
self._createmode = self.store.createmode
|
||||
self.opener.createmode = self.store.createmode
|
||||
|
||||
self.ui = ui.ui(parentui=parentui)
|
||||
try:
|
||||
@ -481,10 +464,6 @@ class localrepository(repo.repository):
|
||||
def join(self, f):
|
||||
return os.path.join(self.path, f)
|
||||
|
||||
def sjoin(self, f):
|
||||
f = self.encodefn(f)
|
||||
return os.path.join(self.spath, f)
|
||||
|
||||
def wjoin(self, f):
|
||||
return os.path.join(self.root, f)
|
||||
|
||||
@ -2061,6 +2040,25 @@ class localrepository(repo.repository):
|
||||
return self.stream_in(remote)
|
||||
return self.pull(remote, heads)
|
||||
|
||||
def storefiles(self):
|
||||
'''get all *.i and *.d files in the store
|
||||
|
||||
Returns (list of (filename, size), total_bytes)'''
|
||||
|
||||
lock = None
|
||||
try:
|
||||
self.ui.debug('scanning\n')
|
||||
entries = []
|
||||
total_bytes = 0
|
||||
# get consistent snapshot of repo, lock during scan
|
||||
lock = self.lock()
|
||||
for name, size in self.store.walk():
|
||||
entries.append((name, size))
|
||||
total_bytes += size
|
||||
return entries, total_bytes
|
||||
finally:
|
||||
del lock
|
||||
|
||||
# used to avoid circular references so destructors work
|
||||
def aftertrans(files):
|
||||
renamefiles = [tuple(t) for t in files]
|
||||
|
@ -55,14 +55,13 @@ class statichttprepository(localrepo.localrepository):
|
||||
|
||||
# setup store
|
||||
if "store" in requirements:
|
||||
self.encodefn = store.encodefilename
|
||||
self.decodefn = store.decodefilename
|
||||
self.spath = self.path + "/store"
|
||||
else:
|
||||
self.encodefn = lambda x: x
|
||||
self.decodefn = lambda x: x
|
||||
self.spath = self.path
|
||||
self.sopener = store.encodedopener(opener(self.spath), self.encodefn)
|
||||
self.encodefn = store.encodefn(requirements)
|
||||
so = opener(self.spath)
|
||||
self.sopener = lambda path, *args, **kw: so(
|
||||
self.encodefn(path), *args, **kw)
|
||||
|
||||
self.manifest = manifest.manifest(self.sopener)
|
||||
self.changelog = changelog.changelog(self.sopener)
|
||||
|
@ -5,6 +5,8 @@
|
||||
# This software may be used and distributed according to the terms
|
||||
# of the GNU General Public License, incorporated herein by reference.
|
||||
|
||||
import os, stat, osutil, util
|
||||
|
||||
def _buildencodefun():
|
||||
e = '_'
|
||||
win_reserved = [ord(x) for x in '\\:*?"<>|']
|
||||
@ -33,7 +35,91 @@ def _buildencodefun():
|
||||
|
||||
encodefilename, decodefilename = _buildencodefun()
|
||||
|
||||
def encodedopener(openerfn, fn):
|
||||
def o(path, *args, **kw):
|
||||
return openerfn(fn(path), *args, **kw)
|
||||
return o
|
||||
def _dirwalk(path, recurse):
|
||||
'''yields (filename, size)'''
|
||||
for e, kind, st in osutil.listdir(path, stat=True):
|
||||
pe = os.path.join(path, e)
|
||||
if kind == stat.S_IFDIR:
|
||||
if recurse:
|
||||
for x in _dirwalk(pe, True):
|
||||
yield x
|
||||
elif kind == stat.S_IFREG:
|
||||
yield pe, st.st_size
|
||||
|
||||
class _store:
|
||||
'''base class for local repository stores'''
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
try:
|
||||
# files in .hg/ will be created using this mode
|
||||
mode = os.stat(self.path).st_mode
|
||||
# avoid some useless chmods
|
||||
if (0777 & ~util._umask) == (0777 & mode):
|
||||
mode = None
|
||||
except OSError:
|
||||
mode = None
|
||||
self.createmode = mode
|
||||
|
||||
def join(self, f):
|
||||
return os.path.join(self.path, f)
|
||||
|
||||
def _revlogfiles(self, relpath='', recurse=False):
|
||||
'''yields (filename, size)'''
|
||||
if relpath:
|
||||
path = os.path.join(self.path, relpath)
|
||||
else:
|
||||
path = self.path
|
||||
striplen = len(self.path) + len(os.sep)
|
||||
filetypes = ('.d', '.i')
|
||||
for f, size in _dirwalk(path, recurse):
|
||||
if (len(f) > 2) and f[-2:] in filetypes:
|
||||
yield util.pconvert(f[striplen:]), size
|
||||
|
||||
def _datafiles(self):
|
||||
for x in self._revlogfiles('data', True):
|
||||
yield x
|
||||
|
||||
def walk(self):
|
||||
'''yields (direncoded filename, size)'''
|
||||
# yield data files first
|
||||
for x in self._datafiles():
|
||||
yield x
|
||||
# yield manifest before changelog
|
||||
meta = util.sort(self._revlogfiles())
|
||||
meta.reverse()
|
||||
for x in meta:
|
||||
yield x
|
||||
|
||||
class directstore(_store):
|
||||
def __init__(self, path):
|
||||
_store.__init__(self, path)
|
||||
self.encodefn = lambda x: x
|
||||
self.opener = util.opener(self.path)
|
||||
self.opener.createmode = self.createmode
|
||||
|
||||
class encodedstore(_store):
|
||||
def __init__(self, path):
|
||||
_store.__init__(self, os.path.join(path, 'store'))
|
||||
self.encodefn = encodefilename
|
||||
op = util.opener(self.path)
|
||||
op.createmode = self.createmode
|
||||
self.opener = lambda f, *args, **kw: op(self.encodefn(f), *args, **kw)
|
||||
|
||||
def _datafiles(self):
|
||||
for f, size in self._revlogfiles('data', True):
|
||||
yield decodefilename(f), size
|
||||
|
||||
def join(self, f):
|
||||
return os.path.join(self.path, self.encodefn(f))
|
||||
|
||||
def encodefn(requirements):
|
||||
if 'store' not in requirements:
|
||||
return lambda x: x
|
||||
else:
|
||||
return encodefilename
|
||||
|
||||
def store(requirements, path):
|
||||
if 'store' not in requirements:
|
||||
return directstore(path)
|
||||
else:
|
||||
return encodedstore(path)
|
||||
|
@ -5,40 +5,12 @@
|
||||
# This software may be used and distributed according to the terms
|
||||
# of the GNU General Public License, incorporated herein by reference.
|
||||
|
||||
import os, osutil, stat, util, lock
|
||||
import util, lock
|
||||
|
||||
# if server supports streaming clone, it advertises "stream"
|
||||
# capability with value that is version+flags of repo it is serving.
|
||||
# client only streams if it can read that repo format.
|
||||
|
||||
def walkrepo(root):
|
||||
'''iterate over metadata files in repository.
|
||||
walk in natural (sorted) order.
|
||||
yields 2-tuples: name of .d or .i file, size of file.'''
|
||||
|
||||
strip_count = len(root) + len(os.sep)
|
||||
def walk(path, recurse):
|
||||
for e, kind, st in osutil.listdir(path, stat=True):
|
||||
pe = os.path.join(path, e)
|
||||
if kind == stat.S_IFDIR:
|
||||
if recurse:
|
||||
for x in walk(pe, True):
|
||||
yield x
|
||||
else:
|
||||
if kind != stat.S_IFREG or len(e) < 2:
|
||||
continue
|
||||
sfx = e[-2:]
|
||||
if sfx in ('.d', '.i'):
|
||||
yield pe[strip_count:], st.st_size
|
||||
# write file data first
|
||||
for x in walk(os.path.join(root, 'data'), True):
|
||||
yield x
|
||||
# write manifest before changelog
|
||||
meta = util.sort(walk(root, False))
|
||||
meta.reverse()
|
||||
for x in meta:
|
||||
yield x
|
||||
|
||||
# stream file format is simple.
|
||||
#
|
||||
# server writes out line that says how many files, how many total
|
||||
@ -59,28 +31,14 @@ def stream_out(repo, fileobj, untrusted=False):
|
||||
fileobj.write('1\n')
|
||||
return
|
||||
|
||||
# get consistent snapshot of repo. lock during scan so lock not
|
||||
# needed while we stream, and commits can happen.
|
||||
repolock = None
|
||||
try:
|
||||
try:
|
||||
repolock = repo.lock()
|
||||
except (lock.LockHeld, lock.LockUnavailable), inst:
|
||||
repo.ui.warn('locking the repository failed: %s\n' % (inst,))
|
||||
fileobj.write('2\n')
|
||||
return
|
||||
|
||||
fileobj.write('0\n')
|
||||
repo.ui.debug('scanning\n')
|
||||
entries = []
|
||||
total_bytes = 0
|
||||
for name, size in walkrepo(repo.spath):
|
||||
name = repo.decodefn(util.pconvert(name))
|
||||
entries.append((name, size))
|
||||
total_bytes += size
|
||||
finally:
|
||||
del repolock
|
||||
entries, total_bytes = repo.storefiles()
|
||||
except (lock.LockHeld, lock.LockUnavailable), inst:
|
||||
repo.ui.warn('locking the repository failed: %s\n' % (inst,))
|
||||
fileobj.write('2\n')
|
||||
return
|
||||
|
||||
fileobj.write('0\n')
|
||||
repo.ui.debug('%d files, %d bytes to transfer\n' %
|
||||
(len(entries), total_bytes))
|
||||
fileobj.write('%d %d\n' % (len(entries), total_bytes))
|
||||
|
Loading…
Reference in New Issue
Block a user