mirror of
https://github.com/facebook/sapling.git
synced 2024-10-07 23:38:50 +03:00
4b17e20f7c
When changesets referencing largefiles are pushed then the corresponding largefiles will be pushed too - unless the target already has them. The client will use statlfile to make sure it only sends largefiles that the target doesn't have. The server would however on every statlfile check that the content of the largefile had the expected hash. What should be cheap thus became an expensive operation that trashed the disk and the cache. Largefile hashes are already checked by putlfile before being stored on the server. A server should thus be able to keep its largefile store free of errors - even more than it can keep revlogs free of errors. Verification should happen when running 'hg verify' locally on the server. Rehashing every largefile on every remote stat is too expensive. Clients will also stat lfiles before downloading them. When the server verified the hash in stat it meant that it had to read the file twice to serve it. With this change the server will assume its own hashes are ok without checking them on every statlfile. Some consequences of this change: - in case of server side corruption the problem will be detected by the existing check on the client side - not on server side - clients that could upload an uncorrupted largefile when pushing will no longer magically heal the server (and break hardlinks) - a client will now only upload its uncorrupted files after the corrupted file has been removed on the server side - client side verify will no longer report corruption in files it doesn't have (Issue3123 discussed related problems - and how they have been fixed.)
159 lines
5.9 KiB
Python
159 lines
5.9 KiB
Python
# Copyright 2011 Fog Creek Software
|
|
#
|
|
# This software may be used and distributed according to the terms of the
|
|
# GNU General Public License version 2 or any later version.
|
|
|
|
import os
|
|
import urllib2
|
|
|
|
from mercurial import error, httppeer, util, wireproto
|
|
from mercurial.wireproto import batchable, future
|
|
from mercurial.i18n import _
|
|
|
|
import lfutil
|
|
|
|
LARGEFILES_REQUIRED_MSG = ('\nThis repository uses the largefiles extension.'
|
|
'\n\nPlease enable it in your Mercurial config '
|
|
'file.\n')
|
|
|
|
def putlfile(repo, proto, sha):
|
|
'''Put a largefile into a repository's local store and into the
|
|
user cache.'''
|
|
proto.redirect()
|
|
|
|
path = lfutil.storepath(repo, sha)
|
|
util.makedirs(os.path.dirname(path))
|
|
tmpfp = util.atomictempfile(path, createmode=repo.store.createmode)
|
|
|
|
try:
|
|
try:
|
|
proto.getfile(tmpfp)
|
|
tmpfp._fp.seek(0)
|
|
if sha != lfutil.hexsha1(tmpfp._fp):
|
|
raise IOError(0, _('largefile contents do not match hash'))
|
|
tmpfp.close()
|
|
lfutil.linktousercache(repo, sha)
|
|
except IOError, e:
|
|
repo.ui.warn(_('largefiles: failed to put %s into store: %s') %
|
|
(sha, e.strerror))
|
|
return wireproto.pushres(1)
|
|
finally:
|
|
tmpfp.discard()
|
|
|
|
return wireproto.pushres(0)
|
|
|
|
def getlfile(repo, proto, sha):
|
|
'''Retrieve a largefile from the repository-local cache or system
|
|
cache.'''
|
|
filename = lfutil.findfile(repo, sha)
|
|
if not filename:
|
|
raise util.Abort(_('requested largefile %s not present in cache') % sha)
|
|
f = open(filename, 'rb')
|
|
length = os.fstat(f.fileno())[6]
|
|
|
|
# Since we can't set an HTTP content-length header here, and
|
|
# Mercurial core provides no way to give the length of a streamres
|
|
# (and reading the entire file into RAM would be ill-advised), we
|
|
# just send the length on the first line of the response, like the
|
|
# ssh proto does for string responses.
|
|
def generator():
|
|
yield '%d\n' % length
|
|
for chunk in f:
|
|
yield chunk
|
|
return wireproto.streamres(generator())
|
|
|
|
def statlfile(repo, proto, sha):
|
|
'''Return '2\n' if the largefile is missing, '0\n' if it seems to be in
|
|
good condition.
|
|
|
|
The value 1 is reserved for mismatched checksum, but that is too expensive
|
|
to be verified on every stat and must be caught be running 'hg verify'
|
|
server side.'''
|
|
filename = lfutil.findfile(repo, sha)
|
|
if not filename:
|
|
return '2\n'
|
|
return '0\n'
|
|
|
|
def wirereposetup(ui, repo):
|
|
class lfileswirerepository(repo.__class__):
|
|
def putlfile(self, sha, fd):
|
|
# unfortunately, httprepository._callpush tries to convert its
|
|
# input file-like into a bundle before sending it, so we can't use
|
|
# it ...
|
|
if issubclass(self.__class__, httppeer.httppeer):
|
|
res = None
|
|
try:
|
|
res = self._call('putlfile', data=fd, sha=sha,
|
|
headers={'content-type':'application/mercurial-0.1'})
|
|
d, output = res.split('\n', 1)
|
|
for l in output.splitlines(True):
|
|
self.ui.warn(_('remote: '), l, '\n')
|
|
return int(d)
|
|
except (ValueError, urllib2.HTTPError):
|
|
self.ui.warn(_('unexpected putlfile response: %s') % res)
|
|
return 1
|
|
# ... but we can't use sshrepository._call because the data=
|
|
# argument won't get sent, and _callpush does exactly what we want
|
|
# in this case: send the data straight through
|
|
else:
|
|
try:
|
|
ret, output = self._callpush("putlfile", fd, sha=sha)
|
|
if ret == "":
|
|
raise error.ResponseError(_('putlfile failed:'),
|
|
output)
|
|
return int(ret)
|
|
except IOError:
|
|
return 1
|
|
except ValueError:
|
|
raise error.ResponseError(
|
|
_('putlfile failed (unexpected response):'), ret)
|
|
|
|
def getlfile(self, sha):
|
|
stream = self._callstream("getlfile", sha=sha)
|
|
length = stream.readline()
|
|
try:
|
|
length = int(length)
|
|
except ValueError:
|
|
self._abort(error.ResponseError(_("unexpected response:"),
|
|
length))
|
|
return (length, stream)
|
|
|
|
@batchable
|
|
def statlfile(self, sha):
|
|
f = future()
|
|
result = {'sha': sha}
|
|
yield result, f
|
|
try:
|
|
yield int(f.value)
|
|
except (ValueError, urllib2.HTTPError):
|
|
# If the server returns anything but an integer followed by a
|
|
# newline, newline, it's not speaking our language; if we get
|
|
# an HTTP error, we can't be sure the largefile is present;
|
|
# either way, consider it missing.
|
|
yield 2
|
|
|
|
repo.__class__ = lfileswirerepository
|
|
|
|
# advertise the largefiles=serve capability
|
|
def capabilities(repo, proto):
|
|
return capabilitiesorig(repo, proto) + ' largefiles=serve'
|
|
|
|
def heads(repo, proto):
|
|
if lfutil.islfilesrepo(repo):
|
|
return wireproto.ooberror(LARGEFILES_REQUIRED_MSG)
|
|
return wireproto.heads(repo, proto)
|
|
|
|
def sshrepocallstream(self, cmd, **args):
|
|
if cmd == 'heads' and self.capable('largefiles'):
|
|
cmd = 'lheads'
|
|
if cmd == 'batch' and self.capable('largefiles'):
|
|
args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
|
|
return ssholdcallstream(self, cmd, **args)
|
|
|
|
def httprepocallstream(self, cmd, **args):
|
|
if cmd == 'heads' and self.capable('largefiles'):
|
|
cmd = 'lheads'
|
|
if cmd == 'batch' and self.capable('largefiles'):
|
|
args['cmds'] = args['cmds'].replace('heads ', 'lheads ')
|
|
return httpoldcallstream(self, cmd, **args)
|