facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2025-01-06 04:43:19 +03:00
Code Issues Projects Releases Wiki Activity

snapshot: use local snapshot storage instead of lfs

Browse Source 
Summary: In the next diff I will replace remote lfs with bundle2

Reviewed By: markbt

Differential Revision: D17132405

fbshipit-source-id: a0dfff3ebad067abb0231cf31de08ae62affe7ce
This commit is contained in:
Aleksei Kulikov 2019-09-06 08:15:36 -07:00 committed by Facebook Github Bot
parent 465e91bbe5
commit 732a5baa9a
6 changed files with 69 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
edenscm/hgext/snapshot/__init__.py
View File

@ -25,7 +25,7 @@ Configs::
from edenscm.mercurial import error, extensions, hg, registrar
from edenscm.mercurial.i18n import _
from . import cmds as snapshotcommands, metadata
from . import blobstore, cmds as snapshotcommands, metadata
cmdtable = snapshotcommands.cmdtable
@ -35,6 +35,14 @@ configitem = registrar.configitem(configtable)
configitem("ui", "allow-checkout-snapshot", default=False)
def reposetup(ui, repo):
# Nothing to do with a remote repo
if not repo.local():
return
repo.svfs.snapshotstore = blobstore.local(repo)
def extsetup(ui):
metadata.extsetup(ui)

24
edenscm/hgext/snapshot/blobstore.py Normal file
View File

@ -0,0 +1,24 @@
# blobstore.py - local blob storage for snapshot metadata
#
# Copyright 2019 Facebook, Inc.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2 or any later version.
from __future__ import absolute_import
from edenscm.mercurial import blobstore, vfs as vfsmod
class local(blobstore.localblobstore):
"""Local blobstore for snapshot metadata contents.
"""
def __init__(self, repo):
fullpath = repo.svfs.join("snapshots/objects")
vfs = vfsmod.blobvfs(fullpath)
cachevfs = None
usercachepath = repo.ui.config("lfs", "usercache")
if usercachepath:
self.cachevfs = vfsmod.blobvfs(usercachepath)
super(local, self).__init__(vfs, cachevfs)

20
edenscm/hgext/snapshot/cmds.py
View File

@ -93,7 +93,7 @@ def createsnapshotcommit(ui, repo, opts):
emptymetadata = snapmetadata.empty
oid = "" # this is better than None because of extra serialization rules
if not emptymetadata:
oid, size = snapmetadata.storetolocallfs()
oid, size = snapmetadata.localstore()
extra = {"snapshotmetadataid": oid}
ui.debug("snapshot extra %s\n" % extra)
# TODO(alexeyqu): deal with unfinished merge state case
@ -152,7 +152,7 @@ def snapshotcheckout(ui, repo, *args, **opts):
repo.setparents(*parents)
snapshotmetadataid = cctx.extra().get("snapshotmetadataid")
if snapshotmetadataid:
snapmetadata = snapshotmetadata.restorefromlfs(repo, snapshotmetadataid)
snapmetadata = snapshotmetadata.getfromlocalstorage(repo, snapshotmetadataid)
checkouttosnapshotmetadata(ui, repo, snapmetadata, force)
ui.status(_("checkout complete\n"))
@ -161,7 +161,7 @@ def snapshotcheckout(ui, repo, *args, **opts):
def debugcreatesnapshotmetadata(ui, repo, *args, **opts):
"""
Creates pseudo metadata for untracked files without committing them.
Loads untracked files and the created metadata into local lfsstore.
Loads untracked files and the created metadata into local blobstore.
Outputs the oid of the created metadata file.
Be careful, snapshot metadata internal structure may change.
@ -175,7 +175,7 @@ def debugcreatesnapshotmetadata(ui, repo, *args, **opts):
)
)
return
oid, size = snapmetadata.storetolocallfs()
oid, size = snapmetadata.localstore()
ui.status(_("metadata oid: %s\n") % oid)
@ -183,13 +183,13 @@ def debugcreatesnapshotmetadata(ui, repo, *args, **opts):
def debuguploadsnapshotmetadata(ui, repo, *args, **opts):
"""
Uploads metadata and all related blobs to remote lfs.
Takes in an oid of the desired metadata in the local lfs.
Takes in an oid of the desired metadata in the local blobstore.
This command does not validate contents of the snapshot metadata.
"""
if not args or len(args) != 1:
raise error.Abort(_("you must specify a metadata oid\n"))
snapmetadata = snapshotmetadata.restorefromlfs(repo, args[0])
snapmetadata = snapshotmetadata.getfromlocalstorage(repo, args[0])
snapmetadata.uploadtoremotelfs()
ui.status(_("upload complete\n"))
@ -210,7 +210,9 @@ def debugcheckoutsnapshotmetadata(ui, repo, *args, **opts):
"""
if not args or len(args) != 1:
raise error.Abort(_("you must specify a metadata oid\n"))
snapmetadata = snapshotmetadata.restorefromlfs(repo, args[0], allow_remote=True)
snapmetadata = snapshotmetadata.getfromlocalstorage(
repo, args[0], allow_remote=True
)
checkouttosnapshotmetadata(ui, repo, snapmetadata, force=opts.get("force"))
ui.status(_("snapshot checkout complete\n"))
@ -232,8 +234,8 @@ def checkouttosnapshotmetadata(ui, repo, snapmetadata, force=True):
ui.warn(_("%s cannot be removed\n") % file.path)
# populating the untracked files
for file in snapmetadata.unknown:
checkaddfile(repo.svfs.lfslocalblobstore, file, repo.wvfs, force)
checkaddfile(repo.svfs.snapshotstore, file, repo.wvfs, force)
# restoring the merge state
with repo.wlock():
for file in snapmetadata.localvfsfiles:
checkaddfile(repo.svfs.lfslocalblobstore, file, repo.localvfs, force)
checkaddfile(repo.svfs.snapshotstore, file, repo.localvfs, force)

22
edenscm/hgext/snapshot/metadata.py
View File

@ -26,7 +26,7 @@ def extsetup(ui):
def checkloadblobbyoid(repo, oid, path, allow_remote=False):
localstore = repo.svfs.lfslocalblobstore
localstore = repo.svfs.snapshotstore
if localstore.has(oid):
return
if allow_remote:
@ -126,10 +126,10 @@ class snapshotmetadata(object):
return metadata
@classmethod
def restorefromlfs(cls, repo, oid, allow_remote=False):
def getfromlocalstorage(cls, repo, oid, allow_remote=False):
metadata = cls(repo, oid)
checkloadblobbyoid(repo, oid, "metadata", allow_remote)
metadata.deserialize(repo.svfs.lfslocalblobstore.read(oid))
metadata.deserialize(repo.svfs.snapshotstore.read(oid))
# validate related files
for file in metadata.unknown:
checkloadblobbyoid(repo, file.oid, file.path, allow_remote)
@ -137,25 +137,25 @@ class snapshotmetadata(object):
checkloadblobbyoid(repo, file.oid, file.path, allow_remote)
return metadata
def storetolocallfs(self):
def storetolfs(repo, data):
def localstore(self):
def store(repo, data):
"""
Util function which uploads data to the local lfs storage.
Returns oid and size of data.
"""
# TODO(alexeyqu): do we care about metadata?
oid = hashlib.sha256(data).hexdigest()
repo.svfs.lfslocalblobstore.write(oid, data)
repo.svfs.snapshotstore.write(oid, data)
return oid, str(len(data))
wctx = self.repo[None]
for f in self.unknown:
f.oid, f.size = storetolfs(self.repo, wctx[f.path].data())
f.oid, f.size = store(self.repo, wctx[f.path].data())
for f in self.localvfsfiles:
f.oid, f.size = storetolfs(
f.oid, f.size = store(
self.repo, self.repo.localvfs.open(path=f.path).read()
)
oid, size = storetolfs(self.repo, self.serialize())
oid, size = store(self.repo, self.serialize())
return oid, size
def uploadtoremotelfs(self):
@ -164,9 +164,13 @@ class snapshotmetadata(object):
pointers.append(lfs.pointer.gitlfspointer(oid=file.oid, size=file.size))
assert self.oid is not None
# TODO(alexeyqu): remove this hack in the next diff
lfslocalstore = self.repo.svfs.lfslocalblobstore
self.repo.svfs.lfslocalblobstore = self.repo.svfs.snapshotstore
pointers = [lfs.pointer.gitlfspointer(oid=self.oid)]
for file in self.unknown:
checkgetpointer(self.repo, file, pointers)
for file in self.localvfsfiles:
checkgetpointer(self.repo, file, pointers)
lfs.wrapper.uploadblobs(self.repo, pointers)
self.repo.svfs.lfslocalblobstore = lfslocalstore

6
tests/test-fb-hgext-snapshot.t
View File

@ -253,7 +253,7 @@
$ METADATAID="$(hg log --hidden -r \"$OID\" -T '{extras % \"{extra}\n\"}' | grep snapshotmetadataid | cut -d'=' -f2)"
$ echo "$METADATAID"
6b32f5f5726caf1b66d313cdd847ad5b4266f14a3480b2acf64a0a173ac14548
$ cat .hg/store/lfs/objects/"${METADATAID:0:2}"/"${METADATAID:2}"
$ cat .hg/store/snapshots/objects/"${METADATAID:0:2}"/"${METADATAID:2}"
{"files": {"deleted": {"foofile": null}, "localvfsfiles": {"merge/fc4ffdcb8ed23cecd44a0e11d23af83b445179b4": {"oid": "0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f", "size": "2"}, "merge/state": {"oid": "fdfea51dfeeae94bd846473c7bef891823af465d33f48e92ed2556bde6b346cb", "size": "166"}, "merge/state2": {"oid": "0e421047ebcf7d0cada48ddd801304725de33da3c4048ccb258041946cd0e81d", "size": "361"}}, "unknown": {"mergefile.orig": {"oid": "0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f", "size": "2"}, "untrackedfile": {"oid": "b05b74c474c1706953bed876a19f146b371ddf51a36474fe0c094922385cc479", "size": "5"}}}, "version": "1"} (no-eol)
# Move back to BASEREV
@ -354,7 +354,7 @@
$ BROKENSNAPSHOTID="$(hg snapshot create --clean | head -n 1 | cut -f2 -d' ')"
$ BROKENMETADATAID="$(hg log --hidden -r \"$BROKENSNAPSHOTID\" -T '{extras % \"{extra}\n\"}' | grep snapshotmetadataid | cut -d'=' -f2)"
# Delete all the related files from the local store
$ find .hg/store/lfs/objects/ -mindepth 1 ! -name "${BROKENMETADATAID:2}" -type f -delete
$ find .hg/store/snapshots/objects/ -mindepth 1 ! -name "${BROKENMETADATAID:2}" -type f -delete
$ hg snapshot checkout $BROKENSNAPSHOTID
will checkout on 06f851b33f41c80be342a09677f43a70ecc4a0f0
abort: file mergefile.orig with oid 0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f not found in local blobstorage
@ -362,7 +362,7 @@
[255]
$ hg status --verbose
# Break the metadata itself
$ echo "break the metadata json" >> .hg/store/lfs/objects/"${BROKENMETADATAID:0:2}"/"${BROKENMETADATAID:2}"
$ echo "break the metadata json" >> .hg/store/snapshots/objects/"${BROKENMETADATAID:0:2}"/"${BROKENMETADATAID:2}"
$ hg snapshot checkout $BROKENSNAPSHOTID
will checkout on 06f851b33f41c80be342a09677f43a70ecc4a0f0
abort: invalid metadata json: {"files": {"deleted": {}, "localvfsfiles": {}, "unknown": {"mergefile.orig": {"oid": "0263829989b6fd954f72baaf2fc64bc2e2f01d692d4de72986ea808f6e99813f", "size": "2"}, "untrackedfile": {"oid": "b05b74c474c1706953bed876a19f146b371ddf51a36474fe0c094922385cc479", "size": "5"}}}, "version": "1"}break the metadata json

18
tests/test-snapshot.t
View File

@ -33,19 +33,19 @@
f62f9175588ac550bc215b56b441de94f6b3c859023f971453057342614db332
# Check that the blobstore is populated
$ find .hg/store/lfs/objects | sort
.hg/store/lfs/objects
.hg/store/lfs/objects/7d
.hg/store/lfs/objects/7d/865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730
.hg/store/lfs/objects/f6
.hg/store/lfs/objects/f6/2f9175588ac550bc215b56b441de94f6b3c859023f971453057342614db332
$ find .hg/store/snapshots/objects | sort
.hg/store/snapshots/objects
.hg/store/snapshots/objects/7d
.hg/store/snapshots/objects/7d/865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730
.hg/store/snapshots/objects/f6
.hg/store/snapshots/objects/f6/2f9175588ac550bc215b56b441de94f6b3c859023f971453057342614db332
# Check the contents of the metadata file
$ cat .hg/store/lfs/objects/"${OID:0:2}"/"${OID:2}"
$ cat .hg/store/snapshots/objects/"${OID:0:2}"/"${OID:2}"
{"files": {"deleted": {"existingfile": null}, "localvfsfiles": {}, "unknown": {"untrackedfile": {"oid": "7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730", "size": "4"}}}, "version": "1"} (no-eol)
# Check that the untracked file is stored in lfs
$ cat .hg/store/lfs/objects/7d/865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730
# Check that the untracked file is stored in local storage
$ cat .hg/store/snapshots/objects/7d/865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730
bar
# Upload the metadata contents to server