snapshot: rename manifest to metadata

Summary: Snapshot `metadata` won't be confused with regular commit manifest.

Reviewed By: markbt

Differential Revision: D16929553

fbshipit-source-id: 466b167aa1cc30f137e0c0b4601d70876509e03c
This commit is contained in:
Aleksei Kulikov 2019-08-21 10:34:34 -07:00 committed by Facebook Github Bot
parent 71a0402c8a
commit e8c5b4830f
3 changed files with 101 additions and 101 deletions

View File

@ -66,7 +66,7 @@ def checkloadblobbyoid(repo, oid, path, allow_remote=False):
class filelfswrapper(object):
"""
Helper class that links files to oids in the lfs blobstorage.
Also does serialization/deserialization for manifest.
Also does serialization/deserialization for metadata.
"""
def __init__(self, path, oid=None, size=None):
@ -87,9 +87,9 @@ class filelfswrapper(object):
raise error.Abort(_("invalid file description: %s\n") % data)
class snapshotmanifest(object):
class snapshotmetadata(object):
"""
Main class that contains snapshot manifest representation.
Main class that contains snapshot metadata representation.
"""
def __init__(self, repo, oid=None):
@ -104,59 +104,59 @@ class snapshotmanifest(object):
return not (self.deleted or self.unknown)
def serialize(self):
manifest = defaultdict(dict)
manifest["deleted"] = {d.path: d.serialize() for d in self.deleted}
manifest["unknown"] = {u.path: u.serialize() for u in self.unknown}
manifest["localvfsfiles"] = {f.path: f.serialize() for f in self.localvfsfiles}
return json.dumps(manifest)
metadata = defaultdict(dict)
metadata["deleted"] = {d.path: d.serialize() for d in self.deleted}
metadata["unknown"] = {u.path: u.serialize() for u in self.unknown}
metadata["localvfsfiles"] = {f.path: f.serialize() for f in self.localvfsfiles}
return json.dumps(metadata)
def deserialize(self, json_string):
try:
manifest = json.loads(json_string)
metadata = json.loads(json_string)
self.deleted = [
filelfswrapper(path) for path in sorted(manifest["deleted"].keys())
filelfswrapper(path) for path in sorted(metadata["deleted"].keys())
]
self.unknown = [
filelfswrapper.deserialize(path, data)
for path, data in sorted(manifest["unknown"].items())
for path, data in sorted(metadata["unknown"].items())
]
self.localvfsfiles = [
filelfswrapper.deserialize(path, data)
for path, data in sorted(manifest["localvfsfiles"].items())
for path, data in sorted(metadata["localvfsfiles"].items())
]
except ValueError:
raise error.Abort(_("invalid manifest json: %s\n") % json_string)
raise error.Abort(_("invalid metadata json: %s\n") % json_string)
@classmethod
def createfromworkingcopy(cls, repo, status=None, include_untracked=True):
manifest = cls(repo)
# populate the manifest
metadata = cls(repo)
# populate the metadata
status = status or repo.status(unknown=include_untracked)
manifest.deleted = [filelfswrapper(path) for path in status.deleted]
manifest.unknown = [filelfswrapper(path) for path in status.unknown]
metadata.deleted = [filelfswrapper(path) for path in status.deleted]
metadata.unknown = [filelfswrapper(path) for path in status.unknown]
# check merge and rebase info
ismergestate = len(repo[None].parents()) > 1
isrebasestate = repo.localvfs.exists("rebasestate")
if ismergestate or isrebasestate:
for root, dirs, files in repo.localvfs.walk(path="merge"):
manifest.localvfsfiles += [
metadata.localvfsfiles += [
filelfswrapper(pathutil.join(root, f)) for f in files
]
if isrebasestate:
manifest.localvfsfiles.append(filelfswrapper("rebasestate"))
return manifest
metadata.localvfsfiles.append(filelfswrapper("rebasestate"))
return metadata
@classmethod
def restorefromlfs(cls, repo, oid, allow_remote=False):
manifest = cls(repo, oid)
checkloadblobbyoid(repo, oid, "manifest", allow_remote)
manifest.deserialize(repo.svfs.lfslocalblobstore.read(oid))
metadata = cls(repo, oid)
checkloadblobbyoid(repo, oid, "metadata", allow_remote)
metadata.deserialize(repo.svfs.lfslocalblobstore.read(oid))
# validate related files
for file in manifest.unknown:
for file in metadata.unknown:
checkloadblobbyoid(repo, file.oid, file.path, allow_remote)
for file in manifest.localvfsfiles:
for file in metadata.localvfsfiles:
checkloadblobbyoid(repo, file.oid, file.path, allow_remote)
return manifest
return metadata
def storetolocallfs(self):
def storetolfs(repo, data):
@ -237,18 +237,18 @@ def debugsnapshot(ui, repo, *args, **opts):
def createsnapshotcommit(ui, repo, opts):
status = repo.status(unknown=True)
snapmanifest = snapshotmanifest.createfromworkingcopy(repo, status=status)
emptymanifest = snapmanifest.empty
snapmetadata = snapshotmetadata.createfromworkingcopy(repo, status=status)
emptymetadata = snapmetadata.empty
oid = "" # this is better than None because of extra serialization rules
if not emptymanifest:
oid, size = snapmanifest.storetolocallfs()
extra = {"snapshotmanifestid": oid}
if not emptymetadata:
oid, size = snapmetadata.storetolocallfs()
extra = {"snapshotmetadataid": oid}
ui.debug("snapshot extra %s\n" % extra)
# TODO(alexeyqu): deal with unfinished merge state case
cctx = context.workingcommitctx(
repo, status, "snapshot", opts.get("user"), opts.get("date"), extra=extra
)
if len(cctx.files()) == 0 and emptymanifest: # don't need an empty snapshot
if len(cctx.files()) == 0 and emptymetadata: # don't need an empty snapshot
return None
with repo.transaction("snapshot"):
return repo.commitctx(cctx, error=True)
@ -263,7 +263,7 @@ def createsnapshotcommit(ui, repo, opts):
def debugcheckoutsnapshot(ui, repo, *args, **opts):
"""
Checks out the working copy to the snapshot state, given its revision id.
Downloads the snapshot manifest from remote lfs if needed.
Downloads the snapshot metadata from remote lfs if needed.
"""
if not args or len(args) != 1:
@ -275,7 +275,7 @@ def debugcheckoutsnapshot(ui, repo, *args, **opts):
except error.RepoLookupError:
ui.status(_("%s is not a valid revision id\n") % node)
raise
if "snapshotmanifestid" not in cctx.extra():
if "snapshotmetadataid" not in cctx.extra():
raise error.Abort(_("%s is not a valid snapshot id\n") % node)
# This is a temporary safety check that WC is clean.
if sum(map(len, repo.status(unknown=True))) != 0 and not force:
@ -301,24 +301,24 @@ def debugcheckoutsnapshot(ui, repo, *args, **opts):
if len(parents) == 2:
with repo.dirstate.parentchange():
repo.setparents(*parents)
snapshotmanifestid = cctx.extra().get("snapshotmanifestid")
if snapshotmanifestid:
snapmanifest = snapshotmanifest.restorefromlfs(repo, snapshotmanifestid)
checkouttosnapshotmanifest(ui, repo, snapmanifest, force)
snapshotmetadataid = cctx.extra().get("snapshotmetadataid")
if snapshotmetadataid:
snapmetadata = snapshotmetadata.restorefromlfs(repo, snapshotmetadataid)
checkouttosnapshotmetadata(ui, repo, snapmetadata, force)
ui.status(_("checkout complete\n"))
@command("debugcreatesnapshotmanifest", inferrepo=True)
def debugcreatesnapshotmanifest(ui, repo, *args, **opts):
@command("debugcreatesnapshotmetadata", inferrepo=True)
def debugcreatesnapshotmetadata(ui, repo, *args, **opts):
"""
Creates pseudo manifest for untracked files without committing them.
Loads untracked files and the created manifest into local lfsstore.
Outputs the oid of the created manifest file.
Creates pseudo metadata for untracked files without committing them.
Loads untracked files and the created metadata into local lfsstore.
Outputs the oid of the created metadata file.
Be careful, snapshot manifest internal structure may change.
Be careful, snapshot metadata internal structure may change.
"""
snapmanifest = snapshotmanifest.createfromworkingcopy(repo)
if snapmanifest.empty:
snapmetadata = snapshotmetadata.createfromworkingcopy(repo)
if snapmetadata.empty:
ui.status(
_(
"Working copy is even with the last commit. "
@ -326,47 +326,47 @@ def debugcreatesnapshotmanifest(ui, repo, *args, **opts):
)
)
return
oid, size = snapmanifest.storetolocallfs()
ui.status(_("manifest oid: %s\n") % oid)
oid, size = snapmetadata.storetolocallfs()
ui.status(_("metadata oid: %s\n") % oid)
@command("debuguploadsnapshotmanifest", [], _("OID"), inferrepo=True)
def debuguploadsnapshotmanifest(ui, repo, *args, **opts):
@command("debuguploadsnapshotmetadata", [], _("OID"), inferrepo=True)
def debuguploadsnapshotmetadata(ui, repo, *args, **opts):
"""
Uploads manifest and all related blobs to remote lfs.
Takes in an oid of the desired manifest in the local lfs.
Uploads metadata and all related blobs to remote lfs.
Takes in an oid of the desired metadata in the local lfs.
This command does not validate contents of the snapshot manifest.
This command does not validate contents of the snapshot metadata.
"""
if not args or len(args) != 1:
raise error.Abort(_("you must specify a manifest oid\n"))
snapmanifest = snapshotmanifest.restorefromlfs(repo, args[0])
snapmanifest.uploadtoremotelfs()
raise error.Abort(_("you must specify a metadata oid\n"))
snapmetadata = snapshotmetadata.restorefromlfs(repo, args[0])
snapmetadata.uploadtoremotelfs()
ui.status(_("upload complete\n"))
@command(
"debugcheckoutsnapshotmanifest",
"debugcheckoutsnapshotmetadata",
[("f", "force", False, _("force checkout"))],
_("OID"),
inferrepo=True,
)
def debugcheckoutsnapshotmanifest(ui, repo, *args, **opts):
def debugcheckoutsnapshotmetadata(ui, repo, *args, **opts):
"""
Checks out the working copy to the snapshot manifest state, given its manifest oid.
Downloads the snapshot manifest from remote lfs if needed.
Takes in an oid of the manifest.
Checks out the working copy to the snapshot metadata state, given its metadata id.
Downloads the snapshot metadata from remote lfs if needed.
Takes in an oid of the metadata.
This command does not validate contents of the snapshot manifest.
This command does not validate contents of the snapshot metadata.
"""
if not args or len(args) != 1:
raise error.Abort(_("you must specify a manifest oid\n"))
snapmanifest = snapshotmanifest.restorefromlfs(repo, args[0], allow_remote=True)
checkouttosnapshotmanifest(ui, repo, snapmanifest, force=opts.get("force"))
raise error.Abort(_("you must specify a metadata oid\n"))
snapmetadata = snapshotmetadata.restorefromlfs(repo, args[0], allow_remote=True)
checkouttosnapshotmetadata(ui, repo, snapmetadata, force=opts.get("force"))
ui.status(_("snapshot checkout complete\n"))
def checkouttosnapshotmanifest(ui, repo, snapmanifest, force=True):
def checkouttosnapshotmetadata(ui, repo, snapmetadata, force=True):
def checkaddfile(store, file, vfs, force):
if not force and vfs.exists(file.path):
ui.note(_("skip adding %s, it exists\n") % file.path)
@ -375,16 +375,16 @@ def checkouttosnapshotmanifest(ui, repo, snapmanifest, force=True):
vfs.write(file.path, store.read(file.oid))
# deleting files that should be missing
for file in snapmanifest.deleted:
for file in snapmetadata.deleted:
try:
ui.note(_("will delete %s\n") % file.path)
util.unlink(repo.wjoin(file.path))
except OSError:
ui.warn(_("%s cannot be removed\n") % file.path)
# populating the untracked files
for file in snapmanifest.unknown:
for file in snapmetadata.unknown:
checkaddfile(repo.svfs.lfslocalblobstore, file, repo.wvfs, force)
# restoring the merge state
with repo.wlock():
for file in snapmanifest.localvfsfiles:
for file in snapmetadata.localvfsfiles:
checkaddfile(repo.svfs.lfslocalblobstore, file, repo.localvfs, force)

View File

@ -32,8 +32,8 @@
# Snapshot test plan:
# 1) Empty snapshot (no changes);
# 2) Snapshot with an empty manifest (changes only in tracked files);
# 3) Snapshot with a manifest (merge state + mixed changes);
# 2) Snapshot with an empty metadata (changes only in tracked files);
# 3) Snapshot with a metadata (merge state + mixed changes);
# 4) Same as 3 but test the --clean flag on creation;
# 5) Same as 3 but test the --force flag on restore;
# 6) TODO(alexeyqu): Same as 3 but sync to the server and another client.
@ -44,7 +44,7 @@
nothing changed
# 2) Snapshot with an empty manifest (changes only in tracked files)
# 2) Snapshot with an empty metadata (changes only in tracked files)
$ hg rm bar/file
$ echo "change" >> foofile
$ echo "another" > bazfile
@ -75,14 +75,14 @@
# Create a snapshot and check the result
$ EMPTYOID="$(hg debugsnapshot | head -n 1 | cut -f2 -d' ')"
$ echo "$EMPTYOID"
509c24c4681a119720ced4260520dce99e1370ee
$ hg log --hidden -r "$EMPTYOID" -T '{extras % \"{extra}\n\"}' | grep snapshotmanifestid
snapshotmanifestid=
9c5c703bba200afd1e7105ef675d68b75d43c6b4
$ hg log --hidden -r "$EMPTYOID" -T '{extras % \"{extra}\n\"}' | grep snapshotmetadataid
snapshotmetadataid=
# The snapshot commit is hidden
$ hg log --hidden -r "not hidden() & $EMPTYOID"
# But it exists!
$ hg show --hidden "$EMPTYOID"
changeset: 1:509c24c4681a
changeset: 1:9c5c703bba20
tag: tip
user: test
date: Thu Jan 01 00:00:00 1970 +0000
@ -91,17 +91,17 @@
snapshot
diff -r 3490593cf53c -r 509c24c4681a bar/file
diff -r 3490593cf53c -r 9c5c703bba20 bar/file
--- a/bar/file Thu Jan 01 00:00:00 1970 +0000
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,1 +0,0 @@
-bar
diff -r 3490593cf53c -r 509c24c4681a bazfile
diff -r 3490593cf53c -r 9c5c703bba20 bazfile
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/bazfile Thu Jan 01 00:00:00 1970 +0000
@@ -0,0 +1,1 @@
+another
diff -r 3490593cf53c -r 509c24c4681a foofile
diff -r 3490593cf53c -r 9c5c703bba20 foofile
--- a/foofile Thu Jan 01 00:00:00 1970 +0000
+++ b/foofile Thu Jan 01 00:00:00 1970 +0000
@@ -1,1 +1,2 @@
@ -112,13 +112,13 @@
$ hg update -q --clean "$BASEREV" && rm bazfile
$ hg status --verbose
$ hg debugcheckoutsnapshot "$EMPTYOID"
will checkout on 509c24c4681a119720ced4260520dce99e1370ee
will checkout on 9c5c703bba200afd1e7105ef675d68b75d43c6b4
checkout complete
$ test "$BEFORESTATUS" = "$(hg status --verbose)"
$ test "$BEFOREDIFF" = "$(hg diff)"
# 3) Snapshot with a manifest + merge conflict!
# 3) Snapshot with a metadata + merge conflict!
$ hg update -q --clean "$BASEREV" && rm bazfile
$ hg status --verbose
$ echo "a" > mergefile
@ -186,14 +186,14 @@
# Create the snapshot
$ OID="$(hg debugsnapshot | cut -f2 -d' ')"
$ echo "$OID"
f788a565a3b78ea7b351a3c8d0bd8622ebeaf52e
34c81e40f10e2df58967bde74f60a35080812037
# hg status/diff are unchanged
$ test "$BEFORESTATUS" = "$(hg status --verbose)"
$ test "$BEFOREDIFF" = "$(hg diff)"
# Check the manifest id and its contents
$ MANIFESTOID="$(hg log --hidden -r \"$OID\" -T '{extras % \"{extra}\n\"}' | grep snapshotmanifestid | cut -d'=' -f2)"
# Check the metadata id and its contents
$ MANIFESTOID="$(hg log --hidden -r \"$OID\" -T '{extras % \"{extra}\n\"}' | grep snapshotmetadataid | cut -d'=' -f2)"
$ echo "$MANIFESTOID"
94d0daf8fadba1a239eca4ddb5cc1a71728097928543dfb02b8be399ae6fcb56
$ cat .hg/store/lfs/objects/"${MANIFESTOID:0:2}"/"${MANIFESTOID:2}"
@ -225,7 +225,7 @@
[255]
# Check out on the snapshot -- positive tests
$ hg debugcheckoutsnapshot "$OID"
will checkout on f788a565a3b78ea7b351a3c8d0bd8622ebeaf52e
will checkout on 34c81e40f10e2df58967bde74f60a35080812037
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
checkout complete
$ test "$BEFORESTATUS" = "$(hg status --verbose)"
@ -239,13 +239,13 @@
.hg/merge/state
.hg/merge/state2
$ hg debugsnapshot --clean
snapshot f788a565a3b78ea7b351a3c8d0bd8622ebeaf52e created
snapshot 34c81e40f10e2df58967bde74f60a35080812037 created
3 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg status --verbose
$ test -d .hg/merge
[1]
$ hg debugcheckoutsnapshot "$OID"
will checkout on f788a565a3b78ea7b351a3c8d0bd8622ebeaf52e
will checkout on 34c81e40f10e2df58967bde74f60a35080812037
checkout complete
@ -260,7 +260,7 @@
[255]
$ hg debugcheckoutsnapshot --force "$OID"
will checkout on f788a565a3b78ea7b351a3c8d0bd8622ebeaf52e
will checkout on 34c81e40f10e2df58967bde74f60a35080812037
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
checkout complete

View File

@ -22,13 +22,13 @@
added 1 changesets with 1 changes to 1 files
# No need to create snapshot now
$ hg debugcreatesnapshotmanifest
$ hg debugcreatesnapshotmetadata
Working copy is even with the last commit. No need to create snapshot.
# Make some changes: add an untracked file and remove the tracked file
$ echo "bar" > untrackedfile
$ rm existingfile
$ OID="$(hg debugcreatesnapshotmanifest | cut -f3 -d' ')"
$ OID="$(hg debugcreatesnapshotmetadata | cut -f3 -d' ')"
$ echo "$OID"
70750cb86e5458ef4c1f6630694943a27bde9f67e838701dc660d92360b290be
@ -40,7 +40,7 @@
.hg/store/lfs/objects/7d
.hg/store/lfs/objects/7d/865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730
# Check the contents of the manifest file
# Check the contents of the metadata file
$ cat .hg/store/lfs/objects/"${OID:0:2}"/"${OID:2}"
{"deleted": {"existingfile": null}, "localvfsfiles": {}, "unknown": {"untrackedfile": {"oid": "7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730", "size": "4"}}} (no-eol)
@ -48,18 +48,18 @@
$ cat .hg/store/lfs/objects/7d/865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730
bar
# Upload the manifest contents to server
# Upload the metadata contents to server
$ cat >> $HGRCPATH << EOF
> [lfs]
> url=file:$TESTTMP/lfsremote/
> EOF
$ hg debuguploadsnapshotmanifest 1f341c81a097100373b4bfe017b80d767d2b74bd434dbfa9ced3c1964024c6b5
abort: file manifest with oid 1f341c81a097100373b4bfe017b80d767d2b74bd434dbfa9ced3c1964024c6b5 not found in local blobstorage
$ hg debuguploadsnapshotmetadata 1f341c81a097100373b4bfe017b80d767d2b74bd434dbfa9ced3c1964024c6b5
abort: file metadata with oid 1f341c81a097100373b4bfe017b80d767d2b74bd434dbfa9ced3c1964024c6b5 not found in local blobstorage
[255]
$ hg debuguploadsnapshotmanifest "$OID"
$ hg debuguploadsnapshotmetadata "$OID"
upload complete
# Check the remote storage
@ -76,7 +76,7 @@
$ cat $TESTTMP/lfsremote/7d/865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730
bar
# Checkout the manifest
# Checkout the metadata
$ cd ../
$ hg clone -q server client2
$ cd client2
@ -85,7 +85,7 @@
$ ls
existingfile
$ hg debugcheckoutsnapshotmanifest --verbose "$OID"
$ hg debugcheckoutsnapshotmetadata --verbose "$OID"
will delete existingfile
will add untrackedfile
snapshot checkout complete
@ -128,9 +128,9 @@
# To continue: hg rebase --continue
# To abort: hg rebase --abort
# So we have an unfinished rebase state, now we will upload it into another manifest
$ MERGEOID="$(hg debugcreatesnapshotmanifest | cut -f3 -d' ')"
$ hg debuguploadsnapshotmanifest "$MERGEOID"
# So we have an unfinished rebase state, now we will upload it into another metadata
$ MERGEOID="$(hg debugcreatesnapshotmetadata | cut -f3 -d' ')"
$ hg debuguploadsnapshotmetadata "$MERGEOID"
upload complete
# Check the result in the second clone
@ -140,7 +140,7 @@
$ hg status
! existingfile
? untrackedfile
$ hg debugcheckoutsnapshotmanifest --verbose "$MERGEOID"
$ hg debugcheckoutsnapshotmetadata --verbose "$MERGEOID"
will add new.orig
skip adding untrackedfile, it exists
will add merge/c2a6b03f190dfb2b4aa91f8af8d477a9bc3401dc