verify LFS blobs were uploaded before push

Summary:
per @[1070541862:quark]'s suggestion, I attempt to add a verification hook that checks LFS blobs were uploaded before push.

The code execution path looks like this:
`command.py: push()` -->
`exchange.py: push()` -->
 ......
 `exchange.py: _pushbundle2()`
 `exchange.py: _pushcheckoutgoing()`
 ......

And I insert the hook in `exchange.py: _pushcheckoutgoing()`

Reviewed By: quark-zju

Differential Revision: D9212199

fbshipit-source-id: 98e016da81f9df85731771b7d486d03848624bcb
This commit is contained in:
Kaley Huang 2018-08-20 16:35:18 -07:00 committed by Facebook Github Bot
parent b04dda41f1
commit 9d54d78caa
10 changed files with 397 additions and 39 deletions

View File

@ -174,6 +174,9 @@ def extsetup(ui):
# when "hg push" uses bundle2, upload related LFS blobs
wrapfunction(exchange, "_pushbundle2", wrapper._pushbundle2)
# verify LFS objects were uploaded when receiving pushes
wrapfunction(changegroup, "checkrevs", wrapper.checkrevs)
@templatekeyword("lfs_files")
def lfsfiles(repo, ctx, **args):

View File

@ -345,6 +345,10 @@ class _gitlfsremote(object):
if self.ui.verbose:
self.ui.write(_("lfs: processed: %s\n") % oid)
def checkblobs(self, pointers):
response = self._batchrequest(pointers, "download")
self._extractobjects(response, pointers, "download")
def __del__(self):
# copied from mercurial/httppeer.py
urlopener = getattr(self, "urlopener", None)
@ -371,6 +375,13 @@ class _dummyremote(object):
content = self.vfs.read(p.oid())
tostore.write(p.oid(), content)
def checkblobs(self, pointers):
for p in pointers:
if not self.vfs.exists(p.oid()):
raise LfsRemoteError(
_("LFS object %s is not uploaded to remote server") % p.oid()
)
class _nullremote(object):
"""Null store storing blobs to /dev/null."""

View File

@ -303,6 +303,15 @@ def _pushbundle2(orig, pushop):
return orig(pushop)
def checkrevs(orig, repo, revs):
"""verify LFS objects were uploaded when receiving pushes"""
pointers = extractpointers(repo, revs)
if len(pointers) > 0:
store = repo.svfs.lfsremoteblobstore
store.checkblobs(pointers)
return orig(repo, revs)
def extractpointers(repo, revs):
"""return a list of lfs pointers added by given revs"""
ui = repo.ui

View File

@ -1111,6 +1111,9 @@ def prepushrebasehooks(op, params, bundle, bundlefile):
else:
op.repo.hook("prepushrebase", throw=True, **prelockrebaseargs)
revs = list(bundle.revs("bundle()"))
changegroup.checkrevs(bundle, revs)
def prefetchcaches(op, params, bundle):
bundlerepocache = {}

View File

@ -97,6 +97,12 @@ def writechunks(ui, chunks, filename, vfs=None):
os.unlink(cleanup)
def checkrevs(repo, revs):
# to be replaced by extensions
# free from extension logic
pass
class cg1unpacker(object):
"""Unpacker for cg1 changegroup streams.
@ -428,6 +434,8 @@ class cg1unpacker(object):
"changegroup-runhooks-%020i" % clstart,
lambda tr: repo._afterlock(runhooks),
)
checkrevs(repo, range(clstart, clend))
finally:
repo.ui.flush()
# never return 0 here:

View File

@ -158,16 +158,8 @@
# Push lfs content to server: succeed
# This should work even if the HG server does not have access to LFS server
$ cat >> $TESTTMP/master/.hg/hgrc << EOF
> [lfs]
> url=null
> EOF
$ mv $TESTTMP/master/.hg/store/lfs{,.bak}
# Push
$ hg push --to master ../master
pushing to ../master
searching for changes
@ -179,14 +171,32 @@
adding file changes
added 2 changesets with 1 changes to 2 files (+1 heads)
# Restore LFS configuration in the master repo
$ mv $TESTTMP/master/.hg/store/lfs{.bak,}
$ rm $TESTTMP/master/.hg/hgrc
# This should work even if the HG server does not have access to LFS server
$ test -d $TESTTMP/master/.hg/store/lfs
[1]
# Check content
$ cd ../master
$ hg log -T '{rev}:{node} {bookmarks} {desc}\n' -G
o 7:* shallow.lfs.commit (glob)
|
@ 6:a5bcdd7fe9f0d300a2dbed2bd81d140547638856 master add-new-file
|
| o 5:949778ec92dd45fafbcc6ab7b8c843fdceb66e24 conflict_master introduce-conflict
|/
o 4:042535657086a5b08463b9210a8f46dc270e51f9 x-lfs-again
|
o 3:c6cc0cd58884b847de39aa817ded71e6051caa9f x-nonlfs
|
o 2:f3dec7f3610207dbf222ec2d7b68df16a5fde0f2 y-nonlfs
|
o 1:799bebfa53189a3db8424680f1a8f9806540e541 y-lfs
|
o 0:0d2948821b2b3b6e58505696145f2215cea2b2cd x-lfs
$ hg log -p -r tip -T '{rev}:{node} {desc}\n'
7:* shallow.lfs.commit (glob)
diff --git a/x b/y
@ -199,6 +209,7 @@
BECOME-LFS-AGAIN
+ADD-A-LINE
$ hg book -f -r tip master
# Verify the server has lfs content after the pushrebase
@ -232,44 +243,34 @@
BECOME-LFS-AGAIN
ADD-A-LINE
$ hg debugindex x
rev offset length delta linkrev nodeid p1 p2
0 0 * -1 0 1ff4e6c9b276 000000000000 000000000000 (glob)
1 * * -1 3 68b9378cf5a1 000000000000 000000000000 (glob)
2 * * -1 4 d33b2f7888d4 68b9378cf5a1 000000000000 (glob)
3 * * -1 5 d587d4396479 d33b2f7888d4 000000000000 (glob)
# pull lfs content from server and update
$ cd ../client2
$ hg pull
$ hg pull --bookmark master
pulling from ssh://user@dummy/master
searching for changes
adding changesets
adding manifests
adding file changes
added 3 changesets with 3 changes to 3 files (+1 heads)
adding remote bookmark conflict_master
added 2 changesets with 2 changes to 2 files
updating bookmark master
new changesets * (glob)
(run 'hg heads' to see heads, 'hg merge' to merge)
(run 'hg update' to get a working copy)
$ hg update tip
2 files updated, 0 files merged, 1 files removed, 0 files unresolved
$ hg log -p -r tip -T '{rev}:{node} {desc}\n'
7:* shallow.lfs.commit (glob)
diff --git a/x b/y
rename from x
rename to y
--- a/x
+++ b/y
@@ -1,2 +1,3 @@
NOTLFS
BECOME-LFS-AGAIN
+ADD-A-LINE
$ hg log -T '{rev}:{node} {bookmarks} {desc}\n' -G
@ 7:* shallow.lfs.commit (glob)
@ 6:* master shallow.lfs.commit (glob)
|
o 6:a5bcdd7fe9f0d300a2dbed2bd81d140547638856 master add-new-file
o 5:a5bcdd7fe9f0d300a2dbed2bd81d140547638856 add-new-file
|
| o 5:949778ec92dd45fafbcc6ab7b8c843fdceb66e24 conflict_master introduce-conflict
|/
o 4:042535657086a5b08463b9210a8f46dc270e51f9 x-lfs-again
|
o 3:c6cc0cd58884b847de39aa817ded71e6051caa9f x-nonlfs
@ -280,3 +281,16 @@
|
o 0:0d2948821b2b3b6e58505696145f2215cea2b2cd x-lfs
$ hg log -p -r tip -T '{rev}:{node} {desc}\n'
6:* shallow.lfs.commit (glob)
diff --git a/x b/y
rename from x
rename to y
--- a/x
+++ b/y
@@ -1,2 +1,3 @@
NOTLFS
BECOME-LFS-AGAIN
+ADD-A-LINE

View File

@ -0,0 +1,277 @@
$ . "$TESTDIR/library.sh"
$ cat >> $HGRCPATH << EOF
> [extensions]
> lfs=
> pushrebase=
> [lfs]
> threshold=10B
> url=file:$TESTTMP/dummy-remote/
> [pushrebase]
> rewritedates = True
> [diff]
> git=1
> EOF
# prepare a full repo with lfs metadata
$ hg init master
$ hg init lfs-upload-trigger
$ cd master
$ echo THIS-IS-LFS-FILE > x
$ hg commit -qAm x-lfs
$ hg mv x y
$ hg commit -m y-lfs
$ echo NOTLFS > y
$ hg commit -m y-nonlfs
$ hg mv y x
$ hg commit -m x-nonlfs
$ echo BECOME-LFS-AGAIN >> x
$ hg commit -m x-lfs-again
$ hg book master
$ hg push -q ../lfs-upload-trigger
$ cd ..
# clone a client
$ hg clone ssh://user@dummy/master client --noupdate
streaming all changes
4 files to transfer, 1.94 KB of data
transferred 1.94 KB in * seconds (*) (glob)
searching for changes
no changes found
$ cd client
$ hg update
1 files updated, 0 files merged, 0 files removed, 0 files unresolved
$ hg log -p -r ::tip -T '{rev}:{node} {desc}\n' -G
@ 4:042535657086a5b08463b9210a8f46dc270e51f9 x-lfs-again
| diff --git a/x b/x
| --- a/x
| +++ b/x
| @@ -1,1 +1,2 @@
| NOTLFS
| +BECOME-LFS-AGAIN
|
o 3:c6cc0cd58884b847de39aa817ded71e6051caa9f x-nonlfs
| diff --git a/y b/x
| rename from y
| rename to x
|
o 2:f3dec7f3610207dbf222ec2d7b68df16a5fde0f2 y-nonlfs
| diff --git a/y b/y
| --- a/y
| +++ b/y
| @@ -1,1 +1,1 @@
| -THIS-IS-LFS-FILE
| +NOTLFS
|
o 1:799bebfa53189a3db8424680f1a8f9806540e541 y-lfs
| diff --git a/x b/y
| rename from x
| rename to y
|
o 0:0d2948821b2b3b6e58505696145f2215cea2b2cd x-lfs
diff --git a/x b/x
new file mode 100644
--- /dev/null
+++ b/x
@@ -0,0 +1,1 @@
+THIS-IS-LFS-FILE
# Clone a second client
$ cp -R . ../client2
# Edit the working copy
$ echo ADD-A-LINE >> x
$ hg mv x y
$ hg diff
diff --git a/x b/y
rename from x
rename to y
--- a/x
+++ b/y
@@ -1,2 +1,3 @@
NOTLFS
BECOME-LFS-AGAIN
+ADD-A-LINE
$ hg status -C
A y
x
R x
$ hg commit -m shallow.lfs.commit
# Push lfs content to server: expect failure, lfs blobs not in server
#if lfs-test-server no-windows
$ LFS_LISTEN="tcp://:$HGPORT"
$ LFS_HOST="localhost:$HGPORT"
$ LFS_PUBLIC=1
$ export LFS_LISTEN LFS_HOST LFS_PUBLIC
$ lfs-test-server &> lfs-server.log &
$ echo $! >> $DAEMON_PIDS
$ cat >> $TESTTMP/master/.hg/hgrc << EOF
> [lfs]
> url=http://test:something@$LFS_HOST/
> EOF
$ hg push --to master ../master
pushing to ../master
searching for changes
abort: LFS server error. Remote object for file unknown not found: *u'oid': u'a2fcdb080e9838f6e1476a494c1d553e6ffefb68b0d146a06f34b535b5198442'* (glob)
[255]
# Reset lfs url
$ cat >> $TESTTMP/master/.hg/hgrc << EOF
> [lfs]
> url=file:$TESTTMP/dummy-remote/
> EOF
# Push lfs content to server: succeed
$ hg push --to master ../master
pushing to ../master
searching for changes
pushing 1 changeset:
515a4dfd2e0c shallow.lfs.commit
1 new changeset from the server will be downloaded
adding changesets
adding manifests
adding file changes
added 1 changesets with 0 changes to 1 files (+1 heads)
# Check content
$ cd ../master
$ hg log -p -r tip -T '{rev}:{node} {desc}\n'
5:* shallow.lfs.commit (glob)
diff --git a/x b/y
rename from x
rename to y
--- a/x
+++ b/y
@@ -1,2 +1,3 @@
NOTLFS
BECOME-LFS-AGAIN
+ADD-A-LINE
$ hg log -T '{rev}:{node} {bookmarks} {desc}\n' -G
o 5:* shallow.lfs.commit (glob)
|
@ 4:042535657086a5b08463b9210a8f46dc270e51f9 master x-lfs-again
|
o 3:c6cc0cd58884b847de39aa817ded71e6051caa9f x-nonlfs
|
o 2:f3dec7f3610207dbf222ec2d7b68df16a5fde0f2 y-nonlfs
|
o 1:799bebfa53189a3db8424680f1a8f9806540e541 y-lfs
|
o 0:0d2948821b2b3b6e58505696145f2215cea2b2cd x-lfs
# Verify the server has lfs content after the pushrebase
$ hg debugindex y
rev offset length delta linkrev nodeid p1 p2
0 0 * -1 1 4ce3392f9979 000000000000 000000000000 (glob)
1 * * -1 2 139f72f7ca98 4ce3392f9979 000000000000 (glob)
2 * * -1 5 f3e0509ec098 000000000000 000000000000 (glob)
$ hg debugdata y 0
version https://git-lfs.github.com/spec/v1
oid sha256:802935f5411aa569948cd326115b3521107250019b5dbadf0f6ab2aa2d1e4639
size 17
x-hg-copy x
x-hg-copyrev 1ff4e6c9b2764057ea0c52f7b4a5a9be2e79c8e0
x-is-binary 0
$ hg debugdata y 1
NOTLFS
$ hg debugdata y 2
version https://git-lfs.github.com/spec/v1
oid sha256:a2fcdb080e9838f6e1476a494c1d553e6ffefb68b0d146a06f34b535b5198442
size 35
x-hg-copy x
x-hg-copyrev d33b2f7888d4f6f9112256d0f1c625af6d188fde
x-is-binary 0
$ hg cat -r 1 y
THIS-IS-LFS-FILE
$ hg cat -r 2 y
NOTLFS
$ hg cat -r 5 y
NOTLFS
BECOME-LFS-AGAIN
ADD-A-LINE
$ hg debugindex x
rev offset length delta linkrev nodeid p1 p2
0 0 126 -1 0 1ff4e6c9b276 000000000000 000000000000
1 126 70 -1 3 68b9378cf5a1 000000000000 000000000000
2 196 126 -1 4 d33b2f7888d4 68b9378cf5a1 000000000000
$ hg debugdata x 0
version https://git-lfs.github.com/spec/v1
oid sha256:802935f5411aa569948cd326115b3521107250019b5dbadf0f6ab2aa2d1e4639
size 17
x-is-binary 0
$ hg debugdata x 1
\x01 (esc)
copy: y
copyrev: 139f72f7ca9816bd6b5fdd8b67331458ba11cc0e
\x01 (esc)
NOTLFS
$ hg debugdata x 2
version https://git-lfs.github.com/spec/v1
oid sha256:080f1dba758e4406ab1e722e16fc18965ab2b183979432957418173bf983427f
size 24
x-is-binary 0
# pull lfs content from server and update
$ cd ../client2
$ hg pull
pulling from ssh://user@dummy/master
searching for changes
adding changesets
adding manifests
adding file changes
added 1 changesets with 1 changes to 1 files
new changesets * (glob)
(run 'hg update' to get a working copy)
$ hg update tip
1 files updated, 0 files merged, 1 files removed, 0 files unresolved
$ hg log -p -r tip -T '{rev}:{node} {desc}\n'
5:* shallow.lfs.commit (glob)
diff --git a/x b/y
rename from x
rename to y
--- a/x
+++ b/y
@@ -1,2 +1,3 @@
NOTLFS
BECOME-LFS-AGAIN
+ADD-A-LINE
$ hg log -T '{rev}:{node} {bookmarks} {desc}\n' -G
@ 5:* shallow.lfs.commit (glob)
|
o 4:042535657086a5b08463b9210a8f46dc270e51f9 master x-lfs-again
|
o 3:c6cc0cd58884b847de39aa817ded71e6051caa9f x-nonlfs
|
o 2:f3dec7f3610207dbf222ec2d7b68df16a5fde0f2 y-nonlfs
|
o 1:799bebfa53189a3db8424680f1a8f9806540e541 y-lfs
|
o 0:0d2948821b2b3b6e58505696145f2215cea2b2cd x-lfs
#endif

View File

@ -92,7 +92,7 @@
new changesets 730e2b7b175c
(run 'hg update' to get a working copy)
prefetching file contents
4 files fetched over 1 fetches - (4 misses, 0.00% hit ratio) over *s (glob)
4 files fetched over * fetches - (4 misses, 0.00% hit ratio) over *s (glob)
$ hg up tip
4 files updated, 0 files merged, 0 files removed, 0 files unresolved

View File

@ -210,10 +210,10 @@
added 1 changesets with 0 changes to 0 files
new changesets 515a4dfd2e0c
(run 'hg update' to get a working copy)
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
$ hg update tip
1 files updated, 0 files merged, 1 files removed, 0 files unresolved
1 files fetched over * (glob)
$ hg log -p -r tip -T '{rev}:{node} {desc}\n'
5:515a4dfd2e0c4c963dcbf4bc48587b9747143598 shallow.lfs.commit
@ -333,7 +333,10 @@
# LFS fast path about binary diff works
$ cd $TESTTMP/master
$ cd ../shallow
$ hg pull -q
$ cd ../shallow2
$ hg up -C tip -q
$ $PYTHON << EOF
> with open('a.bin', 'wb') as f:
@ -354,8 +357,17 @@
> hg commit -m $i a.bin
> done
$ cd $TESTTMP/shallow
$ hg push ../master
pushing to ../master
searching for changes
adding changesets
adding manifests
adding file changes
added 6 changesets with 5 changes to 1 files
$ cd ../shallow
$ hg pull -q
5 files fetched over 5 fetches - (5 misses, 0.00% hit ratio) over *s (glob)
$ hg log --removed a.bin --config diff.nobinary=1 --git -p -T '{desc}\n' -r '::tip' --config lfs.url=null://
binary
diff --git a/a.bin b/a.bin
@ -383,5 +395,4 @@
diff --git a/a.bin b/a.bin
Binary file a.bin has changed
5 files fetched over 5 fetches - (5 misses, 0.00% hit ratio) over * (glob)

View File

@ -13,7 +13,7 @@
> import os
> import subprocess
> import sys
>
>
> for path in os.environ["PATH"].split(os.pathsep):
> exe = os.path.join(path, 'lfs-test-server.exe')
> if os.path.exists(exe):
@ -86,6 +86,28 @@ When the server has some blobs already
adding file changes
added 1 changesets with 3 changes to 3 files
Fail to push if LFS blob is not uploaded to the server
$ echo "[extensions]" >> .hg/hgrc
$ echo "lfs=" >> .hg/hgrc
$ echo "[lfs]" >> .hg/hgrc
$ echo "url=file://$TESTTMP/unused-dummystore" >> .hg/hgrc
$ echo ANOTHER-LFS > f
$ hg commit -m f -A f
$ hg push ../repo1
pushing to ../repo1
searching for changes
adding changesets
adding manifests
adding file changes
added 1 changesets with 1 changes to 1 files
transaction abort!
rollback completed
abort: LFS server error. Remote object for file unknown not found: *u'oid': u'384d99297e974dab7d66361be0276032f5045185d6ce42601f43973e721f1dd9'* (glob)
[255]
$ rm .hg/hgrc
Clear the cache to force a download
$ rm -rf `hg config lfs.usercache`
$ hg --repo ../repo1 update tip -v