sapling/eden/scm/tests/tinit.sh
Xavier Deguillard ed4021b8e3 revisionstore: disallow reading LFS pointers from packfiles
Summary:
For repositories that have the old-style LFS extension enabled, the pointers
are stored in packfiles/indexedlog alongside with a flag that signify to the
upper layers that the blob is externally stored. With the new way of doing LFS,
pointers are stored separately.

When both are enabled, we are observing some interesting behavior where
different get and get_meta calls may return different blobs/metadata for the
same filenode. This may happen if a filenode is stored in both a packfile as an
LFS pointers, and in the LFS store. Guaranteeing that the revisionstore code is
deterministic in this situation is unfortunately way too costly (a get_meta
call would for instance have to fully validate the sha256 of the blob, and this
wouldn't guarantee that it wouldn't become corrupted on disk before calling
get).

The solution take here is to simply ignore all the lfs pointers from
packfiles/indexedlog when remotefilelog.lfs is enabled. This way, there is no
risk of reading the metadata from the packfiles, and the blob from the
LFSStore. This brings however another complication for the user created blobs:
these are stored in packfiles and would thus become unreadable, the solution is
to simply perform a one-time full repack of the local store to make sure that
all the pointers are moved from the packfiles to to LFSStore.

In the code, the Python bindings are using ExtStoredPolicy::Ignore directly as
these are only used in the treemanifest code where no LFS pointers should be
present, the repack code uses ExtStoredPolicy::Use to be able to read the
pointers, it wouldn't be able to otherwise.

Reviewed By: DurhamG

Differential Revision: D22951598

fbshipit-source-id: 0e929708ba5a3bb2a02c0891fd62dae1ccf18204
2020-09-09 18:27:42 -07:00

279 lines
7.0 KiB
Bash

# This file will be sourced by all .t tests. Put general purposed functions
# here.
_repocount=0
if [ -n "$USE_MONONOKE" ] ; then
. "$TESTDIR/../../mononoke/tests/integration/library.sh"
fi
dummysshcmd() {
if [ -n "$DUMMYSSH" ]
then
echo "$DUMMYSSH"
else
echo "$PYTHON $TESTDIR/dummyssh"
fi
}
# Create a new repo
newrepo() {
reponame="$1"
if [ -z "$reponame" ]; then
_repocount=$((_repocount+1))
reponame=repo$_repocount
fi
mkdir "$TESTTMP/$reponame"
cd "$TESTTMP/$reponame"
hg init
}
newserver() {
local reponame="$1"
if [ -n "$USE_MONONOKE" ] ; then
REPONAME=$reponame setup_mononoke_config
mononoke
MONONOKE_START_TIMEOUT=60 wait_for_mononoke "$TESTTMP/$reponame"
else
mkdir "$TESTTMP/$reponame"
cd "$TESTTMP/$reponame"
hg --config extensions.lz4revlog= \
--config extensions.treemanifest= \
--config experimental.narrow-heads=false \
--config visibility.enabled=false \
init
enable lz4revlog remotefilelog remotenames treemanifest
setconfig \
remotefilelog.reponame="$reponame" remotefilelog.server=True \
treemanifest.flatcompat=False treemanifest.rustmanifest=True \
treemanifest.server=True treemanifest.treeonly=True \
infinitepush.server=yes infinitepush.reponame="$reponame" \
infinitepush.indextype=disk infinitepush.storetype=disk \
experimental.narrow-heads=false
fi
}
clone() {
servername="$1"
clientname="$2"
shift 2
cd "$TESTTMP"
remotecmd="hg"
if [ -n "$USE_MONONOKE" ] ; then
remotecmd="$MONONOKE_HGCLI"
fi
hg clone -q --shallow "ssh://user@dummy/$servername" "$clientname" "$@" \
--config "extensions.lz4revlog=" \
--config "extensions.remotefilelog=" \
--config "extensions.remotenames=" \
--config "extensions.treemanifest=" \
--config "remotefilelog.reponame=$servername" \
--config "treemanifest.treeonly=True" \
--config "ui.ssh=$(dummysshcmd)" \
--config "ui.remotecmd=$remotecmd"
cat >> $clientname/.hg/hgrc <<EOF
[extensions]
lz4revlog=
remotefilelog=
remotenames=
treemanifest=
tweakdefaults=
[phases]
publish=False
[remotefilelog]
reponame=$servername
[treemanifest]
flatcompat=False
rustmanifest=True
sendtrees=True
treeonly=True
[ui]
ssh=$(dummysshcmd)
[tweakdefaults]
rebasekeepdate=True
EOF
if [ -n "$USE_MONONOKE" ] ; then
cat >> $clientname/.hg/hgrc <<EOF
[ui]
remotecmd=$MONONOKE_HGCLI
EOF
fi
if [ -n "$COMMITCLOUD" ]; then
hg --cwd $clientname cloud join -q
fi
}
switchrepo() {
reponame="$1"
cd $TESTTMP/$reponame
}
# Set configuration for feature
configure() {
for name in "$@"
do
case "$name" in
dummyssh)
export DUMMYSSH_STABLE_ORDER=1
setconfig ui.ssh="$(dummysshcmd)"
;;
mutation)
setconfig \
experimental.evolution=obsolete \
mutation.enabled=true mutation.record=true mutation.date="0 0" \
visibility.enabled=true
;;
mutation-norecord)
setconfig \
experimental.evolution=obsolete \
mutation.enabled=true mutation.record=false mutation.date="0 0" \
visibility.enabled=true
;;
evolution)
setconfig \
experimental.evolution="createmarkers, allowunstable" \
mutation.enabled=false \
visibility.enabled=false
;;
noevolution)
setconfig \
experimental.evolution=obsolete \
mutation.enabled=false \
visibility.enabled=false
;;
commitcloud)
enable commitcloud infinitepush
setconfig commitcloud.hostname=testhost
setconfig commitcloud.servicetype=local commitcloud.servicelocation=$TESTTMP commitcloud.token_enforced=False
setconfig commitcloud.remotebookmarkssync=True
COMMITCLOUD=1
;;
narrowheads)
configure noevolution mutation-norecord
setconfig experimental.narrow-heads=true
;;
selectivepull)
enable remotenames
setconfig remotenames.selectivepull=True
setconfig remotenames.selectivepulldefault=master
setconfig remotenames.selectivepullaccessedbookmarks=True
;;
modern)
enable amend
setconfig remotenames.rename.default=remote
setconfig remotenames.hoist=remote
setconfig experimental.changegroup3=True
configure dummyssh commitcloud narrowheads selectivepull
;;
esac
done
}
# Enable extensions
enable() {
for name in "$@"
do
setconfig "extensions.$name="
done
}
# Disable extensions
disable() {
for name in "$@"
do
setconfig "extensions.$name=!"
if [[ $name == "treemanifest" ]]; then
setconfig treemanifest.sendtrees=False treemanifest.treeonly=False
fi
done
}
# Like "hg debugdrawdag", but do not leave local tags in the repo and define
# nodes as environment variables.
# This is useful if the test wants to hide those commits because tags would
# make commits visible. The function will set environment variables so
# commits can still be referred as $TAGNAME.
drawdag() {
hg debugdrawdag "$@" --config remotenames.autopullhoistpattern=
eval `hg bookmarks -T '{bookmark}={node}\n'`
BOOKMARKS=$(hg book -T '{bookmark} ')
if [[ -n "${BOOKMARKS}" ]]; then
hg book -fd ${BOOKMARKS}
fi
}
# Simplify error reporting so crash does not show a traceback.
# This is useful to match error messages without the traceback.
shorttraceback() {
enable errorredirect
setconfig errorredirect.script='printf "%s" "$TRACE" | tail -1 1>&2'
}
# Set config items like --config way, instead of using cat >> $HGRCPATH
setconfig() {
python "$RUNTESTDIR/setconfig.py" "$@"
}
# Set config item, but always in the main hgrc
setglobalconfig() {
( cd "$TESTTMP" ; setconfig "$@" )
}
# Set config items that enable modern features.
setmodernconfig() {
enable remotenames amend
setconfig experimental.narrow-heads=true visibility.enabled=true mutation.record=true mutation.enabled=true mutation.date="0 0" experimental.evolution=obsolete remotenames.rename.default=remote
}
# Read config from stdin (usually a heredoc).
readconfig() {
local hgrcpath
if [ -e ".hg" ]
then
hgrcpath=".hg/hgrc"
else
hgrcpath="$HGRCPATH"
fi
cat >> "$hgrcpath"
}
# Read global config from stdin (usually a heredoc).
readglobalconfig() {
cat >> "$HGRCPATH"
}
# Create a new extension
newext() {
extname="$1"
if [ -z "$extname" ]; then
_extcount=$((_extcount+1))
extname=ext$_extcount
fi
cat > "$TESTTMP/$extname.py"
setconfig "extensions.$extname=$TESTTMP/$extname.py"
}
showgraph() {
hg log --graph -T "{rev} {node|short} {desc|firstline}" | sed \$d
}
tglog() {
hg log -G -T "{rev}: {node|short} '{desc}' {bookmarks} {branches}" "$@"
}
tglogp() {
hg log -G -T "{rev}: {node|short} {phase} '{desc}' {bookmarks} {branches}" "$@"
}
tglogm() {
hg log -G -T "{rev}: {node|short} '{desc|firstline}' {bookmarks} {join(mutations % '(Rewritten using {operation} into {join(successors % \'{node|short}\', \', \')})', ' ')}" "$@"
}