metalog: implement bi-directional migration

Summary:
Previously, it's fine to migrate up to metalog, and fine to migrate down once
since we double write data. However, re-enabling metalog can be problematic
since there is no code path to do "migrate up" again.

This diff fixes the issue by tracking what files (or "keys") are using metalog
as the source of truth in metalog. So we can accurately figure out whether to
migrate svfs files to metalog on demand.

Reviewed By: xavierd

Differential Revision: D18864424

fbshipit-source-id: e61e1790c231f9c88de869f413f27bb954a29920

eden/scm/edenscm/mercurial/store.py

@@ -588,13 +588,21 @@ class metavfs(object):
         vfs = self.vfs
         metalog = bindings.metalog.metalog(vfs.join("metalog"))
 
-        # Migrate data from vfs to metalog
-        keys = set(metalog.keys())
-        for name in self.metapaths:
-            if name not in keys:
-                data = vfs.tryread(name)
-                if data is not None:
-                    metalog[name] = data
+        # Keys that are previously tracked in metalog.
+        tracked = set((metalog.get("tracked") or "").split())
+        # Keys that should be tracked (specified by config).
+        desired = set(self.metapaths)
+
+        # Migrate up (from svfs plain files to metalog).
+        for name in desired.difference(tracked):
+            data = vfs.tryread(name)
+            if data is not None:
+                metalog[name] = data
+
+        # Migrating down is a no-op, since we double-write to svfs too.
+
+        metalog["tracked"] = "\n".join(sorted(desired))
+
         try:
             # XXX: This is racy.
             metalog.commit("migrate from vfs", int(util.timer()))

eden/scm/tests/test-metalog-migration-t.py

@@ -0,0 +1,84 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2.
+
+from __future__ import absolute_import
+
+from testutil.autofix import eq
+from testutil.dott import feature, sh, testtmp  # noqa: F401
+
+
+def backup():
+    """Backup .hg/store/{bookmarks,remotenames}"""
+    for name in ["bookmarks", "remotenames"]:
+        path = ".hg/store/%s" % name
+        sh.cp(path, "%s.bak" % path)
+
+
+def restore():
+    """Rewrite .hg/store/{bookmarks,remotenames} with backup"""
+    for name in ["bookmarks", "remotenames"]:
+        path = ".hg/store/%s" % name
+        sh.cp("%s.bak" % path, path)
+
+
+def setbookmarks(name):
+    """Set bookmarks to specified commit"""
+    sh.hg("bookmark", "book", "-r", "desc(%s)" % name)
+    sh.hg("debugremotebookmark", "remotebook", "desc(%s)" % name)
+
+
+def listbookmarks():
+    """List local and remote bookmarks"""
+    local = sh.hg("log", "-r", sh.hg("bookmarks", "-T", "{node}"), "-T{desc}")
+    remote = sh.hg(
+        "log", "-r", sh.hg("bookmarks", "--remote", "-T", "{node}"), "-T{desc}"
+    )
+    return [local, remote]
+
+
+sh.newrepo()
+sh.setconfig("experimental.metalog=0")
+sh.enable("remotenames")
+
+sh % "drawdag" << r"""
+C
+|
+B
+|
+A
+"""
+
+# Prepare bookmarks and remotenames. Set them to A in backup, and B on disk.
+
+setbookmarks("A")
+backup()
+setbookmarks("B")
+
+# Test migrating from disk to metalog.
+# They should migrate "B" from disk to metalog and use it.
+
+sh.setconfig("experimental.metalog=1")
+eq(listbookmarks(), ["B", "B"])
+
+# Metalog is the source of truth. Changes to .hg/store are ignored.
+
+restore()
+eq(listbookmarks(), ["B", "B"])
+
+# Test migrating from metalog to disk.
+# Metalog is not the source of truth. Changes to .hg/store are effective.
+
+sh.setconfig("experimental.metalog=0")
+setbookmarks("C")
+eq(listbookmarks(), ["C", "C"])
+restore()
+eq(listbookmarks(), ["A", "A"])
+
+# Migrate up again.
+# At this time metalog should import "A" from disk to metalog, instead of
+# using "B" that exists in metalog.
+
+sh.setconfig("experimental.metalog=1")
+eq(listbookmarks(), ["A", "A"])