From cfaec40232313162759afd323a15745c06029091 Mon Sep 17 00:00:00 2001 From: Thomas Orozco Date: Mon, 17 Jun 2019 06:16:05 -0700 Subject: [PATCH] infinitepush: add replaybookmarksqueue Summary: This adds a new queue to replay scratch bookmark changes into Mononoke, which will allow us to replay them there. There's not a lot going on in Mercurial (i.e. in this diff) to achieve this: we simply record bookmark changes as they happen in infinitepush, and allow for excluding some bookmarks. Specifically, we'll want to exclude backup branches, which we don't want to copy, since a) there's way too many of them and b) they're deprecated in favor of Commit Cloud. Currently, this does not allow for replaying deletions. That would require further rework of how we delete things, since right now we do it my matching on bookmark names in the DB, which means the Python side of things is not aware of which bookmarks exactly were deleted. I'm not aware of how much use this is currently getting, but I'll research that and add it in if necessary. Finally, one thing that's worth calling out here is the `bookmark_hash` column in this table. This is here in case we need to scale out the replication of bookmarks across multiple workers. Indeed, we'll always want the replication of any given bookmark to happen sequentially, so we should perform it in a single worker. However, if we have too many bookmarks to replicate, then that could become a bottleneck. If that happens, we'll want to scale out workers, which we can do by having each worker operate on separate bookmarks. The `bookmark_hash` column allows us to evenly divide up the space of bookmarks across workers if that becomes necessary (e.g. we could have 16 workers: one for each first hex digit of the hash). We won't use `bookmark_hash` immediately, but since it's very cheap to add (just compute one hash in Mercurial and put it in the table), I'm adding it in this diff now in case we need it later to avoid the friction of having to re-redeploy hg servers for that. Reviewed By: StanislavGlebik Differential Revision: D15778665 fbshipit-source-id: c34898c1a66e5bec08663a0887adca263222300d --- edenscm/hgext/infinitepush/__init__.py | 5 ++ edenscm/hgext/infinitepush/bundleparts.py | 2 +- edenscm/hgext/infinitepush/fileindex.py | 6 +- .../infinitepush/infinitepushcommands.py | 4 +- edenscm/hgext/infinitepush/server.py | 2 +- edenscm/hgext/infinitepush/sqlindex.py | 31 +++++++++- tests/infinitepush/library.sh | 10 +++- tests/infinitepush/schema.sql | 12 ++++ ...epush-replaybookmarksqueue-ignore-backup.t | 60 +++++++++++++++++++ ...sh-replaybookmarksqueue-multiple-updates.t | 57 ++++++++++++++++++ ...tepush-replaybookmarksqueue-one-bookmark.t | 55 +++++++++++++++++ 11 files changed, 234 insertions(+), 10 deletions(-) create mode 100644 tests/test-infinitepush-replaybookmarksqueue-ignore-backup.t create mode 100644 tests/test-infinitepush-replaybookmarksqueue-multiple-updates.t create mode 100644 tests/test-infinitepush-replaybookmarksqueue-one-bookmark.t diff --git a/edenscm/hgext/infinitepush/__init__.py b/edenscm/hgext/infinitepush/__init__.py index 27edaae718..89f131bc62 100644 --- a/edenscm/hgext/infinitepush/__init__.py +++ b/edenscm/hgext/infinitepush/__init__.py @@ -75,6 +75,11 @@ Configs:: # Whether or not to record new bundles into the forwardfillqueue table. forwardfill = False + # Server-side option. Used only if storetype=sql + # Whether or not to record new scratch bookmarks into the + # replaybookmarksqueue table. + replaybookmarks = False + # Client-side option. Used by --list-remote option. List of remote scratch # patterns to list if no patterns are specified. defaultremotepatterns = ['*'] diff --git a/edenscm/hgext/infinitepush/bundleparts.py b/edenscm/hgext/infinitepush/bundleparts.py index 9175d547f9..90447632d5 100644 --- a/edenscm/hgext/infinitepush/bundleparts.py +++ b/edenscm/hgext/infinitepush/bundleparts.py @@ -241,7 +241,7 @@ def bundle2scratchbookmarks(op, part): if todelete: index.deletebookmarks(todelete) if toinsert: - index.addmanybookmarks(toinsert) + index.addmanybookmarks(toinsert, True) @bundle2.parthandler(constants.scratchmutationparttype) diff --git a/edenscm/hgext/infinitepush/fileindex.py b/edenscm/hgext/infinitepush/fileindex.py index 5508d0b036..16ecc2a2ee 100644 --- a/edenscm/hgext/infinitepush/fileindex.py +++ b/edenscm/hgext/infinitepush/fileindex.py @@ -69,15 +69,15 @@ class fileindex(object): nodepath = os.path.join(self._nodemap, node.hex()) self._write(nodepath, bundleid) - def addbookmark(self, bookmark, node): + def addbookmark(self, bookmark, node, _isbackup): """Record a bookmark pointing to a particular node.""" bookmarkpath = os.path.join(self._bookmarkmap, bookmark) self._write(bookmarkpath, node) - def addmanybookmarks(self, bookmarks): + def addmanybookmarks(self, bookmarks, isbackup): """Record the contents of the ``bookmarks`` dict as bookmarks.""" for bookmark, node in bookmarks.items(): - self.addbookmark(bookmark, node) + self.addbookmark(bookmark, node, isbackup) def deletebookmarks(self, patterns): """Delete all bookmarks that match any of the patterns in ``patterns``.""" diff --git a/edenscm/hgext/infinitepush/infinitepushcommands.py b/edenscm/hgext/infinitepush/infinitepushcommands.py index d2d6359117..789d102f46 100644 --- a/edenscm/hgext/infinitepush/infinitepushcommands.py +++ b/edenscm/hgext/infinitepush/infinitepushcommands.py @@ -153,7 +153,7 @@ def debugcreatescratchbookmark(ui, repo, *args, **opts): ) targetnode = _resolvetargetnode(repo, opts.get("rev")) - index.addbookmark(scratchbookmarkname, targetnode) + index.addbookmark(scratchbookmarkname, targetnode, False) @command( @@ -183,4 +183,4 @@ def debugmovescratchbookmark(ui, repo, *args, **opts): targetnode = _resolvetargetnode(repo, opts.get("rev")) index.deletebookmarks([scratchbookmarkname]) - index.addbookmark(scratchbookmarkname, targetnode) + index.addbookmark(scratchbookmarkname, targetnode, False) diff --git a/edenscm/hgext/infinitepush/server.py b/edenscm/hgext/infinitepush/server.py index 47e719f94e..3ec6fe93a9 100644 --- a/edenscm/hgext/infinitepush/server.py +++ b/edenscm/hgext/infinitepush/server.py @@ -712,7 +712,7 @@ def storebundle(op, params, bundlefile): if key: index.addbundle(key, nodesctx) if bookmark and bookmarknode: - index.addbookmark(bookmark, bookmarknode) + index.addbookmark(bookmark, bookmarknode, False) log( constants.scratchbranchparttype, eventtype="success", diff --git a/edenscm/hgext/infinitepush/sqlindex.py b/edenscm/hgext/infinitepush/sqlindex.py index cadcd48950..f8e5a1d6f6 100644 --- a/edenscm/hgext/infinitepush/sqlindex.py +++ b/edenscm/hgext/infinitepush/sqlindex.py @@ -5,6 +5,7 @@ # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. +import hashlib import logging import os import time @@ -81,6 +82,7 @@ class sqlindex(object): "infinitepush", "shorthasholdrevthreshold", 60 ) self.forwardfill = ui.configbool("infinitepush", "forwardfill") + self.replaybookmarks = ui.configbool("infinitepush", "replaybookmarks") def sqlconnect(self): if self.sqlconn: @@ -187,7 +189,7 @@ class sqlindex(object): data, ) - def addbookmark(self, bookmark, node): + def addbookmark(self, bookmark, node, isbackup): """Record a bookmark pointing to a particular node.""" if not self._connected: self.sqlconnect() @@ -200,7 +202,9 @@ class sqlindex(object): params=(bookmark, node, self.reponame), ) - def addmanybookmarks(self, bookmarks): + self.logmanybookmarksforreplay({bookmark: node}, isbackup) + + def addmanybookmarks(self, bookmarks, isbackup): """Record the contents of the ``bookmarks`` dict as bookmarks.""" if not self._connected: self.sqlconnect() @@ -215,6 +219,8 @@ class sqlindex(object): data, ) + self.logmanybookmarksforreplay(bookmarks, isbackup) + def deletebookmarks(self, patterns): """Delete all bookmarks that match any of the patterns in ``patterns``.""" if not self._connected: @@ -404,6 +410,27 @@ class sqlindex(object): params=(jsonmetadata, self.reponame, node), ) + def logmanybookmarksforreplay(self, bookmarks, isbackup): + """Log the contents of the ``bookmarks`` dict for replay.""" + + if isbackup: + # We don't replay backup bookmarks. + return + + if not self.replaybookmarks: + return + + data = [ + (bookmark, node, hashlib.sha1(bookmark).hexdigest(), self.reponame) + for (bookmark, node) in bookmarks.iteritems() + ] + + self.sqlcursor.executemany( + "INSERT INTO replaybookmarksqueue(bookmark, node, bookmark_hash, reponame) " + "VALUES (%s, %s, %s, %s)", + data, + ) + class CustomConverter(mysql.connector.conversion.MySQLConverter): """Ensure that all values being returned are returned as python string diff --git a/tests/infinitepush/library.sh b/tests/infinitepush/library.sh index e7e5f0b625..4ff2606e7c 100644 --- a/tests/infinitepush/library.sh +++ b/tests/infinitepush/library.sh @@ -72,6 +72,13 @@ forwardfill=True EOF } +enablereplaybookmarks() { +cat << EOF >> .hg/hgrc +[infinitepush] +replaybookmarks=True +EOF +} + createdb() { mysql -h $DBHOST -P $DBPORT -u $DBUSER $DBPASSOPT -e "CREATE DATABASE IF NOT EXISTS $DBNAME;" 2>/dev/null mysql -h $DBHOST -P $DBPORT -D $DBNAME -u $DBUSER $DBPASSOPT </dev/null + mysql -h "$DBHOST" -P "$DBPORT" -u "$DBUSER" -D "$DBNAME" "$DBPASSOPT" -e "$1" } setupdb() { diff --git a/tests/infinitepush/schema.sql b/tests/infinitepush/schema.sql index 359aee9b93..661f3df8b9 100644 --- a/tests/infinitepush/schema.sql +++ b/tests/infinitepush/schema.sql @@ -44,4 +44,16 @@ CREATE TABLE `forwardfillerqueue` ( PRIMARY KEY (`id`), KEY `queue_order` (`reponame`, `slice`, `id`), KEY `claim_review` (`claimed_at`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +CREATE TABLE `replaybookmarksqueue` ( + `id` BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + `reponame` varbinary(255) NOT NULL, + `bookmark` varbinary(512) NOT NULL, + `node` varbinary(64) NOT NULL, + `bookmark_hash` varbinary(64) NOT NULL, + `created_at` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `synced` TINYINT(1) NOT NULL DEFAULT 0, + PRIMARY KEY (`id`), + KEY `sync_queue` (`synced`, `reponame`, `bookmark_hash`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 diff --git a/tests/test-infinitepush-replaybookmarksqueue-ignore-backup.t b/tests/test-infinitepush-replaybookmarksqueue-ignore-backup.t new file mode 100644 index 0000000000..c932e37ea5 --- /dev/null +++ b/tests/test-infinitepush-replaybookmarksqueue-ignore-backup.t @@ -0,0 +1,60 @@ +#if no-windows no-osx + $ setconfig extensions.treemanifest=! + $ mkcommit() { + > echo "$1" > "$1" + > hg add "$1" + > hg ci -d "0 0" -m "$1" + > } + $ . "$TESTDIR/infinitepush/library.sh" + $ setupcommon + +Configure the server + $ hg init server + $ cd server + $ setupsqlserverhgrc repo123 + $ setupdb + $ enablereplaybookmarks + $ cd .. + +It should backup many bookmarks + $ hg clone -q ssh://user@dummy/server client + $ cd client + $ setupsqlclienthgrc + $ mkcommit commit0 + $ commit0="$(hg id -i)" + $ hg up -q "$commit0" && mkcommit commit1 + $ hg up -q "$commit0" && mkcommit commit2 + $ hg up -q "$commit0" && mkcommit commit3 + $ hg cloud backup + backing up stack rooted at ace906b76ab4 + remote: pushing 4 commits: + remote: ace906b76ab4 commit0 + remote: b1e07bb9979c commit1 + remote: 33701f08790f commit2 + remote: db45a2d42cf6 commit3 + commitcloud: backed up 4 commits + $ hg push -r . --to scratch/book --create + pushing to ssh://user@dummy/server + searching for changes + remote: pushing 2 commits: + remote: ace906b76ab4 commit0 + remote: db45a2d42cf6 commit3 + $ cd .. + +Backups should be excluded + $ querysqlindex "SELECT * FROM nodestobundle;" + node bundle reponame + 33701f08790f0e038ca262ddb72728754f60ec88 318c3749b9def132140c54c6f84e853855aa5042 repo123 + ace906b76ab45ac794eb67142e1466725def57cb 318c3749b9def132140c54c6f84e853855aa5042 repo123 + b1e07bb9979cf151ae6a05d0cd9008737c77dfea 318c3749b9def132140c54c6f84e853855aa5042 repo123 + db45a2d42cf67d746ba59e17f09df3eb9e8c2f4c 318c3749b9def132140c54c6f84e853855aa5042 repo123 + $ querysqlindex "SELECT reponame, synced, bookmark, node FROM replaybookmarksqueue;" + reponame synced bookmark node + repo123 0 scratch/book db45a2d42cf67d746ba59e17f09df3eb9e8c2f4c + $ querysqlindex "SELECT reponame, bookmark, node FROM bookmarkstonode ORDER BY node ASC;" + reponame bookmark node + repo123 infinitepush/backups/test/*/client/heads/33701f08790f0e038ca262ddb72728754f60ec88 33701f08790f0e038ca262ddb72728754f60ec88 (glob) + repo123 infinitepush/backups/test/*/client/heads/b1e07bb9979cf151ae6a05d0cd9008737c77dfea b1e07bb9979cf151ae6a05d0cd9008737c77dfea (glob) + repo123 infinitepush/backups/test/*/client/heads/db45a2d42cf67d746ba59e17f09df3eb9e8c2f4c db45a2d42cf67d746ba59e17f09df3eb9e8c2f4c (glob) + repo123 scratch/book db45a2d42cf67d746ba59e17f09df3eb9e8c2f4c +#endif diff --git a/tests/test-infinitepush-replaybookmarksqueue-multiple-updates.t b/tests/test-infinitepush-replaybookmarksqueue-multiple-updates.t new file mode 100644 index 0000000000..9516aae31e --- /dev/null +++ b/tests/test-infinitepush-replaybookmarksqueue-multiple-updates.t @@ -0,0 +1,57 @@ +#if no-windows no-osx + $ setconfig extensions.treemanifest=! + $ mkcommit() { + > echo "$1" > "$1" + > hg add "$1" + > hg ci -d "0 0" -m "$1" + > } + $ . "$TESTDIR/infinitepush/library.sh" + $ setupcommon + +Configure the server + $ hg init server + $ cd server + $ setupsqlserverhgrc repo123 + $ setupdb + $ enablereplaybookmarks + $ cd .. + +It should insert an entry for each update + $ hg clone -q ssh://user@dummy/server client2 + $ cd client2 + $ setupsqlclienthgrc + $ mkcommit commit2 + $ hg push -r . --to scratch/123 --create + pushing to ssh://user@dummy/server + searching for changes + remote: pushing 1 commit: + remote: 6fdf683f5af9 commit2 + $ mkcommit commit3 + $ hg push -r . --to scratch/123 + pushing to ssh://user@dummy/server + searching for changes + remote: pushing 2 commits: + remote: 6fdf683f5af9 commit2 + remote: 8e0c8ddac9fb commit3 + $ mkcommit commit4 + $ hg push -r . --to scratch/123 + pushing to ssh://user@dummy/server + searching for changes + remote: pushing 3 commits: + remote: 6fdf683f5af9 commit2 + remote: 8e0c8ddac9fb commit3 + remote: feccf85eaa94 commit4 + $ cd .. + +Proper metadata should have been recorded + $ querysqlindex "SELECT * FROM nodestobundle;" + node bundle reponame + 6fdf683f5af9a2be091b81ef475f335e2624fb0d f47f4ea5c9dade34f2a38376fe371dc6e4c49c1d repo123 + 8e0c8ddac9fb06e5cb0b3ca65a51632a7814f576 f47f4ea5c9dade34f2a38376fe371dc6e4c49c1d repo123 + feccf85eaa94ff5ec0f80b8fd871d0fa3125a09b f47f4ea5c9dade34f2a38376fe371dc6e4c49c1d repo123 + $ querysqlindex "SELECT id, reponame, synced, bookmark, node, bookmark_hash FROM replaybookmarksqueue;" + id reponame synced bookmark node bookmark_hash + 1 repo123 0 scratch/123 6fdf683f5af9a2be091b81ef475f335e2624fb0d 68e2c1170bb6960df6ab9e2c7da427b5d3eca47e + 2 repo123 0 scratch/123 8e0c8ddac9fb06e5cb0b3ca65a51632a7814f576 68e2c1170bb6960df6ab9e2c7da427b5d3eca47e + 3 repo123 0 scratch/123 feccf85eaa94ff5ec0f80b8fd871d0fa3125a09b 68e2c1170bb6960df6ab9e2c7da427b5d3eca47e +#endif diff --git a/tests/test-infinitepush-replaybookmarksqueue-one-bookmark.t b/tests/test-infinitepush-replaybookmarksqueue-one-bookmark.t new file mode 100644 index 0000000000..4758ccf36a --- /dev/null +++ b/tests/test-infinitepush-replaybookmarksqueue-one-bookmark.t @@ -0,0 +1,55 @@ +#if no-windows no-osx + $ setconfig extensions.treemanifest=! + $ mkcommit() { + > echo "$1" > "$1" + > hg add "$1" + > hg ci -d "0 0" -m "$1" + > } + $ . "$TESTDIR/infinitepush/library.sh" + $ setupcommon + +Configure the server + $ hg init server + $ cd server + $ setupsqlserverhgrc repo123 + $ setupdb + $ cd .. + +Without replaybookmarks, it should not insert into the queue + $ hg clone -q ssh://user@dummy/server client1 + $ cd client1 + $ setupsqlclienthgrc + $ mkcommit commit1 + $ hg push -r . --to scratch/book --create + pushing to ssh://user@dummy/server + searching for changes + remote: pushing 1 commit: + remote: cb9a30b04b9d commit1 + $ cd .. + +Enable replaybookmarks on the server + $ cd server + $ enablereplaybookmarks + $ cd .. + +With replaybookmarks, it should insert into the queue + $ hg clone -q ssh://user@dummy/server client2 + $ cd client2 + $ setupsqlclienthgrc + $ mkcommit commit2 + $ hg push -r . --to scratch/book2 --create + pushing to ssh://user@dummy/server + searching for changes + remote: pushing 1 commit: + remote: 6fdf683f5af9 commit2 + $ cd .. + +Proper metadata should have been recorded + $ querysqlindex "SELECT * FROM nodestobundle;" + node bundle reponame + 6fdf683f5af9a2be091b81ef475f335e2624fb0d 8347a06785e3bdd572ebeb7df3aac1356acb4ce5 repo123 + cb9a30b04b9df854f40d21fdac525408f3bd6c78 944fe1c133f63c7711aa15db2dd9216084dacc36 repo123 + $ querysqlindex "SELECT id, reponame, synced, bookmark, node, bookmark_hash FROM replaybookmarksqueue;" + id reponame synced bookmark node bookmark_hash + 1 repo123 0 scratch/book2 6fdf683f5af9a2be091b81ef475f335e2624fb0d bd2df38131efcfd3f7bd81b4307f9e84d8984729 +#endif