From 8b91cccc8bc0ad2b0ea7d69f9ae54e45aeb1dc3a Mon Sep 17 00:00:00 2001
From: Durham Goode <durham@fb.com>
Date: Fri, 4 Sep 2020 14:53:03 -0700
Subject: [PATCH] remotefilelog: log undesired filename fetches

Summary:
Now that the Rust revisionstore records undesired filename fetches,
let's log those results to Scuba in Python.

Reviewed By: StanislavGlebik

Differential Revision: D23462572

fbshipit-source-id: b55f2290e30e3a5c3b67d9f612b24bc3aad403a8
---
 .../hgext/remotefilelog/remotefilelog.py      | 17 ++++++
 .../modules/pyrevisionstore/src/lib.rs        |  5 ++
 .../scm/lib/revisionstore/src/contentstore.rs | 11 +++-
 ...est-remotefilelog-undesired-file-logging.t | 53 +++++++++++++++++++
 4 files changed, 85 insertions(+), 1 deletion(-)
 create mode 100644 eden/scm/tests/test-remotefilelog-undesired-file-logging.t

diff --git a/eden/scm/edenscm/hgext/remotefilelog/remotefilelog.py b/eden/scm/edenscm/hgext/remotefilelog/remotefilelog.py
index 51a1d3fec9..7c8b7ab15d 100644
--- a/eden/scm/edenscm/hgext/remotefilelog/remotefilelog.py
+++ b/eden/scm/edenscm/hgext/remotefilelog/remotefilelog.py
@@ -555,6 +555,8 @@ class remotefileslog(filelog.fileslog):
         additions."""
         if self.contentstore:
             self.contentstore.flush()
+            self.logfetches()
+
         if self.metadatastore:
             self.metadatastore.flush()
         self.commitsharedpacks()
@@ -562,6 +564,21 @@ class remotefileslog(filelog.fileslog):
     def abortpending(self):
         """Used in alternative filelog implementations to throw out pending
         additions."""
+        self.logfetches()
         self.contentstore = None
         self.metadatastore = None
         self._memcachestore = None
+
+    def logfetches(self):
+        if self.contentstore:
+            fetched = self.contentstore.getloggedfetches()
+            if fetched:
+                ui = self.repo.ui
+                for path in fetched:
+                    ui.log(
+                        "undesired_file_fetches",
+                        "",
+                        filename=path,
+                        reponame=self.repo.name,
+                    )
+                ui.metrics.gauge("undesiredfilefetches", len(fetched))
diff --git a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/lib.rs b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/lib.rs
index 33406071be..63cbe9ca80 100644
--- a/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/lib.rs
+++ b/eden/scm/edenscmnative/bindings/modules/pyrevisionstore/src/lib.rs
@@ -935,6 +935,11 @@ py_class!(pub class contentstore |py| {
         let store = self.store(py);
         store.metadata_py(py, name, node)
     }
+
+    def getloggedfetches(&self) -> PyResult<Vec<PyPathBuf>> {
+        let store = self.store(py);
+        Ok(store.get_logged_fetches().into_iter().map(|p| p.into()).collect::<Vec<PyPathBuf>>())
+    }
 });
 
 impl ExtractInnerRef for contentstore {
diff --git a/eden/scm/lib/revisionstore/src/contentstore.rs b/eden/scm/lib/revisionstore/src/contentstore.rs
index 316f777e62..7ec5f6cb83 100644
--- a/eden/scm/lib/revisionstore/src/contentstore.rs
+++ b/eden/scm/lib/revisionstore/src/contentstore.rs
@@ -6,6 +6,7 @@
  */
 
 use std::{
+    collections::HashSet,
     path::{Path, PathBuf},
     sync::Arc,
 };
@@ -19,7 +20,7 @@ use configparser::{
     config::ConfigSet,
     hg::{ByteCount, ConfigSetHgExt},
 };
-use types::Key;
+use types::{Key, RepoPathBuf};
 
 use crate::{
     datastore::{
@@ -76,6 +77,14 @@ impl ContentStore {
             Ok(None)
         }
     }
+
+    pub fn get_logged_fetches(&self) -> HashSet<RepoPathBuf> {
+        if let Some(remote_store) = &self.remote_store {
+            remote_store.take_seen()
+        } else {
+            HashSet::new()
+        }
+    }
 }
 
 // Repack specific methods, not to be used directly but by the repack code.
diff --git a/eden/scm/tests/test-remotefilelog-undesired-file-logging.t b/eden/scm/tests/test-remotefilelog-undesired-file-logging.t
new file mode 100644
index 0000000000..a90222aa74
--- /dev/null
+++ b/eden/scm/tests/test-remotefilelog-undesired-file-logging.t
@@ -0,0 +1,53 @@
+#chg-compatible
+
+  $ . "$TESTDIR/library.sh"
+
+  $ cat >> "$TESTTMP/uilog.py" <<EOF
+  > from edenscm.mercurial import extensions
+  > from edenscm.mercurial import ui as uimod
+  > def uisetup(ui):
+  >   extensions.wrapfunction(uimod.ui, 'log', mylog)
+  > def mylog(orig, self, service, *msg, **opts):
+  >   if service in ['undesired_file_fetches']:
+  >     kw = []
+  >     for k, v in sorted(opts.items()):
+  >       kw.append("%s=%s" % (k, v))
+  >     kwstr = ", ".join(kw)
+  >     msgstr = msg[0] % msg[1:]
+  >     self.warn('%s: %s (%s)\n' % (service, msgstr, kwstr))
+  >     with open('$TESTTMP/undesiredfiles', 'a') as f:
+  >       f.write('%s: %s (%s)\n' % (service, msgstr, kwstr))
+  >   return orig(self, service, *msg, **opts)
+  > EOF
+
+  $ cat >> "$HGRCPATH" <<EOF
+  > [extensions]
+  > uilog=$TESTTMP/uilog.py
+  > EOF
+
+  $ newserver master
+  $ clone master client1
+  $ cd client1
+  $ echo x > x
+  $ hg commit -qAm x
+  $ mkdir dir
+  $ echo y > dir/y
+  $ hg commit -qAm y
+  $ hg push -r tip --to master --create
+  pushing rev 79c51fb96423 to destination ssh://user@dummy/master bookmark master
+  searching for changes
+  exporting bookmark master
+  remote: adding changesets (?)
+  remote: adding manifests (?)
+  remote: adding file changes (?)
+  remote: added 2 changesets with 2 changes to 2 files (?)
+
+  $ cd ..
+  $ clone master shallow --noupdate
+  $ cd shallow
+
+  $ hg update -q master --config remotefilelog.undesiredfileregex=".*" 2>&1 | sort
+  2 trees fetched over 0.00s
+  fetching tree '' 05bd2758dd7a25912490d0633b8975bf52bfab06, found via 79c51fb96423
+  undesired_file_fetches:  (filename=dir/y, reponame=master)
+  undesired_file_fetches:  (filename=x, reponame=master)