fsmonitor: add migration to toggle tracking ignored files

Summary:
It turns out tracking ignored files does have an impact on status performance.
Filtering out the ignored directories is not that fast, and ignored files can
be scattered everywhere (like ".pyc", ".iml" files) that makes them harder to
be filtered out efficiently.

Add code paths to migrate between "ignored tracked" and "ignore untracked".
Store the metadata in treestate.

Reviewed By: phillco

Differential Revision: D12916021

fbshipit-source-id: e02d0c6f3b1a036f70703c11f35381c594e2f8e5

hgext/fsmonitor/__init__.py

@@ -101,9 +101,10 @@ exception if it finds anything. (default: false)
     track-ignore-files = (boolean)
 
 If set to True, fsmonitor will track ignored files in treestate. This behaves
-more correctly if files get unignored, or added to the sparse profile. This
-config option is provided for slowrolling the feature. It will be eventually
-removed.  (default: true)
+more correctly if files get unignored, or added to the sparse profile, at the
+cost of slowing down status command. Turning it off would make things faster,
+at the cast of removing files from ignore patterns (or adding files to sparse
+profiles) won't be detected automatically. (default: True)
 """
 
 # Platforms Supported
@@ -762,8 +763,38 @@ def poststatustreestate(wctx, status):
     # should not affect "status" correctness, even if they are not the latest
     # state. Changing the clock to None would make the next "status" command
     # slower. Therefore avoid doing that.
+    repo = wctx.repo()
     if clock is not None:
-        wctx.repo()._fsmonitorstate.set(clock, hashignore, notefiles)
+        repo._fsmonitorstate.set(clock, hashignore, notefiles)
+
+    dirstate = repo.dirstate
+    oldtrackignored = (dirstate.getmeta("track-ignored") or "1") == "1"
+    newtrackignored = repo.ui.configbool("fsmonitor", "track-ignore-files")
+
+    if oldtrackignored != newtrackignored:
+        if newtrackignored:
+            # Add ignored files to treestate
+            ignored = wctx.status(listignored=True).ignored
+            repo.ui.debug("start tracking %d ignored files\n" % len(ignored))
+            for path in ignored:
+                dirstate.needcheck(path)
+        else:
+            # Remove ignored files from treestate
+            ignore = dirstate._ignore
+            from mercurial.rust.treestate import (
+                NEED_CHECK,
+                EXIST_P1,
+                EXIST_P2,
+                EXIST_NEXT,
+            )
+
+            repo.ui.debug("stop tracking ignored files\n")
+            for path in dirstate._map._tree.walk(
+                NEED_CHECK, EXIST_P1 | EXIST_P2 | EXIST_NEXT
+            ):
+                if ignore(path):
+                    dirstate.delete(path)
+        dirstate.setmeta("track-ignored", str(int(newtrackignored)))
 
 
 class poststatus(object):

mercurial/context.py

@@ -1873,6 +1873,9 @@ class workingctx(committablectx):
         poststatusafter = self._repo.postdsstatus(afterdirstatewrite=True)
         dirstate = self._repo.dirstate
         if fixup or poststatusbefore or poststatusafter or dirstate._dirty:
+            # prevent infinite loop because fsmonitor postfixup might call
+            # wctx.status()
+            self._repo._insidepoststatusfixup = True
             try:
                 oldid = dirstate.identity()
 
@@ -1916,6 +1919,7 @@ class workingctx(committablectx):
             finally:
                 # Even if the wlock couldn't be grabbed, clear out the list.
                 self._repo.clearpostdsstatus()
+                self._repo._insidepoststatusfixup = False
 
     def _dirstatestatus(self, match, ignored=False, clean=False, unknown=False):
         """Gets the status from the dirstate -- internal use only."""
@@ -1936,7 +1940,8 @@ class workingctx(committablectx):
             if fixup and clean:
                 s.clean.extend(fixup)
 
-        self._poststatusfixup(s, fixup)
+        if not getattr(self._repo, "_insidepoststatusfixup", False):
+            self._poststatusfixup(s, fixup)
 
         if match.always():
             # cache for performance

mercurial/dirstate.py

@@ -473,18 +473,27 @@ class dirstate(object):
 
     def setclock(self, clock):
         """Set fsmonitor clock"""
-        if not self._istreestate:
-            raise error.ProgrammingError("setclock is only supported by treestate")
-        clock = clock or ""
-        if clock != self._map._clock:
-            self._map._clock = clock
-            self._dirty = True
+        return self.setmeta("clock", clock)
 
     def getclock(self):
         """Get fsmonitor clock"""
+        return self.getmeta("clock")
+
+    def setmeta(self, name, value):
+        """Set metadata"""
         if not self._istreestate:
-            raise error.ProgrammingError("getclock is only supported by treestate")
-        return self._map._clock or None
+            raise error.ProgrammingError("setmeta is only supported by treestate")
+        value = value or None
+        if value != self.getmeta(name):
+            self._map.updatemetadata({name: value})
+            self._dirty = True
+
+    def getmeta(self, name):
+        """Get metadata"""
+        if not self._istreestate:
+            raise error.ProgrammingError("getmeta is only supported by treestate")
+        # Normalize "" to "None"
+        return self._map.getmetadata().get(name) or None
 
     def _addpath(self, f, state, mode, size, mtime):
         oldstate = self[f]

mercurial/treestate.py

@@ -47,6 +47,8 @@ class _overlaydict(dict):
 def _packmetadata(dictobj):
     result = []
     for k, v in dictobj.iteritems():
+        if not v:
+            continue
         entry = "%s=%s" % (k, v)
         if "=" in k or "\0" in entry:
             raise error.ProgrammingError("illegal metadata entry: %r" % entry)
@@ -138,7 +140,6 @@ class treestatemap(object):
     def clear(self):
         self._threshold = 0
         self._rootid = 0
-        self._clock = ""
         self._parents = (node.nullid, node.nullid)
 
         # use a new file
@@ -374,10 +375,6 @@ class treestatemap(object):
                 raise error.Abort(
                     _("working directory state appears damaged (metadata mismatch)!")
                 )
-            clock = metadata.get("clock")
-        else:
-            clock = ""
-        self._clock = clock
         self._tree = tree
 
     def _setfilename(self, filename=None):
@@ -421,16 +418,8 @@ class treestatemap(object):
 
     def write(self, st, now):
         # write .hg/treestate/<uuid>
-        metadata = {}
-        if self._clock:
-            metadata.update(
-                {
-                    "clock": self._clock,
-                    # for debugging purpose
-                    "pid": os.getpid(),
-                    "now": now,
-                }
-            )
+        metadata = self.getmetadata()
+        metadata.update({"p1": None, "p2": None})
         if self._parents[0] != node.nullid:
             metadata["p1"] = node.hex(self._parents[0])
         if self._parents[1] != node.nullid:
@@ -554,6 +543,18 @@ class treestatemap(object):
 
     # treestate specific methods
 
+    def getmetadata(self):
+        return _unpackmetadata(self._tree.getmetadata())
+
+    def updatemetadata(self, items):
+        metadata = _unpackmetadata(self._tree.getmetadata())
+        metadata.update(items)
+        self._tree.setmetadata(_packmetadata(metadata))
+
+    @property
+    def _clock(self):
+        return self.getmetadata().get("clock") or None
+
     def needcheck(self, path):
         """Mark a file as NEED_CHECK, so it will be included by 'nonnormalset'
 

tests/test-treestate-trackignore.t

@@ -0,0 +1,27 @@
+#require fsmonitor
+
+  $ newrepo
+  $ cat >> .gitignore << EOF
+  > .gitignore
+  > EOF
+
+  $ hg status
+  $ hg debugtree list
+  .gitignore: 0666 -1 -1 NEED_CHECK 
+
+Stop tracking ignored files removes them from treestate. The migration only happens once.
+
+  $ setconfig fsmonitor.track-ignore-files=0
+  $ hg status --debug | grep tracking
+  stop tracking ignored files
+  $ hg status
+  $ hg debugtree list
+
+Start tracking ignored files adds them to treestate. The migration only happens once.
+
+  $ setconfig fsmonitor.track-ignore-files=1
+  $ hg status --debug | grep tracking
+  start tracking 1 ignored files
+  $ hg status
+  $ hg debugtree list
+  .gitignore: 0666 -1 -1 NEED_CHECK