status: support match.bad() for Rust status

Summary:
Some Python commands depend on the "bad" matcher callback getting called by dirstate.status(). Tweak things so this happens when we use the Rust status as well. This also adds support for the "clean" option when using the Rust status since clean files and "bad" are somewhat coupled.

I moved invalid file type detection from filesystem.py into dirstate.py so it covers Rust status as well.

Reviewed By: quark-zju

Differential Revision: D46646006

fbshipit-source-id: 28278af15e02e0f794aefc7bde850ce58b319ac6

eden/scm/edenscm/dirstate.py

@@ -17,7 +17,7 @@ from __future__ import absolute_import
 import contextlib
 import errno
 import os
-import tempfile
+import stat
 import weakref
 from typing import (
     BinaryIO,
@@ -950,18 +950,27 @@ class dirstate(object):
         pass
 
     def _ruststatus(
-        self, match: "Callable[[str], bool]", ignored: bool, clean: bool, unknown: bool
+        self, match: matchmod.basematcher, ignored: bool, clean: bool, unknown: bool
     ) -> "scmutil.status":
-        if ignored or clean:
+        if ignored:
             raise self.FallbackToPythonStatus
 
-        return self._repo._rsrepo.workingcopy().status(
+        status = self._repo._rsrepo.workingcopy().status(
             match, self._lastnormaltime, self._ui._rcfg
         )
 
+        if not unknown:
+            status.unknown.clear()
+
+        self._add_clean_and_trigger_bad_matches(
+            match, status, self._repo[None].p1(), clean
+        )
+
+        return status
+
     @perftrace.tracefunc("Status")
     def status(
-        self, match: "Callable[[str], bool]", ignored: bool, clean: bool, unknown: bool
+        self, match: matchmod.basematcher, ignored: bool, clean: bool, unknown: bool
     ) -> "scmutil.status":
         """Determine the status of the working copy relative to the
         dirstate and return a scmutil.status.
@@ -995,7 +1004,6 @@ class dirstate(object):
         iadd = ignoredpaths.append
         radd = removed.append
         dadd = deleted.append
-        cadd = cleanpaths.append
         ignore = self._ignore
         copymap = self._map.copymap
 
@@ -1153,28 +1161,8 @@ class dirstate(object):
         )
 
         # Step 3: If clean files were requested, add those to the results
-        seenset = set()
-        for files in status:
-            seenset.update(files)
-            seenset.update(util.dirs(files))
-
-        if listclean:
-            for fn in pctx.manifest().matches(match):
-                assert isinstance(fn, str)
-                if fn not in seenset:
-                    cadd(fn)
-            seenset.update(cleanpaths)
-
         # Step 4: Report any explicitly requested files that don't exist
-        # pyre-fixme[16]: Anonymous callable has no attribute `files`.
-        for path in sorted(match.files()):
-            try:
-                if path in seenset:
-                    continue
-                os.lstat(os.path.join(self._root, path))
-            except OSError as ex:
-                # pyre-fixme[16]: Anonymous callable has no attribute `bad`.
-                match.bad(path, encoding.strtolocal(ex.strerror))
+        self._add_clean_and_trigger_bad_matches(match, status, pctx, listclean)
 
         # TODO: fire this inside filesystem. fixup is a list of files that
         # checklookup says are clean
@@ -1188,6 +1176,40 @@ class dirstate(object):
             perftrace.tracevalue("Ignored Files", len(ignoredpaths))
         return status
 
+    def _add_clean_and_trigger_bad_matches(
+        self,
+        match: matchmod.basematcher,
+        status: scmutil.status,
+        pctx: context.changectx,
+        listclean: bool,
+    ) -> None:
+        seenset = set()
+        for files in status:
+            seenset.update(files)
+            seenset.update(util.dirs(files))
+
+        if listclean:
+            clean = status.clean
+            for fn in pctx.manifest().matches(match):
+                assert isinstance(fn, str)
+                if fn not in seenset:
+                    clean.append(fn)
+            seenset.update(clean)
+
+        for path in sorted(match.files()):
+            try:
+                st = os.lstat(os.path.join(self._root, path))
+            except OSError as ex:
+                if path not in seenset:
+                    # This handles does-not-exist, permission error, etc.
+                    match.bad(path, encoding.strtolocal(ex.strerror))
+                continue
+
+            typ = stat.S_IFMT(st.st_mode)
+            if not typ & (stat.S_IFDIR | stat.S_IFREG | stat.S_IFLNK):
+                # This handles invalid types like named pipe.
+                match.bad(path, filesystem.badtype(typ))
+
     def _poststatusfixup(
         self, status: "scmutil.status", wctx: "context.workingctx", oldid: object
     ) -> None:
@@ -1284,7 +1306,7 @@ class dirstate(object):
                 self._repo.clearpostdsstatus()
                 self._repo._insidepoststatusfixup = False
 
-    def matches(self, match: "matchmod.basematcher") -> "Iterable[str]":
+    def matches(self, match: matchmod.basematcher) -> "Iterable[str]":
         """
         return files in the dirstate (in whatever state) filtered by match
         """

eden/scm/edenscm/filesystem.py

@@ -373,11 +373,8 @@ class physicalfilesystem(object):
                                 # unknown file
                                 yield (nf, st)
                         else:
-                            # This can happen for unusual file types, like named
-                            # piped. We treat them as if they were missing, so
-                            # report them as missing. Covered in test-symlinks.t
-                            if nf in explicitfiles:
-                                badfn(nf, badtype(kind))
+                            # Invalid file types invoke match.bad in dirstate.py.
+                            pass
 
     def purge(self, match, removefiles, removedirs, removeignored, dryrun):
         """Deletes untracked files and directories from the filesystem.

eden/scm/edenscm/testing/t/hghave.py

@@ -64,7 +64,9 @@ def checkvers(name, desc, vers):
 
 
 def checkexe(name):
-    f = lambda name=name: os.path.isfile(f"/bin/{name}")
+    f = lambda name=name: os.path.isfile(f"/bin/{name}") or os.path.isfile(
+        f"/usr/bin/{name}"
+    )
     checks[name] = (f, f"{name} executable")
     exes.add(name)
 

eden/scm/lib/workingcopy/src/watchmanfs/treestate.rs

@@ -30,23 +30,28 @@ pub fn mark_needs_check(ts: &mut TreeState, path: &RepoPathBuf) -> Result<bool>
         Some(filestate) => {
             let filestate = filestate.clone();
             if filestate.state.intersects(StateFlags::NEED_CHECK) {
+                tracing::trace!(%path, "already NEED_CHECK");
                 // It's already marked need_check, so return early so we don't mutate the
                 // treestate.
                 return Ok(false);
             }
+            tracing::trace!(%path, "marking NEED_CHECK");
             FileStateV2 {
                 state: filestate.state | StateFlags::NEED_CHECK,
                 ..filestate
             }
         }
         // The file is currently untracked
-        None => FileStateV2 {
-            state: StateFlags::NEED_CHECK,
-            mode: 0o666,
-            size: -1,
-            mtime: -1,
-            copied: None,
-        },
+        None => {
+            tracing::trace!(%path, "inserting NEED_CHECK");
+            FileStateV2 {
+                state: StateFlags::NEED_CHECK,
+                mode: 0o666,
+                size: -1,
+                mtime: -1,
+                copied: None,
+            }
+        }
     };
     ts.insert(path, &filestate)?;
     Ok(true)
@@ -57,6 +62,7 @@ pub fn clear_needs_check(ts: &mut TreeState, path: &RepoPathBuf) -> Result<bool>
     if let Some(filestate) = state {
         let filestate = filestate.clone();
         if !filestate.state.intersects(StateFlags::NEED_CHECK) {
+            tracing::trace!(%path, "already not NEED_CHECK");
             // It's already clear.
             return Ok(false);
         }
@@ -69,8 +75,10 @@ pub fn clear_needs_check(ts: &mut TreeState, path: &RepoPathBuf) -> Result<bool>
             // No other flags means it was ignored/untracked, but now we don't
             // care about it (either it was deleted, or we aren't tracking
             // ignored files anymore).
+            tracing::trace!(%path, "empty after unsetting NEED_CHECK");
             ts.remove(path)?;
         } else {
+            tracing::trace!(%path, "unsetting NEED_CHECK");
             ts.insert(path, &filestate)?;
         }
         return Ok(true);

eden/scm/tests/test-add.t

@@ -1,8 +1,6 @@
-#chg-compatible
 #debugruntest-compatible
 
   $ eagerepo
-  $ setconfig workingcopy.ruststatus=False
   $ hg init a
   $ cd a
   $ echo a > a

eden/scm/tests/test-fileset.t

@@ -1,4 +1,3 @@
-#chg-compatible
 #debugruntest-compatible
 
   $ setconfig devel.segmented-changelog-rev-compat=true

eden/scm/tests/test-rebuildstate.t

@@ -1,6 +1,7 @@
 #chg-compatible
 #debugruntest-compatible
 
+  $ setconfig workingcopy.ruststatus=false
 
   $ eagerepo
   $ newext adddrop <<EOF

eden/scm/tests/test-remove.t

@@ -299,6 +299,7 @@ handling of untracked directories and missing files
   removing d1/a
 
   $ hg rm --after nosuch
+  nosuch: $ENOENT$
   [1]
 
 handling root path in remove with matcher

eden/scm/tests/test-symlinks.t

@@ -73,7 +73,7 @@ it should show a.c, dir/a.o and dir/b.o deleted
   ! dir/a.o
   ? .gitignore
   $ hg status a.c
-  a.c: unsupported file type (type is fifo) (no-fsmonitor !)
+  a.c: unsupported file type (type is fifo)
   ! a.c
   $ cd ..
 

eden/scm/tests/test-walk.t

@@ -1,7 +1,7 @@
-#chg-compatible
-#require mkfifo
 #debugruntest-compatible
 
+  $ eagerepo
+
   $ hg init t
   $ cd t
   $ setconfig experimental.dynmatcher=True
@@ -453,7 +453,7 @@ Test patterns:
   matcher: <patternmatcher patterns='(?:NOEXIST(?:/|$))'>
   NOEXIST: * (glob)
 
-#if fifo
+#if mkfifo
   $ mkfifo fifo
   $ hg debugwalk fifo
   matcher: <patternmatcher patterns='(?:fifo(?:/|$))'>