py3: migrate treedirstate to use utf8 for state and files

Summary:
treedirstate was using bytes for state and for file paths. This does
the appropriate conversions. Note, I don't use strings for all state in rust,
because it's a pain in the butt in some places. We're going to delete
treedirstate eventually anyone, so just I'm getting the test to pass for now.

Reviewed By: quark-zju

Differential Revision: D19715102

fbshipit-source-id: 4f9eac372bee7884d36aa19e3a3ed253392fa7dd

eden/scm/edenscm/mercurial/treedirstate.py

@@ -40,12 +40,15 @@ class _reader(object):
         self.offset += 4
         return v[0]
 
-    def readstr(self):
+    def readbytes(self):
         l = self.readuint()
         v = self.data[self.offset : self.offset + l]
         self.offset += l
         return v
 
+    def readstr(self):
+        return pycompat.decodeutf8(self.readbytes())
+
 
 class _writer(object):
     def __init__(self):
@@ -54,10 +57,13 @@ class _writer(object):
     def writeuint(self, v):
         self.buffer.write(struct.pack(">L", v))
 
-    def writestr(self, v):
+    def writebytes(self, v):
         self.writeuint(len(v))
         self.buffer.write(v)
 
+    def writestr(self, v):
+        self.writebytes(pycompat.encodeutf8(v))
+
 
 # The treedirstatemap iterator uses the getnext method on the dirstatemap
 # to find the next item on each call.  This involves searching down the
@@ -234,7 +240,14 @@ class treedirstatemap(object):
         return self.hastrackeddir(dirname) or self.hasremoveddir(dirname)
 
     def addfile(self, f, oldstate, state, mode, size, mtime):
-        self._rmap.addfile(f, oldstate, state, mode, size, mtime)
+        self._rmap.addfile(
+            f,
+            pycompat.encodeutf8(oldstate),
+            pycompat.encodeutf8(state),
+            mode,
+            size,
+            mtime,
+        )
         if self._nonnormalset is not None:
             if state != "n" or mtime == -1:
                 self._nonnormalset.add(f)
@@ -283,7 +296,14 @@ class treedirstatemap(object):
         for f in files:
             e = self.gettracked(f)
             if e is not None and e[0] == "n" and e[3] == now:
-                self._rmap.addfile(f, e[0], e[0], e[1], e[2], -1)
+                self._rmap.addfile(
+                    f,
+                    pycompat.encodeutf8(e[0]),
+                    pycompat.encodeutf8(e[0]),
+                    e[1],
+                    e[2],
+                    -1,
+                )
                 self.nonnormalset.add(f)
 
     def parents(self):
@@ -585,4 +605,4 @@ def gettreeid(opener, dirstatefile):
     version = r.readuint()
     if version != treedirstateversion:
         return None
-    return r.readstr()
+    return r.readbytes()

eden/scm/edenscmnative/bindings/modules/pytreestate/src/lib.rs

@@ -197,20 +197,20 @@ py_class!(class treedirstatemap |py| {
         Ok(value.is_some())
     }
 
-    def gettracked(&self, filename: PyPathBuf) -> PyResult<Option<(PyBytes, u32, i32, i32)>> {
+    def gettracked(&self, filename: PyPathBuf) -> PyResult<Option<(PyString, u32, i32, i32)>> {
         let mut dirstate = self.dirstate(py).borrow_mut();
         let value = dirstate
                 .get_tracked(filename.as_utf8_bytes())
                 .map_pyerr(py)?;
-        Ok(value.map(|file| (PyBytes::new(py, &[file.state; 1]), file.mode, file.size, file.mtime)))
+        Ok(value.map(|file| (PyString::new(py, unsafe {std::str::from_utf8_unchecked(&[file.state; 1])}), file.mode, file.size, file.mtime)))
     }
 
-    def getremoved(&self, filename: PyPathBuf, default: Option<(PyBytes, u32, i32, i32)>) -> PyResult<Option<(PyBytes, u32, i32, i32)>> {
+    def getremoved(&self, filename: PyPathBuf, default: Option<(PyString, u32, i32, i32)>) -> PyResult<Option<(PyString, u32, i32, i32)>> {
         let mut dirstate = self.dirstate(py).borrow_mut();
         let value = dirstate
                 .get_removed(filename.as_utf8_bytes())
                 .map_pyerr(py)?;
-        Ok(value.map_or(default, |file| Some((PyBytes::new(py, &[file.state; 1]), file.mode, file.size, file.mtime))))
+        Ok(value.map_or(default, |file| Some((PyString::new(py, unsafe {std::str::from_utf8_unchecked(&[file.state; 1])}), file.mode, file.size, file.mtime))))
     }
 
     def hastrackeddir(&self, dirname: PyPathBuf) -> PyResult<bool> {
@@ -256,7 +256,7 @@ py_class!(class treedirstatemap |py| {
     // Get the next dirstate object after the provided filename.  If the filename is None,
     // returns the first file in the tree.  If the provided filename is the last file, returns
     // None.
-    def getnext(&self, filename: Option<PyPathBuf>, removed: bool) -> PyResult<Option<(PyBytes, (PyBytes, u32, i32, i32))>> {
+    def getnext(&self, filename: Option<PyPathBuf>, removed: bool) -> PyResult<Option<(PyBytes, (PyString, u32, i32, i32))>> {
         let mut dirstate = self.dirstate(py).borrow_mut();
         let next = if removed {
             match filename {
@@ -284,7 +284,7 @@ py_class!(class treedirstatemap |py| {
                 }
             }
         };
-        Ok(next.map(|(f, s)| (PyBytes::new(py, &f), (PyBytes::new(py, &[s.state; 1]), s.mode, s.size, s.mtime))))
+        Ok(next.map(|(f, s)| (PyBytes::new(py, &f), (PyString::new(py, unsafe {std::str::from_utf8_unchecked(&[s.state; 1])}), s.mode, s.size, s.mtime))))
     }
 
     def addfile(
@@ -304,7 +304,7 @@ py_class!(class treedirstatemap |py| {
         Ok(PyNone)
     }
 
-    def removefile(&self, filename: PyPathBuf, _old_state: PyBytes, size: i32) -> PyResult<PyNone> {
+    def removefile(&self, filename: PyPathBuf, _old_state: PyString, size: i32) -> PyResult<PyNone> {
         let mut dirstate = self.dirstate(py).borrow_mut();
         dirstate
             .remove_file(filename.as_utf8_bytes(), &FileState::new(b'r', 0, size, 0))

eden/scm/tests/test-debugrebuilddirstate-corrupt.t

@@ -1,4 +1,3 @@
-#require py2
 #chg-compatible
 
 #chg-compatible

eden/scm/tests/test-revset-dirstate-parents.t

@@ -1,5 +1,3 @@
-#require py2
-
 #chg-compatible
 
 #testcases treestate-on treestate-off