facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2025-01-06 04:43:19 +03:00
Code Issues Projects Releases Wiki Activity

define paths as binary rather than strings in the thrift interface

Browse Source 
Summary:
This prevents `hg status` from blowing up with a UTF-8 decode
error inside the generated thrift code.

Push safety concerns:
* This doesn't change the wire representation of the data
* Existing clients that believe it to be a string will continue to have
  the same behavior
* Buck has its own copy of an older version of the thrift spec, so it will
  continue to work "OK".
* When buck resyncs with our thrift file, some changes will likely be needed
  to convert the byte arrays to strings or paths or whatever is appropriate
  for bucks internal API

Work "OK" above means that clients that currently believe that `string` is
utf-8 encoded will have a runtime error if we ever send them a path that
is not utf-8.  This is the behavior prior to this diff and will continue
to be the behavior for clients (like buck) that have an older version
of the thrift file.

Reviewed By: simpkins

Differential Revision: D9270843

fbshipit-source-id: b01135aec9152aaf5199e1c654ddd7f61c03717e
This commit is contained in:
Wez Furlong 2018-08-11 01:34:44 -07:00 committed by Facebook Github Bot
parent 1688b0b40b
commit cfde0c0717
12 changed files with 195 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
eden/cli/config.py
View File

@ -396,7 +396,9 @@ Do you want to run `eden mount %s` instead?"""
self._save_client_config(client_config, config_path)
# Prepare to mount
mount_info = eden_ttypes.MountInfo(mountPoint=path, edenClientPath=client_dir)
mount_info = eden_ttypes.MountInfo(
mountPoint=os.fsencode(path), edenClientPath=os.fsencode(client_dir)
)
with self.get_thrift_client() as client:
client.mount(mount_info)
@ -582,7 +584,9 @@ Do you want to run `eden mount %s` instead?"""
raise
# Ask eden to mount the path
mount_info = eden_ttypes.MountInfo(mountPoint=path, edenClientPath=client_dir)
mount_info = eden_ttypes.MountInfo(
mountPoint=os.fsencode(path), edenClientPath=os.fsencode(client_dir)
)
with self.get_thrift_client() as client:
client.mount(mount_info)

58
eden/cli/debug.py
View File

@ -43,7 +43,7 @@ from .subcmd import Subcmd
debug_cmd = subcmd_mod.Decorator()
def get_mount_path(path: str) -> Tuple[str, str]:
def get_mount_path(path: str) -> Tuple[bytes, bytes]:
"""
Given a path inside an eden mount, find the path to the eden root.
@ -52,26 +52,16 @@ def get_mount_path(path: str) -> Tuple[str, str]:
os.path.join(eden_mount_path, relative_path) refers to the same file as the
original input path.
"""
# TODO: This will probably be easier to do using the special .eden
# directory, once the diff adding .eden lands.
current_path = os.path.realpath(path)
rel_path = ""
while True:
# For now we simply assume that the first mount point we come across is
# the eden mount point. This doesn't handle bind mounts inside the
# eden mount, but that's fine for now.
if os.path.ismount(current_path):
rel_path = os.path.normpath(rel_path)
if rel_path == ".":
rel_path = ""
return (current_path, rel_path)
parent, basename = os.path.split(current_path)
if parent == current_path:
raise Exception("eden mount point not found")
current_path = parent
rel_path = os.path.join(basename, rel_path)
path = os.path.realpath(path)
current_path = path
if not os.path.isdir(current_path):
current_path = os.path.dirname(current_path)
mount_path = os.readlink(os.path.join(current_path, ".eden", "root"))
rel_path = os.path.relpath(path, mount_path)
if rel_path == ".":
rel_path = ""
return (os.fsencode(mount_path), os.fsencode(rel_path))
def escape_path(value: bytes) -> str:
@ -310,7 +300,7 @@ class HgDirstateCmd(Subcmd):
out.writeln(f"Non-normal Files ({len(entries)}):", attr=out.BOLD)
entries.sort(key=lambda entry: entry[0]) # Sort by key.
for path, dirstate_tuple in entries:
_print_hg_nonnormal_file(path, dirstate_tuple, out)
_print_hg_nonnormal_file(os.fsencode(path), dirstate_tuple, out)
out.writeln(f"Copymap ({len(copymap)}):", attr=out.BOLD)
_print_copymap(copymap)
@ -327,7 +317,7 @@ class HgGetDirstateTupleCmd(Subcmd):
def run(self, args: argparse.Namespace) -> int:
mount, rel_path = get_mount_path(args.path)
_parents, dirstate_tuples, _copymap = _get_dirstate_data(mount)
dirstate_tuple = dirstate_tuples.get(rel_path)
dirstate_tuple = dirstate_tuples.get(os.fsdecode(rel_path))
out = ui_mod.get_output()
if dirstate_tuple:
_print_hg_nonnormal_file(rel_path, dirstate_tuple, out)
@ -339,19 +329,19 @@ class HgGetDirstateTupleCmd(Subcmd):
dirstate_tuple = ("n", entry.mode, 0)
_print_hg_nonnormal_file(rel_path, dirstate_tuple, out)
except NoValueForKeyError:
print("No tuple for " + rel_path, file=sys.stderr)
print("No tuple for " + os.fsdecode(rel_path), file=sys.stderr)
return 1
return 0
def _print_hg_nonnormal_file(
rel_path: str, dirstate_tuple: Tuple[str, Any, int], out: ui_mod.Output
rel_path: bytes, dirstate_tuple: Tuple[str, Any, int], out: ui_mod.Output
) -> None:
status = _dirstate_char_to_name(dirstate_tuple[0])
merge_state = _dirstate_merge_state_to_name(dirstate_tuple[2])
out.writeln(f"{rel_path}", fg=out.GREEN)
out.writeln(f"{os.fsdecode(rel_path)}", fg=out.GREEN)
out.writeln(f" status = {status}")
out.writeln(f" mode = {oct(dirstate_tuple[1])}")
out.writeln(f" mergeState = {merge_state}")
@ -384,12 +374,12 @@ def _dirstate_merge_state_to_name(merge_state: int) -> str:
def _get_dirstate_data(
mount: str
mount: bytes
) -> Tuple[Tuple[bytes, bytes], Dict[str, Tuple[str, Any, int]], Dict[str, str]]:
"""Returns a tuple of (parents, dirstate_tuples, copymap).
On error, returns None.
"""
filename = os.path.join(mount, ".hg", "dirstate")
filename = os.path.join(os.fsdecode(mount), ".hg", "dirstate")
with open(filename, "rb") as f:
return eden.dirstate.read(f, filename)
@ -497,18 +487,18 @@ class OverlayCmd(Subcmd):
mount, rel_path = get_mount_path(args.path or os.getcwd())
# Get the path to the overlay directory for this mount point
client_dir = config._get_client_dir_for_mount_point(mount)
client_dir = config._get_client_dir_for_mount_point(os.fsdecode(mount))
self.overlay = overlay_mod.Overlay(os.path.join(client_dir, "local"))
if args.number is not None:
self._display_overlay(args.number, "")
elif rel_path:
rel_path = os.path.normpath(rel_path)
inode_number = self.overlay.lookup_path(rel_path)
inode_number = self.overlay.lookup_path(os.fsdecode(rel_path))
if inode_number is None:
print(f"{rel_path} is not materialized", file=sys.stderr)
return 1
self._display_overlay(inode_number, rel_path)
self._display_overlay(inode_number, os.fsdecode(rel_path))
else:
self._display_overlay(1, "/")
@ -598,7 +588,7 @@ class GetPathCmd(Subcmd):
"%s %s"
% (
"loaded" if inodePathInfo.loaded else "unloaded",
os.path.normpath(os.path.join(mount, inodePathInfo.path))
os.fsdecode(os.path.normpath(os.path.join(mount, inodePathInfo.path)))
if inodePathInfo.linked
else "unlinked",
)
@ -844,7 +834,9 @@ class DebugJournalCmd(Subcmd):
)
params = DebugGetRawJournalParams(
mountPoint=mount, fromPosition=from_position, toPosition=to_position
mountPoint=os.fsencode(mount),
fromPosition=from_position,
toPosition=to_position,
)
raw_journal = client.debugGetRawJournal(params)
if args.pattern:
@ -863,7 +855,7 @@ class DebugJournalCmd(Subcmd):
def print_raw_journal_deltas(
deltas: Iterator[FileDelta], pattern: Optional[Pattern]
) -> None:
matcher: Callable[[str], bool] = (lambda x: True) if pattern is None else cast(
matcher: Callable[[bytes], bool] = (lambda x: True) if pattern is None else cast(
Any, pattern.match
)
for delta in deltas:

4
eden/cli/doctor.py
View File

@ -238,8 +238,8 @@ def run_normal_checks(
fs_util: filesystem.FsUtil,
) -> None:
with config.get_thrift_client() as client:
active_mount_points = [
mount.mountPoint
active_mount_points: List[str] = [
os.fsdecode(mount.mountPoint)
for mount in client.listMounts()
if mount.mountPoint is not None
]

8
eden/cli/main.py
View File

@ -168,7 +168,7 @@ class ListCmd(Subcmd):
try:
with config.get_thrift_client() as client:
active_mount_points: Set[Optional[str]] = {
mount.mountPoint for mount in client.listMounts()
os.fsdecode(mount.mountPoint) for mount in client.listMounts()
}
except EdenNotRunningError:
active_mount_points = set()
@ -441,7 +441,7 @@ class FsckCmd(Subcmd):
mount, rel_path = debug_mod.get_mount_path(args.path or os.getcwd())
# Get the path to the overlay directory for this mount point
client_dir = config._get_client_dir_for_mount_point(mount)
client_dir = config._get_client_dir_for_mount_point(os.fsdecode(mount))
self._overlay_dir = os.path.join(client_dir, "local")
self._overlay = overlay_mod.Overlay(self._overlay_dir)
@ -649,7 +649,7 @@ class PrefetchCmd(Subcmd):
with config.get_thrift_client() as client:
result = client.globFiles(
GlobParams(
mountPoint=repo_root,
mountPoint=os.fsencode(repo_root),
globs=args.PATTERN,
includeDotfiles=False,
prefetchFiles=not args.no_prefetch,
@ -779,7 +779,7 @@ def stop_aux_processes(client: eden.thrift.EdenClient) -> None:
for all mounts"""
active_mount_points: Set[Optional[str]] = {
mount.mountPoint for mount in client.listMounts()
os.fsdecode(mount.mountPoint) for mount in client.listMounts()
}
for repo in active_mount_points:

4
eden/cli/stats.py
View File

@ -10,6 +10,7 @@
import argparse
import io
import logging
import os
import sys
import textwrap
from typing import Dict, List, Optional, Union, cast
@ -73,10 +74,11 @@ def do_stats_general(
inode_info = diag_info.mountPointInfo
for key in inode_info:
info = inode_info[key]
mount_path = os.fsdecode(key)
out.write(
textwrap.dedent(
f"""\
Mount information for {key}
Mount information for {mount_path}
Loaded inodes in memory: {info.loadedInodeCount}
Files: {info.loadedFileCount}
Trees: {info.loadedTreeCount}

2
eden/cli/test/doctor_test.py
View File

@ -1829,7 +1829,7 @@ class FakeConfig:
# Tell the thrift client to report the mount as active
self._fake_client._mounts.append(
eden_ttypes.MountInfo(mountPoint=full_path)
eden_ttypes.MountInfo(mountPoint=os.fsencode(full_path))
)
# Set up directories on disk that look like the mounted checkout

90
eden/fs/service/eden.thrift
View File

@ -25,6 +25,16 @@ typedef i64 /* (cpp.type = "std::uint64_t") */ unsigned64
*/
typedef binary BinaryHash
/**
* So, you thought that a path was a string?
* Paths in posix are arbitrary byte strings with some pre-defined special
* characters. On modern systems they tend to be represented as UTF-8 but
* there is no guarantee. We use the `PathString` type as symbolic way
* to indicate that you may need to perform special processing to safely
* interpret the path data on your system.
*/
typedef binary PathString
exception EdenError {
1: required string message
2: optional i32 errorCode
@ -35,8 +45,8 @@ exception NoValueForKeyError {
}
struct MountInfo {
1: string mountPoint
2: string edenClientPath
1: PathString mountPoint
2: PathString edenClientPath
}
union SHA1Result {
@ -97,9 +107,9 @@ struct FileDelta {
2: JournalPosition toPosition
/** The complete list of paths from both the snapshot and the overlay that
* changed between fromPosition and toPosition */
3: list<string> changedPaths
4: list<string> createdPaths
5: list<string> removedPaths
3: list<PathString> changedPaths
4: list<PathString> createdPaths
5: list<PathString> removedPaths
/** When fromPosition.snapshotHash != toPosition.snapshotHash this holds
* the union of the set of files whose ScmFileStatus differed from the
* committed fromPosition hash before the hash changed, and the set of
@ -108,11 +118,11 @@ struct FileDelta {
* whose state may have changed as part of an update operation, but
* in ways that may not be able to be extracted solely by performing
* source control diff operations on the from/to hashes. */
6: list<string> uncleanPaths
6: list<PathString> uncleanPaths
}
struct DebugGetRawJournalParams {
1: string mountPoint
1: PathString mountPoint
2: JournalPosition fromPosition
3: JournalPosition toPosition
}
@ -151,7 +161,7 @@ enum ScmFileStatus {
}
struct ScmStatus {
1: map<string, ScmFileStatus> entries
1: map<PathString, ScmFileStatus> entries
/**
* A map of { path -> error message }
@ -162,7 +172,7 @@ struct ScmStatus {
*
* This map will be empty if no errors occurred.
*/
2: map<string, string> errors
2: map<PathString, string> errors
}
/** Option for use with checkOutRevision(). */
@ -226,7 +236,7 @@ enum ConflictType {
* Details about conflicts or errors that occurred during a checkout operation
*/
struct CheckoutConflict {
1: string path
1: PathString path
2: ConflictType type
3: string message
}
@ -286,7 +296,7 @@ struct TreeInodeDebugInfo {
}
struct InodePathDebugInfo {
1: string path
1: PathString path
2: bool loaded
3: bool linked
}
@ -322,7 +332,7 @@ struct InternalStats {
* is the details like number of loaded inodes,unloaded inodes in that mount
* and number of materialized inodes in that mountpoint.
*/
3: map<string, MountInodeInfo> mountPointInfo
3: map<PathString, MountInodeInfo> mountPointInfo
/**
* Linux-only: the contents of /proc/self/smaps, to be parsed by the caller.
*/
@ -356,7 +366,7 @@ struct FuseCall {
/** Params for globFiles(). */
struct GlobParams {
1: string mountPoint,
1: PathString mountPoint,
2: list<string> globs,
3: bool includeDotfiles,
// if true, prefetch matching blobs
@ -371,13 +381,13 @@ struct Glob {
* This list cannot contain duplicate values and is not guaranteed to be
* sorted.
*/
1: list<string> matchingFiles,
1: list<PathString> matchingFiles,
}
service EdenService extends fb303.FacebookService {
list<MountInfo> listMounts() throws (1: EdenError ex)
void mount(1: MountInfo info) throws (1: EdenError ex)
void unmount(1: string mountPoint) throws (1: EdenError ex)
void unmount(1: PathString mountPoint) throws (1: EdenError ex)
/**
* Potentially check out the specified snapshot, reporting conflicts (and
@ -402,7 +412,7 @@ service EdenService extends fb303.FacebookService {
* these paths as desired after checkOutRevision() returns.
*/
list<CheckoutConflict> checkOutRevision(
1: string mountPoint,
1: PathString mountPoint,
2: BinaryHash snapshotHash,
3: CheckoutMode checkoutMode)
throws (1: EdenError ex)
@ -414,7 +424,7 @@ service EdenService extends fb303.FacebookService {
* This operation is equivalent to `git reset --soft` or `hg reset --keep`
*/
void resetParentCommits(
1: string mountPoint,
1: PathString mountPoint,
2: WorkingDirectoryParents parents)
throws (1: EdenError ex)
@ -426,19 +436,19 @@ service EdenService extends fb303.FacebookService {
* - path identifies something that is not an ordinary file (e.g., symlink
* or directory).
*/
list<SHA1Result> getSHA1(1: string mountPoint, 2: list<string> paths)
list<SHA1Result> getSHA1(1: PathString mountPoint, 2: list<PathString> paths)
throws (1: EdenError ex)
/**
* Returns a list of paths relative to the mountPoint.
*/
list<string> getBindMounts(1: string mountPoint)
list<PathString> getBindMounts(1: PathString mountPoint)
throws (1: EdenError ex)
/** Returns the sequence position at the time the method is called.
* Returns the instantaneous value of the journal sequence number.
*/
JournalPosition getCurrentJournalPosition(1: string mountPoint)
JournalPosition getCurrentJournalPosition(1: PathString mountPoint)
throws (1: EdenError ex)
/** Returns the set of files (and dirs) that changed since a prior point.
@ -449,7 +459,7 @@ service EdenService extends fb303.FacebookService {
* other available functions in EdenService.
*/
FileDelta getFilesChangedSince(
1: string mountPoint,
1: PathString mountPoint,
2: JournalPosition fromPosition)
throws (1: EdenError ex)
@ -471,8 +481,8 @@ service EdenService extends fb303.FacebookService {
* files in the overlay.
*/
list<FileInformationOrError> getFileInformation(
1: string mountPoint,
2: list<string> paths)
1: PathString mountPoint,
2: list<PathString> paths)
throws (1: EdenError ex)
/**
@ -480,8 +490,8 @@ service EdenService extends fb303.FacebookService {
* Returns a list of files that match the input globs.
* There are no duplicate values in the result.
*/
list<string> glob(
1: string mountPoint,
list<PathString> glob(
1: PathString mountPoint,
2: list<string> globs)
throws (1: EdenError ex)
@ -501,7 +511,7 @@ service EdenService extends fb303.FacebookService {
* SCM system, such as the .git folder in Git and the .hg folder in Mercurial.
*/
ScmStatus getScmStatus(
1: string mountPoint,
1: PathString mountPoint,
2: bool listIgnored,
3: BinaryHash commit,
) throws (1: EdenError ex)
@ -511,7 +521,7 @@ service EdenService extends fb303.FacebookService {
* This does not care about the state of the working copy.
*/
ScmStatus getScmStatusBetweenRevisions(
1: string mountPoint,
1: PathString mountPoint,
2: BinaryHash oldHash,
3: BinaryHash newHash,
) throws (1: EdenError ex)
@ -529,8 +539,8 @@ service EdenService extends fb303.FacebookService {
* parameter rather than assuming the current commit.
*/
ManifestEntry getManifestEntry(
1: string mountPoint
2: string relativePath
1: PathString mountPoint
2: PathString relativePath
) throws (
1: EdenError ex
2: NoValueForKeyError noValueForKeyError
@ -550,7 +560,7 @@ service EdenService extends fb303.FacebookService {
* from the BackingStore if it is not already present in the LocalStore.
*/
list<ScmTreeEntry> debugGetScmTree(
1: string mountPoint,
1: PathString mountPoint,
2: BinaryHash id,
3: bool localStoreOnly,
) throws (1: EdenError ex)
@ -562,7 +572,7 @@ service EdenService extends fb303.FacebookService {
* for the blob, and that the information is correct.
*/
binary debugGetScmBlob(
1: string mountPoint,
1: PathString mountPoint,
2: BinaryHash id,
3: bool localStoreOnly,
) throws (1: EdenError ex)
@ -576,7 +586,7 @@ service EdenService extends fb303.FacebookService {
* debugGetScmBlob() when getting data about extremely large blobs.
*/
ScmBlobMetadata debugGetScmBlobMetadata(
1: string mountPoint,
1: PathString mountPoint,
2: BinaryHash id,
3: bool localStoreOnly,
) throws (1: EdenError ex)
@ -603,8 +613,8 @@ service EdenService extends fb303.FacebookService {
* have outstanding references.
*/
list<TreeInodeDebugInfo> debugInodeStatus(
1: string mountPoint,
2: string path,
1: PathString mountPoint,
2: PathString path,
) throws (1: EdenError ex)
/**
@ -614,7 +624,7 @@ service EdenService extends fb303.FacebookService {
* fuse_in_header.
*/
list<FuseCall> debugOutstandingFuseCalls(
1: string mountPoint,
1: PathString mountPoint,
)
/**
@ -624,7 +634,7 @@ service EdenService extends fb303.FacebookService {
* mountPoint be specified.
*/
InodePathDebugInfo debugGetInodePath(
1: string mountPoint,
1: PathString mountPoint,
2: i64 inodeNumber,
) throws (1: EdenError ex)
@ -667,8 +677,8 @@ service EdenService extends fb303.FacebookService {
* (wall clock) time.
*/
i64 unloadInodeForPath(
1: string mountPoint,
2: string path,
1: PathString mountPoint,
2: PathString path,
3: TimeSpec age,
) throws (1: EdenError ex)
@ -689,8 +699,8 @@ service EdenService extends fb303.FacebookService {
* Invalidate kernel cache for inode.
*/
void invalidateKernelInodeCache(
1: string mountPoint,
2: string path
1: PathString mountPoint,
2: PathString path
)
throws (1: EdenError ex)

2
eden/integration/clone_test.py
View File

@ -61,7 +61,7 @@ class CloneTest(testcase.EdenRepoTest):
with self.get_thrift_client() as client:
active_mount_points: Set[Optional[str]] = {
mount.mountPoint for mount in client.listMounts()
os.fsdecode(mount.mountPoint) for mount in client.listMounts()
}
self.assertIn(
empty_dir, active_mount_points, msg="mounted using the realpath"

65
eden/integration/glob_test.py
View File

@ -42,62 +42,63 @@ class GlobTest(testcase.EdenRepoTest):
self.addCleanup(self.client.close)
def test_exact_path_component_match(self) -> None:
self.assert_glob(["hello"], ["hello"])
self.assert_glob(["ddir/subdir/.dotfile"], ["ddir/subdir/.dotfile"])
self.assert_glob(["hello"], [b"hello"])
self.assert_glob(["ddir/subdir/.dotfile"], [b"ddir/subdir/.dotfile"])
def test_wildcard_path_component_match(self) -> None:
self.assert_glob(["hel*"], ["hello"])
self.assert_glob(["ad*"], ["adir"])
self.assert_glob(["a*/file"], ["adir/file"])
self.assert_glob(["hel*"], [b"hello"])
self.assert_glob(["ad*"], [b"adir"])
self.assert_glob(["a*/file"], [b"adir/file"])
def test_no_accidental_substring_match(self) -> None:
self.assert_glob(["hell"], [], msg="No accidental substring match")
def test_match_all_files_in_directory(self) -> None:
self.assert_glob(["bdir/*"], ["bdir/file", "bdir/otherfile"])
self.assert_glob(["bdir/*"], [b"bdir/file", b"bdir/otherfile"])
def test_match_all_files_in_directory_with_dotfile(self) -> None:
self.assert_glob(["ddir/subdir/*"], ["ddir/subdir/notdotfile"])
self.assert_glob(["ddir/subdir/*"], [b"ddir/subdir/notdotfile"])
def test_overlapping_globs(self) -> None:
self.assert_glob(
["adir/*", "**/file"],
["adir/file", "bdir/file"],
[b"adir/file", b"bdir/file"],
msg="De-duplicate results from multiple globs",
)
def test_recursive_wildcard_prefix(self) -> None:
self.assert_glob(["**/file"], ["adir/file", "bdir/file"])
self.assert_glob(["**/file"], [b"adir/file", b"bdir/file"])
def test_recursive_wildcard_suffix(self) -> None:
self.assert_glob(["adir/**"], ["adir/file"])
self.assert_glob(["adir/**/*"], ["adir/file"])
self.assert_glob(["adir/**"], [b"adir/file"])
self.assert_glob(["adir/**/*"], [b"adir/file"])
def test_recursive_wildcard_suffix_with_dotfile(self) -> None:
self.assert_glob(
["ddir/**"], ["ddir/notdotfile", "ddir/subdir", "ddir/subdir/notdotfile"]
["ddir/**"], [b"ddir/notdotfile", b"ddir/subdir", b"ddir/subdir/notdotfile"]
)
self.assert_glob(
["ddir/**"],
[
"ddir/subdir",
"ddir/subdir/.dotfile",
"ddir/notdotfile",
"ddir/subdir/notdotfile",
b"ddir/subdir",
b"ddir/subdir/.dotfile",
b"ddir/notdotfile",
b"ddir/subdir/notdotfile",
],
include_dotfiles=True,
)
self.assert_glob(
["ddir/**/*"], ["ddir/notdotfile", "ddir/subdir", "ddir/subdir/notdotfile"]
["ddir/**/*"],
[b"ddir/notdotfile", b"ddir/subdir", b"ddir/subdir/notdotfile"],
)
self.assert_glob(
["ddir/**/*"],
[
"ddir/subdir",
"ddir/subdir/.dotfile",
"ddir/notdotfile",
"ddir/subdir/notdotfile",
b"ddir/subdir",
b"ddir/subdir/.dotfile",
b"ddir/notdotfile",
b"ddir/subdir/notdotfile",
],
include_dotfiles=True,
)
@ -106,33 +107,33 @@ class GlobTest(testcase.EdenRepoTest):
self.assert_glob(
["java/com/**/*.java"],
[
"java/com/example/Example.java",
"java/com/example/foo/Foo.java",
"java/com/example/foo/bar/Bar.java",
"java/com/example/foo/bar/baz/Baz.java",
b"java/com/example/Example.java",
b"java/com/example/foo/Foo.java",
b"java/com/example/foo/bar/Bar.java",
b"java/com/example/foo/bar/baz/Baz.java",
],
)
self.assert_glob(
["java/com/example/*/*.java"], ["java/com/example/foo/Foo.java"]
["java/com/example/*/*.java"], [b"java/com/example/foo/Foo.java"]
)
def test_malformed_query(self) -> None:
with self.assertRaises(EdenError) as ctx:
self.client.glob(self.mount, ["adir["])
self.client.glob(self.mount_path_bytes, ["adir["])
self.assertIn("unterminated bracket sequence", str(ctx.exception))
with self.assertRaises(EdenError) as ctx:
self.client.globFiles(GlobParams(self.mount, ["adir["], True))
self.client.globFiles(GlobParams(self.mount_path_bytes, ["adir["], True))
self.assertIn("unterminated bracket sequence", str(ctx.exception))
def assert_glob(
self,
globs: List[str],
expected_matches: List[str],
expected_matches: List[bytes],
include_dotfiles: bool = False,
msg: Optional[str] = None,
) -> None:
params = GlobParams(self.mount, globs, include_dotfiles)
params = GlobParams(self.mount_path_bytes, globs, include_dotfiles)
self.assertCountEqual(
expected_matches, self.client.globFiles(params).matchingFiles, msg=msg
)
@ -140,5 +141,7 @@ class GlobTest(testcase.EdenRepoTest):
# Also verify behavior of legacy Thrift API.
if include_dotfiles:
self.assertCountEqual(
expected_matches, self.client.glob(self.mount, globs), msg=msg
expected_matches,
self.client.glob(self.mount_path_bytes, globs),
msg=msg,
)

4
eden/integration/lib/testcase.py
View File

@ -237,6 +237,10 @@ class EdenTestCase(TestParent):
def mount_path(self) -> pathlib.Path:
return pathlib.Path(self.mount)
@property
def mount_path_bytes(self) -> bytes:
return bytes(self.mount_path)
def get_thrift_client(self) -> eden.thrift.EdenClient:
"""
Get a thrift client to the edenfs daemon.

78
eden/integration/materialized_query_test.py
View File

@ -43,12 +43,12 @@ class MaterializedQueryTest(testcase.EdenRepoTest):
self.addCleanup(self.client.close)
def test_noEntries(self) -> None:
pos = self.client.getCurrentJournalPosition(self.mount)
pos = self.client.getCurrentJournalPosition(self.mount_path_bytes)
# setting up the .eden dir bumps the sequence number
self.assertEqual(INITIAL_SEQ, pos.sequenceNumber)
self.assertNotEqual(0, pos.mountGeneration)
changed = self.client.getFilesChangedSince(self.mount, pos)
changed = self.client.getFilesChangedSince(self.mount_path_bytes, pos)
self.assertEqual(set(), set(changed.changedPaths))
self.assertEqual(set(), set(changed.createdPaths))
self.assertEqual(set(), set(changed.removedPaths))
@ -58,21 +58,33 @@ class MaterializedQueryTest(testcase.EdenRepoTest):
def test_getFileInformation(self) -> None:
""" verify that getFileInformation is consistent with the VFS """
paths = ["", "not-exist", "hello", "adir", "adir/file", "bdir/test.sh", "slink"]
info_list = self.client.getFileInformation(self.mount, paths)
paths = [
b"",
b"not-exist",
b"hello",
b"adir",
b"adir/file",
b"bdir/test.sh",
b"slink",
]
info_list = self.client.getFileInformation(self.mount_path_bytes, paths)
self.assertEqual(len(paths), len(info_list))
for idx, path in enumerate(paths):
try:
st = os.lstat(os.path.join(self.mount, path))
st = os.lstat(os.path.join(self.mount, os.fsdecode(path)))
self.assertEqual(
FileInformationOrError.INFO,
info_list[idx].getType(),
msg="have non-error result for " + path,
msg="have non-error result for " + repr(path),
)
info = info_list[idx].get_info()
self.assertEqual(st.st_mode, info.mode, msg="mode matches for " + path)
self.assertEqual(st.st_size, info.size, msg="size matches for " + path)
self.assertEqual(
st.st_mode, info.mode, msg="mode matches for " + repr(path)
)
self.assertEqual(
st.st_size, info.size, msg="size matches for " + repr(path)
)
self.assertEqual(int(st.st_mtime), info.mtime.seconds)
if not stat.S_ISDIR(st.st_mode):
self.assertNotEqual(0, st.st_mtime)
@ -82,61 +94,63 @@ class MaterializedQueryTest(testcase.EdenRepoTest):
self.assertEqual(
FileInformationOrError.ERROR,
info_list[idx].getType(),
msg="have error result for " + path,
msg="have error result for " + repr(path),
)
err = info_list[idx].get_error()
self.assertEqual(
e.errno, err.errorCode, msg="error code matches for " + path
e.errno, err.errorCode, msg="error code matches for " + repr(path)
)
def test_invalidProcessGeneration(self) -> None:
# Get a candidate position
pos = self.client.getCurrentJournalPosition(self.mount)
pos = self.client.getCurrentJournalPosition(self.mount_path_bytes)
# poke the generation to a value that will never manifest in practice
pos.mountGeneration = 0
with self.assertRaises(EdenService.EdenError) as context:
self.client.getFilesChangedSince(self.mount, pos)
self.client.getFilesChangedSince(self.mount_path_bytes, pos)
self.assertEqual(
errno.ERANGE, context.exception.errorCode, msg="Must return ERANGE"
)
def test_removeFile(self) -> None:
initial_pos = self.client.getCurrentJournalPosition(self.mount)
initial_pos = self.client.getCurrentJournalPosition(self.mount_path_bytes)
self.assertEqual(INITIAL_SEQ, initial_pos.sequenceNumber)
os.unlink(os.path.join(self.mount, "adir", "file"))
changed = self.client.getFilesChangedSince(self.mount, initial_pos)
changed = self.client.getFilesChangedSince(self.mount_path_bytes, initial_pos)
self.assertEqual(set(), set(changed.createdPaths))
self.assertEqual(set(), set(changed.changedPaths))
self.assertEqual({"adir/file"}, set(changed.removedPaths))
self.assertEqual({b"adir/file"}, set(changed.removedPaths))
def test_renameFile(self) -> None:
initial_pos = self.client.getCurrentJournalPosition(self.mount)
initial_pos = self.client.getCurrentJournalPosition(self.mount_path_bytes)
self.assertEqual(INITIAL_SEQ, initial_pos.sequenceNumber)
os.rename(os.path.join(self.mount, "hello"), os.path.join(self.mount, "bye"))
changed = self.client.getFilesChangedSince(self.mount, initial_pos)
self.assertEqual({"bye"}, set(changed.createdPaths))
changed = self.client.getFilesChangedSince(self.mount_path_bytes, initial_pos)
self.assertEqual({b"bye"}, set(changed.createdPaths))
self.assertEqual(set(), set(changed.changedPaths))
self.assertEqual({"hello"}, set(changed.removedPaths))
self.assertEqual({b"hello"}, set(changed.removedPaths))
def test_addFile(self) -> None:
initial_pos = self.client.getCurrentJournalPosition(self.mount)
initial_pos = self.client.getCurrentJournalPosition(self.mount_path_bytes)
self.assertEqual(INITIAL_SEQ, initial_pos.sequenceNumber)
name = os.path.join(self.mount, "overlaid")
with open(name, "w+") as f:
pos = self.client.getCurrentJournalPosition(self.mount)
pos = self.client.getCurrentJournalPosition(self.mount_path_bytes)
self.assertEqual(
INITIAL_SEQ + 1,
pos.sequenceNumber,
msg="creating a file bumps the journal",
)
changed = self.client.getFilesChangedSince(self.mount, initial_pos)
self.assertEqual({"overlaid"}, set(changed.createdPaths))
changed = self.client.getFilesChangedSince(
self.mount_path_bytes, initial_pos
)
self.assertEqual({b"overlaid"}, set(changed.createdPaths))
self.assertEqual(set(), set(changed.changedPaths))
self.assertEqual(set(), set(changed.removedPaths))
self.assertEqual(
@ -147,14 +161,16 @@ class MaterializedQueryTest(testcase.EdenRepoTest):
f.write("NAME!\n")
pos_after_overlaid = self.client.getCurrentJournalPosition(self.mount)
pos_after_overlaid = self.client.getCurrentJournalPosition(
self.mount_path_bytes
)
self.assertEqual(
INITIAL_SEQ + 2,
pos_after_overlaid.sequenceNumber,
msg="writing bumps the journal",
)
changed = self.client.getFilesChangedSince(self.mount, initial_pos)
self.assertEqual({"overlaid"}, set(changed.createdPaths))
changed = self.client.getFilesChangedSince(self.mount_path_bytes, initial_pos)
self.assertEqual({b"overlaid"}, set(changed.createdPaths))
self.assertEqual(set(), set(changed.changedPaths))
self.assertEqual(set(), set(changed.removedPaths))
self.assertEqual(
@ -165,7 +181,7 @@ class MaterializedQueryTest(testcase.EdenRepoTest):
name = os.path.join(self.mount, "adir", "file")
with open(name, "a") as f:
pos = self.client.getCurrentJournalPosition(self.mount)
pos = self.client.getCurrentJournalPosition(self.mount_path_bytes)
self.assertEqual(
INITIAL_SEQ + 2,
pos.sequenceNumber,
@ -173,13 +189,15 @@ class MaterializedQueryTest(testcase.EdenRepoTest):
)
f.write("more stuff on the end\n")
pos = self.client.getCurrentJournalPosition(self.mount)
pos = self.client.getCurrentJournalPosition(self.mount_path_bytes)
self.assertEqual(
INITIAL_SEQ + 3, pos.sequenceNumber, msg="appending bumps the journal"
)
changed = self.client.getFilesChangedSince(self.mount, pos_after_overlaid)
self.assertEqual({"adir/file"}, set(changed.changedPaths))
changed = self.client.getFilesChangedSince(
self.mount_path_bytes, pos_after_overlaid
)
self.assertEqual({b"adir/file"}, set(changed.changedPaths))
self.assertEqual(set(), set(changed.createdPaths))
self.assertEqual(set(), set(changed.removedPaths))
self.assertEqual(

34
eden/integration/thrift_test.py
View File

@ -38,7 +38,7 @@ class ThriftTest(testcase.EdenRepoTest):
self.addCleanup(self.client.close)
def get_loaded_inodes_count(self, path: str) -> int:
result = self.client.debugInodeStatus(self.mount, path)
result = self.client.debugInodeStatus(self.mount_path_bytes, os.fsencode(path))
inode_count = 0
for item in result:
assert item.entries is not None
@ -52,11 +52,11 @@ class ThriftTest(testcase.EdenRepoTest):
self.assertEqual(1, len(mounts))
mount = mounts[0]
self.assertEqual(self.mount, mount.mountPoint)
self.assertEqual(self.mount_path_bytes, mount.mountPoint)
assert mount.edenClientPath is not None
# The client path should always be inside the main eden directory
self.assertTrue(
mount.edenClientPath.startswith(self.eden.eden_dir),
os.fsdecode(mount.edenClientPath).startswith(self.eden.eden_dir),
msg="unexpected client path: %r" % mount.edenClientPath,
)
@ -69,27 +69,27 @@ class ThriftTest(testcase.EdenRepoTest):
self.assertEqual(
[result_for_hello, result_for_adir_file],
self.client.getSHA1(self.mount, ["hello", "adir/file"]),
self.client.getSHA1(self.mount_path_bytes, [b"hello", b"adir/file"]),
)
def test_get_sha1_throws_for_empty_string(self) -> None:
results = self.client.getSHA1(self.mount, [""])
results = self.client.getSHA1(self.mount_path_bytes, [b""])
self.assertEqual(1, len(results))
self.assert_error(results[0], "path cannot be the empty string")
def test_get_sha1_throws_for_directory(self) -> None:
results = self.client.getSHA1(self.mount, ["adir"])
results = self.client.getSHA1(self.mount_path_bytes, [b"adir"])
self.assertEqual(1, len(results))
self.assert_error(results[0], "adir: Is a directory")
def test_get_sha1_throws_for_non_existent_file(self) -> None:
results = self.client.getSHA1(self.mount, ["i_do_not_exist"])
results = self.client.getSHA1(self.mount_path_bytes, [b"i_do_not_exist"])
self.assertEqual(1, len(results))
self.assert_error(results[0], "i_do_not_exist: No such file or directory")
def test_get_sha1_throws_for_symlink(self) -> None:
"""Fails because caller should resolve the symlink themselves."""
results = self.client.getSHA1(self.mount, ["slink"])
results = self.client.getSHA1(self.mount_path_bytes, [b"slink"])
self.assertEqual(1, len(results))
self.assert_error(results[0], "slink: file is a symlink: Invalid argument")
@ -114,7 +114,7 @@ class ThriftTest(testcase.EdenRepoTest):
age = TimeSpec()
age.seconds = 0
age.nanoSeconds = 0
unload_count = self.client.unloadInodeForPath(self.mount, "", age)
unload_count = self.client.unloadInodeForPath(self.mount_path_bytes, b"", age)
self.assertGreaterEqual(
unload_count, 100, "Number of loaded inodes should reduce after unload"
@ -142,7 +142,7 @@ class ThriftTest(testcase.EdenRepoTest):
self.read_file(filename)
reads_2read = self.get_counter("fuse.read_us.count")
self.assertEqual(reads_1read, reads_2read)
self.client.invalidateKernelInodeCache(self.mount, "bdir/file")
self.client.invalidateKernelInodeCache(self.mount_path_bytes, b"bdir/file")
self.read_file(filename)
reads_3read = self.get_counter("fuse.read_us.count")
self.assertEqual(reads_2read + 1, reads_3read)
@ -153,7 +153,7 @@ class ThriftTest(testcase.EdenRepoTest):
lookups_1ls = self.get_counter("fuse.lookup_us.count")
# equal, the file was lookup'ed above.
self.assertEqual(lookups, lookups_1ls)
self.client.invalidateKernelInodeCache(self.mount, "bdir")
self.client.invalidateKernelInodeCache(self.mount_path_bytes, b"bdir")
os.system("ls -hl " + full_dirname + " > /dev/null")
lookups_2ls = self.get_counter("fuse.lookup_us.count")
self.assertEqual(lookups_1ls + 1, lookups_2ls)
@ -171,9 +171,9 @@ class ThriftTest(testcase.EdenRepoTest):
self.assertDictEqual(
diff.entries,
{
"cdir/subdir/new.txt": ScmFileStatus.ADDED,
"bdir/file": ScmFileStatus.MODIFIED,
"README": ScmFileStatus.REMOVED,
b"cdir/subdir/new.txt": ScmFileStatus.ADDED,
b"bdir/file": ScmFileStatus.MODIFIED,
b"README": ScmFileStatus.REMOVED,
},
)
@ -189,8 +189,8 @@ class ThriftTest(testcase.EdenRepoTest):
self.assertDictEqual(
diff.entries,
{
"cdir/subdir/new.txt": ScmFileStatus.ADDED,
"bdir/file": ScmFileStatus.MODIFIED,
"README": ScmFileStatus.REMOVED,
b"cdir/subdir/new.txt": ScmFileStatus.ADDED,
b"bdir/file": ScmFileStatus.MODIFIED,
b"README": ScmFileStatus.REMOVED,
},
)