mirror of
https://github.com/facebook/sapling.git
synced 2024-10-06 23:07:18 +03:00
initial start for an fsck command
Summary: This adds initial code for an `eden debug fsck` command. At the moment this simply scans the overlay and reports errors, but does not correct any issues. In subsequent diffs I'll update it to support correcting some problems. There are also several other issues that I'll improve in subsequent diffs as well. At the moment this code uses the same functions as other CLI commands to find the client directory, and these only work if the checkout is already mounted. For `fsck` we generally want the checkout to be unmounted before checking it. I also plan to clean up some of the output handling as well. Reviewed By: strager Differential Revision: D8813714 fbshipit-source-id: 90238e94540db1cc06c17eabf8ce3a1bb808d4b3
This commit is contained in:
Adam Simpkins
Facebook Github Bot
parent
d2a15f5aff
commit
5fd3e0e6b9
356
eden/cli/fsck.py
Normal file
356
eden/cli/fsck.py
Normal file
@ -0,0 +1,356 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright (c) 2004-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
import binascii
|
||||
import contextlib
|
||||
import enum
|
||||
import os
|
||||
import stat
|
||||
import time
|
||||
import types
|
||||
from typing import ContextManager, Dict, List, NamedTuple, Optional, Tuple, Type
|
||||
|
||||
from . import overlay as overlay_mod
|
||||
|
||||
|
||||
class InodeType(enum.Enum):
|
||||
FILE = enum.auto()
|
||||
DIR = enum.auto()
|
||||
ERROR = enum.auto()
|
||||
DIR_ERROR = enum.auto()
|
||||
|
||||
|
||||
class ChildInfo(NamedTuple):
|
||||
inode_number: int
|
||||
name: str
|
||||
mode: int
|
||||
hash: Optional[bytes]
|
||||
|
||||
|
||||
class InodeInfo:
|
||||
__slots__ = ["inode_number", "type", "parents", "children", "mtime", "error"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
inode_number: int,
|
||||
type: InodeType,
|
||||
children: List[ChildInfo],
|
||||
mtime: Optional[float],
|
||||
error: Optional[Exception],
|
||||
) -> None:
|
||||
self.inode_number = inode_number
|
||||
self.type = type
|
||||
|
||||
# The mtime is the modification time on the overlay file itself, not the
|
||||
# value for the logical file represented by this overlay entry.
|
||||
# This is mainly present for helping identify when a problem was introduced in
|
||||
# the overlay.
|
||||
self.mtime = mtime
|
||||
|
||||
self.error = error
|
||||
|
||||
# The other inode(s) that list this as a child
|
||||
self.parents: List[Tuple[InodeInfo, ChildInfo]] = []
|
||||
self.children = children
|
||||
|
||||
def compute_path(self) -> str:
|
||||
if not self.parents:
|
||||
if self.inode_number == overlay_mod.Overlay.ROOT_INODE_NUMBER:
|
||||
return "/"
|
||||
return "[unlinked]"
|
||||
|
||||
parent, child_entry = self.parents[0]
|
||||
if parent.inode_number == overlay_mod.Overlay.ROOT_INODE_NUMBER:
|
||||
return child_entry.name
|
||||
return parent.compute_path() + os.path.sep + child_entry.name
|
||||
|
||||
|
||||
class ErrorLevel(enum.IntEnum):
|
||||
# WARNING is for issues that do not affect our ability to show file contents
|
||||
# correctly.
|
||||
WARNING = 1
|
||||
|
||||
# ERROR issues are problems that prevent us from being able to read file or
|
||||
# directory contents.
|
||||
ERROR = 2
|
||||
|
||||
@staticmethod
|
||||
def get_label(level: int) -> str:
|
||||
if level == ErrorLevel.WARNING:
|
||||
return "warning"
|
||||
return "error"
|
||||
|
||||
|
||||
class Error:
|
||||
def __init__(self, level: ErrorLevel) -> None:
|
||||
self.level = level
|
||||
|
||||
def detailed_description(self) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
class UnexpectedOverlayFile(Error):
|
||||
def __init__(self, path: str) -> None:
|
||||
super().__init__(ErrorLevel.WARNING)
|
||||
self.path = path
|
||||
self.mtime = None
|
||||
with contextlib.suppress(OSError):
|
||||
self.mtime = os.lstat(path).st_mtime
|
||||
|
||||
def __str__(self) -> str:
|
||||
mtime_str = _get_mtime_str(self.mtime)
|
||||
return f"unexpected file present in overlay: {self.path}{mtime_str}"
|
||||
|
||||
|
||||
class MissingMaterializedInode(Error):
|
||||
def __init__(self, inode: InodeInfo, child: ChildInfo) -> None:
|
||||
super().__init__(ErrorLevel.ERROR)
|
||||
self.inode = inode
|
||||
self.child = child
|
||||
|
||||
def __str__(self) -> str:
|
||||
return (
|
||||
f"missing overlay file for materialized inode "
|
||||
f"{self.inode.compute_path()}/{self.child.name} with "
|
||||
f"file mode {self.child.mode:#o}"
|
||||
)
|
||||
|
||||
|
||||
class InvalidMaterializedInode(Error):
|
||||
def __init__(self, inode: InodeInfo) -> None:
|
||||
super().__init__(ErrorLevel.ERROR)
|
||||
self.inode = inode
|
||||
self.expected_type = self._compute_expected_type()
|
||||
|
||||
def _compute_expected_type(self) -> Optional[InodeType]:
|
||||
# Look at the parents to see if this looks like it should be a file or directory
|
||||
if self.inode.parents:
|
||||
_parent_inode, child_entry = self.inode.parents[0]
|
||||
if stat.S_ISDIR(child_entry.mode):
|
||||
self.expected_type = InodeType.DIR
|
||||
else:
|
||||
self.expected_type = InodeType.FILE
|
||||
elif self.inode.type == InodeType.DIR_ERROR:
|
||||
self.expected_type = InodeType.DIR
|
||||
return None
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self.expected_type is None:
|
||||
type_str = "inode"
|
||||
elif self.expected_type == InodeType.DIR:
|
||||
type_str = "directory inode"
|
||||
else:
|
||||
type_str = "file inode"
|
||||
|
||||
mtime_str = _get_mtime_str(self.inode.mtime)
|
||||
return (
|
||||
f"invalid overlay file for materialized {type_str} "
|
||||
f"{self.inode.compute_path()}{mtime_str}: {self.inode.error}"
|
||||
)
|
||||
|
||||
|
||||
class OrphanInode(Error):
|
||||
def __init__(self, inode: InodeInfo) -> None:
|
||||
super().__init__(ErrorLevel.WARNING)
|
||||
self.inode = inode
|
||||
|
||||
def __str__(self) -> str:
|
||||
if self.inode.type == InodeType.DIR:
|
||||
type_str = "directory inode"
|
||||
elif self.inode.type == InodeType.FILE:
|
||||
type_str = "file inode"
|
||||
else:
|
||||
type_str = f"{self.inode.type} inode"
|
||||
mtime_str = _get_mtime_str(self.inode.mtime)
|
||||
return f"found orphan {type_str} {self.inode.inode_number}{mtime_str}"
|
||||
|
||||
def detailed_description(self) -> Optional[str]:
|
||||
entries = []
|
||||
if self.inode.type == InodeType.DIR:
|
||||
for child in self.inode.children:
|
||||
mode_str = f"{child.mode:#o}"
|
||||
hash_str = binascii.hexlify(child.hash) if child.hash else "-"
|
||||
entries.append(
|
||||
f"{child.name:<30} {mode_str} {child.inode_number} {hash_str}"
|
||||
)
|
||||
return "\n".join(entries)
|
||||
|
||||
|
||||
class HardLinkedInode(Error):
|
||||
def __init__(self, inode: InodeInfo) -> None:
|
||||
super().__init__(ErrorLevel.WARNING)
|
||||
self.inode = inode
|
||||
|
||||
|
||||
class FilesystemChecker:
|
||||
def __init__(self, overlay: overlay_mod.Overlay, verbose: bool = False) -> None:
|
||||
self.overlay = overlay
|
||||
self.errors: List[Error] = []
|
||||
self.verbose = verbose
|
||||
self._overlay_locked: Optional[bool] = None
|
||||
self._overlay_lock: Optional[ContextManager[bool]] = None
|
||||
|
||||
def __enter__(self) -> "FilesystemChecker":
|
||||
self._overlay_lock = self.overlay.try_lock()
|
||||
self._overlay_locked = self._overlay_lock.__enter__()
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_type: Optional[Type[BaseException]],
|
||||
exc_value: Optional[BaseException],
|
||||
exc_traceback: Optional[types.TracebackType],
|
||||
) -> Optional[bool]:
|
||||
assert self._overlay_lock is not None
|
||||
return self._overlay_lock.__exit__(exc_type, exc_value, exc_traceback)
|
||||
|
||||
def _add_error(self, error: Error) -> None:
|
||||
print(f"{ErrorLevel.get_label(error.level)}: {error}")
|
||||
if self.verbose:
|
||||
details = error.detailed_description()
|
||||
if details:
|
||||
print(" " + "\n ".join(details.splitlines()))
|
||||
self.errors.append(error)
|
||||
|
||||
def scan_for_errors(self) -> None:
|
||||
print(f"Checking overlay storage at {self.overlay.path}.")
|
||||
print("This may take some time...")
|
||||
|
||||
if not self._overlay_locked:
|
||||
print("Warning: unable to lock the overlay directory")
|
||||
print(
|
||||
"Will only scan for issues. In order to fix problems you must "
|
||||
"first unmount the checkout."
|
||||
)
|
||||
|
||||
print("Reading materialized inodes...")
|
||||
inodes = self._read_inodes()
|
||||
|
||||
print(f"Found {len(inodes)} materialized inodes")
|
||||
print(f"Computing directory relationships...")
|
||||
self._link_inode_children(inodes)
|
||||
|
||||
print(f"Scanning for inconsistencies...")
|
||||
self._scan_inodes_for_errors(inodes)
|
||||
|
||||
# TODO: Check that the stored max inode number is valid
|
||||
|
||||
def _read_inodes(self) -> Dict[int, InodeInfo]:
|
||||
inodes: Dict[int, InodeInfo] = {}
|
||||
|
||||
for subdir_num in range(256):
|
||||
dir_name = "{:02x}".format(subdir_num)
|
||||
dir_path = os.path.join(self.overlay.path, dir_name)
|
||||
|
||||
# TODO: Handle the error if os.listdir() fails
|
||||
for entry in os.listdir(dir_path):
|
||||
try:
|
||||
inode_number = int(entry, 10)
|
||||
except ValueError as ex:
|
||||
entry_path = os.path.join(dir_path, entry)
|
||||
self._add_error(UnexpectedOverlayFile(entry_path))
|
||||
continue
|
||||
|
||||
# TODO: check if inode_number is actually in the correct subdirectory.
|
||||
# Handle the error if it is in the wrong directory, and if we found
|
||||
# multiple files with the same inode number in different subdirectories
|
||||
|
||||
inode_info = self._load_inode_info(inode_number)
|
||||
inodes[inode_number] = inode_info
|
||||
|
||||
return inodes
|
||||
|
||||
def _link_inode_children(self, inodes: Dict[int, InodeInfo]) -> None:
|
||||
for inode in inodes.values():
|
||||
for child_info in inode.children:
|
||||
if child_info.inode_number == 0:
|
||||
# Older versions of edenfs would leave the inode number set to 0
|
||||
# if the child inode has never been loaded. The child can't be
|
||||
# present in the overlay if it doesn't have an inode number
|
||||
# allocated for it yet.
|
||||
#
|
||||
# Newer versions of edenfs always allocate an inode number for all
|
||||
# children, even if they haven't been loaded yet.
|
||||
continue
|
||||
child_inode = inodes.get(child_info.inode_number, None)
|
||||
if child_inode is None:
|
||||
if child_info.hash is None:
|
||||
# This child is materialized (since it doesn't have a hash
|
||||
# linking it to a source control object). It's a problem if the
|
||||
# materialized data isn't actually present in the overlay.
|
||||
self._add_error(MissingMaterializedInode(inode, child_info))
|
||||
else:
|
||||
child_inode.parents.append((inode, child_info))
|
||||
|
||||
def _scan_inodes_for_errors(self, inodes: Dict[int, InodeInfo]) -> None:
|
||||
for inode in inodes.values():
|
||||
if inode.type in (InodeType.ERROR, InodeType.DIR_ERROR):
|
||||
self._add_error(InvalidMaterializedInode(inode))
|
||||
|
||||
num_parents = len(inode.parents)
|
||||
if (
|
||||
num_parents == 0
|
||||
and inode.inode_number != overlay_mod.Overlay.ROOT_INODE_NUMBER
|
||||
):
|
||||
self._add_error(OrphanInode(inode))
|
||||
elif num_parents > 1:
|
||||
self._add_error(HardLinkedInode(inode))
|
||||
|
||||
def _load_inode_info(self, inode_number: int) -> InodeInfo:
|
||||
dir_data = None
|
||||
stat_info = None
|
||||
error = None
|
||||
try:
|
||||
with self.overlay.open_overlay_file(inode_number) as f:
|
||||
stat_info = os.fstat(f.fileno())
|
||||
header = self.overlay.read_header(f)
|
||||
if header.type == overlay_mod.OverlayHeader.TYPE_DIR:
|
||||
dir_data = f.read()
|
||||
type = InodeType.DIR
|
||||
elif header.type == overlay_mod.OverlayHeader.TYPE_FILE:
|
||||
type = InodeType.FILE
|
||||
else:
|
||||
type = InodeType.ERROR
|
||||
except Exception as ex:
|
||||
# If anything goes wrong trying to open or parse the overlay file
|
||||
# report this as an error, regardless of what type of error it is.
|
||||
type = InodeType.ERROR
|
||||
error = ex
|
||||
|
||||
dir_entries = None
|
||||
children: List[ChildInfo] = []
|
||||
if dir_data is not None:
|
||||
try:
|
||||
parsed_data = self.overlay.parse_dir_inode_data(dir_data)
|
||||
dir_entries = parsed_data.entries
|
||||
except Exception as ex:
|
||||
type = InodeType.DIR_ERROR
|
||||
error = ex
|
||||
|
||||
if dir_entries is not None:
|
||||
for name, entry in dir_entries.items():
|
||||
children.append(
|
||||
ChildInfo(
|
||||
inode_number=entry.inodeNumber or 0,
|
||||
name=name,
|
||||
mode=entry.mode,
|
||||
hash=entry.hash,
|
||||
)
|
||||
)
|
||||
|
||||
mtime = None
|
||||
if stat_info is not None:
|
||||
mtime = stat_info.st_mtime
|
||||
return InodeInfo(inode_number, type, children, mtime, error)
|
||||
|
||||
|
||||
def _get_mtime_str(mtime: Optional[float]) -> str:
|
||||
if mtime is None:
|
||||
return ""
|
||||
return f", with mtime {time.ctime(mtime)}"
|
||||
@ -32,7 +32,9 @@ from . import (
|
||||
debug as debug_mod,
|
||||
doctor as doctor_mod,
|
||||
filesystem,
|
||||
fsck as fsck_mod,
|
||||
mtab,
|
||||
overlay as overlay_mod,
|
||||
rage as rage_mod,
|
||||
stats as stats_mod,
|
||||
subcmd as subcmd_mod,
|
||||
@ -419,6 +421,54 @@ class DoctorCmd(Subcmd):
|
||||
)
|
||||
|
||||
|
||||
@subcmd("fsck", "Perform a filesystem check for Eden")
|
||||
class FsckCmd(Subcmd):
|
||||
def setup_parser(self, parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print more verbose information about issues found.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"path",
|
||||
nargs="?",
|
||||
help="The path to an Eden mount point. Uses `pwd` by default.",
|
||||
)
|
||||
|
||||
def run(self, args: argparse.Namespace) -> int:
|
||||
config = create_config(args)
|
||||
mount, rel_path = debug_mod.get_mount_path(args.path or os.getcwd())
|
||||
|
||||
# Get the path to the overlay directory for this mount point
|
||||
client_dir = config._get_client_dir_for_mount_point(mount)
|
||||
self._overlay_dir = os.path.join(client_dir, "local")
|
||||
self._overlay = overlay_mod.Overlay(self._overlay_dir)
|
||||
|
||||
with fsck_mod.FilesystemChecker(self._overlay, verbose=args.verbose) as checker:
|
||||
checker.scan_for_errors()
|
||||
print("\nReport summary:")
|
||||
if not checker.errors:
|
||||
print(" No issues found")
|
||||
return 0
|
||||
|
||||
num_warnings = 0
|
||||
num_errors = 0
|
||||
for error in checker.errors:
|
||||
if error.level == fsck_mod.ErrorLevel.WARNING:
|
||||
num_warnings += 1
|
||||
else:
|
||||
num_errors += 1
|
||||
|
||||
if num_warnings > 0:
|
||||
print(f" {num_warnings} warnings")
|
||||
print(f" {num_errors} errors")
|
||||
if num_errors == 0:
|
||||
return 2
|
||||
return 1
|
||||
|
||||
|
||||
@subcmd(
|
||||
"mount",
|
||||
"Remount an existing checkout (for instance, after it was manually unmounted)",
|
||||
|
||||
@ -7,11 +7,13 @@
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
import contextlib
|
||||
import errno
|
||||
import fcntl
|
||||
import os
|
||||
import stat
|
||||
import struct
|
||||
from typing import BinaryIO, Optional, Tuple
|
||||
from typing import BinaryIO, Iterator, Optional, Tuple
|
||||
|
||||
from facebook.eden.overlay.ttypes import OverlayDir
|
||||
|
||||
@ -151,6 +153,24 @@ class Overlay:
|
||||
def __init__(self, path: str) -> None:
|
||||
self.path = path
|
||||
|
||||
@contextlib.contextmanager
|
||||
def try_lock(self) -> Iterator[bool]:
|
||||
info_path = os.path.join(self.path, "info")
|
||||
try:
|
||||
lock_file = open(info_path, "rb")
|
||||
except OSError:
|
||||
yield False
|
||||
return
|
||||
|
||||
try:
|
||||
fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
yield True
|
||||
except OSError:
|
||||
yield False
|
||||
finally:
|
||||
# Release the lock once the yield returns
|
||||
lock_file.close()
|
||||
|
||||
def get_path(self, inode_number: int) -> str:
|
||||
dir_name = "{:02x}".format(inode_number % 256)
|
||||
return os.path.join(self.path, dir_name, str(inode_number))
|
||||
|
||||
69
eden/integration/fsck_test.py
Normal file
69
eden/integration/fsck_test.py
Normal file
@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# Copyright (c) 2016-present, Facebook, Inc.
|
||||
# All rights reserved.
|
||||
#
|
||||
# This source code is licensed under the BSD-style license found in the
|
||||
# LICENSE file in the root directory of this source tree. An additional grant
|
||||
# of patent rights can be found in the PATENTS file in the same directory.
|
||||
|
||||
import os
|
||||
from typing import Type
|
||||
|
||||
from .lib import repobase, testcase
|
||||
|
||||
|
||||
class FsckTest(testcase.EdenRepoTest):
|
||||
def populate_repo(self) -> None:
|
||||
self.repo.write_file("README.md", "tbd\n")
|
||||
self.repo.write_file("proj/src/main.c", "int main() { return 0; }\n")
|
||||
self.repo.write_file("proj/src/lib.c", "void foo() {}\n")
|
||||
self.repo.write_file("proj/src/include/lib.h", "#pragma once\nvoid foo();\n")
|
||||
self.repo.write_file(
|
||||
"proj/test/test.sh", "#!/bin/bash\necho test\n", mode=0o755
|
||||
)
|
||||
self.repo.write_file("doc/foo.txt", "foo\n")
|
||||
self.repo.write_file("doc/bar.txt", "bar\n")
|
||||
self.repo.symlink("proj/doc", "../doc")
|
||||
self.repo.commit("Initial commit.")
|
||||
|
||||
def create_repo(self, name: str) -> Type[repobase.Repository]:
|
||||
return self.create_hg_repo("main")
|
||||
|
||||
def setup_eden_test(self) -> None:
|
||||
super().setup_eden_test()
|
||||
self.client_dir = os.readlink(os.path.join(self.mount, ".eden", "client"))
|
||||
self.overlay_dir = os.path.join(self.client_dir, "local")
|
||||
|
||||
def _update_file(self, path: str, contents: str) -> int:
|
||||
"""
|
||||
Update a file by path and return its inode number.
|
||||
|
||||
Updating the file contents ensures it will be materialized and present in the
|
||||
overlay.
|
||||
"""
|
||||
with open(os.path.join(self.mount, path), "w") as f:
|
||||
f.write(contents)
|
||||
stat_info = os.fstat(f.fileno())
|
||||
return stat_info.st_ino
|
||||
|
||||
def _get_overlay_path(self, inode_number: int) -> str:
|
||||
subdir = "{:02x}".format(inode_number % 256)
|
||||
return os.path.join(self.overlay_dir, subdir, str(inode_number))
|
||||
|
||||
def test_fsck_no_issues(self) -> None:
|
||||
output = self.eden.run_cmd("fsck", self.mount)
|
||||
self.assertIn("No issues found", output)
|
||||
|
||||
def test_fsck_empty_overlay_file(self) -> None:
|
||||
inode_number = self._update_file("doc/foo.txt", "new contents\n")
|
||||
self.eden.run_cmd("unmount", self.mount)
|
||||
|
||||
# Truncate the file to 0 length
|
||||
with open(self._get_overlay_path(inode_number), "w"):
|
||||
pass
|
||||
|
||||
self.eden.run_cmd("mount", self.mount)
|
||||
|
||||
cmd_result = self.eden.run_unchecked("fsck", self.mount)
|
||||
self.assertEqual(cmd_result, 1)
|
||||
Loading…
Reference in New Issue
Block a user