sapling/eden/cli/debug.py
Michael Bolin 5d738193e5 Store Hg dirstate data in Hg instead of Eden.
Summary:
This is a major change to how we manage the dirstate in Eden's Hg extension.

Previously, the dirstate information was stored under `$EDEN_CONFIG_DIR`,
which is Eden's private storage. Any time the Mercurial extension wanted to
read or write the dirstate, it had to make a Thrift request to Eden to do so on
its behalf. The upside is that Eden could answer dirstate-related questions
independently of the Python code.

This was sufficiently different than how Mercurial's default dirstate worked
that our subclass, `eden_dirstate`, had to override quite a bit of behavior.
Failing to manage the `.hg/dirstate` file in a way similar to the way Mercurial
does has exposed some "unofficial contracts" that Mercurial has. For example,
tools like Nuclide rely on changes to the `.hg/dirstate` file as a heuristic to
determine when to invalidate its internal caches for Mercurial data.

Today, Mercurial has a well-factored `dirstatemap` abstraction that is primarily
responsible for the transactions with the dirstate's data. With this split, we can
focus on putting most of our customizations in our `eden_dirstate_map` subclass
while our `eden_dirstate` class has to override fewer methods. Because the
data is managed through the `.hg/dirstate` file, transaction logic in Mercurial that
relies on renaming/copying that file will work out-of-the-box. This change
also reduces the number of Thrift calls the Mercurial extension has to make
for operations like `hg status` or `hg add`.

In this revision, we introduce our own binary format for the `.hg/dirstate` file.
The logic to read and write this file is in `eden/py/dirstate.py`. After the first
40 bytes, which are used for the parent hashes, the next four bytes are
reserved for a version number for the file format so we can manage file format
changes going forward.

Admittedly one downside of this change is that it is a breaking change.
Ideally, users should commit all of their local changes in their existing mounts,
shutdown Eden, delete the old mounts, restart Eden, and re-clone.

In the end, this change deletes a number of Mercurial-specific code and Thrift
APIs from Eden. This is a better separation of concerns that makes Eden more
SCM-agnostic. For example, this change removes `Dirstate.cpp` and
`DirstatePersistance.cpp`, replacing them with the much simpler and more
general `Differ.cpp`. The Mercurial-specific logic from `Dirstate.cpp` that turned
a diff into an `hg status` now lives in the Mercurial extension in
`EdenThriftClient.getStatus()`, which is much more appropriate.

Note that this reverts the changes that were recently introduced in D6116105:
we now need to intercept `localrepo.localrepository.dirstate` once again.

Reviewed By: simpkins

Differential Revision: D6179950

fbshipit-source-id: 5b78904909b669c9cc606e2fe1fd118ef6eaab95
2017-11-06 19:56:49 -08:00

618 lines
21 KiB
Python

#!/usr/bin/env python3
#
# Copyright (c) 2004-present, Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD-style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.
import argparse
import binascii
import os
import stat
import sys
from typing import List, IO, Tuple
from facebook.eden.overlay.ttypes import OverlayDir
import eden.dirstate
from facebook.eden.ttypes import NoValueForKeyError, TimeSpec
from . import cmd_util
def get_mount_path(path: str) -> Tuple[str, str]:
'''
Given a path inside an eden mount, find the path to the eden root.
Returns a tuple of (eden_mount_path, relative_path)
where relative_path is the path such that
os.path.join(eden_mount_path, relative_path) refers to the same file as the
original input path.
'''
# TODO: This will probably be easier to do using the special .eden
# directory, once the diff adding .eden lands.
current_path = os.path.realpath(path)
rel_path = ''
while True:
# For now we simply assume that the first mount point we come across is
# the eden mount point. This doesn't handle bind mounts inside the
# eden mount, but that's fine for now.
if os.path.ismount(current_path):
rel_path = os.path.normpath(rel_path)
if rel_path == '.':
rel_path = ''
return (current_path, rel_path)
parent, basename = os.path.split(current_path)
if parent == current_path:
raise Exception('eden mount point not found')
current_path = parent
rel_path = os.path.join(basename, rel_path)
def escape_path(value: bytes) -> str:
'''
Take a binary path value, and return a printable string, with special
characters escaped.
'''
def human_readable_byte(b):
if b < 0x20 or b >= 0x7f:
return '\\x{:02x}'.format(b)
elif b == ord(b'\\'):
return '\\\\'
return chr(b)
return ''.join(human_readable_byte(b) for b in value)
def hash_str(value: bytes) -> str:
'''
Take a hash as a binary value, and return it represented as a hexadecimal
string.
'''
return binascii.hexlify(value).decode('utf-8')
def parse_object_id(value: str) -> bytes:
'''
Parse an object ID as a 40-byte hexadecimal string, and return a 20-byte
binary value.
'''
try:
binary = binascii.unhexlify(value)
if len(binary) != 20:
raise ValueError()
except ValueError:
raise ValueError('blob ID must be a 40-byte hexadecimal value')
return binary
def do_tree(args: argparse.Namespace):
config = cmd_util.create_config(args)
mount, rel_path = get_mount_path(args.mount)
tree_id = parse_object_id(args.id)
local_only = not args.load
with config.get_thrift_client() as client:
entries = client.debugGetScmTree(mount, tree_id,
localStoreOnly=local_only)
for entry in entries:
file_type_flags, perms = _parse_mode(entry.mode)
print('{} {:4o} {:40} {}'.format(
file_type_flags, perms, hash_str(entry.id),
escape_path(entry.name)))
def do_blob(args: argparse.Namespace):
config = cmd_util.create_config(args)
mount, rel_path = get_mount_path(args.mount)
blob_id = parse_object_id(args.id)
local_only = not args.load
with config.get_thrift_client() as client:
data = client.debugGetScmBlob(mount, blob_id,
localStoreOnly=local_only)
sys.stdout.buffer.write(data)
def do_blobmeta(args: argparse.Namespace):
config = cmd_util.create_config(args)
mount, rel_path = get_mount_path(args.mount)
blob_id = parse_object_id(args.id)
local_only = not args.load
with config.get_thrift_client() as client:
info = client.debugGetScmBlobMetadata(mount, blob_id,
localStoreOnly=local_only)
print('Blob ID: {}'.format(args.id))
print('Size: {}'.format(info.size))
print('SHA1: {}'.format(hash_str(info.contentsSha1)))
_FILE_TYPE_FLAGS = {
stat.S_IFREG: 'f',
stat.S_IFDIR: 'd',
stat.S_IFLNK: 'l',
}
def _parse_mode(mode: int) -> Tuple[str, int]:
'''
Take a mode value, and return a tuple of (file_type, permissions)
where file type is a one-character flag indicating if this is a file,
directory, or symbolic link.
'''
file_type_str = _FILE_TYPE_FLAGS.get(stat.S_IFMT(mode), '?')
perms = (mode & 0o7777)
return file_type_str, perms
def do_hg_copy_map_get_all(args: argparse.Namespace):
mount, _ = get_mount_path(args.path)
_parents, _dirstate_tuples, copymap = _get_dirstate_data(mount)
_print_copymap(copymap)
def _print_copymap(copy_map) -> None:
copies = [f'{item[1]} -> {item[0]}' for item in copy_map.items()]
copies.sort()
for copy in copies:
print(copy)
def do_hg_dirstate(args: argparse.Namespace) -> None:
mount, _ = get_mount_path(args.path)
_parents, dirstate_tuples, copymap = _get_dirstate_data(mount)
printer = StdoutPrinter()
entries = list(dirstate_tuples.items())
print(printer.bold('Non-normal Files (%d):' % len(entries)))
entries.sort(key=lambda entry: entry[0]) # Sort by key.
for path, dirstate_tuple in entries:
_print_hg_nonnormal_file(path, dirstate_tuple, printer)
print(printer.bold('Copymap (%d):' % len(copymap)))
_print_copymap(copymap)
def do_hg_get_dirstate_tuple(args: argparse.Namespace):
mount, rel_path = get_mount_path(args.path)
_parents, dirstate_tuples, _copymap = _get_dirstate_data(mount)
dirstate_tuple = dirstate_tuples.get(rel_path)
printer = StdoutPrinter()
if dirstate_tuple:
_print_hg_nonnormal_file(rel_path, dirstate_tuple, printer)
else:
config = cmd_util.create_config(args)
with config.get_thrift_client() as client:
try:
entry = client.getManifestEntry(mount, rel_path)
dirstate_tuple = ('n', entry.mode, 0)
_print_hg_nonnormal_file(rel_path, dirstate_tuple, printer)
except NoValueForKeyError:
print('No tuple for ' + rel_path, file=sys.stderr)
return 1
def _print_hg_nonnormal_file(
rel_path, dirstate_tuple, printer: 'StdoutPrinter'
) -> None:
status = _dirstate_char_to_name(dirstate_tuple[0])
merge_state = _dirstate_merge_state_to_name(dirstate_tuple[2])
print(
f'''\
{printer.green(rel_path)}
status = {status}
mode = {oct(dirstate_tuple[1])}
mergeState = {merge_state}\
'''
)
def _dirstate_char_to_name(state: str) -> str:
if state == 'n':
return 'Normal'
elif state == 'm':
return 'NeedsMerging'
elif state == 'r':
return 'MarkedForRemoval'
elif state == 'a':
return 'MarkedForAddition'
elif state == '?':
return 'NotTracked'
else:
raise Exception(f'Unrecognized dirstate char: {state}')
def _dirstate_merge_state_to_name(merge_state: int) -> str:
if merge_state == 0:
return 'NotApplicable'
elif merge_state == -1:
return 'BothParents'
elif merge_state == -2:
return 'OtherParent'
else:
raise Exception(f'Unrecognized merge_state value: {merge_state}')
def _get_dirstate_data(mount):
'''Returns a tuple of (parents, dirstate_tuples, copymap).
On error, returns None.
'''
filename = os.path.join(mount, '.hg', 'dirstate')
with open(filename, 'rb') as f:
return eden.dirstate.read(f, filename)
def do_inode(args: argparse.Namespace, out: IO[bytes] = None):
if out is None:
out = sys.stdout.buffer
config = cmd_util.create_config(args)
mount, rel_path = get_mount_path(args.path)
with config.get_thrift_client() as client:
results = client.debugInodeStatus(mount, rel_path)
out.write(b'%d loaded TreeInodes\n' % len(results))
for inode_info in results:
_print_inode_info(inode_info, out)
def _print_inode_info(inode_info, out: IO[bytes]):
out.write(inode_info.path + b'\n')
out.write(b' Inode number: %d\n' % inode_info.inodeNumber)
out.write(b' Ref count: %d\n' % inode_info.refcount)
out.write(b' Materialized?: %s\n' % str(inode_info.materialized).encode())
out.write(b' Object ID: %s\n' % hash_str(inode_info.treeHash).encode())
out.write(b' Entries (%d total):\n' % len(inode_info.entries))
for entry in inode_info.entries:
if entry.loaded:
loaded_flag = 'L'
else:
loaded_flag = '-'
file_type_str, perms = _parse_mode(entry.mode)
line = ' {:9} {} {:4o} {} {:40} {}\n'.format(
entry.inodeNumber, file_type_str, perms, loaded_flag,
hash_str(entry.hash), escape_path(entry.name))
out.write(line.encode())
def _load_overlay_tree(overlay_dir: str, inode_number: int) -> OverlayDir:
from thrift.util import Serializer
from thrift.protocol import TCompactProtocol
dir_name = '{:02x}'.format(inode_number % 256)
overlay_file_path = os.path.join(overlay_dir, dir_name, str(inode_number))
with open(overlay_file_path, 'rb') as f:
data = f.read()
assert data[0:4] == b'OVDR'
tree_data = OverlayDir()
protocol_factory = TCompactProtocol.TCompactProtocolFactory()
Serializer.deserialize(protocol_factory, data[64:], tree_data)
return tree_data
def _print_overlay_tree(inode_number: int, path: str, tree_data: OverlayDir):
def hex(binhash):
return binascii.hexlify(binhash).decode('utf-8')
print('Inode {}: {}'.format(inode_number, path))
if not tree_data.entries:
return
name_width = max(len(name) for name in tree_data.entries)
for name, entry in tree_data.entries.items():
perms = entry.mode & 0o7777
file_type = stat.S_IFMT(entry.mode)
if file_type == stat.S_IFREG:
file_type_flag = 'f'
elif file_type == stat.S_IFDIR:
file_type_flag = 'd'
elif file_type == stat.S_IFLNK:
file_type_flag = 'l'
else:
file_type_flag = '?'
print(' {:{name_width}s} : {:12d} {} {:04o} {}'.format(
name, entry.inodeNumber, file_type_flag, perms, hex(entry.hash),
name_width=name_width))
def _find_overlay_tree_rel(
overlay_dir: str,
root: OverlayDir,
path_parts: List[str]) -> int:
desired = path_parts[0]
rest = path_parts[1:]
for name, entry in root.entries.items(): # noqa: ignore=B007
if name == desired:
break
else:
raise Exception('path does not exist')
if stat.S_IFMT(entry.mode) != stat.S_IFDIR:
raise Exception('path does not not refer to a directory')
if entry.hash:
raise Exception('path is not materialized')
if entry.inodeNumber == 0:
raise Exception('path is not loaded')
if rest:
entry_data = _load_overlay_tree(overlay_dir, entry.inodeNumber)
return _find_overlay_tree_rel(overlay_dir, entry_data, rest)
return entry.inodeNumber
def _find_overlay_tree(overlay_dir: str, path: str) -> int:
assert path
assert not os.path.isabs(path)
root = _load_overlay_tree(overlay_dir, 1)
path_parts = path.split(os.sep)
return _find_overlay_tree_rel(overlay_dir, root, path_parts)
def _display_overlay(
args: argparse.Namespace,
overlay_dir: str,
inode_number: int,
path: str,
level: int = 0):
data = _load_overlay_tree(overlay_dir, inode_number)
_print_overlay_tree(inode_number, path, data)
# If args.depth is negative, recurse forever.
# Stop if args.depth is non-negative, and level reaches the maximum
# requested recursion depth.
if args.depth >= 0 and level >= args.depth:
return
for name, entry in data.entries.items():
if entry.hash or entry.inodeNumber == 0:
# This entry is not materialized
continue
if stat.S_IFMT(entry.mode) != stat.S_IFDIR:
# Only display data for directories
continue
print()
entry_path = os.path.join(path, name)
_display_overlay(args, overlay_dir, entry.inodeNumber,
entry_path, level + 1)
def do_overlay(args: argparse.Namespace):
config = cmd_util.create_config(args)
mount, rel_path = get_mount_path(args.path or os.getcwd())
# Get the path to the overlay directory for this mount point
client_dir = config._get_client_dir_for_mount_point(mount)
overlay_dir = os.path.join(client_dir, 'local')
if args.number is not None:
_display_overlay(args, overlay_dir, args.number, '')
elif rel_path:
rel_path = os.path.normpath(rel_path)
inode_number = _find_overlay_tree(overlay_dir, rel_path)
_display_overlay(args, overlay_dir, inode_number, rel_path)
else:
_display_overlay(args, overlay_dir, 1, '/')
def do_getpath(args: argparse.Namespace):
config = cmd_util.create_config(args)
mount, _ = get_mount_path(args.path or os.getcwd())
with config.get_thrift_client() as client:
inodePathInfo = client.debugGetInodePath(mount, args.number)
print('%s %s' %
('loaded' if inodePathInfo.loaded else 'unloaded',
os.path.normpath(os.path.join(mount, inodePathInfo.path)) if
inodePathInfo.linked else 'unlinked'))
def get_loaded_inode_count(inode_info):
count = 0
for tree in inode_info:
for inode in tree.entries:
if inode.loaded:
count += 1
return count
def do_unload_inodes(args: argparse.Namespace):
config = cmd_util.create_config(args)
mount, rel_path = get_mount_path(args.path)
with config.get_thrift_client() as client:
inodeInfo_before_unload = client.debugInodeStatus(mount, rel_path)
inodeCount_before_unload = get_loaded_inode_count(
inodeInfo_before_unload)
# set the age in nanoSeconds
age = TimeSpec()
age.seconds = int(args.age)
age.nanoSeconds = int((args.age - age.seconds) * 10**9)
client.unloadInodeForPath(mount, rel_path, age)
inodeInfo_after_unload = client.debugInodeStatus(mount, rel_path)
inodeCount_after_unload = get_loaded_inode_count(inodeInfo_after_unload)
count = inodeCount_before_unload - inodeCount_after_unload
print('Unloaded %s Inodes under the directory : %s' %
(count, args.path))
def do_flush_cache(args: argparse.Namespace):
config = cmd_util.create_config(args)
mount, rel_path = get_mount_path(args.path)
with config.get_thrift_client() as client:
client.invalidateKernelInodeCache(mount, rel_path)
def do_set_log_level(args: argparse.Namespace):
config = cmd_util.create_config(args)
with config.get_thrift_client() as client:
result = client.debugSetLogLevel(args.category, args.level)
if result.categoryCreated:
print("Warning: New category '{}' created. Did you mistype?".format(
args.category))
class StdoutPrinter:
def __init__(self):
if sys.stdout.isatty():
import curses
curses.setupterm()
self._bold = (curses.tigetstr('bold') or b'').decode()
set_foreground = curses.tigetstr('setaf') or b''
self._red = curses.tparm(set_foreground, curses.COLOR_RED).decode()
self._green = curses.tparm(set_foreground, curses.COLOR_GREEN).decode()
self._reset = (curses.tigetstr('sgr0') or b'').decode()
else:
self._bold = ''
self._red = ''
self._green = ''
self._reset = ''
def bold(self, text: str) -> str:
return self._bold + text + self._reset
def green(self, text: str) -> str:
return self._green + text + self._reset
def red(self, text: str) -> str:
return self._red + text + self._reset
def setup_argparse(parser: argparse.ArgumentParser):
subparsers = parser.add_subparsers(dest='subparser_name')
parser = subparsers.add_parser(
'tree', help='Show eden\'s data for a source control tree')
parser.add_argument('-L', '--load',
action='store_true', default=False,
help='Load data from the backing store if necessary')
parser.add_argument('mount', help='The eden mount point path.')
parser.add_argument('id', help='The tree ID')
parser.set_defaults(func=do_tree)
parser = subparsers.add_parser(
'blob', help='Show eden\'s data for a source control blob')
parser.add_argument('-L', '--load',
action='store_true', default=False,
help='Load data from the backing store if necessary')
parser.add_argument('mount', help='The eden mount point path.')
parser.add_argument('id', help='The blob ID')
parser.set_defaults(func=do_blob)
parser = subparsers.add_parser(
'blobmeta',
help='Show eden\'s metadata about a source control blob')
parser.add_argument('-L', '--load',
action='store_true', default=False,
help='Load data from the backing store if necessary')
parser.add_argument('mount', help='The eden mount point path.')
parser.add_argument('id', help='The blob ID')
parser.set_defaults(func=do_blobmeta)
parser = subparsers.add_parser(
'hg_copy_map_get_all', help='Copymap for dirstate')
parser.add_argument(
'path', nargs='?', default=os.getcwd(),
help='The path to an Eden mount point. Uses `pwd` by default.')
parser.set_defaults(func=do_hg_copy_map_get_all)
parser = subparsers.add_parser(
'hg_dirstate', help='Print full dirstate')
parser.add_argument(
'path', nargs='?', default=os.getcwd(),
help='The path to an Eden mount point. Uses `pwd` by default.')
parser.set_defaults(func=do_hg_dirstate)
parser = subparsers.add_parser(
'hg_get_dirstate_tuple', help='Dirstate status for file')
parser.add_argument(
'path',
help='The path to the file whose status should be queried.')
parser.set_defaults(func=do_hg_get_dirstate_tuple)
parser = subparsers.add_parser(
'inode', help='Show data about loaded inodes')
parser.add_argument(
'path',
help='The path to the eden mount point. If a subdirectory inside '
'a mount point is specified, only data about inodes under the '
'specified subdirectory will be reported.')
parser.set_defaults(func=do_inode)
parser = subparsers.add_parser(
'overlay', help='Show data about the overlay')
parser.add_argument(
'-n', '--number',
type=int,
help='Display information for the specified inode number.')
parser.add_argument(
'-d', '--depth',
type=int, default=0,
help='Recurse to the specified depth.')
parser.add_argument(
'-r', '--recurse',
action='store_const', const=-1, dest='depth', default=0,
help='Recursively print child entries.')
parser.add_argument(
'path', nargs='?',
help='The path to the eden mount point.')
parser.set_defaults(func=do_overlay)
parser = subparsers.add_parser(
'getpath', help='Get the eden path that corresponds to an inode number')
parser.add_argument(
'path', nargs='?',
help='The path to an Eden mount point. Uses `pwd` by default.')
parser.add_argument(
'number',
type=int,
help='Display information for the specified inode number.')
parser.set_defaults(func=do_getpath)
parser = subparsers.add_parser(
'unload', help='Unload unused inodes')
parser.add_argument(
'path',
help='The path to the eden mount point. If a subdirectory inside '
'a mount point is specified, only inodes under the '
'specified subdirectory will be unloaded.')
parser.add_argument(
'age',
type=float,
help='Minimum age of the inodes to be unloaded in seconds'
)
parser.set_defaults(func=do_unload_inodes)
parser = subparsers.add_parser(
'flush_cache', help='Flush kernel cache for inode')
parser.add_argument(
'path',
help='Path to a directory/file inside an eden mount.')
parser.set_defaults(func=do_flush_cache)
parser = subparsers.add_parser(
'set_log_level',
help='Set the log level for a given category in the edenfs daemon.')
parser.add_argument(
'category',
type=str,
help='Period-separated log category.')
parser.add_argument(
'level',
type=str,
help='Log level string as understood by stringToLogLevel.'
)
parser.set_defaults(func=do_set_log_level)