2016-06-27 21:58:55 +03:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
#
|
2017-01-21 09:02:33 +03:00
|
|
|
# Copyright (c) 2016-present, Facebook, Inc.
|
2016-05-12 23:43:17 +03:00
|
|
|
# All rights reserved.
|
|
|
|
#
|
|
|
|
# This source code is licensed under the BSD-style license found in the
|
|
|
|
# LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
# of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
|
2017-08-31 01:07:46 +03:00
|
|
|
import binascii
|
2016-05-12 23:43:17 +03:00
|
|
|
import collections
|
2016-07-07 02:14:18 +03:00
|
|
|
import configparser
|
2016-07-12 04:27:29 +03:00
|
|
|
import errno
|
2016-07-23 03:31:17 +03:00
|
|
|
import fcntl
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
import itertools
|
2016-05-12 23:43:17 +03:00
|
|
|
import json
|
|
|
|
import os
|
2016-07-07 02:14:30 +03:00
|
|
|
import shutil
|
2016-05-12 23:43:17 +03:00
|
|
|
import stat
|
|
|
|
import subprocess
|
2016-07-23 03:31:17 +03:00
|
|
|
import tempfile
|
2016-06-18 01:11:04 +03:00
|
|
|
import time
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
import toml
|
2016-06-18 01:11:04 +03:00
|
|
|
|
2017-11-01 03:04:30 +03:00
|
|
|
from . import configinterpolator, util
|
|
|
|
from .util import print_stderr
|
2016-06-18 01:11:04 +03:00
|
|
|
import eden.thrift
|
2016-05-12 23:43:17 +03:00
|
|
|
import facebook.eden.ttypes as eden_ttypes
|
2016-06-18 01:11:04 +03:00
|
|
|
from fb303.ttypes import fb_status
|
|
|
|
import thrift
|
Write the PID to the lockfile and update `eden health` to use it.
Summary:
We have encountered cases where `eden health` reported
`"edenfs not healthy: edenfs not running"` even though the `edenfs` process is
still running. Because the existing implementation of `eden health` bases its
health check on the output of a `getStatus()` Thrift call, it will erroneously
report `"edenfs not running"` even if Eden is running but its Thrift server is
not running. This type of false negative could occur if `edenfs` has shutdown
the Thrift server, but not the rest of the process (quite possibly, its
shutdown is blocked on calls to `umount2()`).
This is further problematic because `eden daemon` checks `eden health`
before attempting to start the daemon. If it gets a false negative, then
`eden daemon` will forge ahead, trying to launch a new instance of the daemon,
but it will fail with a nasty error like the following:
```
I1017 11:59:25.188414 3064499 main.cpp:81] Starting edenfs. UID=5256, GID=100, PID=3064499
terminate called after throwing an instance of 'std::runtime_error'
what(): another instance of Eden appears to be running for /home/mbolin/local/.eden
*** Aborted at 1508266765 (Unix time, try 'date -d 1508266765') ***
*** Signal 6 (SIGABRT) (0x1488002ec2b3) received by PID 3064499 (pthread TID 0x7fd0d3787d40) (linux TID 3064499) (maybe from PID 30644
99, UID 5256), stack trace: ***
@ 000000000290d3cd folly::symbolizer::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
@ 00007fd0d133cacf (unknown)
@ 00007fd0d093e7c8 __GI_raise
@ 00007fd0d0940590 __GI_abort
@ 00007fd0d1dfeecc __gnu_cxx::__verbose_terminate_handler()
@ 00007fd0d1dfcdc5 __cxxabiv1::__terminate(void (*)())
@ 00007fd0d1dfce10 std::terminate()
@ 00007fd0d1dfd090 __cxa_throw
@ 00000000015fe8ca facebook::eden::EdenServer::acquireEdenLock()
@ 000000000160f27b facebook::eden::EdenServer::prepare()
@ 00000000016107d5 facebook::eden::EdenServer::run()
@ 000000000042c4ee main
@ 00007fd0d0929857 __libc_start_main
@ 0000000000548ad8 _start
Aborted
```
By providing more accurate information to `eden daemon`, if the user tries to
run it while the daemon is already running, they will get a more polite error
like the following:
```
error: edenfs is already running (pid 274205)
```
This revision addresses this issue by writing the PID of `edenfs` in the
lockfile. It updated the implementation of `eden health` to use the PID in the
lockfile to assess the health of Eden if the call to `getStatus()` fails. It
does this by running:
```
ps -p PID -o comm=
```
and applying some heuristics on the output to assess whether the command
associated with that process is the `edenfs` command. If it is, then
`eden health` reports the status as `STOPPED` whereas previously it would report
it as `DEAD`.
Reviewed By: wez
Differential Revision: D6086473
fbshipit-source-id: 825421a6818b56ddd7deea257a92c070c2232bdd
2017-10-18 21:18:43 +03:00
|
|
|
from typing import Optional
|
2016-05-12 23:43:17 +03:00
|
|
|
|
2017-01-24 10:52:46 +03:00
|
|
|
# Use --etcEdenDir to change the value used for a given invocation
|
|
|
|
# of the eden cli.
|
|
|
|
DEFAULT_ETC_EDEN_DIR = '/etc/eden'
|
2016-07-07 02:14:18 +03:00
|
|
|
# These are INI files that hold config data.
|
2017-01-24 10:52:46 +03:00
|
|
|
# CONFIG_DOT_D is relative to DEFAULT_ETC_EDEN_DIR, or whatever the
|
|
|
|
# effective value is for that path
|
|
|
|
CONFIG_DOT_D = 'config.d'
|
|
|
|
# USER_CONFIG is relative to the HOME dir for the user
|
2016-07-26 20:15:43 +03:00
|
|
|
USER_CONFIG = '.edenrc'
|
2016-07-07 02:14:18 +03:00
|
|
|
|
2016-05-12 23:43:17 +03:00
|
|
|
# These paths are relative to the user's client directory.
|
Write the PID to the lockfile and update `eden health` to use it.
Summary:
We have encountered cases where `eden health` reported
`"edenfs not healthy: edenfs not running"` even though the `edenfs` process is
still running. Because the existing implementation of `eden health` bases its
health check on the output of a `getStatus()` Thrift call, it will erroneously
report `"edenfs not running"` even if Eden is running but its Thrift server is
not running. This type of false negative could occur if `edenfs` has shutdown
the Thrift server, but not the rest of the process (quite possibly, its
shutdown is blocked on calls to `umount2()`).
This is further problematic because `eden daemon` checks `eden health`
before attempting to start the daemon. If it gets a false negative, then
`eden daemon` will forge ahead, trying to launch a new instance of the daemon,
but it will fail with a nasty error like the following:
```
I1017 11:59:25.188414 3064499 main.cpp:81] Starting edenfs. UID=5256, GID=100, PID=3064499
terminate called after throwing an instance of 'std::runtime_error'
what(): another instance of Eden appears to be running for /home/mbolin/local/.eden
*** Aborted at 1508266765 (Unix time, try 'date -d 1508266765') ***
*** Signal 6 (SIGABRT) (0x1488002ec2b3) received by PID 3064499 (pthread TID 0x7fd0d3787d40) (linux TID 3064499) (maybe from PID 30644
99, UID 5256), stack trace: ***
@ 000000000290d3cd folly::symbolizer::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
@ 00007fd0d133cacf (unknown)
@ 00007fd0d093e7c8 __GI_raise
@ 00007fd0d0940590 __GI_abort
@ 00007fd0d1dfeecc __gnu_cxx::__verbose_terminate_handler()
@ 00007fd0d1dfcdc5 __cxxabiv1::__terminate(void (*)())
@ 00007fd0d1dfce10 std::terminate()
@ 00007fd0d1dfd090 __cxa_throw
@ 00000000015fe8ca facebook::eden::EdenServer::acquireEdenLock()
@ 000000000160f27b facebook::eden::EdenServer::prepare()
@ 00000000016107d5 facebook::eden::EdenServer::run()
@ 000000000042c4ee main
@ 00007fd0d0929857 __libc_start_main
@ 0000000000548ad8 _start
Aborted
```
By providing more accurate information to `eden daemon`, if the user tries to
run it while the daemon is already running, they will get a more polite error
like the following:
```
error: edenfs is already running (pid 274205)
```
This revision addresses this issue by writing the PID of `edenfs` in the
lockfile. It updated the implementation of `eden health` to use the PID in the
lockfile to assess the health of Eden if the call to `getStatus()` fails. It
does this by running:
```
ps -p PID -o comm=
```
and applying some heuristics on the output to assess whether the command
associated with that process is the `edenfs` command. If it is, then
`eden health` reports the status as `STOPPED` whereas previously it would report
it as `DEAD`.
Reviewed By: wez
Differential Revision: D6086473
fbshipit-source-id: 825421a6818b56ddd7deea257a92c070c2232bdd
2017-10-18 21:18:43 +03:00
|
|
|
LOCK_FILE = 'lock'
|
2016-05-12 23:43:17 +03:00
|
|
|
CLIENTS_DIR = 'clients'
|
|
|
|
STORAGE_DIR = 'storage'
|
|
|
|
ROCKS_DB_DIR = os.path.join(STORAGE_DIR, 'rocks-db')
|
2016-07-07 02:14:30 +03:00
|
|
|
CONFIG_JSON = 'config.json'
|
2016-05-12 23:43:17 +03:00
|
|
|
|
|
|
|
# These are files in a client directory.
|
2017-11-15 01:02:00 +03:00
|
|
|
CLONE_SUCCEEDED = 'clone-succeeded'
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
MOUNT_CONFIG = 'config.toml'
|
2016-05-12 23:43:17 +03:00
|
|
|
SNAPSHOT = 'SNAPSHOT'
|
2017-08-31 01:07:46 +03:00
|
|
|
SNAPSHOT_MAGIC = b'eden\x00\x00\x00\x01'
|
2016-05-12 23:43:17 +03:00
|
|
|
|
2016-07-07 02:08:34 +03:00
|
|
|
# In our test environment, when we need to run as root, we
|
|
|
|
# may need to launch via a helper script that is whitelisted
|
|
|
|
# by the local sudo configuration.
|
|
|
|
SUDO_HELPER = '/var/www/scripts/testinfra/run_eden.sh'
|
|
|
|
|
2016-05-12 23:43:17 +03:00
|
|
|
|
2016-06-18 01:11:04 +03:00
|
|
|
class EdenStartError(Exception):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
2016-07-23 03:31:17 +03:00
|
|
|
class UsageError(Exception):
|
|
|
|
pass
|
|
|
|
|
|
|
|
|
2017-11-17 00:19:59 +03:00
|
|
|
class ClientConfig:
|
|
|
|
'''Configuration for a client. A client stores its config in config.toml
|
|
|
|
under ~/local/.eden/clients/.
|
2017-11-15 01:01:59 +03:00
|
|
|
|
|
|
|
- path real path where the true repo resides on disk
|
2017-11-17 00:19:59 +03:00
|
|
|
- scm_type "hg" or "git"
|
2017-11-15 01:02:00 +03:00
|
|
|
- hooks_path path to where the hooks scripts are for the repo
|
2017-11-15 01:01:59 +03:00
|
|
|
- bind_mounts dict where keys are private pathnames under ~/.eden where the
|
|
|
|
files are actually stored and values are the relative pathnames in the
|
|
|
|
EdenFS mount that maps to them.
|
|
|
|
'''
|
2017-11-17 00:19:57 +03:00
|
|
|
__slots__ = ['path', 'scm_type', 'hooks_path', 'bind_mounts']
|
|
|
|
|
|
|
|
def __init__(self, path, scm_type, hooks_path, bind_mounts):
|
|
|
|
self.path = path
|
|
|
|
self.scm_type = scm_type
|
|
|
|
self.hooks_path = hooks_path
|
|
|
|
self.bind_mounts = bind_mounts
|
|
|
|
|
|
|
|
|
2016-05-12 23:43:17 +03:00
|
|
|
class Config:
|
2017-01-24 10:52:46 +03:00
|
|
|
def __init__(self, config_dir, etc_eden_dir, home_dir):
|
2016-05-12 23:43:17 +03:00
|
|
|
self._config_dir = config_dir
|
2017-01-24 10:52:46 +03:00
|
|
|
self._etc_eden_dir = etc_eden_dir
|
|
|
|
if not self._etc_eden_dir:
|
|
|
|
self._etc_eden_dir = DEFAULT_ETC_EDEN_DIR
|
2016-07-26 20:15:43 +03:00
|
|
|
self._user_config_path = os.path.join(home_dir, USER_CONFIG)
|
2017-01-24 10:52:39 +03:00
|
|
|
self._home_dir = home_dir
|
2016-05-12 23:43:17 +03:00
|
|
|
|
2017-10-26 08:24:49 +03:00
|
|
|
def _loadConfig(self) -> configparser.ConfigParser:
|
2017-01-24 10:52:39 +03:00
|
|
|
''' to facilitate templatizing a centrally deployed config, we
|
|
|
|
allow a limited set of env vars to be expanded.
|
|
|
|
${HOME} will be replaced by the user's home dir,
|
|
|
|
${USER} will be replaced by the user's login name.
|
2017-01-24 10:52:48 +03:00
|
|
|
These are coupled with the equivalent code in
|
|
|
|
eden/fs/config/ClientConfig.cpp and must be kept in sync.
|
2017-01-24 10:52:39 +03:00
|
|
|
'''
|
|
|
|
defaults = {'USER': os.environ.get('USER'),
|
|
|
|
'HOME': self._home_dir}
|
|
|
|
parser = configparser.ConfigParser(
|
|
|
|
interpolation=configinterpolator.EdenConfigInterpolator(defaults))
|
2017-01-24 10:52:37 +03:00
|
|
|
parser.read(self.get_rc_files())
|
|
|
|
return parser
|
|
|
|
|
2016-07-07 02:14:18 +03:00
|
|
|
def get_rc_files(self):
|
2016-08-05 22:49:21 +03:00
|
|
|
result = []
|
2017-01-24 10:52:46 +03:00
|
|
|
config_d = os.path.join(self._etc_eden_dir, CONFIG_DOT_D)
|
|
|
|
if os.path.isdir(config_d):
|
|
|
|
result = os.listdir(config_d)
|
|
|
|
result = [os.path.join(config_d, f) for f in result]
|
2016-08-05 22:49:21 +03:00
|
|
|
result.sort()
|
|
|
|
result.append(self._user_config_path)
|
|
|
|
return result
|
2016-07-07 02:14:18 +03:00
|
|
|
|
2016-07-08 05:10:28 +03:00
|
|
|
def get_repository_list(self, parser=None):
|
2016-07-07 02:14:18 +03:00
|
|
|
result = []
|
2016-07-08 05:10:28 +03:00
|
|
|
if not parser:
|
2017-01-24 10:52:37 +03:00
|
|
|
parser = self._loadConfig()
|
2016-07-07 02:14:18 +03:00
|
|
|
for section in parser.sections():
|
|
|
|
header = section.split(' ')
|
|
|
|
if len(header) == 2 and header[0] == 'repository':
|
|
|
|
result.append(header[1])
|
2016-08-05 22:49:21 +03:00
|
|
|
return sorted(result)
|
2016-07-07 02:14:18 +03:00
|
|
|
|
2017-01-24 10:52:35 +03:00
|
|
|
def get_config_value(self, key):
|
2017-01-24 10:52:37 +03:00
|
|
|
parser = self._loadConfig()
|
2017-01-24 10:52:35 +03:00
|
|
|
section, option = key.split('.', 1)
|
|
|
|
try:
|
|
|
|
return parser.get(section, option)
|
|
|
|
except (configparser.NoOptionError, configparser.NoSectionError) as exc:
|
|
|
|
raise KeyError(str(exc))
|
|
|
|
|
|
|
|
def print_full_config(self):
|
2017-01-24 10:52:37 +03:00
|
|
|
parser = self._loadConfig()
|
2017-01-24 10:52:35 +03:00
|
|
|
for section in parser.sections():
|
|
|
|
print('[%s]' % section)
|
|
|
|
for k, v in parser.items(section):
|
|
|
|
print('%s=%s' % (k, v))
|
|
|
|
|
2017-11-17 00:19:59 +03:00
|
|
|
def find_config_for_alias(self, alias) -> Optional[ClientConfig]:
|
|
|
|
'''Looks through the existing config files and searches for a
|
|
|
|
[repository <alias>] section that defines a config:
|
|
|
|
- If no such section is found, returns None.
|
|
|
|
- If the appropriate section is found, returns a ClientConfig if all of
|
|
|
|
the fields for the config data are present and well-formed.
|
|
|
|
- Otherwise, throws an Exception.
|
2016-07-07 02:14:27 +03:00
|
|
|
'''
|
2017-01-24 10:52:37 +03:00
|
|
|
parser = self._loadConfig()
|
2017-11-17 00:19:59 +03:00
|
|
|
repository_header = f'repository {alias}'
|
|
|
|
if repository_header not in parser:
|
|
|
|
return None
|
|
|
|
repo_data = parser[repository_header]
|
|
|
|
|
|
|
|
bind_mounts_header = f'bindmounts {alias}'
|
2016-08-05 22:49:21 +03:00
|
|
|
if bind_mounts_header in parser:
|
2016-09-09 10:28:49 +03:00
|
|
|
# Convert the ConfigParser section into a dict so it is JSON
|
|
|
|
# serializable for the `eden info` command.
|
2017-11-15 01:01:59 +03:00
|
|
|
bind_mounts = dict(parser[bind_mounts_header].items())
|
2016-08-05 22:49:21 +03:00
|
|
|
else:
|
2017-11-15 01:01:59 +03:00
|
|
|
bind_mounts = {}
|
|
|
|
|
|
|
|
if 'type' not in repo_data:
|
2017-11-17 00:19:59 +03:00
|
|
|
raise Exception(f'repository "{alias}" missing key "type".')
|
2017-11-15 01:01:59 +03:00
|
|
|
elif 'path' not in repo_data:
|
2017-11-17 00:19:59 +03:00
|
|
|
raise Exception(f'repository "{alias}" missing key "path".')
|
2017-11-15 01:01:59 +03:00
|
|
|
|
2017-11-15 01:02:00 +03:00
|
|
|
hooks_path = repo_data.get('hooks')
|
|
|
|
if hooks_path is None:
|
2017-11-17 00:19:59 +03:00
|
|
|
hooks_path = self.get_default_hooks_path()
|
2017-11-15 01:02:00 +03:00
|
|
|
|
2017-11-17 00:19:59 +03:00
|
|
|
return ClientConfig(repo_data['path'], repo_data['type'], hooks_path,
|
|
|
|
bind_mounts)
|
2016-07-07 02:14:30 +03:00
|
|
|
|
2017-11-17 00:19:59 +03:00
|
|
|
def get_default_hooks_path(self) -> str:
|
|
|
|
return os.path.join(self._etc_eden_dir, 'hooks')
|
|
|
|
|
|
|
|
def create_no_such_repository_exception(self, name: str) -> Exception:
|
|
|
|
'''Creates an exception that says no repository is configured with the
|
|
|
|
specified name and suggests other repos that are defined in this Config.
|
2017-10-26 08:24:49 +03:00
|
|
|
'''
|
|
|
|
repos = []
|
|
|
|
prefix = 'repository '
|
2017-11-17 00:19:59 +03:00
|
|
|
config = self._loadConfig()
|
2017-10-26 08:24:49 +03:00
|
|
|
for key in config:
|
|
|
|
if key.startswith(prefix):
|
|
|
|
repos.append(key[len(prefix):])
|
|
|
|
msg = f'No repository configured named "{name}".'
|
|
|
|
if repos:
|
|
|
|
repos.sort()
|
|
|
|
all_repos = ', '.join(map(lambda r: f'"{r}"', repos))
|
|
|
|
msg += f' Try one of: {all_repos}.'
|
2017-11-17 00:19:59 +03:00
|
|
|
return Exception(msg)
|
2017-10-26 08:24:49 +03:00
|
|
|
|
2016-07-07 02:14:30 +03:00
|
|
|
def get_mount_paths(self):
|
|
|
|
'''Return the paths of the set mount points stored in config.json'''
|
|
|
|
return self._get_directory_map().keys()
|
2016-05-12 23:43:17 +03:00
|
|
|
|
|
|
|
def get_all_client_config_info(self):
|
|
|
|
info = {}
|
2016-07-07 02:14:30 +03:00
|
|
|
for path in self.get_mount_paths():
|
|
|
|
info[path] = self.get_client_info(path)
|
2016-05-12 23:43:17 +03:00
|
|
|
|
|
|
|
return info
|
|
|
|
|
|
|
|
def get_thrift_client(self):
|
2016-06-18 01:11:04 +03:00
|
|
|
return eden.thrift.create_thrift_client(self._config_dir)
|
2016-05-12 23:43:17 +03:00
|
|
|
|
2016-07-07 02:14:30 +03:00
|
|
|
def get_client_info(self, path):
|
2016-10-01 05:05:46 +03:00
|
|
|
path = os.path.realpath(path)
|
2016-07-07 02:14:30 +03:00
|
|
|
client_dir = self._get_client_dir_for_mount_point(path)
|
2017-11-17 00:19:57 +03:00
|
|
|
client_config = self._get_client_config(client_dir)
|
2017-11-15 01:02:00 +03:00
|
|
|
snapshot = self._get_snapshot(client_dir)
|
2016-05-12 23:43:17 +03:00
|
|
|
|
|
|
|
return collections.OrderedDict([
|
2017-11-17 00:19:57 +03:00
|
|
|
['bind-mounts', client_config.bind_mounts],
|
2016-07-07 02:14:30 +03:00
|
|
|
['mount', path],
|
2016-05-12 23:43:17 +03:00
|
|
|
['snapshot', snapshot],
|
|
|
|
['client-dir', client_dir],
|
|
|
|
])
|
|
|
|
|
2017-11-15 01:02:00 +03:00
|
|
|
@staticmethod
|
|
|
|
def _get_snapshot(client_dir: str) -> str:
|
|
|
|
'''Return the hex version of the parent hash in the SNAPSHOT file.'''
|
|
|
|
snapshot_file = os.path.join(client_dir, SNAPSHOT)
|
|
|
|
with open(snapshot_file, 'rb') as f:
|
|
|
|
assert f.read(8) == SNAPSHOT_MAGIC
|
|
|
|
return binascii.hexlify(f.read(20)).decode('utf-8')
|
|
|
|
|
2016-07-07 02:14:30 +03:00
|
|
|
def checkout(self, path, snapshot_id):
|
2016-05-12 23:43:17 +03:00
|
|
|
'''Switch the active snapshot id for a given client'''
|
2016-07-23 03:31:20 +03:00
|
|
|
with self.get_thrift_client() as client:
|
2016-07-07 02:14:30 +03:00
|
|
|
client.checkOutRevision(path, snapshot_id)
|
2016-05-12 23:43:17 +03:00
|
|
|
|
2016-07-07 02:14:27 +03:00
|
|
|
def add_repository(self, name, repo_type, source, with_buck=False):
|
2016-07-08 05:10:28 +03:00
|
|
|
# Check if repository already exists
|
2016-07-26 20:15:43 +03:00
|
|
|
with ConfigUpdater(self._user_config_path) as config:
|
2016-07-23 03:31:17 +03:00
|
|
|
if name in self.get_repository_list(config):
|
|
|
|
raise UsageError('''\
|
2016-07-11 23:05:24 +03:00
|
|
|
repository %s already exists. You will need to edit the ~/.edenrc config file \
|
|
|
|
by hand to make changes to the repository or remove it.''' % name)
|
2016-07-08 05:10:28 +03:00
|
|
|
|
2016-07-23 03:31:17 +03:00
|
|
|
# Create a directory for client to store repository metadata
|
|
|
|
bind_mounts = {}
|
|
|
|
if with_buck:
|
|
|
|
bind_mount_name = 'buck-out'
|
|
|
|
bind_mounts[bind_mount_name] = 'buck-out'
|
|
|
|
|
|
|
|
# Add repository to INI file
|
|
|
|
config['repository ' + name] = {'type': repo_type, 'path': source}
|
|
|
|
if bind_mounts:
|
|
|
|
config['bindmounts ' + name] = bind_mounts
|
|
|
|
config.save()
|
2016-07-07 02:14:23 +03:00
|
|
|
|
2017-11-17 00:19:59 +03:00
|
|
|
def clone(self, client_config: ClientConfig, path: str, snapshot_id: str):
|
2016-07-07 02:14:30 +03:00
|
|
|
if path in self._get_directory_map():
|
2017-11-16 06:25:26 +03:00
|
|
|
raise Exception('''\
|
|
|
|
mount path %s is already configured (see `eden list`). \
|
|
|
|
Do you want to run `eden mount %s` instead?''' % (path, path))
|
2016-07-07 02:14:30 +03:00
|
|
|
|
2016-07-12 04:27:29 +03:00
|
|
|
# Make sure that path is a valid destination for the clone.
|
|
|
|
st = None
|
|
|
|
try:
|
|
|
|
st = os.stat(path)
|
|
|
|
except OSError as ex:
|
|
|
|
if ex.errno == errno.ENOENT:
|
|
|
|
# Note that this could also throw if path is /a/b/c and /a
|
|
|
|
# exists, but it is a file.
|
|
|
|
util.mkdir_p(path)
|
|
|
|
else:
|
|
|
|
raise
|
|
|
|
|
|
|
|
# Note that st will be None if `mkdir_p` was run in the catch block.
|
|
|
|
if st:
|
|
|
|
if stat.S_ISDIR(st.st_mode):
|
|
|
|
# If an existing directory was specified, then verify it is
|
|
|
|
# empty.
|
|
|
|
if len(os.listdir(path)) > 0:
|
|
|
|
raise OSError(errno.ENOTEMPTY, os.strerror(errno.ENOTEMPTY),
|
|
|
|
path)
|
|
|
|
else:
|
|
|
|
# Throw because it exists, but it is not a directory.
|
|
|
|
raise OSError(errno.ENOTDIR, os.strerror(errno.ENOTDIR), path)
|
|
|
|
|
2016-07-07 02:14:30 +03:00
|
|
|
# Create client directory
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
clients_dir = self._get_clients_dir()
|
|
|
|
util.mkdir_p(clients_dir) # This directory probably already exists.
|
|
|
|
client_dir = self._create_client_dir_for_path(clients_dir, path)
|
2016-07-07 02:14:30 +03:00
|
|
|
|
|
|
|
# Store snapshot ID
|
2016-07-07 02:14:27 +03:00
|
|
|
if snapshot_id:
|
2016-07-07 02:14:30 +03:00
|
|
|
client_snapshot = os.path.join(client_dir, SNAPSHOT)
|
2017-08-31 01:07:46 +03:00
|
|
|
with open(client_snapshot, 'wb') as f:
|
|
|
|
f.write(SNAPSHOT_MAGIC)
|
|
|
|
f.write(binascii.unhexlify(snapshot_id))
|
2016-07-07 02:14:27 +03:00
|
|
|
else:
|
2016-07-11 23:05:24 +03:00
|
|
|
raise Exception('snapshot id not provided')
|
2016-07-07 02:14:27 +03:00
|
|
|
|
2016-07-07 02:14:30 +03:00
|
|
|
# Create bind mounts directories
|
|
|
|
bind_mounts_dir = os.path.join(client_dir, 'bind-mounts')
|
2016-07-07 02:14:31 +03:00
|
|
|
util.mkdir_p(bind_mounts_dir)
|
2017-11-17 00:19:59 +03:00
|
|
|
for mount in client_config.bind_mounts:
|
2016-07-07 02:14:31 +03:00
|
|
|
util.mkdir_p(os.path.join(bind_mounts_dir, mount))
|
2016-07-07 02:14:24 +03:00
|
|
|
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
config_path = os.path.join(client_dir, MOUNT_CONFIG)
|
2017-11-17 00:19:59 +03:00
|
|
|
self._save_client_config(client_config, config_path)
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
|
2016-07-07 02:14:30 +03:00
|
|
|
# Prepare to mount
|
2016-07-07 02:14:24 +03:00
|
|
|
mount_info = eden_ttypes.MountInfo(mountPoint=path,
|
2016-07-26 20:15:43 +03:00
|
|
|
edenClientPath=client_dir)
|
2016-07-23 03:31:20 +03:00
|
|
|
with self.get_thrift_client() as client:
|
2016-07-07 02:14:24 +03:00
|
|
|
client.mount(mount_info)
|
|
|
|
|
2017-11-17 00:19:59 +03:00
|
|
|
self._run_post_clone_hooks(path, client_dir, client_config)
|
2017-11-15 01:02:00 +03:00
|
|
|
|
2016-07-11 23:13:48 +03:00
|
|
|
# Add mapping of mount path to client directory in config.json
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
self._add_path_to_directory_map(path, os.path.basename(client_dir))
|
|
|
|
|
|
|
|
def _create_client_dir_for_path(self, clients_dir, path) -> str:
|
|
|
|
'''Tries to create a new subdirectory of clients_dir based on the
|
|
|
|
basename of the specified path. Tries appending an increasing sequence
|
|
|
|
of integers to the basename if there is a collision until it finds an
|
|
|
|
available directory name.
|
|
|
|
'''
|
|
|
|
basename = os.path.basename(path)
|
|
|
|
if basename == '':
|
|
|
|
raise Exception('Suspicious attempt to clone into: %s' % path)
|
|
|
|
for i in itertools.count(start=0):
|
|
|
|
if i == 0:
|
|
|
|
dir_name = basename
|
|
|
|
else:
|
|
|
|
dir_name = f'{basename}-{i}'
|
|
|
|
|
|
|
|
client_dir = os.path.join(clients_dir, dir_name)
|
|
|
|
try:
|
|
|
|
os.mkdir(client_dir)
|
|
|
|
return client_dir
|
|
|
|
except OSError as e:
|
|
|
|
if e.errno == errno.EEXIST:
|
|
|
|
# A directory with the specified name already exists: try
|
|
|
|
# again with the next candidate name.
|
|
|
|
continue
|
|
|
|
raise
|
2016-07-11 23:13:48 +03:00
|
|
|
|
2017-11-15 01:02:00 +03:00
|
|
|
def _run_post_clone_hooks(self, eden_mount_path: str, client_dir: str,
|
2017-11-17 00:19:59 +03:00
|
|
|
client_config: ClientConfig):
|
2017-11-15 01:02:00 +03:00
|
|
|
# First, check to see if the post-clone hook has been run successfully
|
|
|
|
# before.
|
|
|
|
clone_success_path = os.path.join(client_dir, CLONE_SUCCEEDED)
|
|
|
|
is_initial_mount = not os.path.isfile(clone_success_path)
|
|
|
|
if is_initial_mount:
|
2017-11-17 00:19:59 +03:00
|
|
|
post_clone = os.path.join(client_config.hooks_path, 'post-clone')
|
2017-11-15 01:02:00 +03:00
|
|
|
snapshot = self._get_snapshot(client_dir)
|
|
|
|
try:
|
|
|
|
subprocess.run([
|
|
|
|
post_clone,
|
2017-11-17 00:19:59 +03:00
|
|
|
client_config.scm_type,
|
2017-11-15 01:02:00 +03:00
|
|
|
eden_mount_path,
|
2017-11-17 00:19:59 +03:00
|
|
|
client_config.path,
|
2017-11-15 01:02:00 +03:00
|
|
|
snapshot,
|
|
|
|
], pass_fds=[1, 2], check=True)
|
|
|
|
except OSError as e:
|
|
|
|
if e.errno != errno.ENOENT:
|
|
|
|
# TODO(T13448173): If clone fails, then we should roll back
|
|
|
|
# the mount.
|
|
|
|
raise
|
|
|
|
print_stderr(f'Did not run post-clone hook "{post_clone}" for '
|
2017-11-17 00:19:59 +03:00
|
|
|
f'{client_config.path} because it was not found.')
|
2017-11-15 01:02:00 +03:00
|
|
|
|
|
|
|
# "touch" the clone_success_path.
|
|
|
|
with open(clone_success_path, 'a'):
|
|
|
|
os.utime(clone_success_path, None)
|
|
|
|
|
2017-11-17 00:19:59 +03:00
|
|
|
def _save_client_config(self, client_config: ClientConfig, config_path: str):
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
# Store information about the mount in the config.toml file.
|
|
|
|
config_data = {
|
|
|
|
'repository': {
|
2017-11-17 00:19:59 +03:00
|
|
|
'path': client_config.path,
|
|
|
|
'type': client_config.scm_type,
|
|
|
|
'hooks': client_config.hooks_path,
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
},
|
2017-11-17 00:19:59 +03:00
|
|
|
'bind-mounts': client_config.bind_mounts,
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
}
|
|
|
|
with open(config_path, 'w') as f:
|
|
|
|
toml.dump(config_data, f)
|
|
|
|
|
|
|
|
# TODO(mbolin): Delete this code in December 2017: Let's just assume
|
|
|
|
# everyone has been migrated by then.
|
|
|
|
def migrate_internal_edenrc_files_to_config_toml_files(self):
|
|
|
|
'''If a client has a legacy edenrc file under ~/local/.eden/clients/*/,
|
|
|
|
migrate it to a config.toml file.'''
|
|
|
|
clients_dir = self._get_clients_dir()
|
|
|
|
if not os.path.isdir(clients_dir):
|
|
|
|
return
|
|
|
|
for entry in os.listdir(clients_dir):
|
|
|
|
edenrc = os.path.join(clients_dir, entry, 'edenrc')
|
|
|
|
if not os.path.isfile(edenrc):
|
|
|
|
continue
|
|
|
|
|
|
|
|
parser = configparser.ConfigParser()
|
|
|
|
parser.read(edenrc)
|
|
|
|
repo_name = parser.get('repository', 'name')
|
2017-11-17 00:19:59 +03:00
|
|
|
client_config = self.get_repo_config(repo_name)
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
config_path = os.path.join(clients_dir, entry, MOUNT_CONFIG)
|
2017-11-17 00:19:59 +03:00
|
|
|
self._save_client_config(client_config, config_path)
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
os.remove(edenrc)
|
|
|
|
|
2016-08-04 02:18:27 +03:00
|
|
|
def mount(self, path):
|
|
|
|
# Load the config info for this client, to make sure we
|
|
|
|
# know about the client.
|
2016-10-01 05:05:46 +03:00
|
|
|
path = os.path.realpath(path)
|
2016-08-04 02:18:27 +03:00
|
|
|
client_dir = self._get_client_dir_for_mount_point(path)
|
2017-11-17 00:19:57 +03:00
|
|
|
|
|
|
|
# Call _get_client_config() for the side-effect of it raising an
|
|
|
|
# Exception if the config is in an invalid state.
|
|
|
|
self._get_client_config(client_dir)
|
2016-08-04 02:18:27 +03:00
|
|
|
|
|
|
|
# Make sure the mount path exists
|
|
|
|
util.mkdir_p(path)
|
|
|
|
|
2017-11-01 03:04:30 +03:00
|
|
|
# Check if it is already mounted.
|
|
|
|
try:
|
|
|
|
root = os.path.join(path, '.eden', 'root')
|
|
|
|
target = os.readlink(root)
|
|
|
|
if target == path:
|
|
|
|
print_stderr('ERROR: Mount point in use! '
|
|
|
|
'{} is already mounted by Eden.', path)
|
|
|
|
return 1
|
|
|
|
else:
|
|
|
|
# If we are here, MOUNT/.eden/root is a symlink, but it does not
|
|
|
|
# point to MOUNT. This suggests `path` is a subdirectory of an
|
|
|
|
# existing mount, though we should never reach this point
|
|
|
|
# because _get_client_dir_for_mount_point() above should have
|
|
|
|
# already thrown an exception. We return non-zero here just in
|
|
|
|
# case.
|
|
|
|
print_stderr('ERROR: Mount point in use! '
|
|
|
|
'{} is already mounted by Eden as part of {}.',
|
|
|
|
path, root)
|
|
|
|
return 1
|
|
|
|
except OSError as ex:
|
|
|
|
err = ex.errno
|
|
|
|
if err != errno.ENOENT and err != errno.EINVAL:
|
|
|
|
raise
|
|
|
|
|
2016-08-04 02:18:27 +03:00
|
|
|
# Ask eden to mount the path
|
|
|
|
mount_info = eden_ttypes.MountInfo(mountPoint=path,
|
|
|
|
edenClientPath=client_dir)
|
|
|
|
with self.get_thrift_client() as client:
|
|
|
|
client.mount(mount_info)
|
|
|
|
|
2017-11-16 20:04:33 +03:00
|
|
|
def unmount(self, path, delete_config):
|
2016-10-01 05:05:46 +03:00
|
|
|
path = os.path.realpath(path)
|
2016-07-23 03:31:20 +03:00
|
|
|
with self.get_thrift_client() as client:
|
|
|
|
client.unmount(path)
|
2016-08-04 02:18:27 +03:00
|
|
|
|
|
|
|
if delete_config:
|
|
|
|
shutil.rmtree(self._get_client_dir_for_mount_point(path))
|
|
|
|
self._remove_path_from_directory_map(path)
|
|
|
|
|
|
|
|
# Delete the now empty mount point
|
|
|
|
os.rmdir(path)
|
2016-05-28 03:40:10 +03:00
|
|
|
|
Write the PID to the lockfile and update `eden health` to use it.
Summary:
We have encountered cases where `eden health` reported
`"edenfs not healthy: edenfs not running"` even though the `edenfs` process is
still running. Because the existing implementation of `eden health` bases its
health check on the output of a `getStatus()` Thrift call, it will erroneously
report `"edenfs not running"` even if Eden is running but its Thrift server is
not running. This type of false negative could occur if `edenfs` has shutdown
the Thrift server, but not the rest of the process (quite possibly, its
shutdown is blocked on calls to `umount2()`).
This is further problematic because `eden daemon` checks `eden health`
before attempting to start the daemon. If it gets a false negative, then
`eden daemon` will forge ahead, trying to launch a new instance of the daemon,
but it will fail with a nasty error like the following:
```
I1017 11:59:25.188414 3064499 main.cpp:81] Starting edenfs. UID=5256, GID=100, PID=3064499
terminate called after throwing an instance of 'std::runtime_error'
what(): another instance of Eden appears to be running for /home/mbolin/local/.eden
*** Aborted at 1508266765 (Unix time, try 'date -d 1508266765') ***
*** Signal 6 (SIGABRT) (0x1488002ec2b3) received by PID 3064499 (pthread TID 0x7fd0d3787d40) (linux TID 3064499) (maybe from PID 30644
99, UID 5256), stack trace: ***
@ 000000000290d3cd folly::symbolizer::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
@ 00007fd0d133cacf (unknown)
@ 00007fd0d093e7c8 __GI_raise
@ 00007fd0d0940590 __GI_abort
@ 00007fd0d1dfeecc __gnu_cxx::__verbose_terminate_handler()
@ 00007fd0d1dfcdc5 __cxxabiv1::__terminate(void (*)())
@ 00007fd0d1dfce10 std::terminate()
@ 00007fd0d1dfd090 __cxa_throw
@ 00000000015fe8ca facebook::eden::EdenServer::acquireEdenLock()
@ 000000000160f27b facebook::eden::EdenServer::prepare()
@ 00000000016107d5 facebook::eden::EdenServer::run()
@ 000000000042c4ee main
@ 00007fd0d0929857 __libc_start_main
@ 0000000000548ad8 _start
Aborted
```
By providing more accurate information to `eden daemon`, if the user tries to
run it while the daemon is already running, they will get a more polite error
like the following:
```
error: edenfs is already running (pid 274205)
```
This revision addresses this issue by writing the PID of `edenfs` in the
lockfile. It updated the implementation of `eden health` to use the PID in the
lockfile to assess the health of Eden if the call to `getStatus()` fails. It
does this by running:
```
ps -p PID -o comm=
```
and applying some heuristics on the output to assess whether the command
associated with that process is the `edenfs` command. If it is, then
`eden health` reports the status as `STOPPED` whereas previously it would report
it as `DEAD`.
Reviewed By: wez
Differential Revision: D6086473
fbshipit-source-id: 825421a6818b56ddd7deea257a92c070c2232bdd
2017-10-18 21:18:43 +03:00
|
|
|
def check_health(self) -> 'HealthStatus':
|
2016-06-18 01:11:04 +03:00
|
|
|
'''
|
|
|
|
Get the status of the edenfs daemon.
|
|
|
|
|
|
|
|
Returns a HealthStatus object containing health information.
|
|
|
|
'''
|
2016-07-23 03:31:20 +03:00
|
|
|
pid = None
|
|
|
|
status = fb_status.DEAD
|
2016-06-18 01:11:04 +03:00
|
|
|
try:
|
2016-07-23 03:31:20 +03:00
|
|
|
with self.get_thrift_client() as client:
|
|
|
|
pid = client.getPid()
|
|
|
|
status = client.getStatus()
|
2016-06-18 01:11:04 +03:00
|
|
|
except eden.thrift.EdenNotRunningError:
|
Write the PID to the lockfile and update `eden health` to use it.
Summary:
We have encountered cases where `eden health` reported
`"edenfs not healthy: edenfs not running"` even though the `edenfs` process is
still running. Because the existing implementation of `eden health` bases its
health check on the output of a `getStatus()` Thrift call, it will erroneously
report `"edenfs not running"` even if Eden is running but its Thrift server is
not running. This type of false negative could occur if `edenfs` has shutdown
the Thrift server, but not the rest of the process (quite possibly, its
shutdown is blocked on calls to `umount2()`).
This is further problematic because `eden daemon` checks `eden health`
before attempting to start the daemon. If it gets a false negative, then
`eden daemon` will forge ahead, trying to launch a new instance of the daemon,
but it will fail with a nasty error like the following:
```
I1017 11:59:25.188414 3064499 main.cpp:81] Starting edenfs. UID=5256, GID=100, PID=3064499
terminate called after throwing an instance of 'std::runtime_error'
what(): another instance of Eden appears to be running for /home/mbolin/local/.eden
*** Aborted at 1508266765 (Unix time, try 'date -d 1508266765') ***
*** Signal 6 (SIGABRT) (0x1488002ec2b3) received by PID 3064499 (pthread TID 0x7fd0d3787d40) (linux TID 3064499) (maybe from PID 30644
99, UID 5256), stack trace: ***
@ 000000000290d3cd folly::symbolizer::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
@ 00007fd0d133cacf (unknown)
@ 00007fd0d093e7c8 __GI_raise
@ 00007fd0d0940590 __GI_abort
@ 00007fd0d1dfeecc __gnu_cxx::__verbose_terminate_handler()
@ 00007fd0d1dfcdc5 __cxxabiv1::__terminate(void (*)())
@ 00007fd0d1dfce10 std::terminate()
@ 00007fd0d1dfd090 __cxa_throw
@ 00000000015fe8ca facebook::eden::EdenServer::acquireEdenLock()
@ 000000000160f27b facebook::eden::EdenServer::prepare()
@ 00000000016107d5 facebook::eden::EdenServer::run()
@ 000000000042c4ee main
@ 00007fd0d0929857 __libc_start_main
@ 0000000000548ad8 _start
Aborted
```
By providing more accurate information to `eden daemon`, if the user tries to
run it while the daemon is already running, they will get a more polite error
like the following:
```
error: edenfs is already running (pid 274205)
```
This revision addresses this issue by writing the PID of `edenfs` in the
lockfile. It updated the implementation of `eden health` to use the PID in the
lockfile to assess the health of Eden if the call to `getStatus()` fails. It
does this by running:
```
ps -p PID -o comm=
```
and applying some heuristics on the output to assess whether the command
associated with that process is the `edenfs` command. If it is, then
`eden health` reports the status as `STOPPED` whereas previously it would report
it as `DEAD`.
Reviewed By: wez
Differential Revision: D6086473
fbshipit-source-id: 825421a6818b56ddd7deea257a92c070c2232bdd
2017-10-18 21:18:43 +03:00
|
|
|
# It is possible that the edenfs process is running, but the Thrift
|
|
|
|
# server is not running. This could be during the startup, shutdown,
|
|
|
|
# or takeover of the edenfs process. As a backup to requesting the
|
|
|
|
# PID from the Thrift server, we read it from the lockfile and try
|
|
|
|
# to deduce the current status of Eden.
|
|
|
|
return self._check_health_using_lockfile()
|
2016-06-18 01:11:04 +03:00
|
|
|
except thrift.Thrift.TException as ex:
|
|
|
|
detail = 'error talking to edenfs: ' + str(ex)
|
|
|
|
return HealthStatus(status, pid, detail)
|
|
|
|
|
|
|
|
status_name = fb_status._VALUES_TO_NAMES.get(status)
|
|
|
|
detail = 'edenfs running (pid {}); status is {}'.format(
|
|
|
|
pid, status_name)
|
|
|
|
return HealthStatus(status, pid, detail)
|
|
|
|
|
Write the PID to the lockfile and update `eden health` to use it.
Summary:
We have encountered cases where `eden health` reported
`"edenfs not healthy: edenfs not running"` even though the `edenfs` process is
still running. Because the existing implementation of `eden health` bases its
health check on the output of a `getStatus()` Thrift call, it will erroneously
report `"edenfs not running"` even if Eden is running but its Thrift server is
not running. This type of false negative could occur if `edenfs` has shutdown
the Thrift server, but not the rest of the process (quite possibly, its
shutdown is blocked on calls to `umount2()`).
This is further problematic because `eden daemon` checks `eden health`
before attempting to start the daemon. If it gets a false negative, then
`eden daemon` will forge ahead, trying to launch a new instance of the daemon,
but it will fail with a nasty error like the following:
```
I1017 11:59:25.188414 3064499 main.cpp:81] Starting edenfs. UID=5256, GID=100, PID=3064499
terminate called after throwing an instance of 'std::runtime_error'
what(): another instance of Eden appears to be running for /home/mbolin/local/.eden
*** Aborted at 1508266765 (Unix time, try 'date -d 1508266765') ***
*** Signal 6 (SIGABRT) (0x1488002ec2b3) received by PID 3064499 (pthread TID 0x7fd0d3787d40) (linux TID 3064499) (maybe from PID 30644
99, UID 5256), stack trace: ***
@ 000000000290d3cd folly::symbolizer::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
@ 00007fd0d133cacf (unknown)
@ 00007fd0d093e7c8 __GI_raise
@ 00007fd0d0940590 __GI_abort
@ 00007fd0d1dfeecc __gnu_cxx::__verbose_terminate_handler()
@ 00007fd0d1dfcdc5 __cxxabiv1::__terminate(void (*)())
@ 00007fd0d1dfce10 std::terminate()
@ 00007fd0d1dfd090 __cxa_throw
@ 00000000015fe8ca facebook::eden::EdenServer::acquireEdenLock()
@ 000000000160f27b facebook::eden::EdenServer::prepare()
@ 00000000016107d5 facebook::eden::EdenServer::run()
@ 000000000042c4ee main
@ 00007fd0d0929857 __libc_start_main
@ 0000000000548ad8 _start
Aborted
```
By providing more accurate information to `eden daemon`, if the user tries to
run it while the daemon is already running, they will get a more polite error
like the following:
```
error: edenfs is already running (pid 274205)
```
This revision addresses this issue by writing the PID of `edenfs` in the
lockfile. It updated the implementation of `eden health` to use the PID in the
lockfile to assess the health of Eden if the call to `getStatus()` fails. It
does this by running:
```
ps -p PID -o comm=
```
and applying some heuristics on the output to assess whether the command
associated with that process is the `edenfs` command. If it is, then
`eden health` reports the status as `STOPPED` whereas previously it would report
it as `DEAD`.
Reviewed By: wez
Differential Revision: D6086473
fbshipit-source-id: 825421a6818b56ddd7deea257a92c070c2232bdd
2017-10-18 21:18:43 +03:00
|
|
|
def _check_health_using_lockfile(self) -> 'HealthStatus':
|
|
|
|
'''Make a best-effort to produce a HealthStatus based on the PID in the
|
|
|
|
Eden lockfile.
|
|
|
|
'''
|
|
|
|
lockfile = os.path.join(self._config_dir, LOCK_FILE)
|
|
|
|
try:
|
|
|
|
with open(lockfile, 'r') as f:
|
|
|
|
lockfile_contents = f.read()
|
|
|
|
pid = lockfile_contents.rstrip()
|
|
|
|
int(pid) # Throw if this does not parse as an integer.
|
2017-11-07 23:31:24 +03:00
|
|
|
except Exception:
|
Write the PID to the lockfile and update `eden health` to use it.
Summary:
We have encountered cases where `eden health` reported
`"edenfs not healthy: edenfs not running"` even though the `edenfs` process is
still running. Because the existing implementation of `eden health` bases its
health check on the output of a `getStatus()` Thrift call, it will erroneously
report `"edenfs not running"` even if Eden is running but its Thrift server is
not running. This type of false negative could occur if `edenfs` has shutdown
the Thrift server, but not the rest of the process (quite possibly, its
shutdown is blocked on calls to `umount2()`).
This is further problematic because `eden daemon` checks `eden health`
before attempting to start the daemon. If it gets a false negative, then
`eden daemon` will forge ahead, trying to launch a new instance of the daemon,
but it will fail with a nasty error like the following:
```
I1017 11:59:25.188414 3064499 main.cpp:81] Starting edenfs. UID=5256, GID=100, PID=3064499
terminate called after throwing an instance of 'std::runtime_error'
what(): another instance of Eden appears to be running for /home/mbolin/local/.eden
*** Aborted at 1508266765 (Unix time, try 'date -d 1508266765') ***
*** Signal 6 (SIGABRT) (0x1488002ec2b3) received by PID 3064499 (pthread TID 0x7fd0d3787d40) (linux TID 3064499) (maybe from PID 30644
99, UID 5256), stack trace: ***
@ 000000000290d3cd folly::symbolizer::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
@ 00007fd0d133cacf (unknown)
@ 00007fd0d093e7c8 __GI_raise
@ 00007fd0d0940590 __GI_abort
@ 00007fd0d1dfeecc __gnu_cxx::__verbose_terminate_handler()
@ 00007fd0d1dfcdc5 __cxxabiv1::__terminate(void (*)())
@ 00007fd0d1dfce10 std::terminate()
@ 00007fd0d1dfd090 __cxa_throw
@ 00000000015fe8ca facebook::eden::EdenServer::acquireEdenLock()
@ 000000000160f27b facebook::eden::EdenServer::prepare()
@ 00000000016107d5 facebook::eden::EdenServer::run()
@ 000000000042c4ee main
@ 00007fd0d0929857 __libc_start_main
@ 0000000000548ad8 _start
Aborted
```
By providing more accurate information to `eden daemon`, if the user tries to
run it while the daemon is already running, they will get a more polite error
like the following:
```
error: edenfs is already running (pid 274205)
```
This revision addresses this issue by writing the PID of `edenfs` in the
lockfile. It updated the implementation of `eden health` to use the PID in the
lockfile to assess the health of Eden if the call to `getStatus()` fails. It
does this by running:
```
ps -p PID -o comm=
```
and applying some heuristics on the output to assess whether the command
associated with that process is the `edenfs` command. If it is, then
`eden health` reports the status as `STOPPED` whereas previously it would report
it as `DEAD`.
Reviewed By: wez
Differential Revision: D6086473
fbshipit-source-id: 825421a6818b56ddd7deea257a92c070c2232bdd
2017-10-18 21:18:43 +03:00
|
|
|
# If we cannot read the PID from the lockfile for any reason, return
|
|
|
|
# DEAD.
|
|
|
|
return self._create_dead_health_status()
|
|
|
|
|
|
|
|
try:
|
|
|
|
stdout = subprocess.check_output(['ps', '-p', pid, '-o', 'comm='])
|
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
# If there is no process with the specified id, return DEAD.
|
|
|
|
return self._create_dead_health_status()
|
|
|
|
|
|
|
|
# Use heuristics to determine that the PID in the lockfile is associated
|
|
|
|
# with an edenfs process as it is possible that edenfs is no longer
|
|
|
|
# running and the PID in the lockfile has been assigned to a new process
|
|
|
|
# unrelated to Eden.
|
|
|
|
comm = stdout.rstrip().decode('utf8')
|
|
|
|
# Note that the command may be just "edenfs" rather than a path, but it
|
|
|
|
# works out fine either way.
|
|
|
|
if os.path.basename(comm) == 'edenfs':
|
|
|
|
return HealthStatus(fb_status.STOPPED, int(pid),
|
|
|
|
'Eden\'s Thrift server does not appear to be '
|
|
|
|
'running, but the process is still alive ('
|
|
|
|
'PID=%s).' % pid)
|
|
|
|
else:
|
|
|
|
return self._create_dead_health_status()
|
|
|
|
|
|
|
|
def _create_dead_health_status(self) -> 'HealthStatus':
|
|
|
|
return HealthStatus(fb_status.DEAD, pid=None,
|
|
|
|
detail='edenfs not running')
|
|
|
|
|
2016-06-16 03:07:34 +03:00
|
|
|
def spawn(self,
|
|
|
|
daemon_binary,
|
2016-06-18 01:11:04 +03:00
|
|
|
extra_args=None,
|
2016-06-16 03:07:34 +03:00
|
|
|
gdb=False,
|
2016-07-26 01:13:43 +03:00
|
|
|
gdb_args=None,
|
2017-09-08 04:54:53 +03:00
|
|
|
strace_file=None,
|
2016-06-16 22:43:25 +03:00
|
|
|
foreground=False):
|
2016-05-12 23:43:17 +03:00
|
|
|
'''
|
2016-06-18 01:11:04 +03:00
|
|
|
Start edenfs.
|
|
|
|
|
|
|
|
If foreground is True this function never returns (edenfs is exec'ed
|
|
|
|
directly in the current process).
|
|
|
|
|
|
|
|
Otherwise, this function waits for edenfs to become healthy, and
|
|
|
|
returns a HealthStatus object. On error an exception will be raised.
|
|
|
|
'''
|
|
|
|
# Check to see if edenfs is already running
|
|
|
|
health_info = self.check_health()
|
|
|
|
if health_info.is_healthy():
|
|
|
|
raise EdenStartError('edenfs is already running (pid {})'.format(
|
|
|
|
health_info.pid))
|
|
|
|
|
2017-09-08 04:54:53 +03:00
|
|
|
if gdb and strace_file is not None:
|
|
|
|
raise EdenStartError('cannot run eden under gdb and '
|
|
|
|
'strace together')
|
|
|
|
|
2016-05-12 23:43:17 +03:00
|
|
|
# Run the eden server.
|
2016-07-26 20:15:43 +03:00
|
|
|
cmd = [daemon_binary, '--edenDir', self._config_dir,
|
2017-01-24 10:52:46 +03:00
|
|
|
'--etcEdenDir', self._etc_eden_dir,
|
2016-07-26 20:15:43 +03:00
|
|
|
'--configPath', self._user_config_path, ]
|
2016-05-12 23:43:17 +03:00
|
|
|
if gdb:
|
2016-07-26 01:13:43 +03:00
|
|
|
gdb_args = gdb_args or []
|
|
|
|
cmd = ['gdb'] + gdb_args + ['--args'] + cmd
|
2016-06-16 22:43:25 +03:00
|
|
|
foreground = True
|
2017-09-08 04:54:53 +03:00
|
|
|
if strace_file is not None:
|
|
|
|
cmd = ['strace', '-fttT', '-o', strace_file] + cmd
|
2016-06-18 01:11:04 +03:00
|
|
|
if extra_args:
|
|
|
|
cmd.extend(extra_args)
|
2016-05-12 23:43:17 +03:00
|
|
|
|
2016-08-10 23:27:21 +03:00
|
|
|
eden_env = self._build_eden_environment()
|
|
|
|
|
2016-05-12 23:43:17 +03:00
|
|
|
# Run edenfs using sudo, unless we already have root privileges,
|
|
|
|
# or the edenfs binary is setuid root.
|
2016-06-11 00:15:26 +03:00
|
|
|
if os.geteuid() != 0:
|
2016-06-16 03:07:34 +03:00
|
|
|
s = os.stat(daemon_binary)
|
2016-06-11 00:15:26 +03:00
|
|
|
if not (s.st_uid == 0 and (s.st_mode & stat.S_ISUID)):
|
2016-06-16 22:43:25 +03:00
|
|
|
# We need to run edenfs under sudo
|
2016-08-10 23:27:21 +03:00
|
|
|
sudo_cmd = ['/usr/bin/sudo']
|
|
|
|
# Add environment variable settings
|
|
|
|
# Depending on the sudo configuration, these may not
|
|
|
|
# necessarily get passed through automatically even when
|
|
|
|
# using "sudo -E".
|
|
|
|
for key, value in eden_env.items():
|
|
|
|
sudo_cmd.append('%s=%s' % (key, value))
|
|
|
|
|
2016-07-07 02:08:34 +03:00
|
|
|
if ('SANDCASTLE' in os.environ) and os.path.exists(SUDO_HELPER):
|
|
|
|
cmd = [SUDO_HELPER] + cmd
|
2016-08-10 23:27:21 +03:00
|
|
|
cmd = sudo_cmd + cmd
|
2016-06-16 22:43:25 +03:00
|
|
|
|
|
|
|
if foreground:
|
2016-06-18 01:11:04 +03:00
|
|
|
# This call does not return
|
2016-06-16 22:43:25 +03:00
|
|
|
os.execve(cmd[0], cmd, eden_env)
|
2016-06-18 01:11:04 +03:00
|
|
|
|
|
|
|
# Open the log file
|
|
|
|
log_path = self.get_log_path()
|
2016-07-07 02:14:31 +03:00
|
|
|
util.mkdir_p(os.path.dirname(log_path))
|
2017-01-24 05:02:39 +03:00
|
|
|
with open(log_path, 'a') as log_file:
|
|
|
|
startup_msg = time.strftime('%Y-%m-%d %H:%M:%S: starting edenfs\n')
|
|
|
|
log_file.write(startup_msg)
|
|
|
|
|
|
|
|
# Start edenfs
|
|
|
|
proc = subprocess.Popen(cmd, env=eden_env, preexec_fn=os.setsid,
|
|
|
|
stdout=log_file, stderr=log_file)
|
2016-06-18 01:11:04 +03:00
|
|
|
|
|
|
|
# Wait for edenfs to start
|
|
|
|
return self._wait_for_daemon_healthy(proc)
|
|
|
|
|
|
|
|
def _wait_for_daemon_healthy(self, proc):
|
|
|
|
'''
|
|
|
|
Wait for edenfs to become healthy.
|
|
|
|
'''
|
|
|
|
def check_health():
|
|
|
|
# Check the thrift status
|
|
|
|
health_info = self.check_health()
|
|
|
|
if health_info.is_healthy():
|
|
|
|
return health_info
|
|
|
|
|
|
|
|
# Make sure that edenfs is still running
|
|
|
|
status = proc.poll()
|
|
|
|
if status is not None:
|
|
|
|
if status < 0:
|
|
|
|
msg = 'terminated with signal {}'.format(-status)
|
|
|
|
else:
|
|
|
|
msg = 'exit status {}'.format(status)
|
|
|
|
raise EdenStartError('edenfs exited before becoming healthy: ' +
|
|
|
|
msg)
|
|
|
|
|
|
|
|
# Still starting
|
|
|
|
return None
|
|
|
|
|
|
|
|
timeout_ex = EdenStartError('timed out waiting for edenfs to become '
|
|
|
|
'healthy')
|
|
|
|
return util.poll_until(check_health, timeout=5, timeout_ex=timeout_ex)
|
|
|
|
|
2017-10-16 21:47:36 +03:00
|
|
|
def get_log_path(self) -> str:
|
2016-06-18 01:11:04 +03:00
|
|
|
return os.path.join(self._config_dir, 'logs', 'edenfs.log')
|
2016-06-16 22:43:25 +03:00
|
|
|
|
|
|
|
def _build_eden_environment(self):
|
|
|
|
# Reset $PATH to the following contents, so that everyone has the
|
|
|
|
# same consistent settings.
|
|
|
|
path_dirs = [
|
|
|
|
'/usr/local/bin',
|
|
|
|
'/bin',
|
|
|
|
'/usr/bin',
|
|
|
|
]
|
|
|
|
|
|
|
|
eden_env = {
|
|
|
|
'PATH': ':'.join(path_dirs),
|
|
|
|
}
|
|
|
|
|
|
|
|
# Preserve the following environment settings
|
|
|
|
preserve = [
|
|
|
|
'USER',
|
|
|
|
'LOGNAME',
|
|
|
|
'HOME',
|
|
|
|
'EMAIL',
|
|
|
|
'NAME',
|
|
|
|
# When we import data from mercurial, the remotefilelog extension
|
|
|
|
# may need to SSH to a remote mercurial server to get the file
|
|
|
|
# contents. Preserve SSH environment variables needed to do this.
|
|
|
|
'SSH_AUTH_SOCK',
|
|
|
|
'SSH_AGENT_PID',
|
2017-02-23 05:34:56 +03:00
|
|
|
'KRB5CCNAME',
|
2016-06-16 22:43:25 +03:00
|
|
|
]
|
|
|
|
|
|
|
|
for name, value in os.environ.items():
|
|
|
|
# Preserve any environment variable starting with "TESTPILOT_".
|
|
|
|
# TestPilot uses a few environment variables to keep track of
|
|
|
|
# processes started during test runs, so it can track down and kill
|
|
|
|
# runaway processes that weren't cleaned up by the test itself.
|
|
|
|
# We want to make sure this behavior works during the eden
|
|
|
|
# integration tests.
|
2017-01-24 10:52:34 +03:00
|
|
|
# Similarly, we want to preserve EDENFS_ env vars which are
|
|
|
|
# populated by our own test infra to relay paths to important
|
|
|
|
# build artifacts in our build tree.
|
|
|
|
if name.startswith('TESTPILOT_') or name.startswith('EDENFS_'):
|
2016-06-16 22:43:25 +03:00
|
|
|
eden_env[name] = value
|
|
|
|
elif name in preserve:
|
|
|
|
eden_env[name] = value
|
|
|
|
else:
|
|
|
|
# Drop any environment variable not matching the above cases
|
2016-06-11 00:15:26 +03:00
|
|
|
pass
|
2016-06-16 22:43:25 +03:00
|
|
|
|
|
|
|
return eden_env
|
2016-05-12 23:43:17 +03:00
|
|
|
|
|
|
|
def get_or_create_path_to_rocks_db(self):
|
|
|
|
rocks_db_dir = os.path.join(self._config_dir, ROCKS_DB_DIR)
|
2016-07-07 02:14:31 +03:00
|
|
|
return util.mkdir_p(rocks_db_dir)
|
2016-05-12 23:43:17 +03:00
|
|
|
|
2017-11-17 00:19:57 +03:00
|
|
|
def _get_client_config(self, client_dir) -> ClientConfig:
|
|
|
|
'''Returns ClientConfig or raises an Exception if the config.toml
|
|
|
|
under the client_dir is not properly formatted or does not exist.
|
|
|
|
'''
|
Change the contents and format for the edenrc file under ~/local/.eden.
Summary:
The headline changes of this revision are:
- Changes the format of the config file from INI to TOML
(the `edenrc` file under `~/local/.eden` has been replaced
with `config.toml`). This revision includes logic for automatically
performing the migration when Eden is restarted.
- Inlines data from `/etc/eden/config.d` into the TOML file.
Historically, the `edenrc` file for a client would contain the
name of the "configuration alias" defined in a config file like
`~/.edenrc` or `/etc/eden/config.d/00-defaults`. When Eden
loaded a client, it would have to first read the `edenrc` and
then reconstitute the rest of the client configuration by
looking up the alias in the set of config files that were used to
create the client in the first place.
This changes things so that all of the data that was being
cross-referenced is now inlined in the client's config file.
This makes loading a config considerably simpler at the cost
of no longer being able to change the config for multiple clients
that were cloned from the same configuration alias in one place.
It was questionable whether being able to modify a client from
a foreign config after it was created was a safe thing to do, anyway.
Eliminating the need for a historic link to the configuration alias
will make it easier to support running `eden clone` on an arbitrary
local Hg or Git repo. So long as `eden clone` can extract enough
information from the local repo to create an appropriate config file
for the new Eden client, there is no need for a configuration alias
to exist a priori.
Since we were already changing the data in the config file, this
seemed like an appropriate time to make the switch from INI to
TOML, as this was something we wanted to do, anyway.
In testing, I discovered a discrepancy between how boost's
`boost::property_tree::ptree` and Python's `ConfigParser` handled
the following section heading:
```
[repository ZtmpZsillyZeden-clone.LIkh32]
```
Apparently `hasSection("repository ZtmpZsillyZeden-clone.LIkh32")`
in boost would fail to find this section. Because
[[https://stackoverflow.com/questions/13109506/are-hyphens-allowed-in-section-definitions-in-ini-files | there is no spec for INI]],
it is not that surprising that boost and `ConfigParser` do not 100% agree
on what they accept. Moving to TOML means we have a configuration
language with the following desirable properties:
- It has a formal spec, unlike INI. This is important because there are parsers
in a wide range of programming languages that, in theory, accept a consistent
input language.
- It is reasonable for humans to write, as it supports comments, unlike JSON.
- It supports nested structures, like maps and arrays, without going crazy
on the input language it supports, unlike YAML.
Eden now depends on the following third-party TOML parsers:
* C++ https://github.com/skystrife/cpptoml
* Python https://github.com/uiri/toml
This revision also changes the organization of `~/local/.eden` slightly. For now,
there is still a `config.json` file, but the values are no longer hashes of the realpath
of the mount. Instead, we take the basename of the realpath and use that as the
name of the directory under `~/local/.eden/clients`. If there is a naming collision, we
add the first available integral suffix. Using the basename makes it easier to
navigate the `~/local/.eden/clients` directory.
Although the `edenrc` file under `~/local/.eden/clients` has been switched from INI
to TOML, the other Eden config files (`~/.edenrc` and `/etc/eden/config.d/*`) still use
INI. Migrating those to TOML will be done in a future revision.
Note this revision allowed us to eliminate `facebook::eden::InterpolatedPropertyTree`
as well as a number of uses of boost due to the elimination of
`ClientConfig::loadConfigData()` in the C++ code. Because `ClientConfig`
no longer does interpolation, a bit of `ClientConfigTest` was deleted as part of
this revision because it is no longer relevant.
Reviewed By: wez
Differential Revision: D6310325
fbshipit-source-id: 2548149c064cdf8e78a3b3ce6fe667ff70f94f84
2017-11-17 00:19:54 +03:00
|
|
|
config_toml = os.path.join(client_dir, MOUNT_CONFIG)
|
|
|
|
with open(config_toml, 'r') as f:
|
|
|
|
config = toml.load(f)
|
|
|
|
repository = config.get('repository')
|
2017-11-17 00:19:57 +03:00
|
|
|
if not isinstance(repository, dict):
|
|
|
|
raise Exception(f'{config_toml} is missing [repository]')
|
|
|
|
|
|
|
|
def get_field(key: str):
|
|
|
|
value = repository.get(key)
|
|
|
|
if not isinstance(value, str):
|
|
|
|
raise Exception(f'{config_toml} is missing {key} in '
|
|
|
|
'[repository]')
|
|
|
|
return value
|
|
|
|
|
|
|
|
path = get_field('path')
|
|
|
|
scm_type = get_field('type')
|
|
|
|
hooks_path = get_field('hooks')
|
|
|
|
|
|
|
|
bind_mounts = {}
|
|
|
|
bind_mounts_dict = config.get('bind-mounts')
|
|
|
|
if bind_mounts_dict is not None:
|
|
|
|
if not isinstance(bind_mounts_dict, dict):
|
|
|
|
raise Exception(f'{config_toml} has an invalid '
|
|
|
|
'[bind-mounts] section')
|
|
|
|
for key, value in bind_mounts_dict.items():
|
|
|
|
if not isinstance(value, str):
|
|
|
|
raise Exception(f'{config_toml} has invalid value in '
|
|
|
|
f'[bind-mounts] for {key}: {value} '
|
|
|
|
'(string expected)')
|
|
|
|
bind_mounts[key] = value
|
|
|
|
|
|
|
|
return ClientConfig(path, scm_type, hooks_path, bind_mounts)
|
2016-07-07 02:14:30 +03:00
|
|
|
|
2017-11-17 00:20:01 +03:00
|
|
|
def get_client_config_for_path(self, path: str) -> Optional[ClientConfig]:
|
|
|
|
directory_map = self._get_directory_map()
|
|
|
|
client_dir_component = directory_map.get(path)
|
|
|
|
if client_dir_component is None:
|
|
|
|
return None
|
|
|
|
client_dir = os.path.join(self._get_clients_dir(), client_dir_component)
|
|
|
|
return self._get_client_config(client_dir)
|
|
|
|
|
2016-07-07 02:14:30 +03:00
|
|
|
def _get_directory_map(self):
|
|
|
|
'''
|
|
|
|
Parse config.json which holds a mapping of mount paths to their
|
|
|
|
respective client directory and return contents in a dictionary.
|
|
|
|
'''
|
|
|
|
directory_map = os.path.join(self._config_dir, CONFIG_JSON)
|
|
|
|
if os.path.isfile(directory_map):
|
|
|
|
with open(directory_map) as f:
|
|
|
|
return json.load(f)
|
|
|
|
return {}
|
|
|
|
|
|
|
|
def _add_path_to_directory_map(self, path, dir_name):
|
|
|
|
config_data = self._get_directory_map()
|
|
|
|
if path in config_data:
|
2016-07-11 23:05:24 +03:00
|
|
|
raise Exception('mount path %s already exists.' % path)
|
2016-07-07 02:14:30 +03:00
|
|
|
config_data[path] = dir_name
|
|
|
|
self._write_directory_map(config_data)
|
|
|
|
|
|
|
|
def _remove_path_from_directory_map(self, path):
|
|
|
|
config_data = self._get_directory_map()
|
|
|
|
if path in config_data:
|
|
|
|
del config_data[path]
|
|
|
|
self._write_directory_map(config_data)
|
|
|
|
|
|
|
|
def _write_directory_map(self, config_data):
|
|
|
|
directory_map = os.path.join(self._config_dir, CONFIG_JSON)
|
|
|
|
with open(directory_map, 'w') as f:
|
|
|
|
json.dump(config_data, f, indent=2, sort_keys=True)
|
|
|
|
f.write('\n')
|
|
|
|
|
|
|
|
def _get_client_dir_for_mount_point(self, path):
|
2016-10-01 05:05:46 +03:00
|
|
|
# The caller is responsible for making sure the path is already
|
|
|
|
# a normalized, absolute path.
|
|
|
|
assert os.path.isabs(path)
|
|
|
|
|
2016-07-07 02:14:30 +03:00
|
|
|
config_data = self._get_directory_map()
|
|
|
|
if path not in config_data:
|
2016-07-11 23:05:24 +03:00
|
|
|
raise Exception('could not find mount path %s' % path)
|
2016-07-07 02:14:30 +03:00
|
|
|
return os.path.join(self._get_clients_dir(), config_data[path])
|
|
|
|
|
2016-05-12 23:43:17 +03:00
|
|
|
def _get_clients_dir(self):
|
|
|
|
return os.path.join(self._config_dir, CLIENTS_DIR)
|
|
|
|
|
|
|
|
|
2016-06-18 01:11:04 +03:00
|
|
|
class HealthStatus(object):
|
Write the PID to the lockfile and update `eden health` to use it.
Summary:
We have encountered cases where `eden health` reported
`"edenfs not healthy: edenfs not running"` even though the `edenfs` process is
still running. Because the existing implementation of `eden health` bases its
health check on the output of a `getStatus()` Thrift call, it will erroneously
report `"edenfs not running"` even if Eden is running but its Thrift server is
not running. This type of false negative could occur if `edenfs` has shutdown
the Thrift server, but not the rest of the process (quite possibly, its
shutdown is blocked on calls to `umount2()`).
This is further problematic because `eden daemon` checks `eden health`
before attempting to start the daemon. If it gets a false negative, then
`eden daemon` will forge ahead, trying to launch a new instance of the daemon,
but it will fail with a nasty error like the following:
```
I1017 11:59:25.188414 3064499 main.cpp:81] Starting edenfs. UID=5256, GID=100, PID=3064499
terminate called after throwing an instance of 'std::runtime_error'
what(): another instance of Eden appears to be running for /home/mbolin/local/.eden
*** Aborted at 1508266765 (Unix time, try 'date -d 1508266765') ***
*** Signal 6 (SIGABRT) (0x1488002ec2b3) received by PID 3064499 (pthread TID 0x7fd0d3787d40) (linux TID 3064499) (maybe from PID 30644
99, UID 5256), stack trace: ***
@ 000000000290d3cd folly::symbolizer::(anonymous namespace)::signalHandler(int, siginfo_t*, void*)
@ 00007fd0d133cacf (unknown)
@ 00007fd0d093e7c8 __GI_raise
@ 00007fd0d0940590 __GI_abort
@ 00007fd0d1dfeecc __gnu_cxx::__verbose_terminate_handler()
@ 00007fd0d1dfcdc5 __cxxabiv1::__terminate(void (*)())
@ 00007fd0d1dfce10 std::terminate()
@ 00007fd0d1dfd090 __cxa_throw
@ 00000000015fe8ca facebook::eden::EdenServer::acquireEdenLock()
@ 000000000160f27b facebook::eden::EdenServer::prepare()
@ 00000000016107d5 facebook::eden::EdenServer::run()
@ 000000000042c4ee main
@ 00007fd0d0929857 __libc_start_main
@ 0000000000548ad8 _start
Aborted
```
By providing more accurate information to `eden daemon`, if the user tries to
run it while the daemon is already running, they will get a more polite error
like the following:
```
error: edenfs is already running (pid 274205)
```
This revision addresses this issue by writing the PID of `edenfs` in the
lockfile. It updated the implementation of `eden health` to use the PID in the
lockfile to assess the health of Eden if the call to `getStatus()` fails. It
does this by running:
```
ps -p PID -o comm=
```
and applying some heuristics on the output to assess whether the command
associated with that process is the `edenfs` command. If it is, then
`eden health` reports the status as `STOPPED` whereas previously it would report
it as `DEAD`.
Reviewed By: wez
Differential Revision: D6086473
fbshipit-source-id: 825421a6818b56ddd7deea257a92c070c2232bdd
2017-10-18 21:18:43 +03:00
|
|
|
def __init__(self, status: fb_status, pid: Optional[int], detail: str) -> None:
|
2016-06-18 01:11:04 +03:00
|
|
|
self.status = status
|
|
|
|
self.pid = pid # The process ID, or None if not running
|
|
|
|
self.detail = detail # a human-readable message
|
|
|
|
|
|
|
|
def is_healthy(self):
|
|
|
|
return self.status == fb_status.ALIVE
|
|
|
|
|
|
|
|
|
2016-07-23 03:31:17 +03:00
|
|
|
class ConfigUpdater(object):
|
|
|
|
'''
|
|
|
|
A helper class to safely update an eden config file.
|
|
|
|
|
|
|
|
This acquires a lock on the config file, reads it in, and then provide APIs
|
|
|
|
to save it back. This ensures that another process cannot change the file
|
|
|
|
in between the time that we read it and when we write it back.
|
|
|
|
|
|
|
|
This also saves the file to a temporary name first, then renames it into
|
|
|
|
place, so that the main config file is always in a good state, and never
|
|
|
|
has partially written contents.
|
|
|
|
'''
|
|
|
|
def __init__(self, path):
|
|
|
|
self.path = path
|
|
|
|
self._lock_path = self.path + '.lock'
|
|
|
|
self._lock_file = None
|
|
|
|
self.config = configparser.ConfigParser()
|
|
|
|
|
|
|
|
# Acquire a lock.
|
|
|
|
# This makes sure that another process can't modify the config in the
|
|
|
|
# middle of a read-modify-write operation. (We can't stop a user
|
|
|
|
# from manually editing the file while we work, but we can stop
|
|
|
|
# other eden CLI processes.)
|
|
|
|
self._acquire_lock()
|
2017-11-13 21:59:37 +03:00
|
|
|
self.config.read(self.path)
|
2016-07-23 03:31:17 +03:00
|
|
|
|
|
|
|
def __enter__(self):
|
|
|
|
return self
|
|
|
|
|
|
|
|
def __exit__(self, exc_type, exc_value, exc_traceback):
|
|
|
|
self.close()
|
|
|
|
|
|
|
|
def __del__(self):
|
|
|
|
self.close()
|
|
|
|
|
|
|
|
def sections(self):
|
|
|
|
return self.config.sections()
|
|
|
|
|
|
|
|
def __getitem__(self, key):
|
|
|
|
return self.config[key]
|
|
|
|
|
|
|
|
def __setitem__(self, key, value):
|
|
|
|
self.config[key] = value
|
|
|
|
|
|
|
|
def _acquire_lock(self):
|
|
|
|
while True:
|
|
|
|
self._lock_file = open(self._lock_path, 'w+')
|
|
|
|
fcntl.flock(self._lock_file.fileno(), fcntl.LOCK_EX)
|
|
|
|
# The original creator of the lock file will unlink it when
|
|
|
|
# it is finished. Make sure we grab the lock on the file still on
|
|
|
|
# disk, and not an unlinked file.
|
|
|
|
st1 = os.fstat(self._lock_file.fileno())
|
|
|
|
st2 = os.lstat(self._lock_path)
|
|
|
|
if st1.st_dev == st2.st_dev and st1.st_ino == st2.st_ino:
|
|
|
|
# We got the real lock
|
|
|
|
return
|
|
|
|
|
|
|
|
# We acquired a lock on an old deleted file.
|
|
|
|
# Close it, and try to acquire the current lock file again.
|
|
|
|
self._lock_file.close()
|
|
|
|
self._lock_file = None
|
|
|
|
continue
|
|
|
|
|
|
|
|
def _unlock(self):
|
|
|
|
# Remove the file on disk before we unlock it.
|
|
|
|
# This way processes currently waiting in _acquire_lock() that already
|
|
|
|
# opened our lock file will see that it isn't the current file on disk
|
|
|
|
# once they acquire the lock.
|
|
|
|
os.unlink(self._lock_path)
|
|
|
|
self._lock_file.close()
|
|
|
|
self._lock_file = None
|
|
|
|
|
|
|
|
def close(self):
|
|
|
|
if self._lock_file is not None:
|
|
|
|
self._unlock()
|
|
|
|
|
|
|
|
def save(self):
|
|
|
|
if self._lock_file is None:
|
|
|
|
raise Exception('Cannot save the config without holding the lock')
|
|
|
|
|
|
|
|
try:
|
|
|
|
st = os.stat(self.path)
|
|
|
|
perms = (st.st_mode & 0o777)
|
|
|
|
except OSError as ex:
|
|
|
|
if ex.errno != errno.ENOENT:
|
|
|
|
raise
|
|
|
|
perms = 0o644
|
|
|
|
|
|
|
|
# Write the contents to a temporary file first, then atomically rename
|
|
|
|
# it to the desired destination. This makes sure the .edenrc file
|
|
|
|
# always has valid contents at all points in time.
|
2016-07-26 20:15:43 +03:00
|
|
|
prefix = USER_CONFIG + '.tmp.'
|
2016-07-23 03:31:17 +03:00
|
|
|
dirname = os.path.dirname(self.path)
|
|
|
|
tmpf = tempfile.NamedTemporaryFile('w', dir=dirname, prefix=prefix,
|
|
|
|
delete=False)
|
|
|
|
try:
|
|
|
|
self.config.write(tmpf)
|
|
|
|
tmpf.close()
|
|
|
|
os.chmod(tmpf.name, perms)
|
|
|
|
os.rename(tmpf.name, self.path)
|
|
|
|
except BaseException:
|
|
|
|
# Remove temporary file on error
|
|
|
|
try:
|
|
|
|
os.unlink(tmpf.name)
|
|
|
|
except Exception:
|
|
|
|
pass
|
|
|
|
raise
|
|
|
|
|
|
|
|
|
2016-05-12 23:43:17 +03:00
|
|
|
def _verify_mount_point(mount_point):
|
|
|
|
if os.path.isdir(mount_point):
|
|
|
|
return
|
|
|
|
parent_dir = os.path.dirname(mount_point)
|
|
|
|
if os.path.isdir(parent_dir):
|
|
|
|
os.mkdir(mount_point)
|
|
|
|
else:
|
|
|
|
raise Exception(
|
|
|
|
('%s must be a directory in order to mount a client at %s. ' +
|
2016-06-07 07:00:41 +03:00
|
|
|
'If this is the correct location, run `mkdir -p %s` to create ' +
|
|
|
|
'the directory.') % (parent_dir, mount_point, parent_dir))
|