#!/usr/bin/env python3 # Copyright (c) Facebook, Inc. and its affiliates. # # This software may be used and distributed according to the terms of the # GNU General Public License version 2. import contextlib import os import signal import sys import time from pathlib import Path from typing import Callable, List, Optional import pexpect from eden.fs.cli import proc_utils as proc_utils_mod from eden.fs.cli.daemon import wait_for_process_exit from eden.fs.cli.util import poll_until from .lib.find_executables import FindExe from .lib.pexpect import PexpectAssertionMixin, wait_for_pexpect_process from .lib.service_test_case import ( ServiceTestCaseBase, SystemdServiceTestCaseMarker, service_test, ) from .lib.testcase import EdenTestCase SHUTDOWN_EXIT_CODE_NORMAL = 0 SHUTDOWN_EXIT_CODE_REQUESTED_SHUTDOWN = 0 SHUTDOWN_EXIT_CODE_NOT_RUNNING_ERROR = 2 SHUTDOWN_EXIT_CODE_TERMINATED_VIA_SIGKILL = 3 class StopTestBase(ServiceTestCaseBase): eden_dir: Path def setUp(self) -> None: super().setUp() self.eden_dir = self.tmp_dir / "eden" self.eden_dir.mkdir() def spawn_stop(self, extra_args: List[str]) -> "pexpect.spawn[str]": return pexpect.spawn( # pyre-ignore[6]: T38947910 FindExe.EDEN_CLI, ["--config-dir", str(self.eden_dir)] + self.get_required_eden_cli_args() + ["stop"] + extra_args, encoding="utf-8", logfile=sys.stderr, ) @service_test class StopTest(StopTestBase, PexpectAssertionMixin): def test_stop_stops_running_daemon(self) -> None: proc_utils = proc_utils_mod.new() with self.spawn_fake_edenfs(self.eden_dir) as daemon_pid: stop_process = self.spawn_stop(["--timeout", "5"]) stop_process.expect_exact("edenfs exited cleanly.") self.assert_process_exit_code(stop_process, SHUTDOWN_EXIT_CODE_NORMAL) self.assertFalse( proc_utils.is_process_alive(daemon_pid), f"Process {daemon_pid} should have died", ) def test_eden_stop_shuts_down_edenfs_cleanly(self) -> None: clean_shutdown_file = self.eden_dir / "clean_shutdown" assert not clean_shutdown_file.exists() with self.spawn_fake_edenfs( self.eden_dir, ["--cleanShutdownFile", str(clean_shutdown_file)] ): self.assertFalse( clean_shutdown_file.exists(), f"{clean_shutdown_file} should not exist after starting EdenFS", ) stop_process = self.spawn_stop([]) wait_for_pexpect_process(stop_process) self.assertTrue( clean_shutdown_file.exists(), f"{clean_shutdown_file} should exist after EdenFS cleanly shuts down", ) def test_stop_sigkill(self) -> None: with self.spawn_fake_edenfs(self.eden_dir, ["--ignoreStop"]): # Ask the CLI to stop edenfs, with a 1 second timeout. # It should have to kill the process with SIGKILL stop_process = self.spawn_stop(["--timeout", "1"]) stop_process.expect_exact("Terminated edenfs with SIGKILL") self.assert_process_exit_code( stop_process, SHUTDOWN_EXIT_CODE_TERMINATED_VIA_SIGKILL ) def test_stop_kill(self) -> None: with self.spawn_fake_edenfs(self.eden_dir, ["--ignoreStop"]): # Run "eden stop --kill" # This should attempt to kill edenfs immediately with SIGKILL. stop_process = self.spawn_stop(["--kill", "--timeout", "1"]) stop_process.expect_exact("Terminated edenfs with SIGKILL") self.assert_process_exit_code(stop_process, SHUTDOWN_EXIT_CODE_NORMAL) def test_async_stop_stops_daemon_eventually(self) -> None: with self.spawn_fake_edenfs(self.eden_dir) as daemon_pid: stop_process = self.spawn_stop(["--timeout", "0"]) stop_process.expect_exact("Sent async shutdown request to edenfs.") self.assert_process_exit_code( stop_process, SHUTDOWN_EXIT_CODE_REQUESTED_SHUTDOWN ) self.assertTrue(wait_for_process_exit(daemon_pid, timeout=10)) def test_stop_not_running(self) -> None: stop_process = self.spawn_stop(["--timeout", "1"]) stop_process.expect_exact("edenfs is not running") self.assert_process_exit_code( stop_process, SHUTDOWN_EXIT_CODE_NOT_RUNNING_ERROR ) def test_stopping_killed_daemon_reports_not_running(self) -> None: daemon = self.spawn_fake_edenfs(self.eden_dir) os.kill(daemon.process_id, signal.SIGKILL) stop_process = self.spawn_stop(["--timeout", "1"]) stop_process.expect_exact("edenfs is not running") self.assert_process_exit_code( stop_process, SHUTDOWN_EXIT_CODE_NOT_RUNNING_ERROR ) def test_killing_hung_daemon_during_stop_makes_stop_finish(self) -> None: with self.spawn_fake_edenfs(self.eden_dir) as daemon_pid: os.kill(daemon_pid, signal.SIGSTOP) try: stop_process = self.spawn_stop(["--timeout", "5"]) time.sleep(2) self.assertTrue( stop_process.isalive(), "'eden stop' should wait while daemon is hung", ) os.kill(daemon_pid, signal.SIGKILL) stop_process.expect_exact("error: edenfs is not running") self.assert_process_exit_code( stop_process, SHUTDOWN_EXIT_CODE_NOT_RUNNING_ERROR ) finally: with contextlib.suppress(ProcessLookupError): os.kill(daemon_pid, signal.SIGCONT) def test_hanging_thrift_call_kills_daemon_with_sigkill(self) -> None: with self.spawn_fake_edenfs(self.eden_dir, ["--sleepBeforeStop=5"]): stop_process = self.spawn_stop(["--timeout", "1"]) self.assert_process_exit_code( stop_process, SHUTDOWN_EXIT_CODE_TERMINATED_VIA_SIGKILL ) def test_stop_succeeds_if_thrift_call_abruptly_kills_daemon(self) -> None: with self.spawn_fake_edenfs(self.eden_dir, ["--exitWithoutCleanupOnStop"]): stop_process = self.spawn_stop(["--timeout", "10"]) self.assert_process_exit_code(stop_process, SHUTDOWN_EXIT_CODE_NORMAL) @service_test class StopWithSystemdTest( SystemdServiceTestCaseMarker, StopTestBase, PexpectAssertionMixin ): def test_stop_stops_systemd_service(self) -> None: with self.spawn_fake_edenfs(self.eden_dir): stop_process = self.spawn_stop(["--timeout", "5"]) stop_process.expect_exact("edenfs exited cleanly.") self.assert_process_exit_code(stop_process, SHUTDOWN_EXIT_CODE_NORMAL) self.assert_systemd_service_is_stopped(eden_dir=self.eden_dir) class AutoStopTest(EdenTestCase): def update_validity_interval(self, interval: str) -> None: config_text = f""" [config] [core] check-validity-interval = "{interval}" """ self.eden.user_rc_path.write_text(config_text) with self.get_thrift_client() as client: client.reloadConfig() def _run_test(self, invalidate_fn: Callable[[], None], timeout: float = 15) -> None: self.update_validity_interval("20ms") # Run the function which will invalidate the state directory invalidate_fn() # EdenFS should exit on its own optional_edenfs = self.eden._process assert optional_edenfs is not None edenfs = optional_edenfs def edenfs_exited() -> Optional[bool]: returncode = edenfs.poll() if returncode is None: return None return True poll_until(edenfs_exited, timeout=timeout) def test_delete_lock_file(self) -> None: def delete_lock_file() -> None: (self.eden.eden_dir / "lock").unlink() self._run_test(delete_lock_file) def test_replace_lock_file(self) -> None: def replace_lock_file() -> None: lock_path = self.eden.eden_dir / "lock" new_lock_path = self.eden.eden_dir / "lock2" new_lock_path.touch() new_lock_path.rename(lock_path) self._run_test(replace_lock_file) def test_move_state_dir(self) -> None: def move_state_dir() -> None: new_path = Path(self.tmp_dir) / "new-eden-dir" self.eden.eden_dir.rename(new_path) self._run_test(move_state_dir) def test_runs_normally(self) -> None: """Make sure that EdenFS continues running normally if the lock file isn't replaced. """ def noop() -> None: pass # Call _run_test(). Since we don't replace the lock file it should time # out waiting for edenfs to exit. with self.assertRaises(TimeoutError): self._run_test(noop, timeout=5) self.assertTrue(self.eden.is_healthy())