sapling/eden/integration/stop_test.py

#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License version 2.

import contextlib
import os
import signal
import sys
import time
from pathlib import Path
from typing import Callable, List, Optional

import pexpect
from eden.fs.cli import proc_utils as proc_utils_mod
from eden.fs.cli.daemon import wait_for_process_exit
from eden.fs.cli.util import poll_until

from .lib.find_executables import FindExe
from .lib.pexpect import (
    PexpectAssertionMixin,
    PexpectSpawnType,
    pexpect_spawn,
    wait_for_pexpect_process,
)
from .lib.service_test_case import (
    ServiceTestCaseBase,
    SystemdServiceTest,
    service_test,
    systemd_test,
)
from .lib.testcase import EdenTestCase


SHUTDOWN_EXIT_CODE_NORMAL = 0
SHUTDOWN_EXIT_CODE_REQUESTED_SHUTDOWN = 0
SHUTDOWN_EXIT_CODE_NOT_RUNNING_ERROR = 2
SHUTDOWN_EXIT_CODE_TERMINATED_VIA_SIGKILL = 3


class StopTestBase(ServiceTestCaseBase):
    eden_dir: Path

    def setUp(self) -> None:
        super().setUp()
        self.eden_dir = self.tmp_dir / "eden"
        self.eden_dir.mkdir()

    def spawn_stop(self, extra_args: List[str]) -> PexpectSpawnType:
        return pexpect_spawn(
            FindExe.EDEN_CLI,
            ["--config-dir", str(self.eden_dir)]
            + self.get_required_eden_cli_args()
            + ["stop"]
            + extra_args,
            encoding="utf-8",
            logfile=sys.stderr,
        )


@service_test
class StopTest(StopTestBase, PexpectAssertionMixin):
    def test_stop_stops_running_daemon(self) -> None:
        proc_utils = proc_utils_mod.new()
        with self.spawn_fake_edenfs(self.eden_dir) as daemon_pid:
            stop_process = self.spawn_stop(["--timeout", "5"])
            stop_process.expect_exact("edenfs exited cleanly.")
            self.assert_process_exit_code(stop_process, SHUTDOWN_EXIT_CODE_NORMAL)
            self.assertFalse(
                proc_utils.is_process_alive(daemon_pid),
                f"Process {daemon_pid} should have died",
            )

    def test_eden_stop_shuts_down_edenfs_cleanly(self) -> None:
        clean_shutdown_file = self.eden_dir / "clean_shutdown"
        assert not clean_shutdown_file.exists()

        with self.spawn_fake_edenfs(
            self.eden_dir, ["--cleanShutdownFile", str(clean_shutdown_file)]
        ):
            self.assertFalse(
                clean_shutdown_file.exists(),
                f"{clean_shutdown_file} should not exist after starting EdenFS",
            )

            stop_process = self.spawn_stop([])
            wait_for_pexpect_process(stop_process)
            self.assertTrue(
                clean_shutdown_file.exists(),
                f"{clean_shutdown_file} should exist after EdenFS cleanly shuts down",
            )

    def test_stop_sigkill(self) -> None:
        with self.spawn_fake_edenfs(self.eden_dir, ["--ignoreStop"]):
            # Ask the CLI to stop edenfs, with a 1 second timeout.
            # It should have to kill the process with SIGKILL
            stop_process = self.spawn_stop(["--timeout", "1"])
            stop_process.expect_exact("Terminated edenfs with SIGKILL")
            self.assert_process_exit_code(
                stop_process, SHUTDOWN_EXIT_CODE_TERMINATED_VIA_SIGKILL
            )

    def test_stop_kill(self) -> None:
        with self.spawn_fake_edenfs(self.eden_dir, ["--ignoreStop"]):
            # Run "eden stop --kill"
            # This should attempt to kill edenfs immediately with SIGKILL.
            stop_process = self.spawn_stop(["--kill", "--timeout", "1"])
            stop_process.expect_exact("Terminated edenfs with SIGKILL")
            self.assert_process_exit_code(stop_process, SHUTDOWN_EXIT_CODE_NORMAL)

    def test_async_stop_stops_daemon_eventually(self) -> None:
        with self.spawn_fake_edenfs(self.eden_dir) as daemon_pid:
            stop_process = self.spawn_stop(["--timeout", "0"])
            stop_process.expect_exact("Sent async shutdown request to edenfs.")
            self.assert_process_exit_code(
                stop_process, SHUTDOWN_EXIT_CODE_REQUESTED_SHUTDOWN
            )

            self.assertTrue(wait_for_process_exit(daemon_pid, timeout=10))

    def test_stop_not_running(self) -> None:
        stop_process = self.spawn_stop(["--timeout", "1"])
        stop_process.expect_exact("edenfs is not running")
        self.assert_process_exit_code(
            stop_process, SHUTDOWN_EXIT_CODE_NOT_RUNNING_ERROR
        )

    def test_stopping_killed_daemon_reports_not_running(self) -> None:
        daemon = self.spawn_fake_edenfs(self.eden_dir)
        os.kill(daemon.process_id, signal.SIGKILL)

        stop_process = self.spawn_stop(["--timeout", "1"])
        stop_process.expect_exact("edenfs is not running")
        self.assert_process_exit_code(
            stop_process, SHUTDOWN_EXIT_CODE_NOT_RUNNING_ERROR
        )

    def test_killing_hung_daemon_during_stop_makes_stop_finish(self) -> None:
        with self.spawn_fake_edenfs(self.eden_dir) as daemon_pid:
            os.kill(daemon_pid, signal.SIGSTOP)
            try:
                stop_process = self.spawn_stop(["--timeout", "5"])

                time.sleep(2)
                self.assertTrue(
                    stop_process.isalive(),
                    "'eden stop' should wait while daemon is hung",
                )

                os.kill(daemon_pid, signal.SIGKILL)

                stop_process.expect_exact("error: edenfs is not running")
                self.assert_process_exit_code(
                    stop_process, SHUTDOWN_EXIT_CODE_NOT_RUNNING_ERROR
                )
            finally:
                with contextlib.suppress(ProcessLookupError):
                    os.kill(daemon_pid, signal.SIGCONT)

    def test_hanging_thrift_call_kills_daemon_with_sigkill(self) -> None:
        with self.spawn_fake_edenfs(self.eden_dir, ["--sleepBeforeStop=5"]):
            stop_process = self.spawn_stop(["--timeout", "1"])
            self.assert_process_exit_code(
                stop_process, SHUTDOWN_EXIT_CODE_TERMINATED_VIA_SIGKILL
            )

    def test_stop_succeeds_if_thrift_call_abruptly_kills_daemon(self) -> None:
        with self.spawn_fake_edenfs(self.eden_dir, ["--exitWithoutCleanupOnStop"]):
            stop_process = self.spawn_stop(["--timeout", "10"])
            self.assert_process_exit_code(stop_process, SHUTDOWN_EXIT_CODE_NORMAL)


@systemd_test
class StopWithSystemdTest(SystemdServiceTest, StopTestBase, PexpectAssertionMixin):
    def test_stop_stops_systemd_service(self) -> None:
        with self.spawn_fake_edenfs(self.eden_dir):
            stop_process = self.spawn_stop(["--timeout", "5"])
            stop_process.expect_exact("edenfs exited cleanly.")
            self.assert_process_exit_code(stop_process, SHUTDOWN_EXIT_CODE_NORMAL)
            self.assert_systemd_service_is_stopped(eden_dir=self.eden_dir)


class AutoStopTest(EdenTestCase):
    def update_validity_interval(self, interval: str) -> None:
        config_text = f"""
[config]
[core]
check-validity-interval = "{interval}"
"""
        self.eden.user_rc_path.write_text(config_text)
        with self.get_thrift_client() as client:
            client.reloadConfig()

    def _run_test(self, invalidate_fn: Callable[[], None], timeout: float = 15) -> None:
        self.update_validity_interval("20ms")

        # Run the function which will invalidate the state directory
        invalidate_fn()

        # EdenFS should exit on its own
        optional_edenfs = self.eden._process
        assert optional_edenfs is not None
        edenfs = optional_edenfs

        def edenfs_exited() -> Optional[bool]:
            returncode = edenfs.poll()
            if returncode is None:
                return None
            return True

        poll_until(edenfs_exited, timeout=timeout)

    def test_delete_lock_file(self) -> None:
        def delete_lock_file() -> None:
            (self.eden.eden_dir / "lock").unlink()

        self._run_test(delete_lock_file)

    def test_replace_lock_file(self) -> None:
        def replace_lock_file() -> None:
            lock_path = self.eden.eden_dir / "lock"
            new_lock_path = self.eden.eden_dir / "lock2"
            new_lock_path.touch()
            new_lock_path.rename(lock_path)

        self._run_test(replace_lock_file)

    def test_move_state_dir(self) -> None:
        def move_state_dir() -> None:
            new_path = Path(self.tmp_dir) / "new-eden-dir"
            self.eden.eden_dir.rename(new_path)

        self._run_test(move_state_dir)

    def test_runs_normally(self) -> None:
        """Make sure that EdenFS continues running normally if the lock file
        isn't replaced.
        """

        def noop() -> None:
            pass

        # Call _run_test().  Since we don't replace the lock file it should time
        # out waiting for edenfs to exit.
        with self.assertRaises(TimeoutError):
            self._run_test(noop, timeout=5)

        self.assertTrue(self.eden.is_healthy())