From 574cef54e36b8afc5a56539f697f16bce47ffdcf Mon Sep 17 00:00:00 2001 From: Matt Glazar Date: Tue, 23 Oct 2018 14:02:52 -0700 Subject: [PATCH] Test restart/stop/status with 'eden start'd and ad-hoc daemon Summary: Right now, 'eden start' spawns edenfs directly (using `os.exec`). In the future, 'eden start' will spawn edenfs using systemd. 'eden restart', 'eden stop', and 'eden status' might behave differently depending on how the edenfs process was spawned. Parameterize some tests on how fake_edenfs was originally started. This flexibility lets us easily spawn fake_edenfs with 'systemctl start' (systemd) in the future. For now, spawn fake_edenfs in ad-hoc mode (i.e. directly without 'eden start') or in managed mode (i.e. indirectly with 'eden start'). Reviewed By: chadaustin Differential Revision: D10414995 fbshipit-source-id: 2ee0d5df8a605ca9d7da8f6eeca1fc171a8342e8 --- eden/integration/health_test.py | 14 ++--- eden/integration/lib/service_test_case.py | 76 +++++++++++++++++++++++ eden/integration/restart_test.py | 8 +-- eden/integration/stop_test.py | 55 +++++++--------- 4 files changed, 108 insertions(+), 45 deletions(-) create mode 100644 eden/integration/lib/service_test_case.py diff --git a/eden/integration/health_test.py b/eden/integration/health_test.py index 359ba787c6..5e2f73c668 100644 --- a/eden/integration/health_test.py +++ b/eden/integration/health_test.py @@ -12,15 +12,14 @@ import pathlib import signal import sys import typing -import unittest import pexpect from eden.cli.daemon import wait_for_shutdown from .lib import edenclient, testcase -from .lib.fake_edenfs import FakeEdenFS from .lib.find_executables import FindExe from .lib.pexpect import PexpectAssertionMixin +from .lib.service_test_case import ServiceTestCaseBase, service_test from .lib.temporary_directory import TemporaryDirectoryMixin @@ -37,21 +36,22 @@ class HealthTest(testcase.EdenTestCase): self.assertFalse(client.is_healthy()) +@service_test class HealthOfFakeEdenFSTest( - unittest.TestCase, PexpectAssertionMixin, TemporaryDirectoryMixin + ServiceTestCaseBase, PexpectAssertionMixin, TemporaryDirectoryMixin ): def setUp(self): super().setUp() self.temp_dir = pathlib.Path(self.make_temporary_directory()) def test_healthy_daemon_is_healthy(self): - with FakeEdenFS.spawn(self.temp_dir): + with self.spawn_fake_edenfs(self.temp_dir): status_process = self.spawn_status([]) status_process.expect_exact("eden running normally") self.assert_process_succeeds(status_process) def test_killed_daemon_is_not_running(self): - with FakeEdenFS.spawn(self.temp_dir) as daemon_pid: + with self.spawn_fake_edenfs(self.temp_dir) as daemon_pid: os.kill(daemon_pid, signal.SIGKILL) wait_for_shutdown(pid=daemon_pid, timeout=5) @@ -60,7 +60,7 @@ class HealthOfFakeEdenFSTest( self.assert_process_fails(status_process, exit_code=1) def test_hanging_thrift_call_reports_daemon_is_unresponsive(self): - with FakeEdenFS.spawn(self.temp_dir, ["--sleepBeforeGetPid=5"]): + with self.spawn_fake_edenfs(self.temp_dir, ["--sleepBeforeGetPid=5"]): status_process = self.spawn_status(["--timeout", "1"]) status_process.expect_exact( "Eden's Thrift server does not appear to be running, but the " @@ -69,7 +69,7 @@ class HealthOfFakeEdenFSTest( self.assert_process_fails(status_process, exit_code=1) def test_slow_thrift_call_reports_daemon_is_healthy(self): - with FakeEdenFS.spawn(self.temp_dir, ["--sleepBeforeGetPid=2"]): + with self.spawn_fake_edenfs(self.temp_dir, ["--sleepBeforeGetPid=2"]): status_process = self.spawn_status(["--timeout", "10"]) status_process.logfile = sys.stderr status_process.expect_exact("eden running normally") diff --git a/eden/integration/lib/service_test_case.py b/eden/integration/lib/service_test_case.py new file mode 100644 index 0000000000..35aac67d28 --- /dev/null +++ b/eden/integration/lib/service_test_case.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2016-present, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. An additional grant +# of patent rights can be found in the PATENTS file in the same directory. + +import abc +import pathlib +import typing +import unittest + +from .fake_edenfs import FakeEdenFS +from .testcase import test_replicator + + +class ServiceTestCaseBase(unittest.TestCase, metaclass=abc.ABCMeta): + """Abstract base class for tests covering 'eden start', 'eden stop', etc. + + Use the @service_test decorator to make a concrete subclass. + """ + + @abc.abstractmethod + def spawn_fake_edenfs( + self, eden_dir: pathlib.Path, extra_arguments: typing.Sequence[str] = () + ) -> FakeEdenFS: + raise NotImplementedError() + + +class AdHocFakeEdenFSMixin: + """Test by spawning fake_edenfs directly. + + Use the @service_test decorator to use this mixin automatically. + """ + + def spawn_fake_edenfs( + self, eden_dir: pathlib.Path, extra_arguments: typing.Sequence[str] = () + ) -> FakeEdenFS: + return FakeEdenFS.spawn(eden_dir=eden_dir, extra_arguments=extra_arguments) + + +class ManagedFakeEdenFSMixin: + """Test by using 'eden start' to spawn fake_edenfs. + + Use the @service_test decorator to use this mixin automatically. + """ + + def spawn_fake_edenfs( + self, eden_dir: pathlib.Path, extra_arguments: typing.Sequence[str] = () + ) -> FakeEdenFS: + return FakeEdenFS.spawn_via_cli( + eden_dir=eden_dir, extra_arguments=extra_arguments + ) + + +def _replicate_service_test( + test_class: typing.Type[ServiceTestCaseBase] +) -> typing.Iterable[typing.Tuple[str, typing.Type[ServiceTestCaseBase]]]: + class ManagedTest(ManagedFakeEdenFSMixin, test_class): + pass + + class AdHocTest(AdHocFakeEdenFSMixin, test_class): + pass + + return [("Managed", ManagedTest), ("AdHoc", AdHocTest)] + + +# A decorator function used to create ServiceTestCaseBase subclasses from a +# given input test class. +# +# Given an input test class named "MyTest", this will create two separate +# classes named "MyTestAdHoc" and "MyTestManaged", which run the tests with +# ad-hoc and managed edenfs processes, respectively. +service_test = test_replicator(_replicate_service_test) diff --git a/eden/integration/restart_test.py b/eden/integration/restart_test.py index 2b4a2e1245..8151cfcf9f 100644 --- a/eden/integration/restart_test.py +++ b/eden/integration/restart_test.py @@ -11,18 +11,18 @@ import os import pathlib import subprocess import sys -import unittest import eden.thrift import eden.thrift.client import pexpect -from .lib.fake_edenfs import FakeEdenFS from .lib.find_executables import FindExe +from .lib.service_test_case import ServiceTestCaseBase, service_test from .lib.temporary_directory import TemporaryDirectoryMixin -class RestartTest(unittest.TestCase, TemporaryDirectoryMixin): +@service_test +class RestartTest(ServiceTestCaseBase, TemporaryDirectoryMixin): def setUp(self) -> None: self.tmp_dir = self.make_temporary_directory() @@ -49,7 +49,7 @@ class RestartTest(unittest.TestCase, TemporaryDirectoryMixin): ) def _start_fake_edenfs(self) -> int: - daemon = FakeEdenFS.spawn_via_cli(eden_dir=pathlib.Path(self.tmp_dir)) + daemon = self.spawn_fake_edenfs(eden_dir=pathlib.Path(self.tmp_dir)) return daemon.process_id def test_restart_starts_edenfs_if_not_running(self) -> None: diff --git a/eden/integration/stop_test.py b/eden/integration/stop_test.py index 74dce90152..87385fc561 100644 --- a/eden/integration/stop_test.py +++ b/eden/integration/stop_test.py @@ -11,19 +11,17 @@ import contextlib import os import pathlib import signal -import subprocess import sys import time import typing -import unittest import pexpect from eden.cli.daemon import did_process_exit from eden.cli.util import poll_until -from .lib.fake_edenfs import FakeEdenFS from .lib.find_executables import FindExe from .lib.pexpect import PexpectAssertionMixin +from .lib.service_test_case import ServiceTestCaseBase, service_test from .lib.temporary_directory import TemporaryDirectoryMixin @@ -33,12 +31,14 @@ SHUTDOWN_EXIT_CODE_NOT_RUNNING_ERROR = 2 SHUTDOWN_EXIT_CODE_TERMINATED_VIA_SIGKILL = 3 -class StopTest(unittest.TestCase, PexpectAssertionMixin, TemporaryDirectoryMixin): +@service_test +class StopTest(ServiceTestCaseBase, PexpectAssertionMixin, TemporaryDirectoryMixin): def setUp(self): + super().setUp() self.tmp_dir = self.make_temporary_directory() def test_stop_stops_running_daemon(self): - with FakeEdenFS.spawn(pathlib.Path(self.tmp_dir)) as daemon_pid: + with self.spawn_fake_edenfs(pathlib.Path(self.tmp_dir)) as daemon_pid: stop_process = self.spawn_stop(["--timeout", "5"]) stop_process.expect_exact("edenfs exited cleanly.") self.assert_process_exit_code(stop_process, SHUTDOWN_EXIT_CODE_NORMAL) @@ -47,32 +47,17 @@ class StopTest(unittest.TestCase, PexpectAssertionMixin, TemporaryDirectoryMixin ) def test_stop_sigkill(self): - # Start eden, using the FAKE_EDENFS binary instead of the real edenfs. - # This binary behaves enough like edenfs to pass health checks, but it refuses - # to ever shut down gracefully. - start_cmd = [ - FindExe.EDEN_CLI, - "--config-dir", - self.tmp_dir, - "start", - "--daemon-binary", - FindExe.FAKE_EDENFS, - "--", - "--ignoreStop", - ] - print("Starting eden: %r" % (start_cmd,)) - subprocess.check_call(start_cmd) - - # Ask the CLI to stop edenfs, with a 1 second timeout. - # It should have to kill the process with SIGKILL - stop_process = self.spawn_stop(["--timeout", "1"]) - stop_process.expect_exact("Terminated edenfs with SIGKILL") - self.assert_process_exit_code( - stop_process, SHUTDOWN_EXIT_CODE_TERMINATED_VIA_SIGKILL - ) + with self.spawn_fake_edenfs(pathlib.Path(self.tmp_dir), ["--ignoreStop"]): + # Ask the CLI to stop edenfs, with a 1 second timeout. + # It should have to kill the process with SIGKILL + stop_process = self.spawn_stop(["--timeout", "1"]) + stop_process.expect_exact("Terminated edenfs with SIGKILL") + self.assert_process_exit_code( + stop_process, SHUTDOWN_EXIT_CODE_TERMINATED_VIA_SIGKILL + ) def test_async_stop_stops_daemon_eventually(self): - with FakeEdenFS.spawn(pathlib.Path(self.tmp_dir)) as daemon_pid: + with self.spawn_fake_edenfs(pathlib.Path(self.tmp_dir)) as daemon_pid: stop_process = self.spawn_stop(["--timeout", "0"]) stop_process.expect_exact("Sent async shutdown request to edenfs.") self.assert_process_exit_code( @@ -95,7 +80,7 @@ class StopTest(unittest.TestCase, PexpectAssertionMixin, TemporaryDirectoryMixin ) def test_stopping_killed_daemon_reports_not_running(self): - daemon = FakeEdenFS.spawn(pathlib.Path(self.tmp_dir)) + daemon = self.spawn_fake_edenfs(pathlib.Path(self.tmp_dir)) os.kill(daemon.process_id, signal.SIGKILL) stop_process = self.spawn_stop(["--timeout", "1"]) @@ -105,7 +90,7 @@ class StopTest(unittest.TestCase, PexpectAssertionMixin, TemporaryDirectoryMixin ) def test_killing_hung_daemon_during_stop_makes_stop_finish(self): - with FakeEdenFS.spawn(pathlib.Path(self.tmp_dir)) as daemon_pid: + with self.spawn_fake_edenfs(pathlib.Path(self.tmp_dir)) as daemon_pid: os.kill(daemon_pid, signal.SIGSTOP) try: stop_process = self.spawn_stop(["--timeout", "5"]) @@ -127,7 +112,7 @@ class StopTest(unittest.TestCase, PexpectAssertionMixin, TemporaryDirectoryMixin os.kill(daemon_pid, signal.SIGCONT) def test_stopping_daemon_stopped_by_sigstop_kills_daemon(self): - with FakeEdenFS.spawn(pathlib.Path(self.tmp_dir)) as daemon_pid: + with self.spawn_fake_edenfs(pathlib.Path(self.tmp_dir)) as daemon_pid: os.kill(daemon_pid, signal.SIGSTOP) try: stop_process = self.spawn_stop(["--timeout", "1"]) @@ -140,14 +125,16 @@ class StopTest(unittest.TestCase, PexpectAssertionMixin, TemporaryDirectoryMixin os.kill(daemon_pid, signal.SIGCONT) def test_hanging_thrift_call_kills_daemon_with_sigkill(self): - with FakeEdenFS.spawn(pathlib.Path(self.tmp_dir), ["--sleepBeforeStop=5"]): + with self.spawn_fake_edenfs( + pathlib.Path(self.tmp_dir), ["--sleepBeforeStop=5"] + ): stop_process = self.spawn_stop(["--timeout", "1"]) self.assert_process_exit_code( stop_process, SHUTDOWN_EXIT_CODE_TERMINATED_VIA_SIGKILL ) def test_stop_succeeds_if_thrift_call_abruptly_kills_daemon(self): - with FakeEdenFS.spawn( + with self.spawn_fake_edenfs( pathlib.Path(self.tmp_dir), ["--exitWithoutCleanupOnStop"] ): stop_process = self.spawn_stop(["--timeout", "10"])