diff --git a/test/make-test.nix b/test/lib/make-test-vm.nix similarity index 100% rename from test/make-test.nix rename to test/lib/make-test-vm.nix diff --git a/test/lib/make-test.nix b/test/lib/make-test.nix new file mode 100644 index 0000000..c3b5edc --- /dev/null +++ b/test/lib/make-test.nix @@ -0,0 +1,41 @@ +scenario: testConfig: + +{ + vm = import ./make-test-vm.nix { + name = "nix-bitcoin-${scenario}"; + + machine = { + imports = [ testConfig ]; + # Needed because duplicity requires 270 MB of free temp space, regardless of backup size + virtualisation.diskSize = 1024; + }; + + testScript = nodes: let + cfg = nodes.nodes.machine.config; + data = { + data = cfg.test.data; + tests = cfg.tests; + }; + dataFile = builtins.toFile "test-data" (builtins.toJSON data); + initData = '' + import json + + with open("${dataFile}") as f: + data = json.load(f) + + enabled_tests = set(test for (test, enabled) in data["tests"].items() if enabled) + test_data = data["data"] + ''; + in + builtins.concatStringsSep "\n\n" [ + initData + (builtins.readFile ./../tests.py) + # Don't run tests in interactive mode. + # is_interactive is set in ../run-tests.sh + '' + if not "is_interactive" in vars(): + run_tests() + '' + ]; + }; +} diff --git a/test/lib/test-lib.nix b/test/lib/test-lib.nix new file mode 100644 index 0000000..5490643 --- /dev/null +++ b/test/lib/test-lib.nix @@ -0,0 +1,30 @@ +{ config, lib, ... }: +with lib; +{ + options = { + test = { + noConnections = mkOption { + type = types.bool; + default = true; + description = '' + Whether services should be configured to not connect to external hosts. + This can silence some warnings while running the test in an offline environment. + ''; + }; + data = mkOption { + type = types.attrs; + default = {}; + description = '' + Attrs that are available in the Python test script under the global + dictionary variable 'test_data'. The data is exported via JSON. + ''; + }; + }; + + tests = mkOption { + type = with types; attrsOf bool; + default = {}; + description = "Python tests that should be run."; + }; + }; +} diff --git a/test/run-tests.sh b/test/run-tests.sh index d1bb80e..13a5d5c 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -8,7 +8,10 @@ # ./run-tests.sh # # Test specific scenario -# ./run-tests.sh --scenario +# ./run-tests.sh --scenario|-s +# +# When is undefined, the test is run with an adhoc scenario +# where services. is enabled. # # Run test and link results to avoid garbage collection # ./run-tests.sh [--scenario ] --out-link-prefix /tmp/nix-bitcoin-test build @@ -19,8 +22,10 @@ # Run interactive test debugging # ./run-tests.sh [--scenario ] debug # -# This starts the testing VM and drops you into a Python REPL where you can -# manually execute the tests from ./test-script.py +# This starts the testing VM and drops you into a Python REPL where you can +# manually execute the tests from ./tests.py +# +# To add custom scenarios, set the environment variable `scenarioOverridesFile`. set -eo pipefail @@ -67,20 +72,14 @@ run() { export TMPDIR=$(mktemp -d /tmp/nix-bitcoin-test.XXX) trap "rm -rf $TMPDIR" EXIT - nix-build --out-link $TMPDIR/driver -E "import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }" -A driver + nix-build --out-link $TMPDIR/driver -E "(import \"$scriptDir/tests.nix\" { scenario = \"$scenario\"; }).vm" -A driver # Variable 'tests' contains the Python code that is executed by the driver on startup if [[ $1 == --interactive ]]; then echo "Running interactive testing environment" tests=$( echo 'is_interactive = True' - # The test script raises an error when 'is_interactive' is defined so - # that it just loads the initial helper functions and stops before - # executing the actual tests - echo 'try:' - echo ' exec(os.environ["testScript"])' - echo 'except:' - echo ' pass' + echo 'exec(os.environ["testScript"])' # Start VM echo 'start_all()' # Start REPL @@ -135,7 +134,7 @@ exprForCI() { vmTestNixExpr() { cat < ]; - security.allowUserNamespaces = true; # re-enable disabled option - }; - - machine = { pkgs, lib, ... }: with lib; { + baseConfig = { imports = [ - ../modules/presets/secure-node.nix + ./lib/test-lib.nix + ../modules/modules.nix ../modules/secrets/generate-secrets.nix - # using the hardened profile increases total test duration by ~50%, so disable it for now - # hardened ]; - # needed because duplicity requires 270 MB of free temp space, regardless of backup size. - virtualisation.diskSize = 1024; + config = { + tests.bitcoind = cfg.bitcoind.enable; + services.bitcoind = { + enable = true; + extraConfig = mkIf config.test.noConnections (mkForce "connect=0"); + }; - nix-bitcoin.netns-isolation.enable = (scenario == "netns"); + tests.clightning = cfg.clightning.enable; - services.bitcoind.extraConfig = mkForce "connect=0"; + tests.spark-wallet = cfg.spark-wallet.enable; - services.clightning.enable = true; - services.spark-wallet.enable = true; - services.lightning-charge.enable = true; - services.nanopos.enable = true; + tests.nanopos = cfg.nanopos.enable; - services.lnd.enable = true; - services.lnd.listenPort = 9736; - services.lightning-loop.enable = true; + tests.lnd = cfg.lnd.enable; + services.lnd.listenPort = 9736; - services.electrs.enable = true; + tests.lightning-loop = cfg.lightning-loop.enable; - services.liquidd = { - enable = true; - listen = mkForce false; - extraConfig = "noconnect=1"; - }; + tests.electrs = cfg.electrs.enable; - services.nix-bitcoin-webindex.enable = true; + tests.liquidd = cfg.liquidd.enable; + services.liquidd = optionalAttrs config.test.noConnections { + listen = mkForce false; + extraConfig = "noconnect=1"; + }; - services.hardware-wallets = { - trezor = true; - ledger = true; - }; + tests.btcpayserver = cfg.btcpayserver.enable; + services.btcpayserver.lightningBackend = "lnd"; + # Needed to test macaroon creation + environment.systemPackages = mkIfTest "btcpayserver" (with pkgs; [ openssl xxd ]); - services.backups.enable = true; + tests.joinmarket = cfg.joinmarket.enable; + services.joinmarket.yieldgenerator = { + enable = config.services.joinmarket.enable; + customParameters = '' + txfee = 200 + cjfee_a = 300 + ''; + }; - services.btcpayserver.enable = true; - services.btcpayserver.lightningBackend = "lnd"; - # needed to test macaroon creation - environment.systemPackages = with pkgs; [ openssl xxd ]; - - services.joinmarket.enable = true; - services.joinmarket.yieldgenerator = { - enable = true; - customParameters = '' - txfee = 200 - cjfee_a = 300 + tests.backups = cfg.backups.enable; + + # To test that unused secrets are made inaccessible by 'setup-secrets' + systemd.services.generate-secrets.postStart = mkIfTest "security" '' + install -o nobody -g nogroup -m777 <(:) /secrets/dummy ''; }; - - # to test that unused secrets are made inaccessible by 'setup-secrets' - systemd.services.generate-secrets.postStart = '' - install -o nobody -g nogroup -m777 <(:) /secrets/dummy - ''; }; - testScript = - builtins.readFile ./tests.py + "\n\n" + builtins.readFile "${./.}/scenarios/${scenario}.py"; -} + + scenarios = { + base = baseConfig; # Included in all scenarios + + default = scenarios.secureNode; + + # All available basic services and tests + full = { + tests.security = true; + + services.clightning.enable = true; + services.spark-wallet.enable = true; + services.lightning-charge.enable = true; + services.nanopos.enable = true; + services.lnd.enable = true; + services.lightning-loop.enable = true; + services.electrs.enable = true; + services.liquidd.enable = true; + services.btcpayserver.enable = true; + services.joinmarket.enable = true; + services.backups.enable = true; + + services.hardware-wallets = { + trezor = true; + ledger = true; + }; + }; + + secureNode = { + imports = [ + scenarios.full + ../modules/presets/secure-node.nix + ]; + services.nix-bitcoin-webindex.enable = true; + tests.secure-node = true; + tests.banlist-and-restart = true; + }; + + netns = { + imports = [ scenarios.secureNode ]; + nix-bitcoin.netns-isolation.enable = true; + tests.netns-isolation = true; + }; + + ## Examples / debug helper + + # Run a selection of tests in scenario 'netns' + selectedTests = { + imports = [ scenarios.netns ]; + tests = mkForce { + btcpayserver = true; + netns-isolation = true; + }; + }; + + adhoc = { + # + # You can also set the env var `scenarioOverridesFile` (used below) to define custom scenarios. + }; + }; +}; +in + let + overrides = builtins.getEnv "scenarioOverridesFile"; + scenarios = testEnv.scenarios // (optionalAttrs (overrides != "") (import overrides { + inherit testEnv config pkgs lib; + })); + autoScenario = { + services.${scenario}.enable = true; + }; + in { + imports = [ + scenarios.base + (scenarios.${scenario} or autoScenario) + ]; + } +) diff --git a/test/tests.py b/test/tests.py index c0f6d8c..e324436 100644 --- a/test/tests.py +++ b/test/tests.py @@ -1,4 +1,4 @@ -is_interactive = "is_interactive" in vars() +from collections import OrderedDict def succeed(*cmds): @@ -28,7 +28,8 @@ def assert_no_failure(unit): def assert_running(unit): - machine.wait_for_unit(unit) + with machine.nested(f"waiting for unit: {unit}"): + machine.wait_for_unit(unit) assert_no_failure(unit) @@ -41,13 +42,63 @@ def wait_for_open_port(address, port): retry(is_port_open) +### Test runner + +tests = OrderedDict() + + +def test(name): + def x(fn): + tests[name] = fn + + return x + + def run_tests(): - # Don't execute the following test suite when this script is running in interactive mode - if is_interactive: - raise Exception() + enabled = enabled_tests.copy() + to_run = [] + for test in tests: + if test in enabled: + enabled.remove(test) + to_run.append(test) + if enabled: + raise RuntimeError(f"The following tests are enabled but not defined: {enabled}") + machine.connect() # Visually separate boot output from the test output + for test in to_run: + with log.nested(f"test: {test}"): + tests[test]() - test_security() +def run_test(test): + tests[test]() + + +### Tests +# All tests are executed in the order they are defined here + + +@test("security") +def _(): + assert_running("setup-secrets") + # Unused secrets should be inaccessible + succeed('[[ $(stat -c "%U:%G %a" /secrets/dummy) = "root:root 440" ]]') + + if "secure-node" in enabled_tests: + # Access to '/proc' should be restricted + machine.succeed("grep -Fq hidepid=2 /proc/mounts") + + machine.wait_for_unit("bitcoind") + # `systemctl status` run by unprivileged users shouldn't leak cgroup info + assert_matches( + "sudo -u electrs systemctl status bitcoind 2>&1 >/dev/null", + "Failed to dump process list for 'bitcoind.service', ignoring: Access denied", + ) + # The 'operator' with group 'proc' has full access + assert_full_match("sudo -u operator systemctl status bitcoind 2>&1 >/dev/null", "") + + +@test("bitcoind") +def _(): assert_running("bitcoind") machine.wait_until_succeeds("bitcoin-cli getnetworkinfo") assert_matches("su operator -c 'bitcoin-cli getnetworkinfo' | jq", '"version"') @@ -59,6 +110,10 @@ def run_tests(): log_has_string("bitcoind", "RPC User public not allowed to call method stop") ) + +# Impure: Stops electrs +@test("electrs") +def _(): assert_running("electrs") wait_for_open_port(ip("electrs"), 4224) # prometeus metrics provider # Check RPC connection to bitcoind @@ -66,18 +121,30 @@ def run_tests(): # Stop electrs from spamming the test log with 'wait for bitcoind sync' messages succeed("systemctl stop electrs") + +@test("liquidd") +def _(): assert_running("liquidd") machine.wait_until_succeeds("elements-cli getnetworkinfo") assert_matches("su operator -c 'elements-cli getnetworkinfo' | jq", '"version"') succeed("su operator -c 'liquidswap-cli --help'") + +@test("clightning") +def _(): assert_running("clightning") assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"') + +@test("lnd") +def _(): assert_running("lnd") assert_matches("su operator -c 'lncli getinfo' | jq", '"version"') assert_no_failure("lnd") + +@test("lightning-loop") +def _(): assert_running("lightning-loop") assert_matches("su operator -c 'loop --version'", "version") # Check that lightning-loop fails with the right error, making sure @@ -89,6 +156,9 @@ def run_tests(): ) ) + +@test("btcpayserver") +def _(): assert_running("nbxplorer") machine.wait_until_succeeds(log_has_string("nbxplorer", "BTC: RPC connection successful")) wait_for_open_port(ip("nbxplorer"), 24444) @@ -103,11 +173,17 @@ def run_tests(): '"version"', ) + +@test("spark-wallet") +def _(): assert_running("spark-wallet") wait_for_open_port(ip("spark-wallet"), 9737) spark_auth = re.search("login=(.*)", succeed("cat /secrets/spark-wallet-login"))[1] assert_matches(f"curl -s {spark_auth}@{ip('spark-wallet')}:9737", "Spark") + +@test("lightning-charge") +def _(): assert_running("lightning-charge") wait_for_open_port(ip("lightning-charge"), 9112) machine.wait_until_succeeds(f"nc -z {ip('lightning-charge')} 9112") @@ -116,12 +192,16 @@ def run_tests(): f"curl -s api-token:{charge_auth}@{ip('lightning-charge')}:9112/info | jq", '"id"' ) + +@test("nanopos") +def _(): assert_running("nanopos") wait_for_open_port(ip("nanopos"), 9116) assert_matches(f"curl {ip('nanopos')}:9116", "tshirt") - assert_running("onion-chef") +@test("joinmarket") +def _(): assert_running("joinmarket") machine.wait_until_succeeds( log_has_string("joinmarket", "P2EPDaemonServerProtocolFactory starting on 27184") @@ -130,6 +210,11 @@ def run_tests(): log_has_string("joinmarket-yieldgenerator", "Failure to get blockheight",) ) + +@test("secure-node") +def _(): + assert_running("onion-chef") + # FIXME: use 'wait_for_unit' because 'create-web-index' always fails during startup due # to incomplete unit dependencies. # 'create-web-index' implicitly tests 'nodeinfo'. @@ -139,30 +224,62 @@ def run_tests(): assert_matches(f"curl {ip('nginx')}", "nix-bitcoin") assert_matches(f"curl -L {ip('nginx')}/store", "tshirt") - machine.wait_until_succeeds(log_has_string("bitcoind-import-banlist", "Importing node banlist")) - assert_no_failure("bitcoind-import-banlist") - ### Additional tests +# Run this test before the following tests that shut down services +# (and their corresponding network namespaces). +@test("netns-isolation") +def _(): + ping_bitcoind = "ip netns exec nb-bitcoind ping -c 1 -w 1" + ping_nanopos = "ip netns exec nb-nanopos ping -c 1 -w 1" + ping_nbxplorer = "ip netns exec nb-nbxplorer ping -c 1 -w 1" - # Current time in µs - pre_restart = succeed("date +%s.%6N").rstrip() - - # Sanity-check system by restarting all services - succeed( - "systemctl restart bitcoind clightning lnd lightning-loop spark-wallet lightning-charge nanopos liquidd" + # Positive ping tests (non-exhaustive) + machine.succeed( + "%s %s &&" % (ping_bitcoind, ip("bitcoind")) + + "%s %s &&" % (ping_bitcoind, ip("clightning")) + + "%s %s &&" % (ping_bitcoind, ip("lnd")) + + "%s %s &&" % (ping_bitcoind, ip("liquidd")) + + "%s %s &&" % (ping_bitcoind, ip("nbxplorer")) + + "%s %s &&" % (ping_nbxplorer, ip("btcpayserver")) + + "%s %s &&" % (ping_nanopos, ip("lightning-charge")) + + "%s %s &&" % (ping_nanopos, ip("nanopos")) + + "%s %s" % (ping_nanopos, ip("nginx")) ) - # Now that the bitcoind restart triggered a banlist import restart, check that - # re-importing already banned addresses works - machine.wait_until_succeeds( - log_has_string(f"bitcoind-import-banlist --since=@{pre_restart}", "Importing node banlist") + # Negative ping tests (non-exhaustive) + machine.fail( + "%s %s ||" % (ping_bitcoind, ip("spark-wallet")) + + "%s %s ||" % (ping_bitcoind, ip("lightning-loop")) + + "%s %s ||" % (ping_bitcoind, ip("lightning-charge")) + + "%s %s ||" % (ping_bitcoind, ip("nanopos")) + + "%s %s ||" % (ping_bitcoind, ip("recurring-donations")) + + "%s %s ||" % (ping_bitcoind, ip("nginx")) + + "%s %s ||" % (ping_nanopos, ip("bitcoind")) + + "%s %s ||" % (ping_nanopos, ip("clightning")) + + "%s %s ||" % (ping_nanopos, ip("lnd")) + + "%s %s ||" % (ping_nanopos, ip("lightning-loop")) + + "%s %s ||" % (ping_nanopos, ip("liquidd")) + + "%s %s ||" % (ping_nanopos, ip("electrs")) + + "%s %s ||" % (ping_nanopos, ip("spark-wallet")) + + "%s %s ||" % (ping_nanopos, ip("recurring-donations")) + + "%s %s" % (ping_nanopos, ip("btcpayserver")) ) - assert_no_failure("bitcoind-import-banlist") - prestop() + # test that netns-exec can't be run for unauthorized namespace + machine.fail("netns-exec nb-electrs ip a") - ### Test duplicity + # test that netns-exec drops capabilities + assert_full_match( + "su operator -c 'netns-exec nb-bitcoind capsh --print | grep Current '", "Current: =\n" + ) + # test that netns-exec can not be executed by users that are not operator + machine.fail("sudo -u clightning netns-exec nb-bitcoind ip a") + + +# Impure: stops bitcoind (and dependent services) +@test("backups") +def _(): succeed("systemctl stop bitcoind") succeed("systemctl start duplicity") machine.wait_until_succeeds(log_has_string("duplicity", "duplicity.service: Succeeded.")) @@ -181,23 +298,50 @@ def run_tests(): assert "var/backup/postgresql/btcpaydb.sql.gz" in files -def test_security(): - assert_running("setup-secrets") - # Unused secrets should be inaccessible - succeed('[[ $(stat -c "%U:%G %a" /secrets/dummy) = "root:root 440" ]]') +# Impure: restarts services +@test("banlist-and-restart") +def _(): + machine.wait_until_succeeds(log_has_string("bitcoind-import-banlist", "Importing node banlist")) + assert_no_failure("bitcoind-import-banlist") - # Access to '/proc' should be restricted - machine.succeed("grep -Fq hidepid=2 /proc/mounts") + # Current time in µs + pre_restart = succeed("date +%s.%6N").rstrip() - machine.wait_for_unit("bitcoind") - # `systemctl status` run by unprivileged users shouldn't leak cgroup info - assert_matches( - "sudo -u electrs systemctl status bitcoind 2>&1 >/dev/null", - "Failed to dump process list for 'bitcoind.service', ignoring: Access denied", + # Sanity-check system by restarting all services + succeed( + "systemctl restart bitcoind clightning lnd lightning-loop spark-wallet lightning-charge nanopos liquidd" ) - # The 'operator' with group 'proc' has full access - assert_full_match("sudo -u operator systemctl status bitcoind 2>&1 >/dev/null", "") + + # Now that the bitcoind restart triggered a banlist import restart, check that + # re-importing already banned addresses works + machine.wait_until_succeeds( + log_has_string(f"bitcoind-import-banlist --since=@{pre_restart}", "Importing node banlist") + ) + assert_no_failure("bitcoind-import-banlist") -def ip(_): - return "127.0.0.1" +if "netns-isolation" in enabled_tests: + netns_ips = { + "bitcoind": "169.254.1.12", + "clightning": "169.254.1.13", + "lnd": "169.254.1.14", + "liquidd": "169.254.1.15", + "electrs": "169.254.1.16", + "spark-wallet": "169.254.1.17", + "lightning-charge": "169.254.1.18", + "nanopos": "169.254.1.19", + "recurring-donations": "169.254.1.20", + "nginx": "169.254.1.21", + "lightning-loop": "169.254.1.22", + "nbxplorer": "169.254.1.23", + "btcpayserver": "169.254.1.24", + } + + def ip(netns): + return netns_ips[netns] + + +else: + + def ip(_): + return "127.0.0.1"