Merge fort-nix/nix-bitcoin#574: Add dev helper and docs

b4d7e1aa8f add dev helper and docs (Erik Arvstedt) b35d08d3f2 docs: move test docs from `examples/README` to `test/README` (Erik Arvstedt) 4d76eb9183 docs/configuration: fix typo (Erik Arvstedt) dc0710f3f4 tests: add example scenario `customTest` (Erik Arvstedt) 9e30d2728b tests: formatting (Erik Arvstedt) c6d85c6fe3 tests: fix broken unit file when clightning is disabled (Erik Arvstedt) a51f7b419e run-tests: use arg instead of env var for scenario overrides (Erik Arvstedt) Pull request description: ACKs for top commit: jonasnick: ACK b4d7e1aa8f Tree-SHA512: f0ed8f8fe326c64eac3b7e9f48597dd00eedb9244333e199d18d1c2c06f369cd015f77aefd48e87235a68aee0b352057249525bf015e0a564fda380bdf7bb9d1
2024-11-26 08:10:41 +03:00 · 2023-01-18 20:53:13 +00:00 · 2023-01-18 20:53:13 +00:00 · 84fc4d48d3
commit 84fc4d48d3
parent dfeff7b17b b4d7e1aa8f
16 changed files with 882 additions and 67 deletions
--- a/README.md
+++ b/README.md
@ -120,6 +120,10 @@ The nix-bitcoin security fund is a 2 of 3 bitcoin multisig address open for dona
 security researchers who discover vulnerabilities in nix-bitcoin or its upstream dependencies.\
 See [Security Fund](./SECURITY.md#nix-bitcoin-security-fund) for details.

+Developing
+---
+See [dev/README](./dev/README.md).
+
 Troubleshooting
 ---
 If you are having problems with nix-bitcoin check the [FAQ](docs/faq.md) or submit an issue.\
--- a/dev/README.md
+++ b/dev/README.md
@ -0,0 +1,104 @@
+This directory contains docs and helper scripts for developing and debugging:
+
+- [`dev.sh`](./dev.sh): misc dev helpers
+- [`dev-features.sh`](./dev-features.sh): helpers for developing specific
+  nix-bitcoin features, like services
+- [`topics`](./topics) features specific topics
+- [`dev-scenarios.nix`](./dev-scenarios.nix): extra test scenarios used in the above scripts
+
+See also: [test/README.md](../test/README.md)
+
+## Run a dev shell
+
+There are two ways to run a dev shell:
+
+### 1. Run command `nix develop`
+
+This starts a shell with [`test/run-tests.sh`](../test/run-tests.sh) and
+the scripts in dir [`helper`](../helper) added to `PATH`.
+
+### 2. Setup and start the `direnv` dev env
+
+This is an opinionated, [direnv](https://direnv.net/)-based dev env, optimized for developer experience.
+
+[`dev-env/create.sh`](./dev-env/create.sh) creates a git repo with the following contents:
+- Dir `src` which contains the nix-bitcoin repo
+- Dir `bin` for helper scripts
+- File `scenarios.nix` for custom test scenarios
+- File `.envrc` that defines a [direnv](https://direnv.net/) environment,
+  mainly for adding nix-bitcoin and helper scripts to `PATH`
+
+#### Installation
+
+1. [Install direnv](https://direnv.net/docs/installation.html).\
+   If you use NixOS (and Bash as the default shell), just add the following to your system config:
+   ```nix
+     environment.systemPackages = [ pkgs.direnv ];
+     programs.bash.interactiveShellInit = ''
+       eval "$(direnv hook bash)"
+     '';
+   ```
+
+2. Create the dev env:
+   ```bash
+   # Set up a dev environment in dir ~/dev/nix-bitcoin.
+   # The dir is created automatically.
+   ./dev-env/create.sh ~/dev/nix-bitcoin
+
+   cd ~/dev/nix-bitcoin
+
+   # Enable direnv
+   direnv allow
+   ```
+
+3. Optional: Editor integration
+   - Add envrc support to your editor
+   - Setup your editor so you can easily execute lines or paragraphs from a shell script
+     file in a shell.\
+     This simplifies using dev helper scripts like [`./dev.sh`](./dev.sh).
+
+#### Explore the dev env
+```bash
+# The direnv is automatically activated when visiting any subdir of ~/dev/nix-bitcoin
+cd ~/dev/nix-bitcoin
+
+ls -al . bin lib
+
+# The direnv config file
+cat .envrc
+
+# You can use this file to define extra scenarios
+cat scenarios.nix
+
+# Binary `dev-run-tests` runs nix-bitcoin's `run-tests.sh` with extra scenarios from ./scenarios.nix
+# Example:
+# Run command `nodeinfo` in `myscenario` (defined in ./scenarios.nix) via a container
+dev-run-tests -s myscenario container --run c nodeinfo
+
+# Equivalent (shorthand)
+te -s myscenario container --run c nodeinfo
+
+# Run the tests for `myscenario` in a VM
+te -s myscenario
+
+# Start an interactive shell inside a VM
+te -s myscenario vm
+```
+
+See also: [test/README.md](../test/README.md)
+
+## Adding a new service
+
+It's easiest to use an existing service as a template:
+- [electrs.nix](../modules/electrs.nix): a basic service
+- [clightning.nix](../modules/clightning.nix): simple, but covers a few more features.\
+  (A `cli` binary and a runtime-composed config to include secrets.)
+- [rtl.nix](../modules/rtl.nix): includes a custom package, defined in [pkgs/rtl](../pkgs/rtl).\
+  Most other services use packages that are already included in nixpkgs.
+
+## Switching to a new NixOS release
+
+- [flake.nix](../flake.nix): update `nixpkgs.url`
+- [cirrus.yml](../.cirrus.yml): update toplevel container -> image attribute
+- [examples/configuration.nix](../examples/configuration.nix): update `system.stateVersion`
+- Treewide: check if any `TODO-EXTERNAL` comments can be resolved
--- a/dev/dev-env/create.sh
+++ b/dev/dev-env/create.sh
@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2016
+set -euo pipefail
+
+destDir=${1:-nix-bitcoin}
+
+scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
+
+mkdir -p "$destDir/"{bin,lib}
+cd "$destDir"
+
+if [[ ! -e src ]]; then
+    echo "Cloning fort-nix/nix-bitcoin"
+    git clone https://github.com/fort-nix/nix-bitcoin src
+fi
+
+echo 'export root=$PWD
+export src=$root/src
+PATH_add bin
+PATH_add src/helper' > .envrc
+
+if [[ ! -e scenarios.nix ]]; then
+    cp "$scriptDir/template-scenarios.nix" scenarios.nix
+fi
+
+install -m 755 <(
+    echo '#!/usr/bin/env bash'
+    echo 'exec run-tests.sh --extra-scenarios "$root/scenarios.nix" "$@"'
+) bin/dev-run-tests
+
+install -m 755 <(
+    echo '#!/usr/bin/env bash'
+    echo 'exec $root/src/test/run-tests.sh --out-link-prefix /tmp/nix-bitcoin/test "$@"'
+) bin/run-tests.sh
+
+ln -sfn dev-run-tests bin/te
+
+## nix-bitcoin-firejail
+
+echo '# Add your shell config files here that should be accessible in the sandbox
+whitelist ${HOME}/.bashrc
+read-only ${HOME}/.bashrc' > lib/nix-bitcoin-firejail.conf
+
+install -m 755 <(
+    echo '#!/usr/bin/env bash'
+    echo '# A sandbox for running shells/binaries in an isolated environment:'
+    echo '# - The sandbox user is the calling user, with all capabilities dropped'
+    echo '#   and with no way to gain new privileges (e.g. via `sudo`).'
+    echo '# - $HOME is bind-mounted to a dir that only contains shell config files and files required by direnv.'
+    echo '#'
+    echo '# You can modify the firejail env by editing `lib/nix-bitcoin-firejail.conf` in your dev env dir.'
+    echo 'exec firejail --profile="$root/lib/nix-bitcoin-firejail.conf" --profile="$root/src/dev/dev-env/nix-bitcoin-firejail.conf" "$@"'
+) bin/nix-bitcoin-firejail
+
+echo "1" > lib/dev-env-version
+
+## git
+
+echo '/src' > .gitignore
+
+if [[ ! -e .git ]]; then
+    git init
+    git add .
+    git commit -a -m init
+fi
--- a/dev/dev-env/dev-shell.nix
+++ b/dev/dev-env/dev-shell.nix
@ -0,0 +1,16 @@
+pkgs:
+
+pkgs.mkShell {
+  shellHook = ''
+    # A known rev from the master branch to test whether `nix develop`
+    # is called inside the nix-bitcoin repo
+    rev=5cafafd02777919c10e559b5686237fdefe920c2
+    if git cat-file -e $rev &>/dev/null; then
+      root=$(git rev-parse --show-toplevel)
+      export PATH=$root/test:$root/helper:$PATH
+    else
+      echo 'Error: `nix develop` must be called inside the nix-bitcoin repo.'
+      exit 1
+    fi
+  '';
+}
--- a/dev/dev-env/nix-bitcoin-firejail.conf
+++ b/dev/dev-env/nix-bitcoin-firejail.conf
@ -0,0 +1,25 @@
+include default.local
+include globals.local
+
+include disable-common.inc
+include disable-programs.inc
+
+caps.drop all
+netfilter
+noinput
+nonewprivs
+noroot
+notv
+novideo
+protocol unix,inet,inet6
+seccomp
+
+## Enable features
+
+allow-debuggers
+
+# Enable direnv configs
+whitelist ${HOME}/.config/direnv
+read-only ${HOME}/.config/direnv
+whitelist ${HOME}/.local/share/direnv
+read-only ${HOME}/.local/share/direnv
--- a/dev/dev-env/template-scenarios.nix
+++ b/dev/dev-env/template-scenarios.nix
@ -0,0 +1,20 @@
+{ pkgs, lib, scenarios, nix-bitcoin }:
+with lib;
+rec {
+  # For more examples, see `scenarios` and `exampleScenarios` in ./src/test/tests.nix
+
+  template = { config, pkgs, lib, ... }: {
+    imports = [
+      (nix-bitcoin + "/modules/presets/secure-node.nix")
+      scenarios.netnsBase
+      scenarios.regtestBase
+    ];
+    test.container.enableWAN = true;
+    test.container.exposeLocalhost = true;
+  };
+
+  myscenario = { config, pkgs, lib, ... }: {
+    services.clightning.enable = true;
+    nix-bitcoin.nodeinfo.enable = true;
+  };
+}
--- a/dev/dev-features.sh
+++ b/dev/dev-features.sh
@ -0,0 +1,290 @@
+# shellcheck disable=SC2086,SC2154
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Run tests
+# See ../test/README.md for a tutorial
+# and ../test/run-tests.sh for a complete documentation
+
+# Start a shell in a container
+run-tests.sh -s electrs container
+
+# Run a command in a container.
+# The container is deleted afterwards.
+run-tests.sh -s electrs container --run c journalctl -u electrs
+
+# Run a bash command
+run-tests.sh -s bitcoind container --run c bash -c "sleep 1; journalctl -u bitcoind"
+
+run-tests.sh -s '{
+  imports = [ scenarios.regtestBase ];
+  services.electrs.enable = true;
+}' container --run c journalctl -u electrs
+
+run-tests.sh -s "{
+  services.electrs.enable = true;
+  nix-bitcoin.nodeinfo.enable = true;
+}" container --run c nodeinfo
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Get generic node infos
+
+# Start container shell
+run-tests.sh -s bitcoind container
+
+# Run commands inside the shell:
+
+# The node's services
+c systemctl status
+
+# Failed units
+c systemctl list-units --failed
+
+# Analyze container boot performance
+c systemd-analyze critical-chain
+
+# Listening TCP sockets
+c netstat -nltp
+# Listening sockets
+c netstat -nlp
+
+# The container root filesystem
+ls -al /var/lib/nixos-containers/nb-test
+
+# The container root filesystem on NixOS systems with stateVersion < 22.05
+ls -al /var/lib/containers/nb-test
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# bitcoind
+run-tests.sh -s bitcoind container
+
+c systemctl status bitcoind
+c systemctl cat bitcoind
+c journalctl --output=short-precise -u bitcoind
+ls -al /var/lib/nixos-containers/nb-test/var/lib/bitcoind
+c bitcoin-cli getpeerinfo
+c bitcoin-cli getnetworkinfo
+c bitcoin-cli getblockchaininfo
+
+run-tests.sh -s '{
+  imports = [ scenarios.regtestBase ];
+  services.bitcoind.enable = true;
+}' container
+
+address=$(c bitcoin-cli getnewaddress)
+echo $address
+c bitcoin-cli generatetoaddress 10 $address
+
+# Run bitcoind with network access
+run-tests.sh -s "{
+  test.container.enableWAN = true;
+  services.bitcoind.enable = true;
+}" container --run c journalctl -u bitcoind -f
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# clightning
+run-tests.sh -s clightning container
+
+c systemctl status clightning
+c journalctl --output=short-precise -u clightning
+c lightning-cli getinfo
+
+# Plugins
+run-tests.sh -s "{
+  services.clightning.enable = true;
+  test.features.clightningPlugins = true;
+}" container
+
+c lightning-cli plugin list
+
+# Show plugin config
+nix eval --raw .#makeTest --apply '
+  makeTest: let
+   config = (makeTest {
+      config = {
+        services.clightning.enable = true;
+        test.features.clightningPlugins = true;
+      };
+    }).nodes.machine;
+  in
+    config.services.clightning.extraConfig
+'
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# clightning-rest
+run-tests.sh -s clightning-rest container
+
+c systemctl status clightning-rest
+c journalctl -u clightning-rest
+c systemctl status clightning-rest-migrate-datadir
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# spark-wallet
+
+run-tests.sh -s "{
+  services.spark-wallet.enable = true;
+  test.container.exposeLocalhost = true;
+}" container
+
+c systemctl status spark-wallet
+c journalctl -u spark-wallet
+
+sparkAuth=$(c cat /secrets/spark-wallet-login | grep -ohP '(?<=login=).*')
+curl -v http://$sparkAuth@$ip:9737
+# Open in browser
+runuser -u "$(logname)" -- xdg-open http://$sparkAuth@$ip:9737
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# electrs
+
+run-tests.sh -s "{
+  imports = [ scenarios.regtestBase ];
+  services.electrs.enable = true;
+}" container
+
+c systemctl status electrs
+c systemctl cat electrs
+c journalctl --output=short-precise -u electrs
+
+electrs_rpc() {
+  echo "$1" | c nc 127.0.0.1 50001 | head -1 | jq
+}
+electrs_rpc '{"method": "server.version", "id": 0, "params": ["electrum/3.3.8", "1.4"]}'
+electrs_rpc '{"method": "blockchain.headers.subscribe", "id": 0}'
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# fulcrum
+
+run-tests.sh -s "{
+  imports = [ scenarios.regtestBase ];
+  services.fulcrum.enable = true;
+}" container
+
+c systemctl status fulcrum
+c systemctl cat fulcrum
+c journalctl --output=short-precise -u fulcrum
+
+fulcrum_rpc() {
+    echo "$1" | c nc 127.0.0.1 50002 | head -1 | jq
+}
+fulcrum_rpc '{"method": "server.version", "id": 0, "params": ["electrum/3.3.8", "1.4"]}'
+fulcrum_rpc '{"method": "blockchain.headers.subscribe", "id": 0}'
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# lnd
+run-tests.sh -s lnd container
+c systemctl status lnd
+c journalctl -u lnd
+c lncli getinfo
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# lightning-loop
+run-tests.sh -s lightning-loop container
+c systemctl status lightning-loop
+c journalctl -u lightning-loop
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# btcpayserver
+# https://docs.btcpayserver.org/Development/GreenFieldExample/
+
+run-tests.sh -s btcpayserver-regtest container
+
+c systemctl status btcpayserver
+c journalctl -u btcpayserver
+c systemctl cat btcpayserver
+
+c systemctl status nbxplorer
+c journalctl -u nbxplorer
+
+## Access the API
+request() {
+    local type=$1
+    local method=$2
+    local body=$3
+    shift; shift; shift
+    curl -sSL -H "Content-Type: application/json" -X $type --user "a@a.a:aaaaaa" \
+         -d "$body" "$@" "$ip:23000/api/v1/$method" | jq
+}
+post() {
+    local method=$1
+    local body=$2
+    shift; shift
+    request post "$method" "$body" "$@"
+}
+get() {
+    local method=$1
+    request get "$method"
+}
+
+# Create new user
+post users '{"email": "a@a.a", "password": "aaaaaa", "isAdministrator": true}'
+
+# Login with:
+# user: a@a.a
+# password: aaaaaa
+runuser -u "$(logname)" -- xdg-open http://$ip:23000
+
+# create store
+post stores '{"name": "a", "defaultPaymentMethod": "BTC_LightningNetwork"}'
+post stores '{"name": "a", "defaultPaymentMethod": "BTC"}'
+
+store=$(get stores | jq -r .[].id)
+echo $store
+get stores/$store
+get stores/$store/payment-methods
+get stores/$store/payment-methods/LightningNetwork
+
+# Connect to internal lightning node (internal API, doesn't work)
+# Lightning must be manually setup via the webinterface.
+post stores/$store/lightning/BTC/setup "" --data-raw 'LightningNodeType=Internal&ConnectionString=&command=save'
+
+nix run --inputs-from . nixpkgs#lynx -- --dump http://$ip:23000/embed/$store/BTC/ln
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# liquid
+run-tests.sh -s liquid container
+
+c systemctl status liquidd
+c elements-cli getpeerinfo
+c elements-cli getnetworkinfo
+c liquidswap-cli --help
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# tor
+run-tests.sh container
+
+c cat /var/lib/tor/state
+c ls -al /var/lib/tor/onion/
+c ls -al /var/lib/tor/onion/bitcoind
+c ls -al /var/lib/tor/onion/clightning-rest
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# joinmarket
+run-tests.sh -s joinmarket container
+
+c systemctl status joinmarket
+c journalctl -u joinmarket
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# joinmarket-ob-watcher
+
+# This starts a container with WAN access, so that jm-ob-watcher
+# can connect to the joinmarket IRC servers over Tor
+run-tests.sh -s jm-ob-watcher container
+
+c systemctl status joinmarket-ob-watcher
+c journalctl -u joinmarket-ob-watcher
+
+# Manually wait for string 'started http server, visit http://127.0.0.1:62601/'
+# This can take >10 minutes when the Tor network is under heavy load.
+# While connecting, errors like `We failed to connect and handshake with ANY directories...`
+# may be shown.
+c journalctl -f -u joinmarket-ob-watcher
+
+# Check webinterface
+c curl localhost:62601
+nix run --inputs-from . nixpkgs#lynx -- --dump $ip:62601
+c curl -s localhost:62601 | grep -i "orders found"
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# rtl
+# see ./topics/rtl.sh
--- a/dev/dev-scenarios.nix
+++ b/dev/dev-scenarios.nix
@ -0,0 +1,45 @@
+# Extra scenarios for developing and debugging
+
+{ lib, scenarios }:
+
+with lib;
+{
+  btcpayserver-regtest = {
+    imports = [ scenarios.regtestBase ];
+    services.btcpayserver.enable = true;
+    test.container.exposeLocalhost = true;
+    # services.btcpayserver.lbtc = false;
+  };
+
+  # A node with internet access to test joinmarket-ob-watcher
+  jm-ob-watcher = {
+    services.joinmarket-ob-watcher.enable = true;
+    # Don't download blocks
+    services.bitcoind.extraConfig = ''
+      connect = 0;
+    '';
+    test.container.exposeLocalhost = true;
+    test.container.enableWAN = true;
+  };
+
+  rtl-dev = { config, pkgs, lib, ... }: {
+    imports = [
+      # scenarios.netnsBase
+      # scenarios.regtestBase
+    ];
+    services.rtl = {
+      enable = true;
+      nodes.clightning = {
+        enable = true;
+        extraConfig.Settings.themeColor = "INDIGO";
+      };
+      # nodes.lnd.enable = false;
+      # services.rtl.nodes.reverseOrder = true;
+      nightTheme = true;
+      extraCurrency = "CHF";
+    };
+    test.container.exposeLocalhost = true;
+    nix-bitcoin.nodeinfo.enable = true;
+    # test.container.enableWAN = true;
+  };
+}
--- a/dev/dev.sh
+++ b/dev/dev.sh
@ -0,0 +1,128 @@
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Access nix-bitcoin flake packages
+
+function nb() {
+    nix build --no-link --print-out-paths --print-build-logs "$@"
+}
+
+# A package defined by nix-bitcoin
+nb .#joinmarket
+# Equivalent
+nb .#modulesPkgs.joinmarket
+
+# A nix-bitcoin python package
+nb .#nbPython3Packages.pyln-client
+
+# A pinned package from nixpkgs(-unstable)
+nb .#pinned.electrs
+# Equivalent
+nb .#modulesPkgs.electrs
+
+## Eval packages
+# Check version
+nix eval .#joinmarket.version
+
+# Eval derivation. --raw is needed due to a Nix bug (https://github.com/NixOS/nix/issues/5731)
+nix eval --raw .#joinmarket; echo
+
+# Check the version of a package in the nixpkgs(-unstable) inputs of the nix-bitcoin flake
+nix eval --inputs-from . nixpkgs#electrs.version
+nix eval --inputs-from . nixpkgs-unstable#electrs.version
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Inspect test systems
+
+# Build a test system
+nix build -o /tmp/system --print-build-logs "$(nix eval --raw .#tests.default --apply '
+  test: test.nodes.machine.system.build.toplevel.drvPath
+')"
+readlink /tmp/system
+# Inspect system files
+cat /tmp/system/activate
+cat /tmp/system/etc/system/bitcoind.service
+
+# Evaluate a config value
+nix eval .#tests.default --apply '
+  test: test.nodes.machine.services.bitcoind.rpc.port
+'
+
+# Evaluate a config value in a custom test
+nix eval .#makeTest --apply '
+  makeTest: let
+   config = (makeTest {
+      config = {
+        services.electrs.port = 10000;
+      };
+    }).nodes.machine;
+  in
+    config.services.electrs.port
+'
+
+# Evaluate a config value in a scenario defined in a file
+nix eval --impure .#getTest --apply '
+  getTest: let
+    config = (getTest {
+      name = "default";
+      extraScenariosFile = builtins.getEnv("root") + "/scenarios.nix";
+    }).nodes.machine;
+  in
+    config.services.bitcoind.port
+'
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Manually run a nix-bitcoin container, without the test framework.
+# This allows sharing directories with the container host via option `bindMounts.`
+
+read -rd '' src <<'EOF' || :
+  let
+    nix-bitcoin = builtins.getFlake "git+file://${toString ../.}";
+  in
+    nix-bitcoin.inputs.extra-container.lib.buildContainers {
+      system = "x86_64-linux";
+      inherit (nix-bitcoin.inputs) nixpkgs;
+      # legacyInstallDirs = true;
+      config = {
+        containers.nb-adhoc = {
+          # bindMounts."/shared" = { hostPath = "/my/hostpath"; isReadOnly = false; };
+          extra.addressPrefix = "10.200.255";
+          config = {
+            imports = [ nix-bitcoin.nixosModules.default ];
+            services.bitcoind.enable = true;
+            nix-bitcoin.generateSecrets = true;
+            nix-bitcoin.nodeinfo.enable = true;
+          };
+        };
+      };
+    }
+EOF
+nix run --impure --expr "$src"
+
+# Run command in container
+nix shell --impure --expr "$src" -c container --run c nodeinfo
+# TODO-EXTERNAL: Use this instead when https://github.com/NixOS/nix/issues/7444 is fixed
+# nix run --impure --expr "$src" -- --run c nodeinfo
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Show build logs from a CI test run
+#
+# If a specific test derivation was already built successfully, the test is not rerun
+# and the CI logs don't show the test output.
+
+# To view the test output:
+# 1. Get the test store path at the end of the CI logs
+# 2.
+fetch_build_log() {
+    local log=$1
+    nix cat-store --store https://nix-bitcoin.cachix.org "$log/output.xml" |
+        nix shell --inputs-from . nixpkgs#html-tidy -c tidy -xml -i - > /tmp/build-output.xml
+    echo
+    echo "Fetched log to /tmp/build-output.xml"
+}
+# Set this to your store path
+fetch_build_log /nix/store/0cdjhvg84jsp47f3357812zjmj2wmz94-vm-test-run-nix-bitcoin-default
+
+# Show runtime
+grep "script finished in" /tmp/build-output.xml
+
+# View XML with node folding
+firefox /tmp/build-output.xml
--- a/dev/topics/rtl.sh
+++ b/dev/topics/rtl.sh
@ -0,0 +1,44 @@
+# shellcheck disable=SC2154
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Docs
+# https://github.com/Ride-The-Lightning/RTL
+# config options: https://github.com/Ride-The-Lightning/RTL/blob/master/.github/docs/Application_configurations.md
+# https://github.com/Ride-The-Lightning/c-lightning-REST
+# local src: ~/s/RTL/
+
+# Browse API docs (in container shell)
+runuser -u "$(logname)" -- xdg-open "http://$ip:4001/api-docs/"
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Debug the service
+run-tests.sh -s rtl-dev container
+
+c systemctl status rtl
+c journalctl -u rtl
+c cat /var/lib/rtl/RTL-Config.json
+
+c systemctl status clightning-rest
+
+# Open webinterface. Password: a
+runuser -u "$(logname)" -- xdg-open "http://$ip:3000"
+
+#―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――
+# Build RTL manually
+# [[~/s/RTL/dockerfiles/Dockerfile][dockerfile]]
+# [[~/s/RTL/package.json][package.json]]
+
+rtl_src=~/s/RTL
+git clone https://github.com/Ride-The-Lightning/RTL "$rtl_src"
+
+nix build -o /tmp/nix-bitcoin-dev/nodejs --inputs-from . nixpkgs#nodejs-16_x
+# Start a shell in a sandbox
+env --chdir "$rtl_src" nix-bitcoin-firejail --whitelist="$rtl_src" --whitelist=/tmp/nix-bitcoin-dev/nodejs
+PATH=/tmp/nix-bitcoin-dev/nodejs/bin:"$PATH"
+
+# Install
+npm ci --omit=dev --omit=optional --no-update-notifier --ignore-scripts
+
+# Run
+node rtl --help
+
+git clean -xdf # Cleanup repo
--- a/docs/configuration.md
+++ b/docs/configuration.md
@ -135,7 +135,7 @@ You can use the same approach to allow connections to other services.
 ## Example: bitcoind

 ```shell
-# 1. Stop bitcoind on your nodes
+# 1. Stop bitcoind on your node
 ssh root@nix-bitcoin-node 'systemctl stop bitcoind'
 # Also stop bitcoind on the node that you'll be copying data from

--- a/examples/README.md
+++ b/examples/README.md
@ -51,60 +51,7 @@ the node:
 ```

 ### Tests
-The internal test suite is also useful for exploring features.\
-The following `run-tests.sh` commands leave no traces (outside of `/nix/store`) on
-the host system.
-
-`run-tests.sh` requires Nix >= 2.10.
-
-```bash
-git clone https://github.com/fort-nix/nix-bitcoin
-cd nix-bitcoin/test
-
-# Run a node in a VM. No tests are executed.
-./run-tests.sh vm
-systemctl status bitcoind
-
-# Run a Python test shell inside a VM node
-./run-tests.sh debug
-print(succeed("systemctl status bitcoind"))
-run_test("bitcoind")
-
-# Run a node in a container. Requires systemd and root privileges.
-./run-tests.sh container
-c systemctl status bitcoind
-
-# Explore a single feature
-./run-tests.sh --scenario electrs container
-
-# Run a command in a container
-./run-tests.sh --scenario '{
-  services.clightning.enable = true;
-  nix-bitcoin.nodeinfo.enable = true;
-}' container --run c nodeinfo
-```
-See [`run-tests.sh`](../test/run-tests.sh) for a complete documentation.
-
-#### Flakes
-Tests can also be directly accessed via Flakes:
-```bash
-# Build test
-nix build --no-link ..#tests.default
-
-# Run a node in a VM. No tests are executed.
-nix run ..#tests.default.vm
-
-# Run a Python test shell inside a VM node
-nix run ..#tests.default.run -- --debug
-
-# Run a node in a container. Requires extra-container, systemd and root privileges
-nix run ..#tests.default.container
-nix run ..#tests.default.containerLegacy # For NixOS with `system.stateVersion` <22.05
-
-# Run a command in a container
-nix run ..#tests.default.container -- --run c nodeinfo
-nix run ..#tests.default.containerLegacy -- --run c nodeinfo # For NixOS with `system.stateVersion` <22.05
-```
+The [nix-bitcoin test suite](../test/README.md) is also useful for exploring features.

 ### Real-world example
 Check the [server repo](https://github.com/fort-nix/nixbitcoin.org) for https://nixbitcoin.org
--- a/flake.nix
+++ b/flake.nix
@ -122,6 +122,8 @@
            program = toString packages.runVM;
          };
        };
+
+        devShells.default = import ./dev/dev-env/dev-shell.nix pkgs;
      }
    ));
 }
--- a/test/README.md
+++ b/test/README.md
@ -0,0 +1,84 @@
+The [`run-tests.sh`](./run-tests.sh) command is most convenient and versatile way to run tests.\
+It leave no traces (outside of `/nix/store`) on the host system.
+
+`run-tests.sh` requires Nix >= 2.10.
+
+### Summary
+```bash
+./run-tests.sh [--scenario|-s <scenario>] [build|vm|debug|container]
+```
+
+See the top of [run-tests.sh](../test/run-tests.sh) for a complete documentation.\
+Test scenarios are defined in [tests.nix](./tests.nix) and [tests.py](tests.py).
+
+### Tutorial
+#### Running tests
+```bash
+# Run the basic set of tests. These tests are also run on the GitHub CI server.
+./run-tests.sh
+
+# Run the test for scenario `regtest`.
+# The test is run via the Nix build system. Successful runs are cached.
+./run-tests.sh -s regtest build
+./run-tests.sh -s regtest # Shorthand, equivalent
+
+# To test a single service, use its name as a scenario.
+./run-tests.sh -s clightning
+
+# When no scenario is specified, scenario `default` is used.
+./run-tests.sh build
+```
+#### Debugging
+```bash
+# Start a shell is inside a test VM. No tests are executed.
+./run-tests.sh -s bitcoind vm
+systemctl status bitcoind
+
+# Run a Python NixOS test shell inside a VM.
+# See https://nixos.org/manual/nixos/stable/#ssec-machine-objects for available commands.
+./run-tests.sh debug
+print(succeed("systemctl status bitcoind"))
+run_test("bitcoind")
+
+# Start a shell in a container node. Requires systemd and root privileges.
+./run-tests.sh container
+
+# In the container shell: Run command in container (with prefix `c`)
+c systemctl status bitcoind
+
+# Explore a single feature
+./run-tests.sh -s electrs container
+
+# Run a command in a container.
+# The container is deleted afterwards.
+./run-tests.sh -s clightning container --run c lightning-cli getinfo
+
+# Define a custom scenario
+./run-tests.sh --scenario '{
+  services.clightning.enable = true;
+  nix-bitcoin.nodeinfo.enable = true;
+}' container --run c nodeinfo
+```
+
+# Running tests with Flakes
+
+Tests can also be accessed via the nix-bitcoin flake:
+
+```bash
+# Build test
+nix build --no-link ..#tests.default
+
+# Run a node in a VM. No tests are executed.
+nix run ..#tests.default.vm
+
+# Run a Python test shell inside a VM node
+nix run ..#tests.default.run -- --debug
+
+# Run a node in a container. Requires extra-container, systemd and root privileges
+nix run ..#tests.default.container
+nix run ..#tests.default.containerLegacy # For NixOS with `system.stateVersion` <22.05
+
+# Run a command in a container
+nix run ..#tests.default.container -- --run c nodeinfo
+nix run ..#tests.default.containerLegacy -- --run c nodeinfo # For NixOS with `system.stateVersion` <22.05
+```
--- a/test/run-tests.sh
+++ b/test/run-tests.sh
@ -4,7 +4,7 @@
 # The tests (defined in ./tests.nix) use the NixOS testing framework and are executed in a VM.
 #
 # Usage:
-#   Run all tests
+#   Run the basic set of tests. These are also run on the CI server.
 #   ./run-tests.sh
 #
 #   Test specific scenario
@ -57,7 +57,8 @@
 #     and reading source files.
 #     Files are copied to /tmp, a caching scheme helps minimizing copies.
 #
-#   To add custom scenarios, set the environment variable `scenarioOverridesFile`.
+#   Add custom scenarios from a file
+#   ./run-tests.sh --extra-scenarios ~/my/scenarios.nix ...

 set -eo pipefail

@ -66,6 +67,7 @@ scriptDir=$(cd "${BASH_SOURCE[0]%/*}" && pwd)
 args=("$@")
 scenario=
 outLinkPrefix=
+scenarioOverridesFile=
 while :; do
    case $1 in
        --scenario|-s)
@ -88,6 +90,16 @@ while :; do
                exit 1
            fi
            ;;
+        --extra-scenarios)
+            if [[ $2 ]]; then
+                scenarioOverridesFile=$2
+                shift
+                shift
+            else
+                >&2 echo "Error: $1 requires an argument."
+                exit 1
+            fi
+            ;;
        --copy-src|-c)
            shift
            if [[ ! $_nixBitcoinInCopiedSrc ]]; then
@ -113,7 +125,7 @@ makeTmpDir() {
 # Support explicit scenario definitions
 if [[ $scenario = *' '* ]]; then
    makeTmpDir
-    export scenarioOverridesFile=$tmpDir/scenario-overrides.nix
+    scenarioOverridesFile=$tmpDir/scenario-overrides.nix
    echo "{ scenarios, pkgs, lib, nix-bitcoin }: with lib; { tmp = $scenario; }" > "$scenarioOverridesFile"
    scenario=tmp
 fi
@ -195,7 +207,7 @@ buildTests() {
 nixInstantiateTest() {
    local attr=$1
    shift
-    if [[ ${scenarioOverridesFile:-} ]]; then
+    if [[ $scenarioOverridesFile ]]; then
        local file="extraScenariosFile = \"$scenarioOverridesFile\";"
    else
        local file=
--- a/test/tests.nix
+++ b/test/tests.nix
@ -53,12 +53,6 @@ let
          clboss.path = "${nbPkgs.clboss}/bin/clboss";
        };
      in map (plugin: pluginPkgs.${plugin}.path) enabled;
-      # Torified 'dig' subprocesses of clboss don't respond to SIGTERM and keep
-      # running for a long time when WAN is disabled, which prevents clightning units
-      # from stopping quickly.
-      # Set TimeoutStopSec for faster stopping.
-      systemd.services.clightning.serviceConfig.TimeoutStopSec =
-        mkIf config.services.clightning.plugins.clboss.enable "500ms";

      tests.clightning-rest = cfg.clightning-rest.enable;

@ -90,6 +84,8 @@ let
        };
      };

+      nix-bitcoin.onionServices.lnd.public = true;
+
      tests.lndconnect-onion-lnd = cfg.lnd.lndconnectOnion.enable;
      tests.lndconnect-onion-clightning = cfg.clightning-rest.lndconnectOnion.enable;

@ -97,7 +93,6 @@ let
      services.lightning-loop.certificate.extraIPs = [ "20.0.0.1" ];

      tests.lightning-pool = cfg.lightning-pool.enable;
-      nix-bitcoin.onionServices.lnd.public = true;

      tests.charge-lnd = cfg.charge-lnd.enable;

@ -140,6 +135,13 @@ let
      # Avoid timeout failures on slow CI nodes
      systemd.services.postgresql.serviceConfig.TimeoutStartSec = "5min";
    }
+    (mkIf config.services.clightning.plugins.clboss.enable {
+      # Torified 'dig' subprocesses of clboss don't respond to SIGTERM and keep
+      # running for a long time when WAN is disabled, which prevents clightning units
+      # from stopping quickly.
+      # Set TimeoutStopSec for faster stopping.
+      systemd.services.clightning.serviceConfig.TimeoutStopSec = "500ms";
+    })
    (mkIf config.test.features.clightningPlugins {
      services.clightning.plugins = {
        clboss.enable = true;
@ -313,7 +315,9 @@ let
      services.lnd.enable = true;
      services.bitcoind.prune = 1000;
    };
-  };
+  } // (import ../dev/dev-scenarios.nix {
+    inherit lib scenarios;
+  });

  ## Example scenarios that showcase extra features
  exampleScenarios = with lib; {
@ -333,6 +337,31 @@ let
      # See ./lib/test-lib.nix for a description
      test.container.exposeLocalhost = true;
    };
+
+    ## Scenarios with a custom Python test
+
+    # Variant 1: Define testing code that always runs
+    customTestSimple = {
+      networking.hostName = "myhost";
+
+      # Variant 1: Define testing code that always runs
+      test.extraTestScript = ''
+        succeed("[[ $(hostname) == myhost ]]")
+      '';
+    };
+
+    # Variant 2: Define a test that can be enabled/disabled
+    # via the Nix module system.
+    customTestExtended = {
+      networking.hostName = "myhost";
+
+      tests.hostName = true;
+      test.extraTestScript = ''
+        @test("hostName")
+        def _():
+            succeed("[[ $(hostname) == myhost ]]")
+      '';
+    };
  };
 in {
  inherit scenarios;