From 1de259485b76d787c5876829fc08da9064e03546 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sun, 6 Aug 2023 21:07:51 +0200 Subject: [PATCH] mempool: add module --- README.md | 7 +- dev/dev-scenarios.nix | 11 ++ dev/topics/mempool.sh | 22 +++ examples/README.md | 6 + examples/configuration.nix | 20 ++ modules/mempool.nix | 331 +++++++++++++++++++++++++++++++++ modules/modules.nix | 1 + modules/netns-isolation.nix | 12 +- modules/nodeinfo.nix | 6 + modules/onion-services.nix | 3 + modules/presets/enable-tor.nix | 1 + test/tests.nix | 7 + test/tests.py | 15 ++ 13 files changed, 435 insertions(+), 7 deletions(-) create mode 100644 dev/topics/mempool.sh create mode 100644 modules/mempool.nix diff --git a/README.md b/README.md index bbef7ef..24b6629 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ NixOS modules ([src](modules/modules.nix)) clightning [via WireGuard](./docs/services.md#use-zeus-mobile-lightning-wallet-via-wireguard) or [Tor](./docs/services.md#use-zeus-mobile-lightning-wallet-via-tor) * [Ride The Lightning](https://github.com/Ride-The-Lightning/RTL): web interface for `lnd` and `clightning` + * [mempool](https://github.com/mempool/mempool): Bitcoin visualizer, explorer, and API service * [electrs](https://github.com/romanz/electrs): Electrum server * [fulcrum](https://github.com/cculianu/Fulcrum): Electrum server (see [the module](modules/fulcrum.nix) for a comparison with electrs) * [btcpayserver](https://github.com/btcpayserver/btcpayserver) @@ -103,12 +104,6 @@ NixOS modules ([src](modules/modules.nix)) * [backups](modules/backups.nix): duplicity backups of all your node's important files * [operator](modules/operator.nix): configures a non-root user who has access to client tools (e.g. `bitcoin-cli`, `lightning-cli`) -### Extension modules -Extension modules are maintained in separate repositories and have their own review -and release process. - -* [Mempool](https://github.com/fort-nix/nix-bitcoin-mempool): Bitcoin visualizer, explorer and API service - Security --- See [SECURITY.md](SECURITY.md) for the security policy and how to report a vulnerability. diff --git a/dev/dev-scenarios.nix b/dev/dev-scenarios.nix index 21f5991..80e2ea2 100644 --- a/dev/dev-scenarios.nix +++ b/dev/dev-scenarios.nix @@ -94,4 +94,15 @@ with lib; test.container.enableWAN = true; }; + + mempool-regtest = { + imports = [ + scenarios.regtestBase + ]; + services.mempool = { + enable = true; + frontend.address = "0.0.0.0"; + }; + nix-bitcoin.nodeinfo.enable = true; + }; } diff --git a/dev/topics/mempool.sh b/dev/topics/mempool.sh new file mode 100644 index 0000000..fb4f4fb --- /dev/null +++ b/dev/topics/mempool.sh @@ -0,0 +1,22 @@ +# Start mempool container +run-tests.sh -s mempool-regtest container + +c systemctl status mempool +c systemctl status mysql +c nodeinfo + +# Check backend +c curl -fsS localhost:8999/api/v1/blocks/1 | jq +c curl -fsS localhost:8999/api/v1/blocks/tip/height | jq +c curl -fsS localhost:8999/api/v1/address/1CGG9qVq2P6F7fo6sZExvNq99Jv2GDpaLE | jq + +# Check frontend +c curl -fsS localhost:60845 +c curl -fsS localhost:60845/api/mempool | jq +c curl -fsS localhost:60845/api/blocks/1 | jq +c curl -fsS localhost:60845/api/v1/blocks/1 | jq +c curl -fsS localhost:60845/api/blocks/tip/height | jq + +# Open frontend +# shellcheck disable=SC2154 +runuser -u "$(logname)" -- xdg-open "http://$ip:60845/" diff --git a/examples/README.md b/examples/README.md index 6825c94..54dd887 100644 --- a/examples/README.md +++ b/examples/README.md @@ -63,3 +63,9 @@ The commands in `shell.nix` allow you to locally run the node in a VM or contain Flakes make it easy to include `nix-bitcoin` in an existing NixOS config. The [flakes example](./flakes/flake.nix) shows how to use `nix-bitcoin` as an input to a system flake. + +### Extending nix-bitcoin with Flakes + +The [mempool extension flake](https://github.com/fort-nix/nix-bitcoin-mempool) shows how to define new +pkgs and modules in a Flake.\ +Since mempool is now a core nix-bitcoin module, this Flake just serves as an example. diff --git a/examples/configuration.nix b/examples/configuration.nix index 5406e78..b6af8e2 100644 --- a/examples/configuration.nix +++ b/examples/configuration.nix @@ -126,6 +126,26 @@ # Automatically enables lightning-loop. # services.rtl.nodes.lnd.loop = true; + ### MEMPOOL + # Set this to enable mempool, a fully featured Bitcoin visualizer, explorer, + # and API service. + # + # services.mempool.enable = true; + # + # Possible options for the Electrum backend server: + # + # - electrs (enabled by default): + # Small database size, slow when querying new addresses. + # + # - fulcrum: + # Large database size, quickly serves arbitrary address queries. + # Enable with: + # services.mempool.electrumServer = "fulcrum"; + # + # Set this to create an onion service to make the mempool web interface + # available via Tor: + # nix-bitcoin.onionServices.mempool-frontend.enable = true; + ### ELECTRS # Set this to enable electrs, an Electrum server implemented in Rust. # services.electrs.enable = true; diff --git a/modules/mempool.nix b/modules/mempool.nix new file mode 100644 index 0000000..de0d0d4 --- /dev/null +++ b/modules/mempool.nix @@ -0,0 +1,331 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + options.services = { + mempool = { + enable = mkOption { + type = types.bool; + default = false; + description = mdDoc '' + Enable Mempool, a fully featured Bitcoin visualizer, explorer, and API service. + + Note: Mempool enables `txindex` in bitcoind (this is a requirement). + + This module has two components: + - A backend service (systemd service `mempool`) + + - An optional web interface run by nginx, defined by options `services.mempool.frontend.*`. + The frontend is enabled by default when mempool is enabled. + For details, see `services.mempool.frontend.enable`. + ''; + }; + + frontend = { + enable = mkOption { + type = types.bool; + default = cfg.enable; + description = mdDoc '' + Enable the mempool frontend (web interface). + This starts a simple nginx instance, configured for local usage with + settings similar to the `mempool/frontend` Docker image. + + IMPORTANT: + If you want to expose the mempool frontend to the internet, you + should create a custom nginx config that includes TLS, backend caching, rate limiting + and performance tuning. + For this task, reuse the config snippets from option `services.mempool.frontend.nginxConfig`. + See also: https://github.com/fort-nix/nixbitcoin.org/blob/master/website/mempool.nix, + which contains a mempool nginx config for public hosting (running at + https://mempool.nixbitcoin.org). + ''; + }; + address = mkOption { + type = types.str; + default = "127.0.0.1"; + description = mdDoc "HTTP server address."; + }; + port = mkOption { + type = types.port; + default = 60845; # A random private port + description = mdDoc "HTTP server port."; + }; + staticContentRoot = mkOption { + type = types.path; + default = nbPkgs.mempool-frontend; + defaultText = "config.nix-bitcoin.pkgs.mempool-frontend"; + description = mdDoc " + Path of the static frontend content root. + "; + }; + nginxConfig = mkOption { + readOnly = true; + default = frontend.nginxConfig; + defaultText = "(See source)"; + description = mdDoc " + An attrset of nginx config snippets for assembling a custom + mempool nginx config. + For details, see the source comments at the point of definition. + "; + }; + }; + + address = mkOption { + type = types.str; + default = "127.0.0.1"; + description = mdDoc "Mempool backend address."; + }; + port = mkOption { + type = types.port; + default = 8999; + description = mdDoc "Mempool backend port."; + }; + electrumServer = mkOption { + type = types.enum [ "electrs" "fulcrum" ]; + default = "electrs"; + description = mdDoc '' + The Electrum server to use for fetching address information. + + Possible options: + - electrs: + Small database size, slow when querying new addresses. + - fulcrum: + Large database size, quickly serves arbitrary address queries. + ''; + }; + settings = mkOption { + type = with types; attrsOf (attrsOf anything); + example = { + MEMPOOL = { + POLL_RATE_MS = 3000; + STDOUT_LOG_MIN_PRIORITY = "debug"; + }; + PRICE_DATA_SERVER = { + CLEARNET_URL = "https://myserver.org/prices"; + }; + }; + description = mdDoc '' + Mempool backend settings. + See here for possible options: + https://github.com/mempool/mempool/blob/master/backend/src/config.ts + ''; + }; + database = { + name = mkOption { + type = types.str; + default = "mempool"; + description = mdDoc "Database name."; + }; + }; + package = mkOption { + type = types.package; + default = nbPkgs.mempool-backend; + defaultText = "config.nix-bitcoin.pkgs.mempool-backend"; + description = mdDoc "The package providing mempool binaries."; + }; + user = mkOption { + type = types.str; + default = "mempool"; + description = mdDoc "The user as which to run Mempool."; + }; + group = mkOption { + type = types.str; + default = cfg.user; + description = mdDoc "The group as which to run Mempool."; + }; + tor = nbLib.tor; + }; + + # Internal read-only options used by `./nodeinfo.nix` and `./onion-services.nix` + mempool-frontend = let + mkAlias = default: mkOption { + internal = true; + readOnly = true; + inherit default; + }; + in { + enable = mkAlias cfg.frontend.enable; + address = mkAlias cfg.frontend.address; + port = mkAlias cfg.frontend.port; + }; + }; + + cfg = config.services.mempool; + nbLib = config.nix-bitcoin.lib; + nbPkgs = config.nix-bitcoin.pkgs; + secretsDir = config.nix-bitcoin.secretsDir; + + configFile = builtins.toFile "mempool-config" (builtins.toJSON cfg.settings); + cacheDir = "/var/cache/mempool"; + + inherit (config.services) + bitcoind + electrs + fulcrum; + + torSocket = config.services.tor.client.socksListenAddress; + + # See the `services.nginx` definition further below below + # on how to use these snippets. + frontend.nginxConfig = { + # This must be added to `services.nginx.commonHttpConfig` when + # `mempool/location-static.conf` is used + httpConfig = '' + include ${nbPkgs.mempool-nginx-conf}/mempool/http-language.conf; + ''; + + # This should be added to `services.nginx.virtualHosts..extraConfig` + staticContent = '' + index index.html; + + add_header Cache-Control "public, no-transform"; + add_header Vary Accept-Language; + add_header Vary Cookie; + + include ${nbPkgs.mempool-nginx-conf}/mempool/location-static.conf; + + # Redirect /api to /docs/api + location = /api { + return 308 https://$host/docs/api; + } + location = /api/ { + return 308 https://$host/docs/api; + } + ''; + + # This should be added to `services.nginx.virtualHosts..extraConfig` + proxyApi = let + backend = "http://${nbLib.addressWithPort cfg.address cfg.port}"; + in '' + location /api/ { + proxy_pass ${backend}/api/v1/; + } + location /api/v1 { + proxy_pass ${backend}; + } + # Websocket API + location /api/v1/ws { + proxy_pass ${backend}; + + # Websocket header settings + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + # Relevant settings from `recommendedProxyConfig` (nixos/nginx/default.nix) + # (In the above api locations, this are inherited from the parent scope) + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + ''; + }; + +in { + inherit options; + + config = mkIf cfg.enable { + services.bitcoind.txindex = true; + services.electrs.enable = mkIf (cfg.electrumServer == "electrs" ) true; + services.fulcrum.enable = mkIf (cfg.electrumServer == "fulcrum" ) true; + services.mysql = { + enable = true; + package = pkgs.mariadb; + ensureDatabases = [ cfg.database.name ]; + ensureUsers = [ + { + name = cfg.user; + ensurePermissions."${cfg.database.name}.*" = "ALL PRIVILEGES"; + } + ]; + }; + + # Available options: + # https://github.com/mempool/mempool/blob/master/backend/src/config.ts + services.mempool.settings = { + MEMPOOL = { + # mempool doesn't support regtest + NETWORK = "mainnet"; + BACKEND = "electrum"; + HTTP_PORT = cfg.port; + CACHE_DIR = "${cacheDir}/cache"; + STDOUT_LOG_MIN_PRIORITY = mkDefault "info"; + }; + CORE_RPC = { + HOST = bitcoind.rpc.address; + PORT = bitcoind.rpc.port; + USERNAME = bitcoind.rpc.users.public.name; + PASSWORD = "@btcRpcPassword@"; + }; + ELECTRUM = let + server = config.services.${cfg.electrumServer}; + in { + HOST = server.address; + PORT = server.port; + TLS_ENABLED = false; + }; + DATABASE = { + ENABLED = true; + DATABASE = cfg.database.name; + SOCKET = "/run/mysqld/mysqld.sock"; + }; + } // optionalAttrs (cfg.tor.proxy) { + # Use Tor for rate fetching + SOCKS5PROXY = { + ENABLED = true; + USE_ONION = true; + HOST = torSocket.addr; + PORT = torSocket.port; + }; + }; + + systemd.services.mempool = { + wantedBy = [ "multi-user.target" ]; + requires = [ "${cfg.electrumServer}.service" ]; + after = [ "${cfg.electrumServer}.service" "mysql.service" ]; + preStart = '' + mkdir -p '${cacheDir}/cache' + <${configFile} sed \ + -e "s|@btcRpcPassword@|$(cat ${secretsDir}/bitcoin-rpcpassword-public)|" \ + > '${cacheDir}/config.json' + ''; + environment.MEMPOOL_CONFIG_FILE = "${cacheDir}/config.json"; + serviceConfig = nbLib.defaultHardening // { + ExecStart = "${cfg.package}/bin/mempool-backend"; + CacheDirectory = "mempool"; + CacheDirectoryMode = "770"; + # Show "mempool" instead of "node" in the journal + SyslogIdentifier = "mempool"; + User = cfg.user; + Restart = "on-failure"; + RestartSec = "10s"; + } // nbLib.allowedIPAddresses cfg.tor.enforce + // nbLib.nodejs; + }; + + services.nginx = mkIf cfg.frontend.enable { + enable = true; + enableReload = true; + recommendedGzipSettings = true; + recommendedOptimisation = true; + recommendedProxySettings = true; + recommendedTlsSettings = true; + commonHttpConfig = frontend.nginxConfig.httpConfig; + virtualHosts."mempool" = { + serverName = "_"; + listen = [ { addr = cfg.frontend.address; port = cfg.frontend.port; } ]; + root = cfg.frontend.staticContentRoot; + extraConfig = + frontend.nginxConfig.staticContent + + frontend.nginxConfig.proxyApi; + }; + }; + + users.users.${cfg.user} = { + isSystemUser = true; + group = cfg.group; + extraGroups = [ "bitcoinrpc-public" ]; + }; + users.groups.${cfg.group} = {}; + }; +} diff --git a/modules/modules.nix b/modules/modules.nix index 1c4d288..1b3c204 100644 --- a/modules/modules.nix +++ b/modules/modules.nix @@ -20,6 +20,7 @@ ./charge-lnd.nix ./lndconnect.nix # Requires onion-addresses.nix ./rtl.nix + ./mempool.nix ./electrs.nix ./fulcrum.nix ./liquid.nix diff --git a/modules/netns-isolation.nix b/modules/netns-isolation.nix index fcd7c99..832bb94 100644 --- a/modules/netns-isolation.nix +++ b/modules/netns-isolation.nix @@ -295,7 +295,14 @@ in { id = 31; connections = [ "bitcoind" ]; }; - # id = 32 reserved for the upcoming mempool module + mempool = { + id = 32; + connections = [ + "bitcoind" + "nginx" + (if (config.services.mempool.electrumServer == "electrs") then "electrs" else "fulcrum") + ]; + }; }; services.bitcoind = { @@ -349,6 +356,9 @@ in { services.rtl.address = netns.rtl.address; services.clightning-rest.address = netns.clightning-rest.address; + + services.mempool.address = netns.mempool.address; + services.mempool.frontend.address = netns.nginx.address; } ]); } diff --git a/modules/nodeinfo.nix b/modules/nodeinfo.nix index d01eaed..f219664 100644 --- a/modules/nodeinfo.nix +++ b/modules/nodeinfo.nix @@ -149,6 +149,12 @@ in { liquidd = mkInfo ""; joinmarket-ob-watcher = mkInfo ""; rtl = mkInfo ""; + mempool = mkInfo ""; + mempool-frontend = name: cfg: mkInfoLong { + inherit name cfg; + systemdServiceName = "nginx"; + extraCode = ""; + }; # Only add sshd when it has an onion service sshd = name: cfg: mkIfOnionPort "sshd" (onionPort: '' add_service("sshd", """info["onion_address"] = get_onion_address("sshd", ${onionPort})""") diff --git a/modules/onion-services.nix b/modules/onion-services.nix index a98b2c0..042ffb5 100644 --- a/modules/onion-services.nix +++ b/modules/onion-services.nix @@ -113,6 +113,9 @@ in { rtl = { externalPort = 80; }; + mempool-frontend = { + externalPort = 80; + }; }; } ]; diff --git a/modules/presets/enable-tor.nix b/modules/presets/enable-tor.nix index 084494b..47766f2 100644 --- a/modules/presets/enable-tor.nix +++ b/modules/presets/enable-tor.nix @@ -27,6 +27,7 @@ in { # disable Tor enforcement until btcpayserver can fetch rates over Tor # btcpayserver = defaultEnableTorProxy; lightning-pool = defaultEnableTorProxy; + mempool = defaultEnableTorProxy; # These services don't make outgoing connections # (or use Tor by default in case of joinmarket) diff --git a/test/tests.nix b/test/tests.nix index cf2cdfd..578a3f8 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -78,6 +78,9 @@ let echo a > rtl-password ''); + tests.mempool = cfg.mempool.enable; + services.mempool.electrumServer = "fulcrum"; + tests.lnd = cfg.lnd.enable; services.lnd = { port = 9736; @@ -195,6 +198,7 @@ let services.lightning-loop.enable = true; services.lightning-pool.enable = true; services.charge-lnd.enable = true; + services.mempool.enable = true; services.electrs.enable = true; services.fulcrum.enable = true; services.liquidd.enable = true; @@ -219,6 +223,8 @@ let tests.secure-node = true; tests.restart-bitcoind = true; + nix-bitcoin.onionServices.mempool-frontend.enable = true; + # Stop electrs from spamming the test log with 'WARN - wait until IBD is over' messages tests.stop-electrs = true; }; @@ -237,6 +243,7 @@ let services.clightning-rest.enable = true; services.liquidd.enable = true; services.rtl.enable = true; + services.mempool.enable = true; services.lnd.enable = true; services.lightning-loop.enable = true; services.lightning-pool.enable = true; diff --git a/test/tests.py b/test/tests.py index a106942..784a5e6 100644 --- a/test/tests.py +++ b/test/tests.py @@ -251,6 +251,15 @@ def _(): log_has_string("clightning-rest", "cl-rest api server is ready and listening") ) +@test("mempool") +def _(): + assert_running("mempool") + assert_running("nginx") + machine.wait_until_succeeds( + log_has_string("mempool", "Mempool Server is running on port 8999") + ) + assert_matches(f"curl -L {ip('nginx')}:60845", "mempool - Bitcoin Explorer") + @test("joinmarket") def _(): assert_running("joinmarket") @@ -427,6 +436,12 @@ def _(): if enabled("btcpayserver"): machine.wait_until_succeeds(log_has_string("nbxplorer", f"At height: {num_blocks}")) + if enabled("mempool"): + assert_running("nginx") + assert_full_match( + f"curl -fsS http://{ip('nginx')}:60845/api/v1/blocks/tip/height", str(num_blocks) + ) + @test("trustedcoin") def _(): def expect_clightning_log(str):