mempool: add module

README.md

@@ -90,6 +90,7 @@ NixOS modules ([src](modules/modules.nix))
     clightning [via WireGuard](./docs/services.md#use-zeus-mobile-lightning-wallet-via-wireguard) or
     [Tor](./docs/services.md#use-zeus-mobile-lightning-wallet-via-tor)
   * [Ride The Lightning](https://github.com/Ride-The-Lightning/RTL): web interface for `lnd` and `clightning`
+  * [mempool](https://github.com/mempool/mempool): Bitcoin visualizer, explorer, and API service
   * [electrs](https://github.com/romanz/electrs): Electrum server
   * [fulcrum](https://github.com/cculianu/Fulcrum): Electrum server (see [the module](modules/fulcrum.nix) for a comparison with electrs)
   * [btcpayserver](https://github.com/btcpayserver/btcpayserver)
@@ -103,12 +104,6 @@ NixOS modules ([src](modules/modules.nix))
   * [backups](modules/backups.nix): duplicity backups of all your node's important files
   * [operator](modules/operator.nix): configures a non-root user who has access to client tools (e.g. `bitcoin-cli`, `lightning-cli`)
 
-### Extension modules
-Extension modules are maintained in separate repositories and have their own review
-and release process.
-
-* [Mempool](https://github.com/fort-nix/nix-bitcoin-mempool): Bitcoin visualizer, explorer and API service
-
 Security
 ---
 See [SECURITY.md](SECURITY.md) for the security policy and how to report a vulnerability.

dev/dev-scenarios.nix

@@ -94,4 +94,15 @@ with lib;
 
     test.container.enableWAN = true;
   };
+
+  mempool-regtest = {
+    imports = [
+      scenarios.regtestBase
+    ];
+    services.mempool = {
+      enable = true;
+      frontend.address = "0.0.0.0";
+    };
+    nix-bitcoin.nodeinfo.enable = true;
+  };
 }

dev/topics/mempool.sh

@@ -0,0 +1,22 @@
+# Start mempool container
+run-tests.sh -s mempool-regtest container
+
+c systemctl status mempool
+c systemctl status mysql
+c nodeinfo
+
+# Check backend
+c curl -fsS localhost:8999/api/v1/blocks/1 | jq
+c curl -fsS localhost:8999/api/v1/blocks/tip/height | jq
+c curl -fsS localhost:8999/api/v1/address/1CGG9qVq2P6F7fo6sZExvNq99Jv2GDpaLE | jq
+
+# Check frontend
+c curl -fsS localhost:60845
+c curl -fsS localhost:60845/api/mempool | jq
+c curl -fsS localhost:60845/api/blocks/1 | jq
+c curl -fsS localhost:60845/api/v1/blocks/1 | jq
+c curl -fsS localhost:60845/api/blocks/tip/height | jq
+
+# Open frontend
+# shellcheck disable=SC2154
+runuser -u "$(logname)" -- xdg-open "http://$ip:60845/"

examples/README.md

@@ -63,3 +63,9 @@ The commands in `shell.nix` allow you to locally run the node in a VM or contain
 
 Flakes make it easy to include `nix-bitcoin` in an existing NixOS config.
 The [flakes example](./flakes/flake.nix) shows how to use `nix-bitcoin` as an input to a system flake.
+
+### Extending nix-bitcoin with Flakes
+
+The [mempool extension flake](https://github.com/fort-nix/nix-bitcoin-mempool) shows how to define new
+pkgs and modules in a Flake.\
+Since mempool is now a core nix-bitcoin module, this Flake just serves as an example.

examples/configuration.nix

@@ -126,6 +126,26 @@
   # Automatically enables lightning-loop.
   # services.rtl.nodes.lnd.loop = true;
 
+  ### MEMPOOL
+  # Set this to enable mempool, a fully featured Bitcoin visualizer, explorer,
+  # and API service.
+  #
+  # services.mempool.enable = true;
+  #
+  # Possible options for the Electrum backend server:
+  #
+  # - electrs (enabled by default):
+  #   Small database size, slow when querying new addresses.
+  #
+  # - fulcrum:
+  #   Large database size, quickly serves arbitrary address queries.
+  #   Enable with:
+  #     services.mempool.electrumServer = "fulcrum";
+  #
+  # Set this to create an onion service to make the mempool web interface
+  # available via Tor:
+  # nix-bitcoin.onionServices.mempool-frontend.enable = true;
+
   ### ELECTRS
   # Set this to enable electrs, an Electrum server implemented in Rust.
   # services.electrs.enable = true;

modules/mempool.nix

@@ -0,0 +1,331 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  options.services = {
+    mempool = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = mdDoc ''
+          Enable Mempool, a fully featured Bitcoin visualizer, explorer, and API service.
+
+          Note: Mempool enables `txindex` in bitcoind (this is a requirement).
+
+          This module has two components:
+          - A backend service (systemd service `mempool`)
+
+          - An optional web interface run by nginx, defined by options `services.mempool.frontend.*`.
+            The frontend is enabled by default when mempool is enabled.
+            For details, see `services.mempool.frontend.enable`.
+        '';
+      };
+
+      frontend = {
+        enable = mkOption {
+          type = types.bool;
+          default = cfg.enable;
+          description = mdDoc ''
+            Enable the mempool frontend (web interface).
+            This starts a simple nginx instance, configured for local usage with
+            settings similar to the `mempool/frontend` Docker image.
+
+            IMPORTANT:
+            If you want to expose the mempool frontend to the internet, you
+            should create a custom nginx config that includes TLS, backend caching, rate limiting
+            and performance tuning.
+            For this task, reuse the config snippets from option `services.mempool.frontend.nginxConfig`.
+            See also: https://github.com/fort-nix/nixbitcoin.org/blob/master/website/mempool.nix,
+            which contains a mempool nginx config for public hosting (running at
+            https://mempool.nixbitcoin.org).
+          '';
+        };
+        address = mkOption {
+          type = types.str;
+          default = "127.0.0.1";
+          description = mdDoc "HTTP server address.";
+        };
+        port = mkOption {
+          type = types.port;
+          default = 60845; # A random private port
+          description = mdDoc "HTTP server port.";
+        };
+        staticContentRoot = mkOption {
+          type = types.path;
+          default = nbPkgs.mempool-frontend;
+          defaultText = "config.nix-bitcoin.pkgs.mempool-frontend";
+          description = mdDoc "
+            Path of the static frontend content root.
+          ";
+        };
+        nginxConfig = mkOption {
+          readOnly = true;
+          default = frontend.nginxConfig;
+          defaultText = "(See source)";
+          description = mdDoc "
+            An attrset of nginx config snippets for assembling a custom
+            mempool nginx config.
+            For details, see the source comments at the point of definition.
+          ";
+        };
+      };
+
+      address = mkOption {
+        type = types.str;
+        default = "127.0.0.1";
+        description = mdDoc "Mempool backend address.";
+      };
+      port = mkOption {
+        type = types.port;
+        default = 8999;
+        description = mdDoc "Mempool backend port.";
+      };
+      electrumServer = mkOption {
+        type = types.enum [ "electrs" "fulcrum" ];
+        default = "electrs";
+        description = mdDoc ''
+          The Electrum server to use for fetching address information.
+
+          Possible options:
+          - electrs:
+            Small database size, slow when querying new addresses.
+          - fulcrum:
+            Large database size, quickly serves arbitrary address queries.
+        '';
+      };
+      settings = mkOption {
+        type = with types; attrsOf (attrsOf anything);
+        example = {
+          MEMPOOL = {
+            POLL_RATE_MS = 3000;
+            STDOUT_LOG_MIN_PRIORITY = "debug";
+          };
+          PRICE_DATA_SERVER = {
+            CLEARNET_URL = "https://myserver.org/prices";
+          };
+        };
+        description = mdDoc ''
+          Mempool backend settings.
+          See here for possible options:
+          https://github.com/mempool/mempool/blob/master/backend/src/config.ts
+        '';
+      };
+      database = {
+        name = mkOption {
+          type = types.str;
+          default = "mempool";
+          description = mdDoc "Database name.";
+        };
+      };
+      package = mkOption {
+        type = types.package;
+        default = nbPkgs.mempool-backend;
+        defaultText = "config.nix-bitcoin.pkgs.mempool-backend";
+        description = mdDoc "The package providing mempool binaries.";
+      };
+      user = mkOption {
+        type = types.str;
+        default = "mempool";
+        description = mdDoc "The user as which to run Mempool.";
+      };
+      group = mkOption {
+        type = types.str;
+        default = cfg.user;
+        description = mdDoc "The group as which to run Mempool.";
+      };
+      tor = nbLib.tor;
+    };
+
+    # Internal read-only options used by `./nodeinfo.nix` and `./onion-services.nix`
+    mempool-frontend = let
+      mkAlias = default: mkOption {
+        internal = true;
+        readOnly = true;
+        inherit default;
+      };
+    in {
+      enable = mkAlias cfg.frontend.enable;
+      address = mkAlias cfg.frontend.address;
+      port = mkAlias cfg.frontend.port;
+    };
+  };
+
+  cfg = config.services.mempool;
+  nbLib = config.nix-bitcoin.lib;
+  nbPkgs = config.nix-bitcoin.pkgs;
+  secretsDir = config.nix-bitcoin.secretsDir;
+
+  configFile = builtins.toFile "mempool-config" (builtins.toJSON cfg.settings);
+  cacheDir = "/var/cache/mempool";
+
+  inherit (config.services)
+    bitcoind
+    electrs
+    fulcrum;
+
+  torSocket = config.services.tor.client.socksListenAddress;
+
+  # See the `services.nginx` definition further below below
+  # on how to use these snippets.
+  frontend.nginxConfig = {
+    # This must be added to `services.nginx.commonHttpConfig` when
+    # `mempool/location-static.conf` is used
+    httpConfig = ''
+      include ${nbPkgs.mempool-nginx-conf}/mempool/http-language.conf;
+    '';
+
+    # This should be added to `services.nginx.virtualHosts.<mempool server name>.extraConfig`
+    staticContent = ''
+      index index.html;
+
+      add_header Cache-Control "public, no-transform";
+      add_header Vary Accept-Language;
+      add_header Vary Cookie;
+
+      include ${nbPkgs.mempool-nginx-conf}/mempool/location-static.conf;
+
+      # Redirect /api to /docs/api
+      location = /api {
+        return 308 https://$host/docs/api;
+      }
+      location = /api/ {
+        return 308 https://$host/docs/api;
+      }
+    '';
+
+    # This should be added to `services.nginx.virtualHosts.<mempool server name>.extraConfig`
+    proxyApi = let
+      backend = "http://${nbLib.addressWithPort cfg.address cfg.port}";
+    in ''
+      location /api/ {
+          proxy_pass ${backend}/api/v1/;
+      }
+      location /api/v1 {
+          proxy_pass ${backend};
+      }
+      # Websocket API
+      location /api/v1/ws {
+          proxy_pass ${backend};
+
+          # Websocket header settings
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "Upgrade";
+
+          # Relevant settings from `recommendedProxyConfig` (nixos/nginx/default.nix)
+          # (In the above api locations, this are inherited from the parent scope)
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header X-Forwarded-Proto $scheme;
+      }
+    '';
+  };
+
+in {
+  inherit options;
+
+  config = mkIf cfg.enable {
+    services.bitcoind.txindex = true;
+    services.electrs.enable = mkIf (cfg.electrumServer == "electrs" ) true;
+    services.fulcrum.enable = mkIf (cfg.electrumServer == "fulcrum" ) true;
+    services.mysql = {
+      enable = true;
+      package = pkgs.mariadb;
+      ensureDatabases = [ cfg.database.name ];
+      ensureUsers = [
+        {
+          name = cfg.user;
+          ensurePermissions."${cfg.database.name}.*" = "ALL PRIVILEGES";
+        }
+      ];
+    };
+
+    # Available options:
+    # https://github.com/mempool/mempool/blob/master/backend/src/config.ts
+    services.mempool.settings = {
+      MEMPOOL = {
+        # mempool doesn't support regtest
+        NETWORK = "mainnet";
+        BACKEND = "electrum";
+        HTTP_PORT = cfg.port;
+        CACHE_DIR = "${cacheDir}/cache";
+        STDOUT_LOG_MIN_PRIORITY = mkDefault "info";
+      };
+      CORE_RPC = {
+        HOST = bitcoind.rpc.address;
+        PORT = bitcoind.rpc.port;
+        USERNAME = bitcoind.rpc.users.public.name;
+        PASSWORD = "@btcRpcPassword@";
+      };
+      ELECTRUM = let
+        server = config.services.${cfg.electrumServer};
+      in {
+        HOST = server.address;
+        PORT = server.port;
+        TLS_ENABLED = false;
+      };
+      DATABASE = {
+        ENABLED = true;
+        DATABASE = cfg.database.name;
+        SOCKET = "/run/mysqld/mysqld.sock";
+      };
+    } // optionalAttrs (cfg.tor.proxy) {
+      # Use Tor for rate fetching
+      SOCKS5PROXY = {
+        ENABLED = true;
+        USE_ONION = true;
+        HOST = torSocket.addr;
+        PORT = torSocket.port;
+      };
+    };
+
+    systemd.services.mempool = {
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "${cfg.electrumServer}.service" ];
+      after = [ "${cfg.electrumServer}.service" "mysql.service" ];
+      preStart = ''
+        mkdir -p '${cacheDir}/cache'
+        <${configFile} sed \
+          -e "s|@btcRpcPassword@|$(cat ${secretsDir}/bitcoin-rpcpassword-public)|" \
+          > '${cacheDir}/config.json'
+      '';
+      environment.MEMPOOL_CONFIG_FILE = "${cacheDir}/config.json";
+      serviceConfig = nbLib.defaultHardening // {
+        ExecStart = "${cfg.package}/bin/mempool-backend";
+        CacheDirectory = "mempool";
+        CacheDirectoryMode = "770";
+        # Show "mempool" instead of "node" in the journal
+        SyslogIdentifier = "mempool";
+        User = cfg.user;
+        Restart = "on-failure";
+        RestartSec = "10s";
+      } // nbLib.allowedIPAddresses cfg.tor.enforce
+        // nbLib.nodejs;
+    };
+
+    services.nginx = mkIf cfg.frontend.enable {
+      enable = true;
+      enableReload = true;
+      recommendedGzipSettings = true;
+      recommendedOptimisation = true;
+      recommendedProxySettings = true;
+      recommendedTlsSettings = true;
+      commonHttpConfig = frontend.nginxConfig.httpConfig;
+      virtualHosts."mempool" = {
+        serverName = "_";
+        listen = [ { addr = cfg.frontend.address; port = cfg.frontend.port; } ];
+        root = cfg.frontend.staticContentRoot;
+        extraConfig =
+          frontend.nginxConfig.staticContent +
+          frontend.nginxConfig.proxyApi;
+      };
+    };
+
+    users.users.${cfg.user} = {
+      isSystemUser = true;
+      group = cfg.group;
+      extraGroups = [ "bitcoinrpc-public" ];
+    };
+    users.groups.${cfg.group} = {};
+  };
+}

modules/modules.nix

@@ -20,6 +20,7 @@
     ./charge-lnd.nix
     ./lndconnect.nix # Requires onion-addresses.nix
     ./rtl.nix
+    ./mempool.nix
     ./electrs.nix
     ./fulcrum.nix
     ./liquid.nix

modules/netns-isolation.nix

@@ -295,7 +295,14 @@ in {
         id = 31;
         connections = [ "bitcoind" ];
       };
-      # id = 32 reserved for the upcoming mempool module
+      mempool = {
+        id = 32;
+        connections = [
+          "bitcoind"
+          "nginx"
+          (if (config.services.mempool.electrumServer == "electrs") then "electrs" else "fulcrum")
+        ];
+      };
     };
 
     services.bitcoind = {
@@ -349,6 +356,9 @@ in {
     services.rtl.address = netns.rtl.address;
 
     services.clightning-rest.address = netns.clightning-rest.address;
+
+    services.mempool.address = netns.mempool.address;
+    services.mempool.frontend.address = netns.nginx.address;
   }
   ]);
 }

modules/nodeinfo.nix

@@ -149,6 +149,12 @@ in {
       liquidd = mkInfo "";
       joinmarket-ob-watcher = mkInfo "";
       rtl = mkInfo "";
+      mempool = mkInfo "";
+      mempool-frontend = name: cfg: mkInfoLong {
+        inherit name cfg;
+        systemdServiceName = "nginx";
+        extraCode = "";
+      };
       # Only add sshd when it has an onion service
       sshd = name: cfg: mkIfOnionPort "sshd" (onionPort: ''
         add_service("sshd", """info["onion_address"] = get_onion_address("sshd", ${onionPort})""")

modules/onion-services.nix

@@ -113,6 +113,9 @@ in {
         rtl = {
           externalPort = 80;
         };
+        mempool-frontend = {
+          externalPort = 80;
+        };
       };
     }
   ];

modules/presets/enable-tor.nix

@@ -27,6 +27,7 @@ in {
     # disable Tor enforcement until btcpayserver can fetch rates over Tor
     # btcpayserver = defaultEnableTorProxy;
     lightning-pool = defaultEnableTorProxy;
+    mempool = defaultEnableTorProxy;
 
     # These services don't make outgoing connections
     # (or use Tor by default in case of joinmarket)

test/tests.nix

@@ -78,6 +78,9 @@ let
         echo a > rtl-password
       '');
 
+      tests.mempool = cfg.mempool.enable;
+      services.mempool.electrumServer = "fulcrum";
+
       tests.lnd = cfg.lnd.enable;
       services.lnd = {
         port = 9736;
@@ -195,6 +198,7 @@ let
       services.lightning-loop.enable = true;
       services.lightning-pool.enable = true;
       services.charge-lnd.enable = true;
+      services.mempool.enable = true;
       services.electrs.enable = true;
       services.fulcrum.enable = true;
       services.liquidd.enable = true;
@@ -219,6 +223,8 @@ let
       tests.secure-node = true;
       tests.restart-bitcoind = true;
 
+      nix-bitcoin.onionServices.mempool-frontend.enable = true;
+
       # Stop electrs from spamming the test log with 'WARN - wait until IBD is over' messages
       tests.stop-electrs = true;
     };
@@ -237,6 +243,7 @@ let
       services.clightning-rest.enable = true;
       services.liquidd.enable = true;
       services.rtl.enable = true;
+      services.mempool.enable = true;
       services.lnd.enable = true;
       services.lightning-loop.enable = true;
       services.lightning-pool.enable = true;

test/tests.py

@@ -251,6 +251,15 @@ def _():
         log_has_string("clightning-rest", "cl-rest api server is ready and listening")
     )
 
+@test("mempool")
+def _():
+    assert_running("mempool")
+    assert_running("nginx")
+    machine.wait_until_succeeds(
+        log_has_string("mempool", "Mempool Server is running on port 8999")
+    )
+    assert_matches(f"curl -L {ip('nginx')}:60845", "mempool - Bitcoin Explorer")
+
 @test("joinmarket")
 def _():
     assert_running("joinmarket")
@@ -427,6 +436,12 @@ def _():
     if enabled("btcpayserver"):
         machine.wait_until_succeeds(log_has_string("nbxplorer", f"At height: {num_blocks}"))
 
+    if enabled("mempool"):
+        assert_running("nginx")
+        assert_full_match(
+            f"curl -fsS http://{ip('nginx')}:60845/api/v1/blocks/tip/height", str(num_blocks)
+        )
+
 @test("trustedcoin")
 def _():
     def expect_clightning_log(str):