{ config, lib, pkgs, ... }: with lib; let options.nix-bitcoin.netns-isolation = { enable = mkEnableOption "netns isolation"; addressblock = mkOption { type = types.ints.u8; default = 1; description = mdDoc '' The address block N in 169.254.N.0/24, used as the prefix for netns addresses. ''; }; services = mkOption { default = {}; type = types.attrsOf (types.submodule { options = { id = mkOption { # TODO: Assert uniqueness type = types.ints.between 11 255; description = mdDoc '' id for the netns, used for the IP address host part and for naming the interfaces. Must be unique. Must be greater than 10. ''; }; connections = mkOption { type = with types; listOf str; default = []; }; }; }); }; allowedUser = mkOption { type = types.str; description = mdDoc '' User that is allowed to execute commands in the service network namespaces. The user's group is also authorized. ''; default = config.nix-bitcoin.operator.name; }; netns = mkOption { readOnly = true; default = netns; description = mdDoc "Exposes netns parameters."; }; bridgeIp = mkOption { readOnly = true; default = bridgeIp; description = mdDoc "IP of the netns bridge interface."; }; }; cfg = config.nix-bitcoin.netns-isolation; netns = builtins.mapAttrs (n: v: { inherit (v) id; address = "169.254.${toString cfg.addressblock}.${toString v.id}"; availableNetns = availableNetns.${n}; netnsName = "nb-${n}"; }) enabledServices; # Symmetric netns connection matrix # if clightning.connections = [ "bitcoind" ]; then # availableNetns.bitcoind = [ "clighting" ]; # and # availableNetns.clighting = [ "bitcoind" ]; # # TODO-EXTERNAL: # Although negligible for our purposes, this calculation's runtime # is in the order of (number of connections * number of services), # because attrsets and lists are fully copied on each update with '//' or '++'. # This can only be improved with an update in the nix language. # availableNetns = let # base = { clightning = [ "bitcoind" ]; ... } base = builtins.mapAttrs (n: v: builtins.filter isEnabled v.connections ) enabledServices; in foldl (xs: s1: foldl (xs: s2: xs // { "${s2}" = xs.${s2} ++ [ s1 ]; } ) xs cfg.services.${s1}.connections ) base (builtins.attrNames base); enabledServices = filterAttrs (n: v: isEnabled n) cfg.services; isEnabled = x: config.services.${x}.enable; ip = "${pkgs.iproute}/bin/ip"; iptables = "${config.networking.firewall.package}/bin/iptables"; bridgeIp = "169.254.${toString cfg.addressblock}.10"; mkCliExec = service: "exec netns-exec ${netns.${service}.netnsName}"; in { inherit options; config = mkIf cfg.enable (mkMerge [ # Base infrastructure { networking.dhcpcd.denyInterfaces = [ "nb-br" "nb-veth*" ]; services.tor.client.socksListenAddress = { addr = bridgeIp; # Default NixOS values. These must be repeated when redefining this option. port = 9050; IsolateDestAddr = true; }; services.i2pd.proto.sam.address = bridgeIp; networking.firewall.interfaces.nb-br.allowedTCPPorts = [ config.services.tor.client.socksListenAddress.port config.services.i2pd.proto.sam.port ]; boot.kernel.sysctl."net.ipv4.ip_forward" = true; security.wrappers.netns-exec = { source = config.nix-bitcoin.pkgs.netns-exec; capabilities = "cap_sys_admin=ep"; owner = cfg.allowedUser; group = ""; # Set to the group of `owner` permissions = "550"; }; systemd.services = { # Due to a NixOS bug we can't currently use option `networking.bridges` to # setup the bridge while `networking.useDHCP` is enabled. nb-netns-bridge = { description = "nix-bitcoin netns bridge"; wantedBy = [ "network-setup.service" ]; partOf = [ "network-setup.service" ]; before = [ "network-setup.service" ]; after = [ "network-pre.target" ]; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; }; script = '' ${ip} link add name nb-br type bridge ${ip} link set nb-br up ${ip} addr add ${bridgeIp}/24 brd + dev nb-br ${iptables} -w -t nat -A POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE ''; preStop = '' ${iptables} -w -t nat -D POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE ${ip} link del nb-br ''; }; } // (let makeNetnsServices = n: v: let veth = "nb-veth-${toString v.id}"; peer = "nb-veth-br-${toString v.id}"; inherit (v) netnsName; nsenter = "${pkgs.utillinux}/bin/nsenter"; allowedNetnsAddresses = map (available: netns.${available}.address) v.availableNetns; allowedAddresses = concatStringsSep "," ([ "127.0.0.1,${bridgeIp},${v.address}" ] ++ allowedNetnsAddresses); setup = '' ${ip} netns add ${netnsName} ${ip} link add ${veth} type veth peer name ${peer} ${ip} link set ${veth} netns ${netnsName} # The peer link is never used directly, so don't auto-assign an IPv6 address echo 1 > /proc/sys/net/ipv6/conf/${peer}/disable_ipv6 ${ip} link set ${peer} up ${ip} link set ${peer} master nb-br exec ${nsenter} --net=/run/netns/${netnsName} ${script "in-netns" setupInNetns} ''; setupInNetns = '' ${ip} link set lo up ${ip} addr add ${v.address}/24 dev ${veth} ${ip} link set ${veth} up ${ip} route add default via ${bridgeIp} ${iptables} -w -P INPUT DROP # allow return traffic to outgoing connections initiated by the service itself ${iptables} -w -A INPUT -m conntrack --ctstate ESTABLISHED -j ACCEPT ${iptables} -w -A INPUT -s ${allowedAddresses} -j ACCEPT '' + optionalString (config.services.${n}.tor.enforce or false) '' ${iptables} -w -P OUTPUT DROP ${iptables} -w -A OUTPUT -d ${allowedAddresses} -j ACCEPT ''; script = name: src: pkgs.writers.writeDash name '' set -e ${src} ''; in { "${n}".serviceConfig.NetworkNamespacePath = "/var/run/netns/${netnsName}"; "netns-${n}" = rec { requires = [ "nb-netns-bridge.service" ]; after = [ "nb-netns-bridge.service" ]; requiredBy = [ "${n}.service" ]; before = requiredBy; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; ExecStart = script "setup" setup; }; # Link deletion is implicit in netns deletion, but it sometimes only happens # after `netns delete` finishes. Add an extra `link del` to ensure that # the link is deleted before the service stops, which is needed for service # restart to succeed. preStop = '' ${ip} netns delete ${netnsName} ${ip} link del ${peer} 2> /dev/null || true ''; }; }; in foldl (services: n: services // (makeNetnsServices n netns.${n}) ) {} (builtins.attrNames netns) ); } # Service-specific config { nix-bitcoin.netns-isolation.services = { bitcoind = { id = 12; }; clightning = { id = 13; connections = [ "bitcoind" ]; }; lnd = { id = 14; connections = [ "bitcoind" ]; }; liquidd = { id = 15; connections = [ "bitcoind" ]; }; electrs = { id = 16; connections = [ "bitcoind" ]; }; nginx = { id = 21; }; lightning-loop = { id = 22; connections = [ "lnd" ]; }; nbxplorer = { id = 23; connections = [ "bitcoind" ] ++ optional config.services.btcpayserver.lbtc "liquidd"; }; btcpayserver = { id = 24; connections = [ "nbxplorer" ] ++ optional (config.services.btcpayserver.lightningBackend == "lnd") "lnd" ++ optional config.services.btcpayserver.lbtc "liquidd"; # communicates with clightning over rpc socket }; joinmarket = { id = 25; connections = [ "bitcoind" ]; }; joinmarket-ob-watcher = { id = 26; connections = [ "bitcoind" ]; }; lightning-pool = { id = 27; connections = [ "lnd" ]; }; charge-lnd = { id = 28; connections = [ "lnd" "electrs" ]; }; rtl = { id = 29; connections = let nodes = config.services.rtl.nodes; in optional nodes.lnd.enable "lnd" ++ optional (nodes.lnd.enable && nodes.lnd.loop) "lightning-loop" ++ optional nodes.clightning.enable "clightning-rest"; }; clightning-rest = { id = 30; }; fulcrum = { id = 31; connections = [ "bitcoind" ]; }; mempool = { id = 32; connections = [ "bitcoind" "nginx" (if (config.services.mempool.electrumServer == "electrs") then "electrs" else "fulcrum") ]; }; }; services.bitcoind = { address = netns.bitcoind.address; rpc.address = netns.bitcoind.address; rpc.allowip = [ bridgeIp # For operator user netns.bitcoind.address ] ++ map (n: netns.${n}.address) netns.bitcoind.availableNetns; }; services.clightning.address = netns.clightning.address; services.lnd = { address = netns.lnd.address; rpcAddress = netns.lnd.address; restAddress = netns.lnd.address; }; services.liquidd = { address = netns.liquidd.address; rpc.address = netns.liquidd.address; rpcallowip = [ bridgeIp # For operator user netns.liquidd.address ] ++ map (n: netns.${n}.address) netns.liquidd.availableNetns; }; services.electrs.address = netns.electrs.address; services.fulcrum.address = netns.fulcrum.address; services.lightning-loop.rpcAddress = netns.lightning-loop.address; services.nbxplorer.address = netns.nbxplorer.address; services.btcpayserver.address = netns.btcpayserver.address; services.joinmarket = { payjoinAddress = netns.joinmarket.address; messagingAddress = netns.joinmarket.address; cliExec = mkCliExec "joinmarket"; }; systemd.services.joinmarket-yieldgenerator = mkIf config.services.joinmarket.yieldgenerator.enable { serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-joinmarket"; }; services.joinmarket-ob-watcher.address = netns.joinmarket-ob-watcher.address; services.lightning-pool.rpcAddress = netns.lightning-pool.address; services.rtl.address = netns.rtl.address; services.clightning-rest.address = netns.clightning-rest.address; services.mempool.address = netns.mempool.address; services.mempool.frontend.address = netns.nginx.address; } ]); }