{ config, lib, pkgs, ... }: with lib; let cfg = config.nix-bitcoin.netns-isolation; netns = builtins.mapAttrs (n: v: { inherit (v) id; address = "169.254.${toString cfg.addressblock}.${toString v.id}"; availableNetns = availableNetns.${n}; }) enabledServices; # Symmetric netns connection matrix # if clightning.connections = [ "bitcoind" ]; then # availableNetns.bitcoind = [ "clighting" ]; # and # availableNetns.clighting = [ "bitcoind" ]; # # FIXME: Although negligible for our purposes, this calculation's runtime # is in the order of (number of connections * number of services), # because attrsets and lists are fully copied on each update with '//' or '++'. # This can only be improved with an update in the nix language. # availableNetns = let # base = { clightning = [ "bitcoind" ]; ... } base = builtins.mapAttrs (n: v: builtins.filter isEnabled v.connections ) enabledServices; in foldl (xs: s1: foldl (xs: s2: xs // { "${s2}" = xs.${s2} ++ [ s1 ]; } ) xs cfg.services.${s1}.connections ) base (builtins.attrNames base); enabledServices = filterAttrs (n: v: isEnabled n) cfg.services; isEnabled = x: config.services.${x}.enable; ip = "${pkgs.iproute}/bin/ip"; iptables = "${config.networking.firewall.package}/bin/iptables"; bridgeIp = "169.254.${toString cfg.addressblock}.10"; in { options.nix-bitcoin.netns-isolation = { enable = mkEnableOption "netns isolation"; addressblock = mkOption { type = types.ints.u8; default = "1"; description = '' The address block N in 169.254.N.0/24, used as the prefix for netns addresses. ''; }; services = mkOption { default = {}; type = types.attrsOf (types.submodule { options = { id = mkOption { # TODO: Assert uniqueness type = types.ints.between 11 255; description = '' id for the netns, used for the IP address host part and for naming the interfaces. Must be unique. Must be greater than 10. ''; }; connections = mkOption { type = with types; listOf str; default = []; }; }; }); }; }; config = mkIf cfg.enable (mkMerge [ # Base infrastructure { networking.dhcpcd.denyInterfaces = [ "br0" "br-nb*" "nb-veth*" ]; services.tor.client.socksListenAddress = "${bridgeIp}:9050"; networking.firewall.interfaces.br0.allowedTCPPorts = [ 9050 ]; boot.kernel.sysctl."net.ipv4.ip_forward" = true; security.wrappers.netns-exec = { source = "${pkgs.nix-bitcoin.netns-exec}/netns-exec"; capabilities = "cap_sys_admin=ep"; owner = "${config.nix-bitcoin.operatorName}"; permissions = "u+rx,g+rx,o-rwx"; }; systemd.services = { netns-bridge = { description = "Create bridge"; requiredBy = [ "tor.service" ]; before = [ "tor.service" ]; script = '' ${ip} link add name br0 type bridge ${ip} link set br0 up ${ip} addr add ${bridgeIp}/24 brd + dev br0 ${iptables} -w -t nat -A POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE ''; preStop = '' ${iptables} -w -t nat -D POSTROUTING -s 169.254.${toString cfg.addressblock}.0/24 -j MASQUERADE ${ip} link del br0 ''; serviceConfig = { Type = "oneshot"; RemainAfterExit = "yes"; }; }; } // (let makeNetnsServices = n: v: let vethName = "nb-veth-${toString v.id}"; netnsName = "nb-${n}"; ipNetns = "${ip} -n ${netnsName}"; netnsIptables = "${ip} netns exec ${netnsName} ${config.networking.firewall.package}/bin/iptables"; in { "${n}".serviceConfig.NetworkNamespacePath = "/var/run/netns/${netnsName}"; "netns-${n}" = rec { requires = [ "netns-bridge.service" ]; after = [ "netns-bridge.service" ]; bindsTo = [ "${n}.service" ]; requiredBy = bindsTo; before = bindsTo; script = '' ${ip} netns add ${netnsName} ${ipNetns} link set lo up ${ip} link add ${vethName} type veth peer name br-${vethName} ${ip} link set ${vethName} netns ${netnsName} ${ipNetns} addr add ${v.address}/24 dev ${vethName} ${ip} link set br-${vethName} up ${ipNetns} link set ${vethName} up ${ip} link set br-${vethName} master br0 ${ipNetns} route add default via ${bridgeIp} ${netnsIptables} -w -P INPUT DROP ${netnsIptables} -w -A INPUT -s 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT '' + (optionalString (config.services.${n}.enforceTor or false)) '' ${netnsIptables} -w -P OUTPUT DROP ${netnsIptables} -w -A OUTPUT -d 127.0.0.1,${bridgeIp},${v.address} -j ACCEPT '' + concatMapStrings (otherNetns: let other = netns.${otherNetns}; in '' ${netnsIptables} -w -A INPUT -s ${other.address} -j ACCEPT ${netnsIptables} -w -A OUTPUT -d ${other.address} -j ACCEPT '') v.availableNetns; preStop = '' ${ip} netns delete ${netnsName} ${ip} link del br-${vethName} ''; serviceConfig = { Type = "oneshot"; RemainAfterExit = "yes"; ExecStartPre = "-${ip} netns delete ${netnsName}"; }; }; }; in foldl (services: n: services // (makeNetnsServices n netns.${n}) ) {} (builtins.attrNames netns)); } # Service-specific config { nix-bitcoin.netns-isolation.services = { bitcoind = { id = 12; }; clightning = { id = 13; connections = [ "bitcoind" ]; }; lnd = { id = 14; connections = [ "bitcoind" ]; }; liquidd = { id = 15; connections = [ "bitcoind" ]; }; electrs = { id = 16; connections = [ "bitcoind" ]; }; spark-wallet = { id = 17; # communicates with clightning over lightning-rpc socket }; lightning-charge = { id = 18; # communicates with clightning over lightning-rpc socket }; nanopos = { id = 19; connections = [ "nginx" "lightning-charge" ]; }; recurring-donations = { id = 20; # communicates with clightning over lightning-rpc socket }; nginx = { id = 21; }; lightning-loop = { id = 22; connections = [ "lnd" ]; }; }; services.bitcoind = { bind = netns.bitcoind.address; rpcbind = [ "${netns.bitcoind.address}" "127.0.0.1" ]; rpcallowip = [ "127.0.0.1" ] ++ map (n: "${netns.${n}.address}") netns.bitcoind.availableNetns; cli = pkgs.writeScriptBin "bitcoin-cli" '' netns-exec nb-bitcoind ${config.services.bitcoind.package}/bin/bitcoin-cli -datadir='${config.services.bitcoind.dataDir}' "$@" ''; }; systemd.services.bitcoind-import-banlist.serviceConfig.NetworkNamespacePath = "/var/run/netns/nb-bitcoind"; services.clightning = { bitcoin-rpcconnect = netns.bitcoind.address; bind-addr = netns.clightning.address; }; services.lnd = { listen = netns.lnd.address; rpclisten = [ "${netns.lnd.address}" "127.0.0.1" ]; restlisten = [ "${netns.lnd.address}" "127.0.0.1" ]; bitcoind-host = netns.bitcoind.address; cli = pkgs.writeScriptBin "lncli" # Switch user because lnd makes datadir contents readable by user only '' netns-exec nb-lnd sudo -u lnd ${config.services.lnd.package}/bin/lncli --tlscertpath ${config.nix-bitcoin.secretsDir}/lnd-cert \ --macaroonpath '${config.services.lnd.dataDir}/chain/bitcoin/mainnet/admin.macaroon' "$@" ''; }; services.liquidd = { bind = netns.liquidd.address; rpcbind = [ "${netns.liquidd.address}" "127.0.0.1" ]; rpcallowip = [ "127.0.0.1" ] ++ map (n: "${netns.${n}.address}") netns.liquidd.availableNetns; mainchainrpchost = netns.bitcoind.address; cli = pkgs.writeScriptBin "elements-cli" '' netns-exec nb-liquidd ${pkgs.nix-bitcoin.elementsd}/bin/elements-cli -datadir='${config.services.liquidd.dataDir}' "$@" ''; swap-cli = pkgs.writeScriptBin "liquidswap-cli" '' netns-exec nb-liquidd ${pkgs.nix-bitcoin.liquid-swap}/bin/liquidswap-cli -c '${config.services.liquidd.dataDir}/elements.conf' "$@" ''; }; services.electrs = { address = netns.electrs.address; daemonrpc = "${netns.bitcoind.address}:${toString config.services.bitcoind.rpc.port}"; }; services.spark-wallet = { host = netns.spark-wallet.address; extraArgs = "--no-tls"; }; services.lightning-charge.host = netns.lightning-charge.address; services.nanopos = { charged-url = "http://${netns.lightning-charge.address}:9112"; host = netns.nanopos.address; }; services.nix-bitcoin-webindex.host = netns.nginx.address; services.lightning-loop = { cli = pkgs.writeScriptBin "loop" # Switch user because lnd makes datadir contents readable by user only '' netns-exec nb-lightning-loop sudo -u lnd ${config.services.lightning-loop.package}/bin/loop "$@" ''; }; } ]); }