add curated clightning plugins

.travis.yml

@@ -33,6 +33,7 @@ env:
   - PKG=nixops19_09 STABLE=1
   - PKG=joinmarket STABLE=1
   - PKG=joinmarket STABLE=0
+  - PKG=clightning-plugins-all STABLE=1
 script:
   - printf '%s (%s)\n' "$NIX_PATH" "$VER"
   - |

README.md

@@ -91,7 +91,7 @@ By default the `configuration.nix` provides:
 * adds non-root user "operator" which has access to bitcoin-cli and lightning-cli
 
 In `configuration.nix` the user can enable:
-* a clightning hidden service
+* a clightning hidden service with [plugins](https://github.com/lightningd/plugins)
 * [liquid](https://github.com/elementsproject/elements)
 * [lightning charge](https://github.com/ElementsProject/lightning-charge)
 * [nanopos](https://github.com/ElementsProject/nanopos)

docs/usage.md

@@ -338,3 +338,31 @@ See [here](https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/master
     ```
 
 3. Profit
+
+clightning
+---
+
+## Plugins
+
+There are a number of [plugins](https://github.com/lightningd/plugins) available for clightning. Currently `nix-bitcoin` supports:
+
+- helpme
+- monitor
+- prometheus
+- rebalance
+- summary
+- zmq
+
+You can activate and configure these plugins like so:
+
+```nix
+services.clightning = {
+    enable = true;
+    plugins = {
+        prometheus.enable = true;
+        prometheus.listen = "0.0.0.0:9900";
+    };
+};
+```
+
+Please have a look at the module for a plugin (e.g. [prometheus.nix](../modules/clightning-plugins/prometheus.nix)) to learn its configuration options.

examples/configuration.nix

@@ -38,10 +38,14 @@
   # Enable this module to use clightning, a Lightning Network implementation
   # in C.
   services.clightning.enable = true;
+  # == TOR
   # Enable this option to announce our Tor Hidden Service. By default clightning
   # offers outgoing functionality, but doesn't announce the Tor Hidden Service
   # under which peers can reach us.
   # services.clightning.announce-tor = true;
+  # == Plugins
+  # See ../docs/usage.md for the list of available plugins.
+  # services.clightning.plugins.prometheus.enable = true;
 
   ### LND
   # Uncomment the following line in order to enable lnd, a lightning

modules/clightning-plugins/default.nix

@@ -0,0 +1,27 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.services.clightning.plugins;
+  pluginPkgs = config.nix-bitcoin.pkgs.clightning-plugins;
+in {
+  imports = [
+    ./prometheus.nix
+    ./summary.nix
+    ./zmq.nix
+  ];
+
+  options.services.clightning.plugins = {
+    helpme.enable = mkEnableOption "Help me (clightning plugin)";
+    monitor.enable = mkEnableOption "Monitor (clightning plugin)";
+    rebalance.enable = mkEnableOption "Rebalance (clightning plugin)";
+  };
+
+  config = {
+    services.clightning.extraConfig = mkMerge [
+      (mkIf cfg.helpme.enable "plugin=${pluginPkgs.helpme.path}")
+      (mkIf cfg.monitor.enable "plugin=${pluginPkgs.monitor.path}")
+      (mkIf cfg.rebalance.enable "plugin=${pluginPkgs.rebalance.path}")
+    ];
+  };
+}

modules/clightning-plugins/prometheus.nix

@@ -0,0 +1,21 @@
+{ config, lib, ... }:
+
+with lib;
+let cfg = config.services.clightning.plugins.prometheus; in
+{
+  options.services.clightning.plugins.prometheus = {
+    enable = mkEnableOption "Prometheus (clightning plugin)";
+    listen = mkOption {
+      type = types.str;
+      default = "0.0.0.0:9750";
+      description = "Address and port to bind to.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.clightning.extraConfig = ''
+      plugin=${config.nix-bitcoin.pkgs.clightning-plugins.prometheus.path}
+      prometheus-listen=${cfg.listen}
+    '';
+  };
+}

modules/clightning-plugins/summary.nix

@@ -0,0 +1,39 @@
+{ config, lib, ... }:
+
+with lib;
+let cfg = config.services.clightning.plugins.summary; in
+{
+  options.services.clightning.plugins.summary = {
+    enable = mkEnableOption "Summary (clightning plugin)";
+    currency = mkOption {
+      type = types.str;
+      default = "USD";
+      description = "The currency to look up on btcaverage.";
+    };
+    currencyPrefix = mkOption {
+      type = types.str;
+      default = "USD $";
+      description = "The prefix to use for the currency.";
+    };
+    availabilityInterval = mkOption {
+      type = types.int;
+      default = 300;
+      description = "How often in seconds the availability should be calculated.";
+    };
+    availabilityWindow = mkOption {
+      type = types.int;
+      default = 72;
+      description = "How many hours the availability should be averaged over.";
+    };
+  };
+
+  config = mkIf cfg.enable {
+    services.clightning.extraConfig = ''
+      plugin=${config.nix-bitcoin.pkgs.clightning-plugins.summary.path}
+      summary-currency="${cfg.currency}"
+      summary-currency-prefix="${cfg.currencyPrefix}"
+      summary-availability-interval=${toString cfg.availabilityInterval}
+      summary-availability-window=${toString cfg.availabilityWindow}
+    '';
+  };
+}

modules/clightning-plugins/zmq.nix

@@ -0,0 +1,42 @@
+{ config, lib, ... }:
+
+with lib;
+let
+  cfg = config.services.clightning.plugins.zmq;
+
+  endpoints = [
+    "channel-opened"
+    "connect"
+    "disconnect"
+    "invoice-payment"
+    "warning"
+    "forward-event"
+    "sendpay-success"
+    "sendpay-failure"
+  ];
+
+  mkEndpointOption = name:
+    mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = "Endpoint for ${name}";
+    };
+
+  setEndpoint = ep:
+    let value = builtins.getAttr ep cfg; in
+    optionalString (value != null) ''
+      zmq-pub-${ep}=${value}
+    '';
+in
+{
+  options.services.clightning.plugins.zmq = {
+    enable = mkEnableOption "ZMQ (clightning plugin)";
+  } // lib.genAttrs endpoints mkEndpointOption;
+
+  config = mkIf cfg.enable {
+    services.clightning.extraConfig = ''
+      plugin=${config.nix-bitcoin.pkgs.clightning-plugins.zmq.path}
+      ${concatStrings (map setEndpoint endpoints)}
+    '';
+  };
+}

modules/modules.nix

@@ -9,6 +9,7 @@
     # Main features
     ./bitcoind.nix
     ./clightning.nix
+    ./clightning-plugins
     ./lightning-charge.nix
     ./nanopos.nix
     ./spark-wallet.nix

pkgs/clightning-plugins/default.nix

@@ -0,0 +1,63 @@
+pkgs: nbPython3Packages:
+
+let
+  inherit (pkgs) lib;
+
+  src = pkgs.fetchFromGitHub {
+    owner = "lightningd";
+    repo = "plugins";
+    rev = "6cd472636926f05a9c472139fabe1ff11c90aa6a";
+    sha256 = "1lisx85vzsfzjhdc6zdz0l6bcrdgg6rp5xbc5jmx93mv8qqg2cns";
+  };
+
+  version = builtins.substring 0 7 src.rev;
+
+  plugins = with nbPython3Packages; {
+    helpme = {};
+    monitor = {};
+    prometheus = {
+      extraPkgs = [ prometheus_client ];
+      patchRequirements = "--replace prometheus-client==0.6.0 prometheus-client==0.8.0";
+    };
+    rebalance = {};
+    summary = {
+      extraPkgs = [ packaging requests ];
+    };
+    zmq = {
+      scriptName = "cl-zmq";
+      extraPkgs = [ twisted txzmq ];
+    };
+  };
+
+  basePkgs = [ nbPython3Packages.pyln-client ];
+
+  mkPlugin = name: plugin: let
+    python = pkgs.python3.withPackages (_: basePkgs ++ (plugin.extraPkgs or []));
+    script = "${plugin.scriptName or name}.py";
+    drv = pkgs.stdenv.mkDerivation {
+      pname = "clightning-plugin-${name}";
+      inherit version;
+
+      buildInputs = [ python ];
+
+      buildCommand = ''
+        cp --no-preserve=mode -r ${src}/${name} $out
+        cd $out
+        ${lib.optionalString (plugin ? patchRequirements) ''
+          substituteInPlace requirements.txt ${plugin.patchRequirements}
+        ''}
+
+        # Check that requirements are met
+        PYTHONPATH=${toString python}/${python.sitePackages} \
+          ${pkgs.python3Packages.pip}/bin/pip install -r requirements.txt --no-cache --no-index
+
+        chmod +x ${script}
+        patchShebangs ${script}
+      '';
+
+      passthru.path = "${drv}/${script}";
+    };
+  in drv;
+
+in
+  builtins.mapAttrs mkPlugin plugins

pkgs/clightning-plugins/get-sha256.sh

@@ -0,0 +1,14 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p git
+set -euo pipefail
+
+archive_hash () {
+    repo=$1
+    rev=$2
+    nix-prefetch-url --unpack "https://github.com/${repo}/archive/${rev}.tar.gz" 2> /dev/null | tail -n 1
+}
+
+echo "Fetching latest lightningd/plugins release"
+latest=$(git ls-remote https://github.com/lightningd/plugins master | cut -f 1)
+echo "rev: ${latest}"
+echo "sha256: $(archive_hash lightningd/plugins $latest)"

pkgs/default.nix

@@ -13,6 +13,7 @@ let self = {
   netns-exec = pkgs.callPackage ./netns-exec { };
   lightning-loop = pkgs.callPackage ./lightning-loop { };
   extra-container = pkgs.callPackage ./extra-container { };
+  clightning-plugins = import ./clightning-plugins pkgs self.nbPython3Packages;
 
   nbPython3Packages = (pkgs.python3.override {
     packageOverrides = pySelf: super: import ./python-packages self pySelf;
@@ -23,4 +24,8 @@ let self = {
   lib = import ./lib.nix { inherit (pkgs) lib; };
 
   modulesPkgs = self // self.pinned;
+
+  # Used in ../.travis.yml
+  clightning-plugins-all = pkgs.writeText "clightning-plugins"
+    (pkgs.lib.concatMapStringsSep "\n" toString (builtins.attrValues self.clightning-plugins));
 }; in self

test/tests.nix

@@ -21,7 +21,11 @@ let testEnv = rec {
       }
     ];
 
-    config = {
+    options.test.features = {
+      clightningPlugins = mkEnableOption "all clightning plugins";
+    };
+
+    config = mkMerge [{
       tests.bitcoind = cfg.bitcoind.enable;
       services.bitcoind = {
         enable = true;
@@ -31,6 +35,11 @@ let testEnv = rec {
       tests.clightning = cfg.clightning.enable;
       # When WAN is disabled, DNS bootstrapping slows down service startup by ~15 s.
       services.clightning.extraConfig = mkIf config.test.noConnections "disable-dns";
+      test.data.clightning-plugins = let
+        plugins = config.services.clightning.plugins;
+        enabled = builtins.filter (plugin: plugins.${plugin}.enable) (builtins.attrNames plugins);
+        pluginPkgs = config.nix-bitcoin.pkgs.clightning-plugins;
+      in map (plugin: pluginPkgs.${plugin}.path) enabled;
 
       tests.spark-wallet = cfg.spark-wallet.enable;
 
@@ -67,7 +76,28 @@ let testEnv = rec {
       systemd.services.generate-secrets.postStart = mkIfTest "security" ''
         install -o nobody -g nogroup -m777 <(:) /secrets/dummy
       '';
-    };
+    }
+    (mkIf config.test.features.clightningPlugins {
+      services.clightning.plugins = {
+        helpme.enable = true;
+        monitor.enable = true;
+        prometheus.enable = true;
+        rebalance.enable = true;
+        summary.enable = true;
+        zmq = let tcpEndpoint = "tcp://127.0.0.1:5501"; in {
+          enable = true;
+          channel-opened = tcpEndpoint;
+          connect = tcpEndpoint;
+          disconnect = tcpEndpoint;
+          invoice-payment = tcpEndpoint;
+          warning = tcpEndpoint;
+          forward-event = tcpEndpoint;
+          sendpay-success = tcpEndpoint;
+          sendpay-failure = tcpEndpoint;
+        };
+      };
+    })
+    ];
   };
 
   scenarios = {
@@ -80,6 +110,7 @@ let testEnv = rec {
       tests.security = true;
 
       services.clightning.enable = true;
+      test.features.clightningPlugins = true;
       services.spark-wallet.enable = true;
       services.lightning-charge.enable = true;
       services.nanopos.enable = true;
@@ -120,6 +151,7 @@ let testEnv = rec {
     regtest = {
       imports = [ scenarios.regtestBase ];
       services.clightning.enable = true;
+      test.features.clightningPlugins = true;
       services.spark-wallet.enable = true;
       services.lnd.enable = true;
       services.lightning-loop.enable = true;

test/tests.py

@@ -1,4 +1,5 @@
 from collections import OrderedDict
+import json
 
 
 def succeed(*cmds):
@@ -138,6 +139,20 @@ def _():
 def _():
     assert_running("clightning")
     assert_matches("su operator -c 'lightning-cli getinfo' | jq", '"id"')
+    if test_data["clightning-plugins"]:
+        plugin_list = succeed("lightning-cli plugin list")
+        plugins = json.loads(plugin_list)["plugins"]
+        active = set(plugin["name"] for plugin in plugins if plugin["active"])
+        failed = set(test_data["clightning-plugins"]).difference(active)
+        if failed:
+            raise Exception(
+                f"The following clightning plugins are inactive:\n{failed}.\n\n"
+                f"Output of 'lightning-cli plugin list':\n{plugin_list}"
+            )
+        else:
+            log.log("Active clightning plugins:")
+            for p in test_data["clightning-plugins"]:
+                log.log(os.path.basename(p))
 
 
 @test("lnd")