mempool: init at 2.5.0

helper/update-fixed-output-derivation.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 set -euo pipefail
 
 # The file that defines the derivation that should be updated

pkgs/default.nix

@@ -17,6 +17,10 @@ let self = {
   lndinit = pkgs.callPackage ./lndinit { };
   liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
   rtl = pkgs.callPackage ./rtl { inherit (self) fetchNodeModules; };
+  inherit (pkgs.callPackage ./mempool { inherit (self) fetchNodeModules; })
+    mempool-backend
+    mempool-frontend
+    mempool-nginx-conf;
   # The secp256k1 version used by joinmarket
   secp256k1 = pkgs.callPackage ./secp256k1 { };
   trustedcoin = pkgs.callPackage ./trustedcoin { };

pkgs/mempool/default.nix

@@ -0,0 +1,143 @@
+{ lib
+, stdenvNoCC
+, nodejs-18_x
+, nodejs-slim-18_x
+, fetchFromGitHub
+, fetchNodeModules
+, runCommand
+, makeWrapper
+, curl
+, cacert
+, rsync
+}:
+rec {
+  nodejs = nodejs-18_x;
+  nodejsRuntime = nodejs-slim-18_x;
+
+  src = fetchFromGitHub {
+    owner = "mempool";
+    repo = "mempool";
+    rev = "v2.5.0";
+    hash = "sha256-8HmfytxRte3fQ0QKOljUVk9YAuaXhQQWuv3EFNmOgfQ=";
+  };
+
+  nodeModules = {
+    frontend = fetchNodeModules {
+      inherit src nodejs;
+      preBuild = "cd frontend";
+      hash = "sha256-/Z0xNvob7eMGpzdUWolr47vljpFiIutZpGwd0uYhPWI=";
+    };
+    backend = fetchNodeModules {
+      inherit src nodejs;
+      preBuild = "cd backend";
+      hash = "sha256-HpzzSTuSRWDWGbctVhTcUA01if/7OTI4xN3DAbAAX+U=";
+    };
+  };
+
+  frontendAssets = fetchFiles {
+    name = "mempool-frontend-assets";
+    hash = "sha256-3TmulAfzJJMf0UFhnHEqjAnzc1TNC5DM2XcsU7eyinY=";
+    fetcher = ./frontend-assets-fetch.sh;
+  };
+
+  mempool-backend = mkDerivationMempool {
+    pname = "mempool-backend";
+
+    buildPhase = ''
+      cd backend
+      ${sync} --chmod=+w ${nodeModules.backend}/lib/node_modules .
+      patchShebangs node_modules
+
+      npm run package
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      mkdir -p $out/lib/mempool-backend
+      ${sync} package/ $out/lib/mempool-backend
+
+      makeWrapper ${nodejsRuntime}/bin/node $out/bin/mempool-backend \
+        --add-flags $out/lib/mempool-backend/index.js
+
+      runHook postInstall
+    '';
+
+    passthru = {
+      inherit nodejs nodejsRuntime;
+    };
+  };
+
+  mempool-frontend = mkDerivationMempool {
+    pname = "mempool-frontend";
+
+    buildPhase = ''
+      cd frontend
+
+      ${sync} --chmod=+w ${nodeModules.frontend}/lib/node_modules .
+      patchShebangs node_modules
+
+      # sync-assets.js is called during `npm run build` and downloads assets from the
+      # internet. Disable this script and instead add the assets manually after building.
+      : > sync-assets.js
+
+      # If this produces incomplete output (when run in a different build setup),
+      # see https://github.com/mempool/mempool/issues/1256
+      npm run build
+
+      # Add assets that would otherwise be downloaded by sync-assets.js
+      ${sync} ${frontendAssets}/ dist/mempool/browser/resources
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      ${sync} dist/mempool/browser/ $out
+
+      runHook postInstall
+    '';
+
+    passthru = { assets = frontendAssets; };
+  };
+
+  mempool-nginx-conf = runCommand "mempool-nginx-conf" {} ''
+    ${sync} --chmod=u+w ${./nginx-conf}/ $out
+    ${sync} ${src}/production/nginx/http-language.conf $out/mempool
+  '';
+
+  sync = "${rsync}/bin/rsync -a --inplace";
+
+  mkDerivationMempool = args: stdenvNoCC.mkDerivation ({
+    version = src.rev;
+    inherit src meta;
+
+    nativeBuildInputs = [
+      makeWrapper
+      nodejs
+      rsync
+    ];
+
+    phases = "unpackPhase patchPhase buildPhase installPhase";
+  } // args);
+
+  fetchFiles = { name, hash, fetcher }: stdenvNoCC.mkDerivation {
+    inherit name;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+    outputHash = hash;
+    nativeBuildInputs = [ curl cacert ];
+    buildCommand = ''
+      mkdir $out
+      cd $out
+      ${builtins.readFile fetcher}
+    '';
+  };
+
+  meta = with lib; {
+    description = "Bitcoin blockchain and mempool explorer";
+    homepage = "https://github.com/mempool/mempool/";
+    license = licenses.agpl3Plus;
+    maintainers = with maintainers; [ erikarvstedt ];
+    platforms = platforms.unix;
+  };
+}

pkgs/mempool/frontend-assets-fetch.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Fetch hash-locked versions of assets that are dynamically fetched via
+# https://github.com/mempool/mempool/blob/master/frontend/sync-assets.js
+# when running `npm run build` in the frontend.
+#
+# This file is updated by ./frontend-assets-update.sh
+
+declare -A revs=(
+    ["mempool/mining-pools"]=e889230b0924d7d72eb28186db6f96ef94361fa5
+    ["mempool/mining-pool-logos"]=9cb443035878c3f112af97384d624de245afe72d
+)
+
+fetchFile() {
+    repo=$1
+    file=$2
+    rev=${revs["$repo"]}
+    curl -fsS "https://raw.githubusercontent.com/$repo/$rev/$file"
+}
+
+fetchRepo() {
+    repo=$1
+    rev=${revs["$repo"]}
+    curl -fsSL "https://github.com/$repo/archive/$rev.tar.gz"
+}
+
+# shellcheck disable=SC2094
+fetchFile "mempool/mining-pools" pools.json > pools.json
+mkdir mining-pools
+fetchRepo "mempool/mining-pool-logos" | tar xz --strip-components=1 -C mining-pools

pkgs/mempool/frontend-assets-update.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+updateRepoHash() {
+    repo=$1
+    echo -n "Fetching latest rev for $repo: "
+    hash=$(curl -fsS "https://api.github.com/repos/$repo/commits/master" | jq -r '.sha')
+    echo "$hash"
+    sed -i -E "s|( +)\[\"$repo(.*)|\1[\"$repo\"]=$hash|" frontend-assets-fetch.sh
+}
+
+<frontend-assets-fetch.sh sed -nE 's| +\["([^"]+).*|\1|p' | while read -r repo; do
+    updateRepoHash "$repo"
+done

pkgs/mempool/generate.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p gnupg gnused jq
+set -euo pipefail
+
+# Use this to start a debug shell at the location of this statement
+# . "${BASH_SOURCE[0]%/*}/../../helper/start-bash-session.sh"
+
+version=2.5.0
+# You can also specify a rev instead:
+# rev=57eddac7f0b99b4fe84d91c0f4a50a4f7ccfe55f
+owner=mempool
+repo=https://github.com/$owner/mempool
+
+cd "${BASH_SOURCE[0]%/*}"
+
+updateSrc() {
+    TMPDIR="$(mktemp -d /tmp/mempool.XXX)"
+    trap 'rm -rf $TMPDIR' EXIT
+
+    # Fetch and verify source
+    src=$TMPDIR/src
+    mkdir -p "$src"
+    if [[ -v rev ]]; then
+        # Fetch revision
+        git -C "$src" init
+        git -C "$src" fetch --depth 1 "$repo" "$rev:src"
+        git -C "$src" checkout src
+    else
+        tag=v$version
+        # Fetch and GPG-verify version tag
+        git clone --depth 1 --branch "$tag" -c advice.detachedHead=false $repo "$src"
+        git -C "$src" checkout tags/$tag
+        export GNUPGHOME=$TMPDIR
+        # Fetch wiz' key
+        gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 913C5FF1F579B66CA10378DBA394E332255A6173 2> /dev/null
+        git -C "$src" verify-tag $tag
+        rev=$tag
+    fi
+    rm -rf "$src"/.git
+    hash=$(nix hash path "$src")
+
+    sed -i "
+      s|\bowner = .*;|owner = \"$owner\";|
+      s|\brev = .*;|rev = \"$rev\";|
+      s|\bhash = .*;|hash = \"$hash\";|
+    " default.nix
+}
+
+updateNodeModulesHash() {
+    component=$1
+    echo
+    echo "Fetching node modules for mempool-$component"
+    ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-"$component" "cd $component"
+}
+
+updateFrontendAssets() {
+  . ./frontend-assets-update.sh
+  echo
+  echo "Fetching frontend assets"
+  ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-frontend.assets "frontendAssets"
+}
+
+if [[ $# == 0 ]]; then
+    # Each of these can be run separately
+    updateSrc
+    updateFrontendAssets
+    updateNodeModulesHash backend
+    updateNodeModulesHash frontend
+else
+    "$@"
+fi

pkgs/mempool/nginx-conf/mempool/location-static.conf

@@ -0,0 +1,44 @@
+# see order of nginx location rules
+# https://stackoverflow.com/questions/5238377/nginx-location-priority
+
+# for exact / requests, redirect based on $lang
+# cache redirect for 5 minutes
+location = / {
+	if ($lang != '') {
+		return 302 $scheme://$host/$lang/;
+	}
+	try_files /en-US/index.html =404;
+	expires 5m;
+}
+
+# cache /<lang>/main.f40e91d908a068a2.js forever since they never change
+location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) {
+	try_files $uri =404;
+	expires 1y;
+}
+# cache everything else for 5 minutes
+location ~ ^/([a-z][a-z])$ {
+	try_files $uri /$1/index.html /en-US/index.html =404;
+	expires 5m;
+}
+location ~ ^/([a-z][a-z])/ {
+	try_files $uri /$1/index.html /en-US/index.html =404;
+	expires 5m;
+}
+
+# cache /resources/** for 1 week since they don't change often
+location /resources {
+	try_files $uri /en-US/index.html;
+	expires 1w;
+}
+# cache /main.f40e91d908a068a2.js forever since they never change
+location ~* ^/.+\..+\.(js|css) {
+	try_files /$lang/$uri /en-US/$uri =404;
+	expires 1y;
+}
+# catch-all for all URLs i.e. /address/foo /tx/foo /block/000
+# cache 5 minutes since they change frequently
+location / {
+	try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404;
+	expires 5m;
+}

pkgs/mempool/nginx-conf/mempool/mempool.conf

@@ -0,0 +1,44 @@
+access_log /var/log/nginx/access_mempool.log;
+error_log /var/log/nginx/error_mempool.log;
+
+root /var/www/mempool/browser;
+
+index index.html;
+
+# enable browser and proxy caching
+add_header Cache-Control "public, no-transform";
+
+# vary cache if user changes language preference
+add_header Vary Accept-Language;
+add_header Vary Cookie;
+
+include mempool/location-static.conf;
+
+# static API docs
+location = /api {
+	try_files $uri $uri/ /en-US/index.html =404;
+}
+location = /api/ {
+	try_files $uri $uri/ /en-US/index.html =404;
+}
+
+location /api/v1/ws {
+	proxy_pass http://127.0.0.1:8999/;
+	proxy_http_version 1.1;
+	proxy_set_header Upgrade $http_upgrade;
+	proxy_set_header Connection "Upgrade";
+}
+location /api/v1 {
+	proxy_pass http://127.0.0.1:8999/api/v1;
+}
+location /api/ {
+	proxy_pass http://127.0.0.1:8999/api/v1/;
+}
+
+# mainnet API
+location /ws {
+	proxy_pass http://127.0.0.1:8999/;
+	proxy_http_version 1.1;
+	proxy_set_header Upgrade $http_upgrade;
+	proxy_set_header Connection "Upgrade";
+}

pkgs/mempool/nginx-conf/nginx.conf

@@ -0,0 +1,82 @@
+user nobody;
+pid /var/run/nginx.pid;
+
+worker_processes auto;
+worker_rlimit_nofile 100000;
+
+events {
+    worker_connections 9000;
+    multi_accept on;
+}
+
+http {
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+
+    server_tokens off;
+    server_name_in_redirect off;
+
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
+
+    # reset timed out connections freeing ram
+    reset_timedout_connection on;
+    # maximum time between packets the client can pause when sending nginx any data
+    client_body_timeout 10s;
+    # maximum time the client has to send the entire header to nginx
+    client_header_timeout 10s;
+    # timeout which a single keep-alive client connection will stay open
+    keepalive_timeout 69s;
+    # maximum time between packets nginx is allowed to pause when sending the client data
+    send_timeout 69s;
+
+    # number of requests per connection, does not affect SPDY
+    keepalive_requests 1337;
+
+    # enable gzip compression
+    gzip on;
+    gzip_vary on;
+    gzip_comp_level 6;
+    gzip_min_length 1000;
+    gzip_proxied expired no-cache no-store private auth;
+    # text/html is always compressed by gzip module
+    gzip_types application/javascript application/json application/ld+json application/manifest+json application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard;
+
+    # limit request body size
+    client_max_body_size 10m;
+
+    # proxy cache
+    proxy_cache off;
+    proxy_cache_path /var/cache/nginx keys_zone=cache:20m levels=1:2 inactive=600s max_size=500m;
+    types_hash_max_size 2048;
+
+    # exempt localhost from rate limit
+    geo $limited_ip {
+        default        1;
+        127.0.0.1    0;
+    }
+    map $limited_ip $limited_ip_key {
+        1 $binary_remote_addr;
+        0 '';
+    }
+
+    # rate limit requests
+    limit_req_zone $limited_ip_key zone=api:5m rate=200r/m;
+    limit_req_zone $limited_ip_key zone=electrs:5m rate=2000r/m;
+    limit_req_status 429;
+
+    # rate limit connections
+    limit_conn_zone $limited_ip_key zone=websocket:10m;
+    limit_conn_status 429;
+
+    include mempool/http-language.conf;
+
+    server {
+        listen 127.0.0.1:80;
+        include mempool/mempool.conf;
+    }
+}