diff --git a/helper/update-fixed-output-derivation.sh b/helper/update-fixed-output-derivation.sh index dafceb4..cc28b9f 100755 --- a/helper/update-fixed-output-derivation.sh +++ b/helper/update-fixed-output-derivation.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + set -euo pipefail # The file that defines the derivation that should be updated diff --git a/pkgs/default.nix b/pkgs/default.nix index 0cdd059..c0e9fdf 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -17,6 +17,10 @@ let self = { lndinit = pkgs.callPackage ./lndinit { }; liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { }; rtl = pkgs.callPackage ./rtl { inherit (self) fetchNodeModules; }; + inherit (pkgs.callPackage ./mempool { inherit (self) fetchNodeModules; }) + mempool-backend + mempool-frontend + mempool-nginx-conf; # The secp256k1 version used by joinmarket secp256k1 = pkgs.callPackage ./secp256k1 { }; trustedcoin = pkgs.callPackage ./trustedcoin { }; diff --git a/pkgs/mempool/default.nix b/pkgs/mempool/default.nix new file mode 100644 index 0000000..1a45785 --- /dev/null +++ b/pkgs/mempool/default.nix @@ -0,0 +1,143 @@ +{ lib +, stdenvNoCC +, nodejs-18_x +, nodejs-slim-18_x +, fetchFromGitHub +, fetchNodeModules +, runCommand +, makeWrapper +, curl +, cacert +, rsync +}: +rec { + nodejs = nodejs-18_x; + nodejsRuntime = nodejs-slim-18_x; + + src = fetchFromGitHub { + owner = "mempool"; + repo = "mempool"; + rev = "v2.5.0"; + hash = "sha256-8HmfytxRte3fQ0QKOljUVk9YAuaXhQQWuv3EFNmOgfQ="; + }; + + nodeModules = { + frontend = fetchNodeModules { + inherit src nodejs; + preBuild = "cd frontend"; + hash = "sha256-/Z0xNvob7eMGpzdUWolr47vljpFiIutZpGwd0uYhPWI="; + }; + backend = fetchNodeModules { + inherit src nodejs; + preBuild = "cd backend"; + hash = "sha256-HpzzSTuSRWDWGbctVhTcUA01if/7OTI4xN3DAbAAX+U="; + }; + }; + + frontendAssets = fetchFiles { + name = "mempool-frontend-assets"; + hash = "sha256-3TmulAfzJJMf0UFhnHEqjAnzc1TNC5DM2XcsU7eyinY="; + fetcher = ./frontend-assets-fetch.sh; + }; + + mempool-backend = mkDerivationMempool { + pname = "mempool-backend"; + + buildPhase = '' + cd backend + ${sync} --chmod=+w ${nodeModules.backend}/lib/node_modules . + patchShebangs node_modules + + npm run package + + runHook postBuild + ''; + + installPhase = '' + mkdir -p $out/lib/mempool-backend + ${sync} package/ $out/lib/mempool-backend + + makeWrapper ${nodejsRuntime}/bin/node $out/bin/mempool-backend \ + --add-flags $out/lib/mempool-backend/index.js + + runHook postInstall + ''; + + passthru = { + inherit nodejs nodejsRuntime; + }; + }; + + mempool-frontend = mkDerivationMempool { + pname = "mempool-frontend"; + + buildPhase = '' + cd frontend + + ${sync} --chmod=+w ${nodeModules.frontend}/lib/node_modules . + patchShebangs node_modules + + # sync-assets.js is called during `npm run build` and downloads assets from the + # internet. Disable this script and instead add the assets manually after building. + : > sync-assets.js + + # If this produces incomplete output (when run in a different build setup), + # see https://github.com/mempool/mempool/issues/1256 + npm run build + + # Add assets that would otherwise be downloaded by sync-assets.js + ${sync} ${frontendAssets}/ dist/mempool/browser/resources + + runHook postBuild + ''; + + installPhase = '' + ${sync} dist/mempool/browser/ $out + + runHook postInstall + ''; + + passthru = { assets = frontendAssets; }; + }; + + mempool-nginx-conf = runCommand "mempool-nginx-conf" {} '' + ${sync} --chmod=u+w ${./nginx-conf}/ $out + ${sync} ${src}/production/nginx/http-language.conf $out/mempool + ''; + + sync = "${rsync}/bin/rsync -a --inplace"; + + mkDerivationMempool = args: stdenvNoCC.mkDerivation ({ + version = src.rev; + inherit src meta; + + nativeBuildInputs = [ + makeWrapper + nodejs + rsync + ]; + + phases = "unpackPhase patchPhase buildPhase installPhase"; + } // args); + + fetchFiles = { name, hash, fetcher }: stdenvNoCC.mkDerivation { + inherit name; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = hash; + nativeBuildInputs = [ curl cacert ]; + buildCommand = '' + mkdir $out + cd $out + ${builtins.readFile fetcher} + ''; + }; + + meta = with lib; { + description = "Bitcoin blockchain and mempool explorer"; + homepage = "https://github.com/mempool/mempool/"; + license = licenses.agpl3Plus; + maintainers = with maintainers; [ erikarvstedt ]; + platforms = platforms.unix; + }; +} diff --git a/pkgs/mempool/frontend-assets-fetch.sh b/pkgs/mempool/frontend-assets-fetch.sh new file mode 100755 index 0000000..f114e8c --- /dev/null +++ b/pkgs/mempool/frontend-assets-fetch.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Fetch hash-locked versions of assets that are dynamically fetched via +# https://github.com/mempool/mempool/blob/master/frontend/sync-assets.js +# when running `npm run build` in the frontend. +# +# This file is updated by ./frontend-assets-update.sh + +declare -A revs=( + ["mempool/mining-pools"]=e889230b0924d7d72eb28186db6f96ef94361fa5 + ["mempool/mining-pool-logos"]=9cb443035878c3f112af97384d624de245afe72d +) + +fetchFile() { + repo=$1 + file=$2 + rev=${revs["$repo"]} + curl -fsS "https://raw.githubusercontent.com/$repo/$rev/$file" +} + +fetchRepo() { + repo=$1 + rev=${revs["$repo"]} + curl -fsSL "https://github.com/$repo/archive/$rev.tar.gz" +} + +# shellcheck disable=SC2094 +fetchFile "mempool/mining-pools" pools.json > pools.json +mkdir mining-pools +fetchRepo "mempool/mining-pool-logos" | tar xz --strip-components=1 -C mining-pools diff --git a/pkgs/mempool/frontend-assets-update.sh b/pkgs/mempool/frontend-assets-update.sh new file mode 100755 index 0000000..33fdf2d --- /dev/null +++ b/pkgs/mempool/frontend-assets-update.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +updateRepoHash() { + repo=$1 + echo -n "Fetching latest rev for $repo: " + hash=$(curl -fsS "https://api.github.com/repos/$repo/commits/master" | jq -r '.sha') + echo "$hash" + sed -i -E "s|( +)\[\"$repo(.*)|\1[\"$repo\"]=$hash|" frontend-assets-fetch.sh +} + + /dev/null + git -C "$src" verify-tag $tag + rev=$tag + fi + rm -rf "$src"/.git + hash=$(nix hash path "$src") + + sed -i " + s|\bowner = .*;|owner = \"$owner\";| + s|\brev = .*;|rev = \"$rev\";| + s|\bhash = .*;|hash = \"$hash\";| + " default.nix +} + +updateNodeModulesHash() { + component=$1 + echo + echo "Fetching node modules for mempool-$component" + ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-"$component" "cd $component" +} + +updateFrontendAssets() { + . ./frontend-assets-update.sh + echo + echo "Fetching frontend assets" + ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-frontend.assets "frontendAssets" +} + +if [[ $# == 0 ]]; then + # Each of these can be run separately + updateSrc + updateFrontendAssets + updateNodeModulesHash backend + updateNodeModulesHash frontend +else + "$@" +fi diff --git a/pkgs/mempool/nginx-conf/mempool/location-static.conf b/pkgs/mempool/nginx-conf/mempool/location-static.conf new file mode 100644 index 0000000..2afed65 --- /dev/null +++ b/pkgs/mempool/nginx-conf/mempool/location-static.conf @@ -0,0 +1,44 @@ +# see order of nginx location rules +# https://stackoverflow.com/questions/5238377/nginx-location-priority + +# for exact / requests, redirect based on $lang +# cache redirect for 5 minutes +location = / { + if ($lang != '') { + return 302 $scheme://$host/$lang/; + } + try_files /en-US/index.html =404; + expires 5m; +} + +# cache //main.f40e91d908a068a2.js forever since they never change +location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) { + try_files $uri =404; + expires 1y; +} +# cache everything else for 5 minutes +location ~ ^/([a-z][a-z])$ { + try_files $uri /$1/index.html /en-US/index.html =404; + expires 5m; +} +location ~ ^/([a-z][a-z])/ { + try_files $uri /$1/index.html /en-US/index.html =404; + expires 5m; +} + +# cache /resources/** for 1 week since they don't change often +location /resources { + try_files $uri /en-US/index.html; + expires 1w; +} +# cache /main.f40e91d908a068a2.js forever since they never change +location ~* ^/.+\..+\.(js|css) { + try_files /$lang/$uri /en-US/$uri =404; + expires 1y; +} +# catch-all for all URLs i.e. /address/foo /tx/foo /block/000 +# cache 5 minutes since they change frequently +location / { + try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404; + expires 5m; +} diff --git a/pkgs/mempool/nginx-conf/mempool/mempool.conf b/pkgs/mempool/nginx-conf/mempool/mempool.conf new file mode 100644 index 0000000..78d2813 --- /dev/null +++ b/pkgs/mempool/nginx-conf/mempool/mempool.conf @@ -0,0 +1,44 @@ +access_log /var/log/nginx/access_mempool.log; +error_log /var/log/nginx/error_mempool.log; + +root /var/www/mempool/browser; + +index index.html; + +# enable browser and proxy caching +add_header Cache-Control "public, no-transform"; + +# vary cache if user changes language preference +add_header Vary Accept-Language; +add_header Vary Cookie; + +include mempool/location-static.conf; + +# static API docs +location = /api { + try_files $uri $uri/ /en-US/index.html =404; +} +location = /api/ { + try_files $uri $uri/ /en-US/index.html =404; +} + +location /api/v1/ws { + proxy_pass http://127.0.0.1:8999/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; +} +location /api/v1 { + proxy_pass http://127.0.0.1:8999/api/v1; +} +location /api/ { + proxy_pass http://127.0.0.1:8999/api/v1/; +} + +# mainnet API +location /ws { + proxy_pass http://127.0.0.1:8999/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; +} diff --git a/pkgs/mempool/nginx-conf/nginx.conf b/pkgs/mempool/nginx-conf/nginx.conf new file mode 100644 index 0000000..cce77d7 --- /dev/null +++ b/pkgs/mempool/nginx-conf/nginx.conf @@ -0,0 +1,82 @@ +user nobody; +pid /var/run/nginx.pid; + +worker_processes auto; +worker_rlimit_nofile 100000; + +events { + worker_connections 9000; + multi_accept on; +} + +http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + + server_tokens off; + server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + # reset timed out connections freeing ram + reset_timedout_connection on; + # maximum time between packets the client can pause when sending nginx any data + client_body_timeout 10s; + # maximum time the client has to send the entire header to nginx + client_header_timeout 10s; + # timeout which a single keep-alive client connection will stay open + keepalive_timeout 69s; + # maximum time between packets nginx is allowed to pause when sending the client data + send_timeout 69s; + + # number of requests per connection, does not affect SPDY + keepalive_requests 1337; + + # enable gzip compression + gzip on; + gzip_vary on; + gzip_comp_level 6; + gzip_min_length 1000; + gzip_proxied expired no-cache no-store private auth; + # text/html is always compressed by gzip module + gzip_types application/javascript application/json application/ld+json application/manifest+json application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard; + + # limit request body size + client_max_body_size 10m; + + # proxy cache + proxy_cache off; + proxy_cache_path /var/cache/nginx keys_zone=cache:20m levels=1:2 inactive=600s max_size=500m; + types_hash_max_size 2048; + + # exempt localhost from rate limit + geo $limited_ip { + default 1; + 127.0.0.1 0; + } + map $limited_ip $limited_ip_key { + 1 $binary_remote_addr; + 0 ''; + } + + # rate limit requests + limit_req_zone $limited_ip_key zone=api:5m rate=200r/m; + limit_req_zone $limited_ip_key zone=electrs:5m rate=2000r/m; + limit_req_status 429; + + # rate limit connections + limit_conn_zone $limited_ip_key zone=websocket:10m; + limit_conn_status 429; + + include mempool/http-language.conf; + + server { + listen 127.0.0.1:80; + include mempool/mempool.conf; + } +}