Merge pull request #251950 from erictapen/mastodon

nixos/doc/manual/release-notes/rl-2311.section.md

@@ -154,6 +154,8 @@
 
 - The latest version of `clonehero` now stores custom content in `~/.clonehero`. See the [migration instructions](https://clonehero.net/2022/11/29/v23-to-v1-migration-instructions.html). Typically, these content files would exist along side the binary, but the previous build used a wrapper script that would store them in `~/.config/unity3d/srylain Inc_/Clone Hero`.
 
+- `services.mastodon` doesn't support providing a TCP port to its `streaming` component anymore, as upstream implemented parallelization by running multiple instances instead of running multiple processes in one instance. Please create a PR if you are interested in this feature.
+
 - The `services.hostapd` module was rewritten to support `passwordFile` like options, WPA3-SAE, and management of multiple interfaces. This breaks compatibility with older configurations.
   - `hostapd` is now started with additional systemd sandbox/hardening options for better security.
   - `services.hostapd.interface` was replaced with a per-radio and per-bss configuration scheme using [services.hostapd.radios](#opt-services.hostapd.radios).

nixos/modules/services/web-apps/mastodon.nix

@@ -17,9 +17,6 @@ let
     WEB_CONCURRENCY = toString cfg.webProcesses;
     MAX_THREADS = toString cfg.webThreads;
 
-    # mastodon-streaming concurrency.
-    STREAMING_CLUSTER_NUM = toString cfg.streamingProcesses;
-
     DB_USER = cfg.database.user;
 
     REDIS_HOST = cfg.redis.host;
@@ -141,8 +138,44 @@ let
     })
   ) cfg.sidekiqProcesses;
 
+  streamingUnits = builtins.listToAttrs
+      (map (i: {
+        name = "mastodon-streaming-${toString i}";
+        value = {
+          after = [ "network.target" "mastodon-init-dirs.service" ]
+            ++ lib.optional databaseActuallyCreateLocally "postgresql.service"
+            ++ lib.optional cfg.automaticMigrations "mastodon-init-db.service";
+          requires = [ "mastodon-init-dirs.service" ]
+            ++ lib.optional databaseActuallyCreateLocally "postgresql.service"
+            ++ lib.optional cfg.automaticMigrations "mastodon-init-db.service";
+          wantedBy = [ "mastodon.target" "mastodon-streaming.target" ];
+          description = "Mastodon streaming ${toString i}";
+          environment = env // { SOCKET = "/run/mastodon-streaming/streaming-${toString i}.socket"; };
+          serviceConfig = {
+            ExecStart = "${cfg.package}/run-streaming.sh";
+            Restart = "always";
+            RestartSec = 20;
+            EnvironmentFile = [ "/var/lib/mastodon/.secrets_env" ] ++ cfg.extraEnvFiles;
+            WorkingDirectory = cfg.package;
+            # Runtime directory and mode
+            RuntimeDirectory = "mastodon-streaming";
+            RuntimeDirectoryMode = "0750";
+            # System Call Filtering
+            SystemCallFilter = [ ("~" + lib.concatStringsSep " " (systemCallsList ++ [ "@memlock" "@resources" ])) "pipe" "pipe2" ];
+          } // cfgService;
+        };
+      })
+      (lib.range 1 cfg.streamingProcesses));
+
 in {
 
+  imports = [
+    (lib.mkRemovedOptionModule
+      [ "services" "mastodon" "streamingPort" ]
+      "Mastodon currently doesn't support streaming via TCP ports. Please open a PR if you need this."
+    )
+  ];
+
   options = {
     services.mastodon = {
       enable = lib.mkEnableOption (lib.mdDoc "Mastodon, a federated social network server");
@@ -191,18 +224,13 @@ in {
         default = "mastodon";
       };
 
-      streamingPort = lib.mkOption {
-        description = lib.mdDoc "TCP port used by the mastodon-streaming service.";
-        type = lib.types.port;
-        default = 55000;
-      };
       streamingProcesses = lib.mkOption {
         description = lib.mdDoc ''
-          Processes used by the mastodon-streaming service.
-          Defaults to the number of CPU cores minus one.
+          Number of processes used by the mastodon-streaming service.
+          Recommended is the amount of your CPU cores minus one.
         '';
-        type = lib.types.nullOr lib.types.int;
-        default = null;
+        type = lib.types.ints.positive;
+        example = 3;
       };
 
       webPort = lib.mkOption {
@@ -603,6 +631,12 @@ in {
       after = [ "network.target" ];
     };
 
+    systemd.targets.mastodon-streaming = {
+      description = "Target for all Mastodon streaming services";
+      wantedBy = [ "multi-user.target" "mastodon.target" ];
+      after = [ "network.target" ];
+    };
+
     systemd.services.mastodon-init-dirs = {
       script = ''
         umask 077
@@ -688,33 +722,6 @@ in {
         ++ lib.optional databaseActuallyCreateLocally "postgresql.service";
     };
 
-    systemd.services.mastodon-streaming = {
-      after = [ "network.target" "mastodon-init-dirs.service" ]
-        ++ lib.optional databaseActuallyCreateLocally "postgresql.service"
-        ++ lib.optional cfg.automaticMigrations "mastodon-init-db.service";
-      requires = [ "mastodon-init-dirs.service" ]
-        ++ lib.optional databaseActuallyCreateLocally "postgresql.service"
-        ++ lib.optional cfg.automaticMigrations "mastodon-init-db.service";
-      wantedBy = [ "mastodon.target" ];
-      description = "Mastodon streaming";
-      environment = env // (if cfg.enableUnixSocket
-        then { SOCKET = "/run/mastodon-streaming/streaming.socket"; }
-        else { PORT = toString(cfg.streamingPort); }
-      );
-      serviceConfig = {
-        ExecStart = "${cfg.package}/run-streaming.sh";
-        Restart = "always";
-        RestartSec = 20;
-        EnvironmentFile = [ "/var/lib/mastodon/.secrets_env" ] ++ cfg.extraEnvFiles;
-        WorkingDirectory = cfg.package;
-        # Runtime directory and mode
-        RuntimeDirectory = "mastodon-streaming";
-        RuntimeDirectoryMode = "0750";
-        # System Call Filtering
-        SystemCallFilter = [ ("~" + lib.concatStringsSep " " (systemCallsList ++ [ "@memlock" "@resources" ])) "pipe" "pipe2" ];
-      } // cfgService;
-    };
-
     systemd.services.mastodon-web = {
       after = [ "network.target" "mastodon-init-dirs.service" ]
         ++ lib.optional databaseActuallyCreateLocally "postgresql.service"
@@ -780,10 +787,20 @@ in {
         };
 
         locations."/api/v1/streaming/" = {
-          proxyPass = (if cfg.enableUnixSocket then "http://unix:/run/mastodon-streaming/streaming.socket" else "http://127.0.0.1:${toString(cfg.streamingPort)}/");
+          proxyPass = "http://mastodon-streaming";
           proxyWebsockets = true;
         };
       };
+      upstreams.mastodon-streaming = {
+        extraConfig = ''
+          least_conn;
+        '';
+        servers = builtins.listToAttrs
+          (map (i: {
+            name = "unix:/run/mastodon-streaming/streaming-${toString i}.socket";
+            value = { };
+          }) (lib.range 1 cfg.streamingProcesses));
+      };
     };
 
     services.postfix = lib.mkIf (cfg.smtp.createLocally && cfg.smtp.host == "127.0.0.1") {
@@ -819,7 +836,7 @@ in {
 
     users.groups.${cfg.group}.members = lib.optional cfg.configureNginx config.services.nginx.user;
   }
-  { systemd.services = sidekiqUnits; }
+  { systemd.services = lib.mkMerge [ sidekiqUnits streamingUnits ]; }
   ]);
 
   meta.maintainers = with lib.maintainers; [ happy-river erictapen ];

nixos/tests/web-apps/mastodon/remote-postgresql.nix

@@ -16,7 +16,7 @@ in
   meta.maintainers = with pkgs.lib.maintainers; [ erictapen izorkin ];
 
   nodes = {
-    database = {
+    database = { config, ... }: {
       networking = {
         interfaces.eth1 = {
           ipv4.addresses = [
@@ -24,11 +24,13 @@ in
           ];
         };
         extraHosts = hosts;
-        firewall.allowedTCPPorts = [ 5432 ];
+        firewall.allowedTCPPorts = [ config.services.postgresql.port ];
       };
 
       services.postgresql = {
         enable = true;
+        # TODO remove once https://github.com/NixOS/nixpkgs/pull/266270 is resolved.
+        package = pkgs.postgresql_14;
         enableTCPIP = true;
         authentication = ''
           hostnossl mastodon_local mastodon_test 192.168.2.201/32 md5
@@ -41,7 +43,7 @@ in
       };
     };
 
-    nginx = {
+    nginx = { nodes, ... }: {
       networking = {
         interfaces.eth1 = {
           ipv4.addresses = [
@@ -69,18 +71,14 @@ in
             tryFiles = "$uri @proxy";
           };
           locations."@proxy" = {
-            proxyPass = "http://192.168.2.201:55001";
-            proxyWebsockets = true;
-          };
-          locations."/api/v1/streaming/" = {
-            proxyPass = "http://192.168.2.201:55002";
+            proxyPass = "http://192.168.2.201:${toString nodes.server.services.mastodon.webPort}";
             proxyWebsockets = true;
           };
         };
       };
     };
 
-    server = { pkgs, ... }: {
+    server = { config, pkgs, ... }: {
       virtualisation.memorySize = 2048;
 
       environment = {
@@ -98,7 +96,10 @@ in
           ];
         };
         extraHosts = hosts;
-        firewall.allowedTCPPorts = [ 55001 55002 ];
+        firewall.allowedTCPPorts = [
+          config.services.mastodon.webPort
+          config.services.mastodon.sidekiqPort
+        ];
       };
 
       services.mastodon = {
@@ -106,6 +107,7 @@ in
         configureNginx = false;
         localDomain = "mastodon.local";
         enableUnixSocket = false;
+        streamingProcesses = 2;
         database = {
           createLocally = false;
           host = "192.168.2.102";

nixos/tests/web-apps/mastodon/script.nix

@@ -10,9 +10,8 @@
 
   server.wait_for_unit("redis-mastodon.service")
   server.wait_for_unit("mastodon-sidekiq-all.service")
-  server.wait_for_unit("mastodon-streaming.service")
+  server.wait_for_unit("mastodon-streaming.target")
   server.wait_for_unit("mastodon-web.service")
-  server.wait_for_open_port(55000)
   server.wait_for_open_port(55001)
 
   # Check that mastodon-media-auto-remove is scheduled

nixos/tests/web-apps/mastodon/standard.nix

@@ -40,11 +40,15 @@ in
         port = 31637;
       };
 
+      # TODO remove once https://github.com/NixOS/nixpkgs/pull/266270 is resolved.
+      services.postgresql.package = pkgs.postgresql_14;
+
       services.mastodon = {
         enable = true;
         configureNginx = true;
         localDomain = "mastodon.local";
         enableUnixSocket = false;
+        streamingProcesses = 2;
         smtp = {
           createLocally = false;
           fromAddress = "mastodon@mastodon.local";

pkgs/servers/mastodon/default.nix

@@ -1,5 +1,5 @@
 { lib, stdenv, nodejs-slim, bundlerEnv, nixosTests
-, yarn, callPackage, imagemagick, ffmpeg, file, ruby_3_0, writeShellScript
+, yarn, callPackage, imagemagick, ffmpeg, file, ruby, writeShellScript
 , fetchYarnDeps, fixup_yarn_lock
 , brotli
 
@@ -19,8 +19,7 @@ stdenv.mkDerivation rec {
 
   mastodonGems = bundlerEnv {
     name = "${pname}-gems-${version}";
-    inherit version gemset;
-    ruby = ruby_3_0;
+    inherit version gemset ruby;
     gemdir = src;
     # This fix (copied from https://github.com/NixOS/nixpkgs/pull/76765) replaces the gem
     # symlinks with directories, resolving this error when running rake:

pkgs/servers/mastodon/source.nix

@@ -1,16 +1,18 @@
 # This file was generated by pkgs.mastodon.updateScript.
 { fetchFromGitHub, applyPatches }:
 let
-  version = "4.1.9";
+  version = "4.2.1";
 in
-applyPatches {
+(
+  applyPatches {
+    src = fetchFromGitHub {
+      owner = "mastodon";
+      repo = "mastodon";
+      rev = "v${version}";
+      hash = "sha256-SM9WdD+xpxo+gfBft9DARV6QjwNbF2Y9McVrrdDT3fw=";
+    };
+    patches = [];
+  }) // {
   inherit version;
-  src = fetchFromGitHub {
-    owner = "mastodon";
-    repo = "mastodon";
-    rev = "v${version}";
-    hash = "sha256-xpE/mg2AeioW6NThUjLS+SBxGavG4w1xtp3BOMADfYo=";
-  };
-  patches = [];
-  yarnHash = "sha256-e3rl/WuKXaUdeDEYvo1sSubuIwtBjkbguCYdAijwXOA=";
+  yarnHash = "sha256-qoLesubmSvRsXhKwMEWHHXcpcqRszqcdZgHQqnTpNPE=";
 }

pkgs/servers/mastodon/update.sh

@@ -53,9 +53,10 @@ fi
 
 if [[ -z "$REVISION" ]]; then
     REVISION="$(curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s "https://api.github.com/repos/$OWNER/$REPO/releases" | jq -r  'map(select(.prerelease == false)) | .[0].tag_name')"
-    VERSION="$(echo "$REVISION" | cut -c2-)"
 fi
 
+VERSION="$(echo "$REVISION" | cut -c2-)"
+
 rm -f gemset.nix source.nix
 cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
 
@@ -85,15 +86,17 @@ cat > source.nix << EOF
 let
   version = "$VERSION";
 in
-applyPatches {
+(
+  applyPatches {
+    src = fetchFromGitHub {
+      owner = "$OWNER";
+      repo = "$REPO";
+      rev = "v\${version}";
+      hash = "$HASH";
+    };
+    patches = [$PATCHES];
+  }) // {
   inherit version;
-  src = fetchFromGitHub {
-    owner = "$OWNER";
-    repo = "$REPO";
-    rev = "v\${version}";
-    hash = "$HASH";
-  };
-  patches = [$PATCHES];
   yarnHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
 }
 EOF

pkgs/top-level/all-packages.nix

@@ -26587,7 +26587,10 @@ with pkgs;
 
   maker-panel = callPackage ../tools/misc/maker-panel { };
 
-  mastodon = callPackage ../servers/mastodon { };
+  mastodon = callPackage ../servers/mastodon {
+    nodejs-slim = nodejs-slim_20;
+    ruby = ruby_3_2;
+  };
 
   gotosocial = callPackage ../servers/gotosocial { };