From 552043a34dcf84a327295a30b92034292f4d8975 Mon Sep 17 00:00:00 2001 From: Alexander Tomokhov Date: Tue, 3 Oct 2023 22:19:36 +0400 Subject: [PATCH] nixos/castopod: fix startup, displaying images, uploads up to 500 MiB - new maxUploadSize option - new dataDir option (with ReadWritePaths systemd support) - admin page reports correct free disk space (instead of /nix/store) - fix example configuration in documentation - now podcast creation and file upload are tested during NixOS test - move castopod from audio to web-apps folder - verbose logging from the browser test --- nixos/modules/module-list.nix | 2 +- .../services/{audio => web-apps}/castopod.md | 1 + .../services/{audio => web-apps}/castopod.nix | 60 ++-- nixos/tests/castopod.nix | 272 ++++++++++++++---- pkgs/applications/audio/castopod/default.nix | 13 +- 5 files changed, 260 insertions(+), 88 deletions(-) rename nixos/modules/services/{audio => web-apps}/castopod.md (89%) rename nixos/modules/services/{audio => web-apps}/castopod.nix (80%) diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 0a15360f6ea5..71ef594809a8 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -342,7 +342,6 @@ ./services/amqp/rabbitmq.nix ./services/audio/alsa.nix ./services/audio/botamusique.nix - ./services/audio/castopod.nix ./services/audio/gmediarender.nix ./services/audio/gonic.nix ./services/audio/goxlr-utility.nix @@ -1302,6 +1301,7 @@ ./services/web-apps/bookstack.nix ./services/web-apps/c2fmzq-server.nix ./services/web-apps/calibre-web.nix + ./services/web-apps/castopod.nix ./services/web-apps/coder.nix ./services/web-apps/changedetection-io.nix ./services/web-apps/chatgpt-retrieval-plugin.nix diff --git a/nixos/modules/services/audio/castopod.md b/nixos/modules/services/web-apps/castopod.md similarity index 89% rename from nixos/modules/services/audio/castopod.md rename to nixos/modules/services/web-apps/castopod.md index 40838cc77aa6..5ecd807686fd 100644 --- a/nixos/modules/services/audio/castopod.md +++ b/nixos/modules/services/web-apps/castopod.md @@ -4,6 +4,7 @@ Castopod is an open-source hosting platform made for podcasters who want to enga ## Quickstart {#module-services-castopod-quickstart} +Configure ACME (https://nixos.org/manual/nixos/unstable/#module-security-acme). Use the following configuration to start a public instance of Castopod on `castopod.example.com` domain: ```nix diff --git a/nixos/modules/services/audio/castopod.nix b/nixos/modules/services/web-apps/castopod.nix similarity index 80% rename from nixos/modules/services/audio/castopod.nix rename to nixos/modules/services/web-apps/castopod.nix index b782b5489147..f195482a0450 100644 --- a/nixos/modules/services/audio/castopod.nix +++ b/nixos/modules/services/web-apps/castopod.nix @@ -4,7 +4,6 @@ let fpm = config.services.phpfpm.pools.castopod; user = "castopod"; - stateDirectory = "/var/lib/castopod"; # https://docs.castopod.org/getting-started/install.html#requirements phpPackage = pkgs.php.withExtensions ({ enabled, all }: with all; [ @@ -29,6 +28,15 @@ in defaultText = lib.literalMD "pkgs.castopod"; description = lib.mdDoc "Which Castopod package to use."; }; + dataDir = lib.mkOption { + type = lib.types.path; + default = "/var/lib/castopod"; + description = lib.mdDoc '' + The path where castopod stores all data. This path must be in sync + with the castopod package (where it is hardcoded during the build in + accordance with its own `dataDir` argument). + ''; + }; database = { createLocally = lib.mkOption { type = lib.types.bool; @@ -111,6 +119,18 @@ in Options for Castopod's PHP pool. See the documentation on `php-fpm.conf` for details on configuration directives. ''; }; + maxUploadSize = lib.mkOption { + type = lib.types.str; + default = "512M"; + description = lib.mdDoc '' + Maximum supported size for a file upload in. Maximum HTTP body + size is set to this value for nginx and PHP (because castopod doesn't + support chunked uploads yet: + https://code.castopod.org/adaures/castopod/-/issues/330). Note, that + practical upload size limit is smaller. For example, with 512 MiB + setting - around 500 MiB is possible. + ''; + }; }; }; @@ -120,13 +140,13 @@ in sslEnabled = with config.services.nginx.virtualHosts.${cfg.localDomain}; addSSL || forceSSL || onlySSL || enableACME || useACMEHost != null; baseURL = "http${lib.optionalString sslEnabled "s"}://${cfg.localDomain}"; in - lib.mapAttrs (name: lib.mkDefault) { + lib.mapAttrs (_: lib.mkDefault) { "app.forceGlobalSecureRequests" = sslEnabled; "app.baseURL" = baseURL; - "media.baseURL" = "/"; + "media.baseURL" = baseURL; "media.root" = "media"; - "media.storage" = stateDirectory; + "media.storage" = cfg.dataDir; "admin.gateway" = "admin"; "auth.gateway" = "auth"; @@ -142,13 +162,13 @@ in services.phpfpm.pools.castopod = { inherit user; group = config.services.nginx.group; - phpPackage = phpPackage; + inherit phpPackage; phpOptions = '' - # https://code.castopod.org/adaures/castopod/-/blob/main/docker/production/app/uploads.ini + # https://code.castopod.org/adaures/castopod/-/blob/develop/docker/production/common/uploads.template.ini file_uploads = On memory_limit = 512M - upload_max_filesize = 500M - post_max_size = 512M + upload_max_filesize = ${cfg.maxUploadSize} + post_max_size = ${cfg.maxUploadSize} max_execution_time = 300 max_input_time = 300 ''; @@ -165,25 +185,25 @@ in path = [ pkgs.openssl phpPackage ]; script = let - envFile = "${stateDirectory}/.env"; + envFile = "${cfg.dataDir}/.env"; media = "${cfg.settings."media.storage"}/${cfg.settings."media.root"}"; in '' - mkdir -p ${stateDirectory}/writable/{cache,logs,session,temp,uploads} + mkdir -p ${cfg.dataDir}/writable/{cache,logs,session,temp,uploads} if [ ! -d ${lib.escapeShellArg media} ]; then cp --no-preserve=mode,ownership -r ${cfg.package}/share/castopod/public/media ${lib.escapeShellArg media} fi - if [ ! -f ${stateDirectory}/salt ]; then - openssl rand -base64 33 > ${stateDirectory}/salt + if [ ! -f ${cfg.dataDir}/salt ]; then + openssl rand -base64 33 > ${cfg.dataDir}/salt fi cat <<'EOF' > ${envFile} ${lib.generators.toKeyValue { } cfg.settings} EOF - echo "analytics.salt=$(cat ${stateDirectory}/salt)" >> ${envFile} + echo "analytics.salt=$(cat ${cfg.dataDir}/salt)" >> ${envFile} ${if (cfg.database.passwordFile != null) then '' echo "database.default.password=$(cat ${lib.escapeShellArg cfg.database.passwordFile})" >> ${envFile} @@ -192,10 +212,10 @@ in ''} ${lib.optionalString (cfg.environmentFile != null) '' - cat ${lib.escapeShellArg cfg.environmentFile}) >> ${envFile} + cat ${lib.escapeShellArg cfg.environmentFile} >> ${envFile} ''} - php spark castopod:database-update + php ${cfg.package}/share/castopod/spark castopod:database-update ''; serviceConfig = { StateDirectory = "castopod"; @@ -204,6 +224,7 @@ in RemainAfterExit = true; User = user; Group = config.services.nginx.group; + ReadWritePaths = cfg.dataDir; }; }; @@ -212,9 +233,7 @@ in wantedBy = [ "multi-user.target" ]; path = [ phpPackage ]; script = '' - php public/index.php scheduled-activities - php public/index.php scheduled-websub-publish - php public/index.php scheduled-video-clips + php ${cfg.package}/share/castopod/spark tasks:run ''; serviceConfig = { StateDirectory = "castopod"; @@ -222,6 +241,8 @@ in Type = "oneshot"; User = user; Group = config.services.nginx.group; + ReadWritePaths = cfg.dataDir; + LogLevelMax = "notice"; # otherwise periodic tasks flood the journal }; }; @@ -251,6 +272,7 @@ in extraConfig = '' try_files $uri $uri/ /index.php?$args; index index.php index.html; + client_max_body_size ${cfg.maxUploadSize}; ''; locations."^~ /${cfg.settings."media.root"}/" = { @@ -278,7 +300,7 @@ in }; }; - users.users.${user} = lib.mapAttrs (name: lib.mkDefault) { + users.users.${user} = lib.mapAttrs (_: lib.mkDefault) { description = "Castopod user"; isSystemUser = true; group = config.services.nginx.group; diff --git a/nixos/tests/castopod.nix b/nixos/tests/castopod.nix index 4435ec617d4e..241705aebd72 100644 --- a/nixos/tests/castopod.nix +++ b/nixos/tests/castopod.nix @@ -4,74 +4,218 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: meta = with lib.maintainers; { maintainers = [ alexoundos misuzu ]; }; + nodes.castopod = { nodes, ... }: { + # otherwise 500 MiB file upload fails! + virtualisation.diskSize = 512 + 3 * 512; + networking.firewall.allowedTCPPorts = [ 80 ]; - networking.extraHosts = '' - 127.0.0.1 castopod.example.com - ''; + networking.extraHosts = + lib.strings.concatStringsSep "\n" + (lib.attrsets.mapAttrsToList + (name: _: "127.0.0.1 ${name}") + nodes.castopod.services.nginx.virtualHosts); + services.castopod = { enable = true; database.createLocally = true; localDomain = "castopod.example.com"; + maxUploadSize = "512M"; }; - environment.systemPackages = - let - username = "admin"; - email = "admin@castood.example.com"; - password = "v82HmEp5"; - testRunner = pkgs.writers.writePython3Bin "test-runner" - { - libraries = [ pkgs.python3Packages.selenium ]; - flakeIgnore = [ - "E501" - ]; - } '' - from selenium.webdriver.common.by import By - from selenium.webdriver import Firefox - from selenium.webdriver.firefox.options import Options - from selenium.webdriver.support.ui import WebDriverWait - from selenium.webdriver.support import expected_conditions as EC - - options = Options() - options.add_argument('--headless') - driver = Firefox(options=options) - try: - driver.implicitly_wait(20) - driver.get('http://castopod.example.com/cp-install') - - wait = WebDriverWait(driver, 10) - - wait.until(EC.title_contains("installer")) - - driver.find_element(By.CSS_SELECTOR, '#username').send_keys( - '${username}' - ) - driver.find_element(By.CSS_SELECTOR, '#email').send_keys( - '${email}' - ) - driver.find_element(By.CSS_SELECTOR, '#password').send_keys( - '${password}' - ) - driver.find_element(By.XPATH, "//button[contains(., 'Finish install')]").click() - - wait.until(EC.title_contains("Auth")) - - driver.find_element(By.CSS_SELECTOR, '#email').send_keys( - '${email}' - ) - driver.find_element(By.CSS_SELECTOR, '#password').send_keys( - '${password}' - ) - driver.find_element(By.XPATH, "//button[contains(., 'Login')]").click() - - wait.until(EC.title_contains("Admin dashboard")) - finally: - driver.close() - driver.quit() - ''; - in - [ pkgs.firefox-unwrapped pkgs.geckodriver testRunner ]; }; + + nodes.client = { nodes, pkgs, lib, ... }: + let + domain = nodes.castopod.services.castopod.localDomain; + + getIP = node: + (builtins.head node.networking.interfaces.eth1.ipv4.addresses).address; + + targetPodcastSize = 500 * 1024 * 1024; + lameMp3Bitrate = 348300; + lameMp3FileAdjust = -800; + targetPodcastDuration = toString + ((targetPodcastSize + lameMp3FileAdjust) / (lameMp3Bitrate / 8)); + bannerWidth = 3000; + banner = pkgs.runCommand "gen-castopod-cover.jpg" { } '' + ${pkgs.imagemagick}/bin/magick ` + `-background green -bordercolor white -gravity northwest xc:black ` + `-duplicate 99 ` + `-seed 1 -resize "%[fx:rand()*72+24]" ` + `-seed 0 -rotate "%[fx:rand()*360]" -border 6x6 -splice 16x36 ` + `-seed 0 -rotate "%[fx:floor(rand()*4)*90]" -resize "150x50!" ` + `+append -crop 10x1@ +repage -roll "+%[fx:(t%2)*72]+0" -append ` + `-resize ${toString bannerWidth} -quality 1 $out + ''; + + coverWidth = toString 3000; + cover = pkgs.runCommand "gen-castopod-banner.jpg" { } '' + ${pkgs.imagemagick}/bin/magick ` + `-background white -bordercolor white -gravity northwest xc:black ` + `-duplicate 99 ` + `-seed 1 -resize "%[fx:rand()*72+24]" ` + `-seed 0 -rotate "%[fx:rand()*360]" -border 6x6 -splice 36x36 ` + `-seed 0 -rotate "%[fx:floor(rand()*4)*90]" -resize "144x144!" ` + `+append -crop 10x1@ +repage -roll "+%[fx:(t%2)*72]+0" -append ` + `-resize ${coverWidth} -quality 1 $out + ''; + in + { + networking.extraHosts = + lib.strings.concatStringsSep "\n" + (lib.attrsets.mapAttrsToList + (name: _: "${getIP nodes.castopod} ${name}") + nodes.castopod.services.nginx.virtualHosts); + + environment.systemPackages = + let + username = "admin"; + email = "admin@${domain}"; + password = "Abcd1234"; + podcastTitle = "Some Title"; + episodeTitle = "Episode Title"; + browser-test = pkgs.writers.writePython3Bin "browser-test" + { + libraries = [ pkgs.python3Packages.selenium ]; + flakeIgnore = [ "E124" "E501" ]; + } '' + from selenium.webdriver.common.by import By + from selenium.webdriver import Firefox + from selenium.webdriver.firefox.options import Options + from selenium.webdriver.firefox.service import Service + from selenium.webdriver.support.ui import WebDriverWait + from selenium.webdriver.support import expected_conditions as EC + from subprocess import STDOUT + import logging + + selenium_logger = logging.getLogger("selenium") + selenium_logger.setLevel(logging.DEBUG) + selenium_logger.addHandler(logging.StreamHandler()) + + options = Options() + options.add_argument('--headless') + service = Service(log_output=STDOUT) + driver = Firefox(options=options, service=service) + driver = Firefox(options=options) + driver.implicitly_wait(30) + + # install ########################################################## + + driver.get('http://${domain}/cp-install') + + wait = WebDriverWait(driver, 10) + + wait.until(EC.title_contains("installer")) + + driver.find_element(By.CSS_SELECTOR, '#username').send_keys( + '${username}' + ) + driver.find_element(By.CSS_SELECTOR, '#email').send_keys( + '${email}' + ) + driver.find_element(By.CSS_SELECTOR, '#password').send_keys( + '${password}' + ) + driver.find_element(By.XPATH, + "//button[contains(., 'Finish install')]" + ).click() + + wait.until(EC.title_contains("Auth")) + + driver.find_element(By.CSS_SELECTOR, '#email').send_keys( + '${email}' + ) + driver.find_element(By.CSS_SELECTOR, '#password').send_keys( + '${password}' + ) + driver.find_element(By.XPATH, + "//button[contains(., 'Login')]" + ).click() + + wait.until(EC.title_contains("Admin dashboard")) + + # create podcast ################################################### + + driver.get('http://${domain}/admin/podcasts/new') + + wait.until(EC.title_contains("Create podcast")) + + driver.find_element(By.CSS_SELECTOR, '#cover').send_keys( + '${cover}' + ) + driver.find_element(By.CSS_SELECTOR, '#banner').send_keys( + '${banner}' + ) + driver.find_element(By.CSS_SELECTOR, '#title').send_keys( + '${podcastTitle}' + ) + driver.find_element(By.CSS_SELECTOR, '#handle').send_keys( + 'some_handle' + ) + driver.find_element(By.CSS_SELECTOR, '#description').send_keys( + 'Some description' + ) + driver.find_element(By.CSS_SELECTOR, '#owner_name').send_keys( + 'Owner Name' + ) + driver.find_element(By.CSS_SELECTOR, '#owner_email').send_keys( + 'owner@email.xyz' + ) + driver.find_element(By.XPATH, + "//button[contains(., 'Create podcast')]" + ).click() + + wait.until(EC.title_contains("${podcastTitle}")) + + driver.find_element(By.XPATH, + "//span[contains(., 'Add an episode')]" + ).click() + + wait.until(EC.title_contains("Add an episode")) + + # upload podcast ################################################### + + driver.find_element(By.CSS_SELECTOR, '#audio_file').send_keys( + '/tmp/podcast.mp3' + ) + driver.find_element(By.CSS_SELECTOR, '#cover').send_keys( + '${cover}' + ) + driver.find_element(By.CSS_SELECTOR, '#description').send_keys( + 'Episode description' + ) + driver.find_element(By.CSS_SELECTOR, '#title').send_keys( + '${episodeTitle}' + ) + driver.find_element(By.XPATH, + "//button[contains(., 'Create episode')]" + ).click() + + wait.until(EC.title_contains("${episodeTitle}")) + + driver.close() + driver.quit() + ''; + in + [ + pkgs.firefox-unwrapped + pkgs.geckodriver + browser-test + (pkgs.writeShellApplication { + name = "build-mp3"; + runtimeInputs = with pkgs; [ sox lame ]; + text = '' + out=/tmp/podcast.mp3 + sox -n -r 48000 -t wav - synth ${targetPodcastDuration} sine 440 ` + `| lame --noreplaygain -cbr -q 9 -b 320 - $out + FILESIZE="$(stat -c%s $out)" + [ "$FILESIZE" -gt 0 ] + [ "$FILESIZE" -le "${toString targetPodcastSize}" ] + ''; + }) + ]; + }; + testScript = '' start_all() castopod.wait_for_unit("castopod-setup.service") @@ -79,9 +223,11 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: castopod.wait_for_unit("nginx.service") castopod.wait_for_open_port(80) castopod.wait_until_succeeds("curl -sS -f http://castopod.example.com") - castopod.succeed("curl -s http://localhost/cp-install | grep 'Create your Super Admin account' > /dev/null") - with subtest("Create superadmin and log in"): - castopod.succeed("PYTHONUNBUFFERED=1 systemd-cat -t test-runner test-runner") + client.succeed("build-mp3") + + with subtest("Create superadmin, log in, create and upload a podcast"): + client.succeed(\ + "PYTHONUNBUFFERED=1 systemd-cat -t browser-test browser-test") ''; }) diff --git a/pkgs/applications/audio/castopod/default.nix b/pkgs/applications/audio/castopod/default.nix index 801368a131cf..13bb4afe8e2d 100644 --- a/pkgs/applications/audio/castopod/default.nix +++ b/pkgs/applications/audio/castopod/default.nix @@ -3,7 +3,7 @@ , ffmpeg-headless , lib , nixosTests -, stateDirectory ? "/var/lib/castopod" +, dataDir ? "/var/lib/castopod" }: stdenv.mkDerivation { pname = "castopod"; @@ -20,13 +20,16 @@ stdenv.mkDerivation { postPatch = '' # not configurable at runtime unfortunately: substituteInPlace app/Config/Paths.php \ - --replace "__DIR__ . '/../../writable'" "'${stateDirectory}/writable'" + --replace "__DIR__ . '/../../writable'" "'${dataDir}/writable'" - # configuration file must be writable, place it to ${stateDirectory} + substituteInPlace modules/Admin/Controllers/DashboardController.php \ + --replace "disk_total_space('./')" "disk_total_space('${dataDir}')" + + # configuration file must be writable, place it to ${dataDir} substituteInPlace modules/Install/Controllers/InstallController.php \ - --replace "ROOTPATH" "'${stateDirectory}/'" + --replace "ROOTPATH" "'${dataDir}/'" substituteInPlace public/index.php spark \ - --replace "DotEnv(ROOTPATH)" "DotEnv('${stateDirectory}')" + --replace "DotEnv(ROOTPATH)" "DotEnv('${dataDir}')" # ffmpeg is required for Video Clips feature substituteInPlace modules/MediaClipper/VideoClipper.php \