Merge pull request #136993 from sbruder/invidious

invidious: init

doc/languages-frameworks/crystal.section.md

@@ -4,12 +4,12 @@
 
 This section uses [Mint](https://github.com/mint-lang/mint) as an example for how to build a Crystal package.
 
-If the Crystal project has any dependencies, the first step is to get a `shards.nix` file encoding those. Get a copy of the project and go to its root directory such that its `shard.lock` file is in the current directory, then run `crystal2nix` in it
+If the Crystal project has any dependencies, the first step is to get a `shards.nix` file encoding those. Get a copy of the project and go to its root directory such that its `shard.lock` file is in the current directory. Executable projects should usually commit the `shard.lock` file, but sometimes that's not the case, which means you need to generate it yourself. With an existing `shard.lock` file, `crystal2nix` can be run.
-
 ```bash
 $ git clone https://github.com/mint-lang/mint
 $ cd mint
 $ git checkout 0.5.0
+$ if [ ! -f shard.lock ]; then nix-shell -p shards --run "shards lock"; fi
 $ nix-shell -p crystal2nix --run crystal2nix
 ```
 

nixos/modules/module-list.nix

@@ -992,6 +992,7 @@
   ./services/web-apps/jitsi-meet.nix
   ./services/web-apps/keycloak.nix
   ./services/web-apps/lemmy.nix
+  ./services/web-apps/invidious.nix
   ./services/web-apps/limesurvey.nix
   ./services/web-apps/mastodon.nix
   ./services/web-apps/mattermost.nix

nixos/modules/services/web-apps/invidious.nix

@@ -0,0 +1,263 @@
+{ lib, config, pkgs, options, ... }:
+let
+  cfg = config.services.invidious;
+  # To allow injecting secrets with jq, json (instead of yaml) is used
+  settingsFormat = pkgs.formats.json { };
+  inherit (lib) types;
+
+  settingsFile = settingsFormat.generate "invidious-settings" cfg.settings;
+
+  serviceConfig = {
+    systemd.services.invidious = {
+      description = "Invidious (An alternative YouTube front-end)";
+      wants = [ "network-online.target" ];
+      after = [ "syslog.target" "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      script =
+        let
+          jqFilter = "."
+            + lib.optionalString (cfg.database.host != null) "[0].db.password = \"'\"'\"$(cat ${lib.escapeShellArg cfg.database.passwordFile})\"'\"'\""
+            + " | .[0]"
+            + lib.optionalString (cfg.extraSettingsFile != null) " * .[1]";
+          jqFiles = [ settingsFile ] ++ lib.optional (cfg.extraSettingsFile != null) cfg.extraSettingsFile;
+        in
+        ''
+          export INVIDIOUS_CONFIG="$(${pkgs.jq}/bin/jq -s "${jqFilter}" ${lib.escapeShellArgs jqFiles})"
+          exec ${cfg.package}/bin/invidious
+        '';
+
+      serviceConfig = {
+        RestartSec = "2s";
+        DynamicUser = true;
+
+        CapabilityBoundingSet = "";
+        PrivateDevices = true;
+        PrivateUsers = true;
+        ProtectHome = true;
+        ProtectKernelLogs = true;
+        ProtectProc = "invisible";
+        RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ];
+        RestrictNamespaces = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ];
+      };
+    };
+
+    services.invidious.settings = {
+      inherit (cfg) port;
+
+      # Automatically initialises and migrates the database if necessary
+      check_tables = true;
+
+      db = {
+        user = lib.mkDefault "kemal";
+        dbname = lib.mkDefault "invidious";
+        port = cfg.database.port;
+        # Blank for unix sockets, see
+        # https://github.com/will/crystal-pg/blob/1548bb255210/src/pq/conninfo.cr#L100-L108
+        host = if cfg.database.host == null then "" else cfg.database.host;
+        # Not needed because peer authentication is enabled
+        password = lib.mkIf (cfg.database.host == null) "";
+      };
+    } // (lib.optionalAttrs (cfg.domain != null) {
+      inherit (cfg) domain;
+    });
+
+    assertions = [{
+      assertion = cfg.database.host != null -> cfg.database.passwordFile != null;
+      message = "If database host isn't null, database password needs to be set";
+    }];
+  };
+
+  # Settings necessary for running with an automatically managed local database
+  localDatabaseConfig = lib.mkIf cfg.database.createLocally {
+    # Default to using the local database if we create it
+    services.invidious.database.host = lib.mkDefault null;
+
+    services.postgresql = {
+      enable = true;
+      ensureDatabases = lib.singleton cfg.settings.db.dbname;
+      ensureUsers = lib.singleton {
+        name = cfg.settings.db.user;
+        ensurePermissions = {
+          "DATABASE ${cfg.settings.db.dbname}" = "ALL PRIVILEGES";
+        };
+      };
+      # This is only needed because the unix user invidious isn't the same as
+      # the database user. This tells postgres to map one to the other.
+      identMap = ''
+        invidious invidious ${cfg.settings.db.user}
+      '';
+      # And this specifically enables peer authentication for only this
+      # database, which allows passwordless authentication over the postgres
+      # unix socket for the user map given above.
+      authentication = ''
+        local ${cfg.settings.db.dbname} ${cfg.settings.db.user} peer map=invidious
+      '';
+    };
+
+    systemd.services.invidious-db-clean = {
+      description = "Invidious database cleanup";
+      documentation = [ "https://docs.invidious.io/Database-Information-and-Maintenance.md" ];
+      startAt = lib.mkDefault "weekly";
+      path = [ config.services.postgresql.package ];
+      script = ''
+        psql ${cfg.settings.db.dbname} ${cfg.settings.db.user} -c "DELETE FROM nonces * WHERE expire < current_timestamp"
+        psql ${cfg.settings.db.dbname} ${cfg.settings.db.user} -c "TRUNCATE TABLE videos"
+      '';
+      serviceConfig = {
+        DynamicUser = true;
+        User = "invidious";
+      };
+    };
+
+    systemd.services.invidious = {
+      requires = [ "postgresql.service" ];
+      after = [ "postgresql.service" ];
+
+      serviceConfig = {
+        User = "invidious";
+      };
+    };
+  };
+
+  nginxConfig = lib.mkIf cfg.nginx.enable {
+    services.invidious.settings = {
+      https_only = config.services.nginx.virtualHosts.${cfg.domain}.forceSSL;
+      external_port = 80;
+    };
+
+    services.nginx = {
+      enable = true;
+      virtualHosts.${cfg.domain} = {
+        locations."/".proxyPass = "http://127.0.0.1:${toString cfg.port}";
+
+        enableACME = lib.mkDefault true;
+        forceSSL = lib.mkDefault true;
+      };
+    };
+
+    assertions = [{
+      assertion = cfg.domain != null;
+      message = "To use services.invidious.nginx, you need to set services.invidious.domain";
+    }];
+  };
+in
+{
+  options.services.invidious = {
+    enable = lib.mkEnableOption "Invidious";
+
+    package = lib.mkOption {
+      type = types.package;
+      default = pkgs.invidious;
+      defaultText = "pkgs.invidious";
+      description = "The Invidious package to use.";
+    };
+
+    settings = lib.mkOption {
+      type = settingsFormat.type;
+      default = { };
+      description = ''
+        The settings Invidious should use.
+
+        See <link xlink:href="https://github.com/iv-org/invidious/blob/master/config/config.example.yml">config.example.yml</link> for a list of all possible options.
+      '';
+    };
+
+    extraSettingsFile = lib.mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = ''
+        A file including Invidious settings.
+
+        It gets merged with the setttings specified in <option>services.invidious.settings</option>
+        and can be used to store secrets like <literal>hmac_key</literal> outside of the nix store.
+      '';
+    };
+
+    # This needs to be outside of settings to avoid infinite recursion
+    # (determining if nginx should be enabled and therefore the settings
+    # modified).
+    domain = lib.mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = ''
+        The FQDN Invidious is reachable on.
+
+        This is used to configure nginx and for building absolute URLs.
+      '';
+    };
+
+    port = lib.mkOption {
+      type = types.port;
+      # Default from https://docs.invidious.io/Configuration.md
+      default = 3000;
+      description = ''
+        The port Invidious should listen on.
+
+        To allow access from outside,
+        you can use either <option>services.invidious.nginx</option>
+        or add <literal>config.services.invidious.port</literal> to <option>networking.firewall.allowedTCPPorts</option>.
+      '';
+    };
+
+    database = {
+      createLocally = lib.mkOption {
+        type = types.bool;
+        default = true;
+        description = ''
+          Whether to create a local database with PostgreSQL.
+        '';
+      };
+
+      host = lib.mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = ''
+          The database host Invidious should use.
+
+          If <literal>null</literal>, the local unix socket is used. Otherwise
+          TCP is used.
+        '';
+      };
+
+      port = lib.mkOption {
+        type = types.port;
+        default = options.services.postgresql.port.default;
+        description = ''
+          The port of the database Invidious should use.
+
+          Defaults to the the default postgresql port.
+        '';
+      };
+
+      passwordFile = lib.mkOption {
+        type = types.nullOr types.str;
+        apply = lib.mapNullable toString;
+        default = null;
+        description = ''
+          Path to file containing the database password.
+        '';
+      };
+    };
+
+    nginx.enable = lib.mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to configure nginx as a reverse proxy for Invidious.
+
+        It serves it under the domain specified in <option>services.invidious.settings.domain</option> with enabled TLS and ACME.
+        Further configuration can be done through <option>services.nginx.virtualHosts.''${config.services.invidious.settings.domain}.*</option>,
+        which can also be used to disable AMCE and TLS.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable (lib.mkMerge [
+    serviceConfig
+    localDatabaseConfig
+    nginxConfig
+  ]);
+}

nixos/tests/all-tests.nix

@@ -173,6 +173,7 @@ in
   hedgedoc = handleTest ./hedgedoc.nix {};
   herbstluftwm = handleTest ./herbstluftwm.nix {};
   installed-tests = pkgs.recurseIntoAttrs (handleTest ./installed-tests {});
+  invidious = handleTest ./invidious.nix {};
   oci-containers = handleTestOn ["x86_64-linux"] ./oci-containers.nix {};
   # 9pnet_virtio used to mount /nix partition doesn't support
   # hibernation. This test happens to work on x86_64-linux but

nixos/tests/invidious.nix

@@ -0,0 +1,81 @@
+import ./make-test-python.nix ({ pkgs, ... }: {
+  name = "invidious";
+
+  meta = with pkgs.lib.maintainers; {
+    maintainers = [ sbruder ];
+  };
+
+  machine = { config, lib, pkgs, ... }: {
+    services.invidious = {
+      enable = true;
+    };
+
+    specialisation = {
+      nginx.configuration = {
+        services.invidious = {
+          nginx.enable = true;
+          domain = "invidious.example.com";
+        };
+        services.nginx.virtualHosts."invidious.example.com" = {
+          forceSSL = false;
+          enableACME = false;
+        };
+        networking.hosts."127.0.0.1" = [ "invidious.example.com" ];
+      };
+      postgres-tcp.configuration = {
+        services.invidious = {
+          database = {
+            createLocally = false;
+            host = "127.0.0.1";
+            passwordFile = toString (pkgs.writeText "database-password" "correct horse battery staple");
+          };
+        };
+        # Normally not needed because when connecting to postgres over TCP/IP
+        # the database is most likely on another host.
+        systemd.services.invidious = {
+          after = [ "postgresql.service" ];
+          requires = [ "postgresql.service" ];
+        };
+        services.postgresql =
+          let
+            inherit (config.services.invidious.settings.db) dbname user;
+          in
+          {
+            enable = true;
+            initialScript = pkgs.writeText "init-postgres-with-password" ''
+              CREATE USER kemal WITH PASSWORD 'correct horse battery staple';
+              CREATE DATABASE invidious;
+              GRANT ALL PRIVILEGES ON DATABASE invidious TO kemal;
+            '';
+          };
+      };
+    };
+  };
+
+  testScript = { nodes, ... }: ''
+    def curl_assert_status_code(url, code, form=None):
+        assert int(machine.succeed(f"curl -s -o /dev/null -w %{{http_code}} {'-F ' + form + ' ' if form else '''}{url}")) == code
+
+
+    def activate_specialisation(name: str):
+        machine.succeed(f"${nodes.machine.config.system.build.toplevel}/specialisation/{name}/bin/switch-to-configuration test >&2")
+
+
+    url = "http://localhost:${toString nodes.machine.config.services.invidious.port}"
+    port = ${toString nodes.machine.config.services.invidious.port}
+
+    machine.wait_for_open_port(port)
+    curl_assert_status_code(f"{url}/search", 200)
+
+    activate_specialisation("nginx")
+    machine.wait_for_open_port(80)
+    curl_assert_status_code("http://invidious.example.com/search", 200)
+
+    # Remove the state so the `initialScript` gets run
+    machine.succeed("systemctl stop postgresql")
+    machine.succeed("rm -r /var/lib/postgresql")
+    activate_specialisation("postgres-tcp")
+    machine.wait_for_open_port(port)
+    curl_assert_status_code(f"{url}/search", 200)
+  '';
+})

pkgs/servers/invidious/default.nix

@@ -0,0 +1,99 @@
+{ lib, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, sqlite, lsquic, nixosTests }:
+let
+  # When updating, always update the following:
+  #  * the git revision
+  #  * the version attribute
+  #  * the source hash (sha256)
+  # If the shards.lock file changed, also the following:
+  #  * shards.nix (by running `crystal2nix` in invidious’ source tree)
+  #  * If the lsquic.cr dependency changed: lsquic in lsquic.nix (version, sha256)
+  #  * If the lsquic version changed: boringssl' in lsquic.nix (version, sha256)
+  rev = "21b96a31599e890fe063e3e24cf5f3a995779a69";
+in
+crystal.buildCrystalPackage rec {
+  pname = "invidious";
+  version = "unstable-2021-10-15";
+
+  src = fetchFromGitHub {
+    owner = "iv-org";
+    repo = pname;
+    inherit rev;
+    sha256 = "sha256-Rp3YqjHbP6szohlaEpgopFNdLK31yrcHtyKCeVz76CA=";
+  };
+
+  postPatch =
+    let
+      # Replacing by the value (templates) of the variables ensures that building
+      # fails if upstream changes the way the metadata is formatted.
+      branchTemplate = ''{{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}'';
+      commitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}'';
+      versionTemplate = ''{{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}'';
+      # This always uses the latest commit which invalidates the cache even if
+      # the assets were not changed
+      assetCommitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }}'';
+    in
+    ''
+      # Use the version metadata from the derivation instead of using git at
+      # build-time
+      substituteInPlace src/invidious.cr \
+          --replace ${lib.escapeShellArg branchTemplate} '"master"' \
+          --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 rev}"' \
+          --replace ${lib.escapeShellArg versionTemplate} '"${lib.replaceChars ["-"] ["."] (lib.substring 9 10 version)}"' \
+          --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 rev}"'
+
+      # Patch the assets and locales paths to be absolute
+      substituteInPlace src/invidious.cr \
+          --replace 'public_folder "assets"' 'public_folder "${placeholder "out"}/share/invidious/assets"'
+      substituteInPlace src/invidious/helpers/i18n.cr \
+          --replace 'File.read("locales/' 'File.read("${placeholder "out"}/share/invidious/locales/'
+
+      # Reference sql initialisation/migration scripts by absolute path
+      substituteInPlace src/invidious/helpers/helpers.cr \
+            --replace 'config/sql' '${placeholder "out"}/share/invidious/config/sql'
+
+      substituteInPlace src/invidious/users.cr \
+          --replace 'Process.run(%(rsvg-convert' 'Process.run(%(${lib.getBin librsvg}/bin/rsvg-convert'
+    '';
+
+  nativeBuildInputs = [ pkg-config ];
+  buildInputs = [ libxml2 openssl sqlite ];
+
+  format = "crystal";
+  shardsFile = ./shards.nix;
+  crystalBinaries.invidious.src = "src/invidious.cr";
+
+  postConfigure = ''
+    # lib includes nix store paths which can’t be patched, so the links have to
+    # be dereferenced first.
+    cp -rL lib lib2
+    rm -r lib
+    mv lib2 lib
+    chmod +w -R lib
+    cp ${lsquic}/lib/liblsquic.a lib/lsquic/src/lsquic/ext
+  '';
+
+  postInstall = ''
+    mkdir -p $out/share/invidious/config
+
+    # Copy static parts
+    cp -r assets locales $out/share/invidious
+    cp -r config/sql $out/share/invidious/config
+  '';
+
+  # Invidious tries to open config/config.yml and connect to the database, even
+  # when running --help. This specifies a minimal configuration in an
+  # environment variable. Even though the database is bogus, --help still
+  # works.
+  installCheckPhase = ''
+    INVIDIOUS_CONFIG="database_url: sqlite3:///dev/null" $out/bin/invidious --help
+  '';
+
+  passthru.tests = { inherit (nixosTests) invidious; };
+
+  meta = with lib; {
+    description = "An open source alternative front-end to YouTube";
+    homepage = "https://invidious.io/";
+    license = licenses.agpl3;
+    maintainers = with maintainers; [ infinisil sbruder ];
+  };
+}

pkgs/servers/invidious/lsquic.nix

@@ -0,0 +1,58 @@
+{ lib, boringssl, stdenv, fetchgit, fetchFromGitHub, cmake, zlib, perl, libevent }:
+let
+  # lsquic requires a specific boringssl version (noted in its README)
+  boringssl' = boringssl.overrideAttrs (old: rec {
+    version = "251b5169fd44345f455438312ec4e18ae07fd58c";
+    src = fetchgit {
+      url = "https://boringssl.googlesource.com/boringssl";
+      rev = version;
+      sha256 = "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A=";
+    };
+  });
+in
+stdenv.mkDerivation rec {
+  pname = "lsquic";
+  version = "2.18.1";
+
+  src = fetchFromGitHub {
+    owner = "litespeedtech";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM=";
+    fetchSubmodules = true;
+  };
+
+  nativeBuildInputs = [ cmake perl ];
+  buildInputs = [ boringssl' libevent zlib ];
+
+  cmakeFlags = [
+    "-DBORINGSSL_DIR=${boringssl'}"
+    "-DBORINGSSL_LIB_crypto=${boringssl'}/lib/libcrypto.a"
+    "-DBORINGSSL_LIB_ssl=${boringssl'}/lib/libssl.a"
+    "-DZLIB_LIB=${zlib}/lib/libz.so"
+  ];
+
+  # adapted from lsquic.cr’s Dockerfile
+  # (https://github.com/iv-org/lsquic.cr/blob/master/docker/Dockerfile)
+  installPhase = ''
+    runHook preInstall
+
+    mkdir combinedlib
+    cd combinedlib
+    ar -x ${boringssl'}/lib/libssl.a
+    ar -x ${boringssl'}/lib/libcrypto.a
+    ar -x ../src/liblsquic/liblsquic.a
+    ar rc liblsquic.a *.o
+    ranlib liblsquic.a
+    install -D liblsquic.a $out/lib/liblsquic.a
+
+    runHook postInstall
+  '';
+
+  meta = with lib; {
+    description = "A library for QUIC and HTTP/3 (version for Invidious)";
+    homepage = "https://github.com/litespeedtech/lsquic";
+    maintainers = with maintainers; [ infinisil sbruder ];
+    license = with licenses; [ openssl isc mit bsd3 ]; # statically links against boringssl, so has to include its licenses
+  };
+}

pkgs/servers/invidious/shards.nix

@@ -0,0 +1,68 @@
+{
+  athena-negotiation = {
+    owner = "athena-framework";
+    repo = "negotiation";
+    rev = "v0.1.1";
+    sha256 = "1vkk59lqrxb0l8kyzs114i3c18zb2bdiah2xhazkk8q7x6fz4yzk";
+  };
+  backtracer = {
+    owner = "sija";
+    repo = "backtracer.cr";
+    rev = "v1.2.1";
+    sha256 = "02r1l7rn2wsljkx495s5s7j04zgn73m2kx0hkzs7620camvlwbqq";
+  };
+  db = {
+    owner = "crystal-lang";
+    repo = "crystal-db";
+    rev = "v0.10.1";
+    sha256 = "03c5h14z6h2mxnx949lihnyqjd19hcj38iasdwq9fp95h8cld376";
+  };
+  exception_page = {
+    owner = "crystal-loot";
+    repo = "exception_page";
+    rev = "v0.2.0";
+    sha256 = "0nlgnh5iykbr1v2132342k2mz6s2laws6nkgqsqlwhhcr4gb4jcx";
+  };
+  kemal = {
+    owner = "kemalcr";
+    repo = "kemal";
+    rev = "v1.1.0";
+    sha256 = "07vlvddy4mba9li2bvskzqzywwq55cyvlgkz13q6dsl4zfgc96ca";
+  };
+  kilt = {
+    owner = "jeromegn";
+    repo = "kilt";
+    rev = "v0.6.1";
+    sha256 = "0dpc15y9m8c5l9zdfif6jlf7zmkrlm9w4m2igi5xa22fdjwamwfp";
+  };
+  lsquic = {
+    owner = "iv-org";
+    repo = "lsquic.cr";
+    rev = "v2.18.1-2";
+    sha256 = "0bljk0pwbjb813dfwrhgi00w2ai09k868xvak4hfzdkbmpc7id6y";
+  };
+  pg = {
+    owner = "will";
+    repo = "crystal-pg";
+    rev = "v0.24.0";
+    sha256 = "07i5bqkv5j6y6f8v5cpqdxc5wzzrvgv3ds24znv4mzv6nc84csn4";
+  };
+  protodec = {
+    owner = "iv-org";
+    repo = "protodec";
+    rev = "v0.1.4";
+    sha256 = "15azh9izxqgwpgkpicmivfdz31wkibnwy09rwhxsg0lyc4wf8xj9";
+  };
+  radix = {
+    owner = "luislavena";
+    repo = "radix";
+    rev = "v0.4.1";
+    sha256 = "1l08cydkdidq9yyil1wl240hvk41iycv04jrg6nx5mkvzw4z1bzg";
+  };
+  sqlite3 = {
+    owner = "crystal-lang";
+    repo = "crystal-sqlite3";
+    rev = "v0.18.0";
+    sha256 = "03nnvpchhq9f9ywsm3pk2rrj4a3figw7xs96zdziwgr5znkz6x93";
+  };
+}

pkgs/top-level/all-packages.nix

@@ -6401,6 +6401,11 @@ with pkgs;
 
   intermodal = callPackage ../tools/misc/intermodal { };
 
+  invidious = callPackage ../servers/invidious {
+    # needs a specific version of lsquic
+    lsquic = callPackage ../servers/invidious/lsquic.nix { };
+  };
+
   invoice2data  = callPackage ../tools/text/invoice2data  { };
 
   inxi = callPackage ../tools/system/inxi { };