redpanda: 22.3.11 -> 22.3.13

Build rpk from source. Add redpanda server.

maintainers/maintainer-list.nix

@@ -1417,6 +1417,12 @@
     githubId = 12958979;
     name = "Mika Naylor";
   };
+  avakhrenev = {
+    email = "avakhrenev@gmail.com";
+    github = "avakhrenev";
+    githubId = 1060224;
+    name = "Alexey Vakhrenev";
+  };
   avaq = {
     email = "nixpkgs@account.avaq.it";
     github = "Avaq";

pkgs/servers/redpanda/base64.nix

@@ -0,0 +1,26 @@
+{ clangStdenv
+, cmake
+, fetchFromGitHub
+, lib
+}:
+let
+  pname = "base64";
+  version = "0.5.0";
+in
+clangStdenv.mkDerivation {
+  inherit pname version;
+  src = fetchFromGitHub {
+    owner = "aklomp";
+    repo = "base64";
+    rev = "v${version}";
+    sha256 = "sha256-2HNI9ycT9f+NLwLElEuR61qmTguOsI+kNxv01ipxSqQ=";
+  };
+  nativeBuildInputs = [ cmake ];
+  meta = with lib; {
+    description = "Fast Base64 stream encoder/decoder in C99, with SIMD acceleration";
+    license = licenses.bsd2;
+    homepage = "https://github.com/aklomp/base64";
+    maintainers = with maintainers; [ avakhrenev ];
+    platforms = platforms.unix;
+  };
+}

pkgs/servers/redpanda/default.nix

@@ -1,48 +1,52 @@
-{ lib, stdenv, fetchzip }:
-
+{ buildGoModule
+, callPackage
+, doCheck ? !stdenv.isDarwin # Can't start localhost test server in MacOS sandbox.
+, fetchFromGitHub
+, installShellFiles
+, lib
+, stdenv
+}:
 let
-  version = "22.3.11";
-  platform = if stdenv.isLinux then "linux" else "darwin";
-  arch = if stdenv.isAarch64 then "arm" else "amd";
-  sha256s = {
-    darwin.amd = "sha256-kwAKxFg7BSNInvsQvFqgtpq8EEwSnmDeDyaF5b8L8SQ=";
-    darwin.arm = "sha256-kH5Ii672SeAIiRcWuAO3oVJVSBWp+r78RmTiR3BaDbg=";
-    linux.amd = "sha256-EKgkRKBrM4+X2YGoP2LpWRHL+fdHu44LYwCZ+O+c5ZY=";
-    linux.arm = "sha256-9b4oerRXjUVUYoswJWtnMBJSQDoCKClf673VjDQFUAw=";
+  version = "22.3.13";
+  src = fetchFromGitHub {
+    owner = "redpanda-data";
+    repo = "redpanda";
+    rev = "v${version}";
+    sha256 = "sha256-cUQFDXWnQYSLcfKFYg6BLrxF77iX+Yx3hcul4tMxdoc=";
   };
-in stdenv.mkDerivation rec {
-  pname = "redpanda";
-  inherit version;
+  server = callPackage ./server.nix { inherit src version; };
+in
+buildGoModule rec {
+  pname = "redpanda-rpk";
+  inherit doCheck src version;
+  modRoot = "./src/go/rpk";
+  runVend = false;
+  vendorSha256 = "sha256-JVZuHRh3gavIGArxDkqUQsL5oBjz35EKGsC75Sy+cMo=";
 
-  src = fetchzip {
-    url = "https://github.com/redpanda-data/redpanda/releases/download/v${version}/rpk-${platform}-${arch}64.zip";
-    sha256 = sha256s.${platform}.${arch};
-  };
+  ldflags = [
+    ''-X "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/version.version=${version}"''
+    ''-X "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/version.rev=v${version}"''
+    ''-X "github.com/redpanda-data/redpanda/src/go/rpk/pkg/cli/cmd/container/common.tag=v${version}"''
+  ];
 
-  installPhase = ''
-    runHook preInstall
+  nativeBuildInputs = [ installShellFiles ];
 
-    mkdir -p $out/bin
-    cp rpk $out/bin
-
-    ${lib.optionalString stdenv.isLinux ''
-        patchelf \
-          --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" \
-          $out/bin/rpk
-    ''}
-
-    runHook postInstall
+  postInstall = ''
+    for shell in bash fish zsh; do
+      $out/bin/rpk generate shell-completion $shell > rpk.$shell
+      installShellCompletion rpk.$shell
+    done
   '';
 
-  # stripping somehow completely breaks it
-  dontStrip = true;
+  passthru = {
+    inherit server;
+  };
 
   meta = with lib; {
-    description = "Redpanda is a streaming data platform for developers. Kafka API compatible. 10x faster. No ZooKeeper. No JVM! ";
-    sourceProvenance = with sourceTypes; [ binaryNativeCode ];
-    license = licenses.bsl11;
+    description = "Redpanda client";
     homepage = "https://redpanda.com/";
-    maintainers = with maintainers; [ happysalada ];
+    license = licenses.bsl11;
+    maintainers = with maintainers; [ avakhrenev happysalada ];
     platforms = platforms.all;
   };
 }

pkgs/servers/redpanda/hdr-histogram.nix

@@ -0,0 +1,28 @@
+{ clangStdenv
+, cmake
+, fetchFromGitHub
+, lib
+, zlib
+}:
+let
+  pname = "HdrHistogram_c";
+  version = "0.11.5";
+in
+clangStdenv.mkDerivation {
+  inherit pname version;
+  src = fetchFromGitHub {
+    owner = "HdrHistogram";
+    repo = "HdrHistogram_c";
+    rev = version;
+    sha256 = "sha256-29if+0H8wdpQBN48lt0ylGgtUCv/tJYZnG5LzcIqXDs=";
+  };
+  nativeBuildInputs = [ cmake ];
+  buildInputs = [ zlib ];
+  meta = with lib; {
+    description = "C port of the HdrHistogram";
+    license = licenses.bsd2;
+    homepage = "https://github.com/HdrHistogram/HdrHistogram_c";
+    maintainers = with maintainers; [ avakhrenev ];
+    platforms = platforms.unix;
+  };
+}

pkgs/servers/redpanda/rapidjson.nix

@@ -0,0 +1,32 @@
+# rapidjson used in nixpkgs is too old. Although it is technically a latest release, it was made in 2016.
+# Redpanda uses its own version
+{ clangStdenv
+, cmake
+, fetchFromGitHub
+, lib
+, pkg-config
+}:
+
+clangStdenv.mkDerivation rec {
+  pname = "rapidjson";
+  version = "1.1.1";
+
+  src = fetchFromGitHub {
+    owner = "redpanda-data";
+    repo = "rapidjson";
+    rev = "27c3a8dc0e2c9218fe94986d249a12b5ed838f1d";
+    sha256 = "sha256-wggyCL5uEsnJDxkYAUsXOjoO1MNQBGB05E6aSpsNcl0=";
+  };
+
+  nativeBuildInputs = [ pkg-config cmake ];
+
+  doCheck = false;
+
+  meta = with lib; {
+    description = "Fast JSON parser/generator for C++ with both SAX/DOM style API";
+    homepage = "http://rapidjson.org/";
+    maintainers = with maintainers; [ avakhrenev ];
+    license = licenses.mit;
+    platforms = platforms.unix;
+  };
+}

pkgs/servers/redpanda/redpanda.patch

@@ -0,0 +1,73 @@
+diff --git a/cmake/main.cmake b/cmake/main.cmake
+index 8c60c4214..194f33a21 100644
+--- a/cmake/main.cmake
++++ b/cmake/main.cmake
+@@ -15,15 +15,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
+ set(CMAKE_UNITY_BUILD_BATCH_SIZE 10)
+ set(CMAKE_CXX_EXTENSIONS OFF)
+ set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+-list(APPEND BASE_LD_FLAGS_LIST
+-  -L${REDPANDA_DEPS_INSTALL_DIR}/lib
+-  -L${REDPANDA_DEPS_INSTALL_DIR}/lib64
+-  -fuse-ld=lld)
+-set(PKG_CONFIG_PATH_LIST
+-  ${REDPANDA_DEPS_INSTALL_DIR}/lib64/pkgconfig
+-  ${REDPANDA_DEPS_INSTALL_DIR}/share/pkgconfig
+-  ${REDPANDA_DEPS_INSTALL_DIR}/lib/pkgconfig
+-  )
+ 
+ list(APPEND BASE_CXX_FLAGS_LIST -fPIC)
+ list(APPEND BASE_C_FLAGS_LIST -fPIC)
+diff --git a/cmake/testing.cmake b/cmake/testing.cmake
+index 7f149dc82..7c57aa3dd 100644
+--- a/cmake/testing.cmake
++++ b/cmake/testing.cmake
+@@ -24,6 +24,7 @@ message(STATUS "RP_ENABLE_BENCHMARK_TESTS=${RP_ENABLE_BENCHMARK_TESTS}")
+ message(STATUS "RP_ENABLE_HONEY_BADGER_TESTS=${RP_ENABLE_HONEY_BADGER_TESTS}")
+ 
+ function (rp_test)
++  return()
+   set(options
+     INTEGRATION_TEST UNIT_TEST BENCHMARK_TEST HBADGER_TEST)
+   set(oneValueArgs BINARY_NAME TIMEOUT PREPARE_COMMAND POST_COMMAND)
+diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
+index eecd145ed..b9efa89b5 100644
+--- a/src/CMakeLists.txt
++++ b/src/CMakeLists.txt
+@@ -5,4 +5,5 @@ find_package(Boost REQUIRED
+     unit_test_framework)
+ find_package(absl REQUIRED)
+ add_subdirectory(v)
+-add_subdirectory(go/kreq-gen)
++# Don't build kafka-request-generator, it is needed only for tests
++# add_subdirectory(go/kreq-gen)
+diff --git a/src/v/CMakeLists.txt b/src/v/CMakeLists.txt
+index 075da485e..af7ede2bc 100644
+--- a/src/v/CMakeLists.txt
++++ b/src/v/CMakeLists.txt
+@@ -58,7 +58,9 @@ else()
+   if(${ENABLE_GIT_HASH})
+     message(FATAL_ERROR "ENABLE_GIT_HASH cannot be 'on' when ENABLE_GIT_VERSION is 'off'")
+   endif()
+-  set(GIT_VER "no_version")
++  if(NOT GIT_VER)
++    set(GIT_VER "no_version")
++  endif()
+ endif()
+ 
+ if(${ENABLE_GIT_HASH})
+@@ -71,8 +73,12 @@ if(${ENABLE_GIT_HASH})
+     set(GIT_CLEAN_DIRTY "")
+   endif()
+ else()
+-  set(GIT_SHA1 "000")
+-  set(GIT_CLEAN_DIRTY "-dev")
++  if(NOT GIT_SHA1)
++    set(GIT_SHA1 "000")
++  endif()
++  if(NOT GIT_CLEAN_DIRTY)
++    set(GIT_CLEAN_DIRTY "-dev")
++  endif()
+ endif()
+ 
+ configure_file(version.h.in version.h @ONLY)

pkgs/servers/redpanda/seastar-fixes.patch

@@ -0,0 +1,13 @@
+diff --git a/include/seastar/core/std-coroutine.hh b/include/seastar/core/std-coroutine.hh
+index ea364bee..57474529 100644
+--- a/include/seastar/core/std-coroutine.hh
++++ b/include/seastar/core/std-coroutine.hh
+@@ -87,7 +87,7 @@ class coroutine_handle<void> {
+ 
+     explicit operator bool() const noexcept { return _pointer; }
+ 
+-    static coroutine_handle from_address(void* ptr) noexcept {
++    static constexpr coroutine_handle from_address(void* ptr) noexcept {
+         coroutine_handle hndl;
+         hndl._pointer = ptr;
+         return hndl;

pkgs/servers/redpanda/seastar.nix

@@ -0,0 +1,84 @@
+{ boost175
+, c-ares
+, cmake
+, cryptopp
+, fetchFromGitHub
+, fmt_8
+, gnutls
+, hwloc
+, lib
+, libsystemtap
+, libtasn1
+, liburing
+, libxfs
+, lksctp-tools
+, llvmPackages_14
+, lz4
+, ninja
+, numactl
+, openssl
+, pkg-config
+, python3
+, ragel
+, valgrind
+, yaml-cpp
+}:
+let
+  pname = "seastar";
+  version = "22.11.0";
+in
+llvmPackages_14.stdenv.mkDerivation {
+  inherit pname version;
+  strictDeps = true;
+  src = fetchFromGitHub {
+    owner = "redpanda-data";
+    repo = "seastar";
+    rev = "30d3a28bde08d2228b4e560c173b89fdd94c3f05";
+    sha256 = "sha256-Xzu7AJMkvE++BGEqluod3fwMEIpDnbCczmlEad0/4v4=";
+  };
+  nativeBuildInputs = [
+    cmake
+    ninja
+    openssl
+    pkg-config
+    python3
+    ragel
+  ];
+  buildInputs = [
+    libsystemtap
+    libxfs
+  ];
+  propagatedBuildInputs = [
+    boost175
+    c-ares
+    gnutls
+    cryptopp
+    fmt_8
+    hwloc
+    libtasn1
+    liburing
+    lksctp-tools
+    lz4
+    numactl
+    valgrind
+    yaml-cpp
+  ];
+  patches = [
+    ./seastar-fixes.patch
+  ];
+  postPatch = ''
+    patchShebangs ./scripts/seastar-json2code.py
+  '';
+  cmakeFlags = [
+    "-DSeastar_EXCLUDE_DEMOS_FROM_ALL=ON"
+    "-DSeastar_EXCLUDE_TESTS_FROM_ALL=ON"
+  ];
+  doCheck = false;
+  meta = with lib; {
+    description = "High performance server-side application framework.";
+    license = licenses.asl20;
+    homepage = "https://seastar.io/";
+    maintainers = with maintainers; [ avakhrenev ];
+    platforms = platforms.unix;
+  };
+}

pkgs/servers/redpanda/server.nix

@@ -0,0 +1,123 @@
+{ abseil-cpp
+, avro-cpp
+, callPackage
+, ccache
+, cmake
+, crc32c
+, croaring
+, ctre
+, curl
+, dpdk
+, git
+, lib
+, llvmPackages_14
+, llvm_14
+, ninja
+, p11-kit
+, pkg-config
+, procps
+, protobuf3_21
+, python3
+, snappy
+, src
+, unzip
+, version
+, writeShellScriptBin
+, xxHash
+, zip
+, zstd
+}:
+let
+  pname = "redpanda";
+  pythonPackages = p: with p; [ jinja2 ];
+  seastar = callPackage ./seastar.nix { };
+  base64 = callPackage ./base64.nix { };
+  hdr-histogram = callPackage ./hdr-histogram.nix { };
+  kafka-codegen-venv = python3.withPackages (ps: [
+    ps.jinja2
+    ps.jsonschema
+  ]);
+  rapidjson = callPackage ./rapidjson.nix { };
+in
+llvmPackages_14.stdenv.mkDerivation rec {
+  inherit pname version src;
+
+  preConfigure = ''
+    # setup sccache
+    export CCACHE_DIR=$TMPDIR/sccache-redpanda
+    mkdir -p $CCACHE_DIR
+  '';
+  patches = [
+    ./redpanda.patch
+  ];
+  postPatch = ''
+    # Fix 'error: use of undeclared identifier 'roaring'; did you mean 'Roaring
+    #      qualified reference to 'Roaring' is a constructor name rather than a type in this context'
+    substituteInPlace \
+        ./src/v/storage/compacted_offset_list.h \
+        ./src/v/storage/compaction_reducers.cc \
+        ./src/v/storage/compaction_reducers.h \
+        ./src/v/storage/segment_utils.h \
+        ./src/v/storage/segment_utils.cc \
+        --replace 'roaring::Roaring' 'Roaring'
+
+    patchShebangs ./src/v/rpc/rpc_compiler.py
+  '';
+
+  doCheck = false;
+
+  nativeBuildInputs = [
+    (python3.withPackages pythonPackages)
+    (writeShellScriptBin "kafka-codegen-venv" "exec -a $0 ${kafka-codegen-venv}/bin/python3 $@")
+    ccache
+    cmake
+    curl
+    git
+    llvm_14
+    ninja
+    pkg-config
+    procps
+    seastar
+    unzip
+    zip
+  ];
+
+  cmakeFlags = [
+    "-DREDPANDA_DEPS_SKIP_BUILD=ON"
+    "-DRP_ENABLE_TESTS=OFF"
+    "-Wno-dev"
+    "-DGIT_VER=${version}"
+    "-DGIT_CLEAN_DIRTY=\"\""
+  ];
+
+  buildInputs = [
+    abseil-cpp
+    avro-cpp
+    base64
+    crc32c
+    croaring
+    ctre
+    dpdk
+    hdr-histogram
+    p11-kit
+    protobuf3_21
+    rapidjson
+    seastar
+    snappy
+    xxHash
+    zstd
+  ];
+
+  meta = with lib; {
+    description = "Kafka-compatible streaming platform.";
+    license = licenses.bsl11;
+    longDescription = ''
+      Redpanda is a Kafka-compatible streaming data platform that is
+      proven to be 10x faster and 6x lower in total costs. It is also JVM-free,
+      ZooKeeper-free, Jepsen-tested and source available.
+    '';
+    homepage = "https://redpanda.com/";
+    maintainers = with maintainers; [ avakhrenev happysalada ];
+    platforms = platforms.linux;
+  };
+}

pkgs/top-level/all-packages.nix

@@ -11397,6 +11397,8 @@ with pkgs;
 
   redpanda = callPackage ../servers/redpanda { };
 
+  redpanda-server = redpanda.server;
+
   redsocks = callPackage ../tools/networking/redsocks { };
 
   renpy = callPackage ../development/interpreters/renpy { };