diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix index 63ed17ec8d70..61d2ef8b3052 100644 --- a/maintainers/maintainer-list.nix +++ b/maintainers/maintainer-list.nix @@ -18466,6 +18466,12 @@ github = "zmitchell"; githubId = 10246891; }; + znewman01 = { + email = "znewman01@gmail.com"; + github = "znewman01"; + githubId = 873857; + name = "Zack Newman"; + }; zoedsoupe = { github = "zoedsoupe"; githubId = 44469426; diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 5ccaa92914e1..38fd9a3ff03d 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -26,6 +26,8 @@ - [trust-dns](https://trust-dns.org/), a Rust based DNS server built to be safe and secure from the ground up. Available as [services.trust-dns](#opt-services.trust-dns.enable). +- [osquery](https://www.osquery.io/), a SQL powered operating system instrumentation, monitoring, and analytics. + ## Backward Incompatibilities {#sec-release-23.11-incompatibilities} - The `boot.loader.raspberryPi` options have been marked deprecated, with intent for removal for NixOS 24.11. They had a limited use-case, and do not work like people expect. They required either very old installs ([before mid-2019](https://github.com/NixOS/nixpkgs/pull/62462)) or customized builds out of scope of the standard and generic AArch64 support. That option set never supported the Raspberry Pi 4 family of devices. diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 17e3acba948a..1e1c42c26301 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -764,6 +764,7 @@ ./services/monitoring/nagios.nix ./services/monitoring/netdata.nix ./services/monitoring/opentelemetry-collector.nix + ./services/monitoring/osquery.nix ./services/monitoring/parsedmarc.nix ./services/monitoring/prometheus/alertmanager-irc-relay.nix ./services/monitoring/prometheus/alertmanager.nix diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix index 0e8b823c2bdd..a32d433bbdde 100644 --- a/nixos/modules/rename.nix +++ b/nixos/modules/rename.nix @@ -72,7 +72,6 @@ in (mkRemovedOptionModule [ "services" "mesos" ] "The corresponding package was removed from nixpkgs.") (mkRemovedOptionModule [ "services" "moinmoin" ] "The corresponding package was removed from nixpkgs.") (mkRemovedOptionModule [ "services" "mwlib" ] "The corresponding package was removed from nixpkgs.") - (mkRemovedOptionModule [ "services" "osquery" ] "The osquery module has been removed") (mkRemovedOptionModule [ "services" "pantheon" "files" ] '' This module was removed, please add pkgs.pantheon.elementary-files to environment.systemPackages directly. '') diff --git a/nixos/modules/services/monitoring/osquery.nix b/nixos/modules/services/monitoring/osquery.nix new file mode 100644 index 000000000000..98524d0c700c --- /dev/null +++ b/nixos/modules/services/monitoring/osquery.nix @@ -0,0 +1,97 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + cfg = config.services.osquery; + dirname = path: with lib.strings; with lib.lists; concatStringsSep "/" + (init (splitString "/" (normalizePath path))); + + # conf is the osquery configuration file used when the --config_plugin=filesystem. + # filesystem is the osquery default value for the config_plugin flag. + conf = pkgs.writeText "osquery.conf" (builtins.toJSON cfg.settings); + + # flagfile is the file containing osquery command line flags to be + # provided to the application using the special --flagfile option. + flagfile = pkgs.writeText "osquery.flags" + (concatStringsSep "\n" + (mapAttrsToList (name: value: "--${name}=${value}") + # Use the conf derivation if not otherwise specified. + ({ config_path = conf; } // cfg.flags))); + + osqueryi = pkgs.runCommand "osqueryi" { nativeBuildInputs = [ pkgs.makeWrapper ]; } '' + mkdir -p $out/bin + makeWrapper ${pkgs.osquery}/bin/osqueryi $out/bin/osqueryi \ + --add-flags "--flagfile ${flagfile}" + ''; +in +{ + options.services.osquery = { + enable = mkEnableOption (mdDoc "osqueryd daemon"); + + settings = mkOption { + default = { }; + description = mdDoc '' + Configuration to be written to the osqueryd JSON configuration file. + To understand the configuration format, refer to https://osquery.readthedocs.io/en/stable/deployment/configuration/#configuration-components. + ''; + example = { + options.utc = false; + }; + type = types.attrs; + }; + + flags = mkOption { + default = { }; + description = mdDoc '' + Attribute set of flag names and values to be written to the osqueryd flagfile. + For more information, refer to https://osquery.readthedocs.io/en/stable/installation/cli-flags. + ''; + example = { + config_refresh = "10"; + }; + type = with types; + submodule { + freeformType = attrsOf str; + options = { + database_path = mkOption { + default = "/var/lib/osquery/osquery.db"; + readOnly = true; + description = mdDoc "Path used for the database file."; + type = path; + }; + logger_path = mkOption { + default = "/var/log/osquery"; + readOnly = true; + description = mdDoc "Base directory used for logging."; + type = path; + }; + pidfile = mkOption { + default = "/run/osquery/osqueryd.pid"; + readOnly = true; + description = mdDoc "Path used for pid file."; + type = path; + }; + }; + }; + }; + }; + + config = mkIf cfg.enable { + environment.systemPackages = [ osqueryi ]; + systemd.services.osqueryd = { + after = [ "network.target" "syslog.service" ]; + description = "The osquery daemon"; + serviceConfig = { + ExecStart = "${pkgs.osquery}/bin/osqueryd --flagfile ${flagfile}"; + PIDFile = cfg.flags.pidfile; + LogsDirectory = cfg.flags.logger_path; + StateDirectory = dirname cfg.flags.database_path; + Restart = "always"; + }; + wantedBy = [ "multi-user.target" ]; + }; + systemd.tmpfiles.rules = [ + "d ${dirname (cfg.flags.pidfile)} 0755 root root -" + ]; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 5f4aefad890f..a64f7c5ccbcf 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -577,6 +577,7 @@ in { openvscode-server = handleTest ./openvscode-server.nix {}; orangefs = handleTest ./orangefs.nix {}; os-prober = handleTestOn ["x86_64-linux"] ./os-prober.nix {}; + osquery = handleTestOn ["x86_64-linux"] ./osquery.nix {}; osrm-backend = handleTest ./osrm-backend.nix {}; overlayfs = handleTest ./overlayfs.nix {}; pacemaker = handleTest ./pacemaker.nix {}; diff --git a/nixos/tests/osquery.nix b/nixos/tests/osquery.nix new file mode 100644 index 000000000000..dc55fa82f8d1 --- /dev/null +++ b/nixos/tests/osquery.nix @@ -0,0 +1,56 @@ +import ./make-test-python.nix ({ lib, pkgs, ... }: + +with lib; + +let + config_refresh = "10"; + nullvalue = "NULL"; + utc = false; +in +{ + name = "osquery"; + meta = with maintainers; { + maintainers = [ znewman01 lewo ]; + }; + + nodes.machine = { config, pkgs, ... }: { + services.osquery = { + enable = true; + + settings.options = { inherit nullvalue utc; }; + flags = { + inherit config_refresh; + nullvalue = "IGNORED"; + }; + }; + }; + + testScript = { nodes, ... }: + let + cfg = nodes.machine.services.osquery; + in + '' + machine.start() + machine.wait_for_unit("osqueryd.service") + + # Stop the osqueryd service so that we can use osqueryi to check information stored in the database. + machine.wait_until_succeeds("systemctl stop osqueryd.service") + + # osqueryd was able to query information about the host. + machine.succeed("echo 'SELECT address FROM etc_hosts LIMIT 1;' | osqueryi | tee /dev/console | grep -q '127.0.0.1'") + + # osquery binaries respect configuration from the Nix config option. + machine.succeed("echo 'SELECT value FROM osquery_flags WHERE name = \"utc\";' | osqueryi | tee /dev/console | grep -q ${boolToString utc}") + + # osquery binaries respect configuration from the Nix flags option. + machine.succeed("echo 'SELECT value FROM osquery_flags WHERE name = \"config_refresh\";' | osqueryi | tee /dev/console | grep -q ${config_refresh}") + + # Demonstrate that osquery binaries prefer configuration plugin options over CLI flags. + # https://osquery.readthedocs.io/en/latest/deployment/configuration/#options. + machine.succeed("echo 'SELECT value FROM osquery_flags WHERE name = \"nullvalue\";' | osqueryi | tee /dev/console | grep -q ${nullvalue}") + + # Module creates directories for default database_path and pidfile flag values. + machine.succeed("test -d $(dirname ${cfg.flags.database_path})") + machine.succeed("test -d $(dirname ${cfg.flags.pidfile})") + ''; +}) diff --git a/pkgs/tools/system/osquery/Remove-circular-definition-of-AUDIT_FILTER_EXCLUDE.patch b/pkgs/tools/system/osquery/Remove-circular-definition-of-AUDIT_FILTER_EXCLUDE.patch new file mode 100644 index 000000000000..d3aeca1c3007 --- /dev/null +++ b/pkgs/tools/system/osquery/Remove-circular-definition-of-AUDIT_FILTER_EXCLUDE.patch @@ -0,0 +1,25 @@ +From: Jack Baldry +Date: Tue, 15 Nov 2022 15:40:31 -0400 +Subject: [PATCH] Remove circular definition of AUDIT_FILTER_EXCLUDE + +https://github.com/osquery/osquery/issues/6551 + +Signed-off-by: Jack Baldry +--- + libraries/cmake/source/libaudit/src/lib/libaudit.h | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/libraries/cmake/source/libaudit/src/lib/libaudit.h b/libraries/cmake/source/libaudit/src/libaudit.h +--- a/libraries/cmake/source/libaudit/src/lib/libaudit.h ++++ b/libraries/cmake/source/libaudit/src/lib/libaudit.h +@@ -260,7 +260,6 @@ extern "C" { + #define AUDIT_KEY_SEPARATOR 0x01 + + /* These are used in filter control */ +-#define AUDIT_FILTER_EXCLUDE AUDIT_FILTER_TYPE + #define AUDIT_FILTER_MASK 0x07 /* Mask to get actual filter */ + #define AUDIT_FILTER_UNSET 0x80 /* This value means filter is unset */ + +-- +2.38.1 + diff --git a/pkgs/tools/system/osquery/Remove-git-reset.patch b/pkgs/tools/system/osquery/Remove-git-reset.patch new file mode 100644 index 000000000000..af5b165a851e --- /dev/null +++ b/pkgs/tools/system/osquery/Remove-git-reset.patch @@ -0,0 +1,37 @@ +From: Jack Baldry +Date: Tue, 15 Nov 2022 13:48:07 -0400 +Subject: [PATCH] Remove git reset + +This is not required for nixpkgs builds because we are not working in +the source repository and therefore do not need to be careful about +updating submodule content. + +Signed-off-by: Jack Baldry +--- + libraries/cmake/source/modules/utils.cmake | 11 ----------- + 1 file changed, 11 deletions(-) + +diff --git a/libraries/cmake/source/modules/utils.cmake b/libraries/cmake/source/modules/utils.cmake +--- a/libraries/cmake/source/modules/utils.cmake ++++ b/libraries/cmake/source/modules/utils.cmake +@@ -102,17 +102,6 @@ function(patchSubmoduleSourceCode library_name patches_dir source_dir apply_to_d + file(COPY "${source_dir}" DESTINATION "${parent_dir}") + endif() + +- # We need to restore the source code to its original state, pre patch +- execute_process( +- COMMAND "${GIT_EXECUTABLE}" reset --hard HEAD +- RESULT_VARIABLE process_exit_code +- WORKING_DIRECTORY "${source_dir}" +- ) +- +- if(NOT ${process_exit_code} EQUAL 0) +- message(FATAL_ERROR "Failed to git reset the following submodule: \"${source_dir}\"") +- endif() +- + set(patchSubmoduleSourceCode_Patched TRUE PARENT_SCOPE) + endfunction() + +-- +2.38.1 + diff --git a/pkgs/tools/system/osquery/Remove-system-controls-table.patch b/pkgs/tools/system/osquery/Remove-system-controls-table.patch new file mode 100644 index 000000000000..e448f70a3d1b --- /dev/null +++ b/pkgs/tools/system/osquery/Remove-system-controls-table.patch @@ -0,0 +1,157 @@ +From: Jack Baldry +Date: Wed, 16 Nov 2022 22:00:06 -0400 +Subject: [PATCH] Remove system controls table + +Relies on which is not present in glibc since 2.32. + +Signed-off-by: Jack Baldry +--- + osquery/tables/system/CMakeLists.txt | 4 -- + specs/CMakeLists.txt | 1 - + specs/posix/system_controls.table | 21 ------- + tests/integration/tables/system_controls.cpp | 61 -------------------- + 4 files changed, 87 deletions(-) + delete mode 100644 specs/posix/system_controls.table + delete mode 100644 tests/integration/tables/system_controls.cpp + +diff --git a/osquery/tables/system/CMakeLists.txt b/osquery/tables/system/CMakeLists.txt +--- a/osquery/tables/system/CMakeLists.txt ++++ b/osquery/tables/system/CMakeLists.txt +@@ -43,7 +43,6 @@ function(generateOsqueryTablesSystemSystemtable) + posix/smbios_utils.cpp + posix/sudoers.cpp + posix/suid_bin.cpp +- posix/system_controls.cpp + posix/ulimit_info.cpp + ) + endif() +@@ -82,7 +81,6 @@ function(generateOsqueryTablesSystemSystemtable) + linux/shared_memory.cpp + linux/smbios_tables.cpp + linux/startup_items.cpp +- linux/sysctl_utils.cpp + linux/system_info.cpp + linux/usb_devices.cpp + linux/user_groups.cpp +@@ -156,7 +154,6 @@ function(generateOsqueryTablesSystemSystemtable) + darwin/smbios_tables.cpp + darwin/smc_keys.cpp + darwin/startup_items.cpp +- darwin/sysctl_utils.cpp + darwin/system_extensions.mm + darwin/system_info.cpp + darwin/time_machine.cpp +@@ -326,7 +323,6 @@ function(generateOsqueryTablesSystemSystemtable) + posix/shell_history.h + posix/ssh_keys.h + posix/sudoers.h +- posix/sysctl_utils.h + posix/last.h + posix/openssl_utils.h + posix/authorized_keys.h +diff --git a/specs/CMakeLists.txt b/specs/CMakeLists.txt +--- a/specs/CMakeLists.txt ++++ b/specs/CMakeLists.txt +@@ -246,7 +246,6 @@ function(generateNativeTables) + "posix/socket_events.table:linux,macos" + "posix/sudoers.table:linux,macos,freebsd" + "posix/suid_bin.table:linux,macos,freebsd" +- "posix/system_controls.table:linux,macos,freebsd" + "posix/ulimit_info.table:linux,macos,freebsd" + "posix/usb_devices.table:linux,macos" + "posix/user_events.table:linux,macos,freebsd" +diff --git a/specs/posix/system_controls.table b/specs/posix/system_controls.table +deleted file mode 100644 +--- a/specs/posix/system_controls.table ++++ /dev/null +@@ -1,21 +0,0 @@ +-table_name("system_controls") +-description("sysctl names, values, and settings information.") +-schema([ +- Column("name", TEXT, "Full sysctl MIB name", index=True), +- Column("oid", TEXT, "Control MIB", additional=True), +- Column("subsystem", TEXT, "Subsystem ID, control type", additional=True), +- Column("current_value", TEXT, "Value of setting"), +- Column("config_value", TEXT, "The MIB value set in /etc/sysctl.conf"), +- Column("type", TEXT, "Data type"), +-]) +-extended_schema(DARWIN, [ +- Column("field_name", TEXT, "Specific attribute of opaque type"), +-]) +- +-implementation("system_controls@genSystemControls") +-fuzz_paths([ +- "/run/sysctl.d/", +- "/usr/lib/sysctl.d/", +- "/lib/sysctl.d/", +- "/sys" +-]) +diff --git a/tests/integration/tables/system_controls.cpp b/tests/integration/tables/system_controls.cpp +deleted file mode 100644 +--- a/tests/integration/tables/system_controls.cpp ++++ /dev/null +@@ -1,61 +0,0 @@ +-/** +- * Copyright (c) 2014-present, The osquery authors +- * +- * This source code is licensed as defined by the LICENSE file found in the +- * root directory of this source tree. +- * +- * SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only) +- */ +- +-// Sanity check integration test for system_controls +-// Spec file: specs/posix/system_controls.table +- +-#include +- +-namespace osquery { +-namespace table_tests { +-namespace { +- +-class SystemControlsTest : public testing::Test { +- protected: +- void SetUp() override { +- setUpEnvironment(); +- } +-}; +- +-TEST_F(SystemControlsTest, test_sanity) { +- auto const rows = execute_query("select * from system_controls"); +- auto const row_map = ValidationMap{ +- {"name", NonEmptyString}, +- {"oid", NormalType}, +- {"subsystem", +- SpecificValuesCheck{"", +- "abi", +- "debug", +- "dev", +- "fs", +- "fscache", +- "hw", +- "kern", +- "kernel", +- "machdep", +- "net", +- "sunrpc", +- "user", +- "vfs", +- "vm"}}, +- {"current_value", NormalType}, +- {"config_value", NormalType}, +- {"type", +- SpecificValuesCheck{ +- "", "node", "int", "string", "quad", "opaque", "struct"}}, +-#ifdef __APPLE__ +- {"field_name", NormalType}, +-#endif +- }; +- validate_rows(rows, row_map); +-} +- +-} // namespace +-} // namespace table_tests +-} // namespace osquery +-- +2.38.1 + diff --git a/pkgs/tools/system/osquery/Use-locale.h-instead-of-removed-xlocale.h-header.patch b/pkgs/tools/system/osquery/Use-locale.h-instead-of-removed-xlocale.h-header.patch new file mode 100644 index 000000000000..63dd5a387f30 --- /dev/null +++ b/pkgs/tools/system/osquery/Use-locale.h-instead-of-removed-xlocale.h-header.patch @@ -0,0 +1,29 @@ +From: Jack Baldry +Date: Tue, 15 Nov 2022 14:34:33 -0400 +Subject: [PATCH] Use locale.h instead of removed xlocale.h header + +https://sourceware.org/glibc/wiki/Release/2.26#Removal_of_.27xlocale.h.27 + +Signed-off-by: Jack Baldry +--- + libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h b/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h +--- a/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h ++++ b/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h +@@ -48,9 +48,9 @@ + /* NetBSD 5.0 mis-defines NULL. */ + #include + +-/* Mac OS X 10.5 defines the locale_t type in . */ ++/* Mac OS X 10.5 defines the locale_t type in . */ + #if 1 +-# include ++# include + #endif + + /* The definitions of _GL_FUNCDECL_RPL etc. are copied here. */ +-- +2.38.1 + diff --git a/pkgs/tools/system/osquery/default.nix b/pkgs/tools/system/osquery/default.nix new file mode 100644 index 000000000000..6c6d0b45e2a1 --- /dev/null +++ b/pkgs/tools/system/osquery/default.nix @@ -0,0 +1,85 @@ +{ lib +, cmake +, fetchFromGitHub +, git +, llvmPackages +, nixosTests +, overrideCC +, perl +, python3 +, stdenv +, openssl_1_1 +}: + +let + buildStdenv = overrideCC stdenv llvmPackages.clangUseLLVM; +in +buildStdenv.mkDerivation rec { + pname = "osquery"; + version = "5.5.1"; + + src = fetchFromGitHub { + owner = "osquery"; + repo = "osquery"; + rev = version; + fetchSubmodules = true; + sha256 = "sha256-Q6PQVnBjAjAlR725fyny+RhQFUNwxWGjLDuS5p9JKlU="; + }; + + patches = [ + ./Remove-git-reset.patch + ./Use-locale.h-instead-of-removed-xlocale.h-header.patch + ./Remove-circular-definition-of-AUDIT_FILTER_EXCLUDE.patch + # For current state of compilation against glibc in the clangWithLLVM toolchain, refer to the upstream issue in https://github.com/osquery/osquery/issues/7823. + ./Remove-system-controls-table.patch + ]; + + + buildInputs = [ + llvmPackages.libunwind + ]; + nativeBuildInputs = [ + cmake + git + perl + python3 + ]; + + postPatch = '' + substituteInPlace cmake/install_directives.cmake --replace "/control" "control" + # This is required to build libarchive with our glibc version + # which provides the ARC4RANDOM_BUF function + substituteInPlace libraries/cmake/source/libarchive/CMakeLists.txt --replace " target_compile_definitions(thirdparty_libarchive PRIVATE" " target_compile_definitions(thirdparty_libarchive PRIVATE HAVE_ARC4RANDOM_BUF" + # We need to override this hash because we use our own openssl 1.1 version + substituteInPlace libraries/cmake/formula/openssl/CMakeLists.txt --replace "d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca" "e2f8d84b523eecd06c7be7626830370300fbcc15386bf5142d72758f6963ebc6" + cat libraries/cmake/formula/openssl/CMakeLists.txt + ''; + + # For explanation of these deletions, refer to the ./Use-locale.h-instead-of-removed-xlocale.h-header.patch file. + preConfigure = '' + find libraries/cmake/source -name 'config.h' -exec sed -i '/#define HAVE_XLOCALE_H 1/d' {} \; + ''; + + cmakeFlags = [ + "-DOSQUERY_VERSION=${version}" + "-DOSQUERY_OPENSSL_ARCHIVE_PATH=${openssl_1_1.src}" + ]; + + postFixup = '' + patchelf --set-rpath "${llvmPackages.libunwind}/lib:$(patchelf --print-rpath $out/bin/osqueryd)" "$out/bin/osqueryd" + ''; + + passthru.tests.osquery = nixosTests.osquery; + + meta = with lib; { + description = "SQL powered operating system instrumentation, monitoring, and analytics."; + longDescription = '' + The system controls table is not included as it does not presently compile with glibc >= 2.32. + For more information, refer to https://github.com/osquery/osquery/issues/7823 + ''; + homepage = "https://osquery.io"; + license = licenses.bsd3; + platforms = platforms.linux; + maintainers = with maintainers; [ znewman01 lewo ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index ba780295452e..c1420841ff84 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1789,6 +1789,8 @@ with pkgs; openbugs = pkgsi686Linux.callPackage ../applications/science/machine-learning/openbugs { }; + osquery = callPackage ../tools/system/osquery { }; + paperview = callPackage ../tools/X11/paperview { }; pferd = callPackage ../tools/misc/pferd { };