diff --git a/nixos/doc/manual/release-notes/rl-1909.xml b/nixos/doc/manual/release-notes/rl-1909.xml
index c0b7cc48a461..78e911fa5d93 100644
--- a/nixos/doc/manual/release-notes/rl-1909.xml
+++ b/nixos/doc/manual/release-notes/rl-1909.xml
@@ -710,6 +710,13 @@
nix-shell -p altcoins.dogecoin, etc.
+
+ Ceph has been upgraded to v14.2.1.
+ See the release notes for details.
+ The mgr dashboard as well as osds backed by loop-devices is no longer explicitly supported by the package and module.
+ Note: There's been some issues with python-cherrypy, which is used by the dashboard
+ and prometheus mgr modules (and possibly others), hence 0000-dont-check-cherrypy-version.patch.
+
diff --git a/nixos/modules/services/network-filesystems/ceph.nix b/nixos/modules/services/network-filesystems/ceph.nix
index 54841861c081..aef284c31a5a 100644
--- a/nixos/modules/services/network-filesystems/ceph.nix
+++ b/nixos/modules/services/network-filesystems/ceph.nix
@@ -3,18 +3,18 @@
with lib;
let
- ceph = pkgs.ceph;
cfg = config.services.ceph;
+
# function that translates "camelCaseOptions" to "camel case options", credits to tilpner in #nixos@freenode
- translateOption = replaceStrings upperChars (map (s: " ${s}") lowerChars);
- generateDaemonList = (daemonType: daemons: extraServiceConfig:
- mkMerge (
- map (daemon:
- { "ceph-${daemonType}-${daemon}" = generateServiceFile daemonType daemon cfg.global.clusterName ceph extraServiceConfig; }
- ) daemons
- )
- );
- generateServiceFile = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: {
+ expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars);
+ expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value);
+
+ makeServices = (daemonType: daemonIds: extraServiceConfig:
+ mkMerge (map (daemonId:
+ { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph extraServiceConfig; })
+ daemonIds));
+
+ makeService = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: {
enable = true;
description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
@@ -34,23 +34,29 @@ let
Restart = "on-failure";
StartLimitBurst = "5";
StartLimitInterval = "30min";
- ExecStart = "${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} -f --cluster ${clusterName} --id ${if daemonType == "rgw" then "client.${daemonId}" else daemonId} --setuser ceph --setgroup ceph";
+ ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
+ -f --cluster ${clusterName} --id ${daemonId} --setuser ceph \
+ --setgroup ${if daemonType == "osd" then "disk" else "ceph"}'';
} // extraServiceConfig
- // optionalAttrs (daemonType == "osd") { ExecStartPre = "${ceph.out}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}"; };
- } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) { preStart = ''
+ // optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh \
+ --id ${daemonId} --cluster ${clusterName}''; };
+ } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) {
+ preStart = ''
daemonPath="/var/lib/ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"
- if [ ! -d ''$daemonPath ]; then
- mkdir -m 755 -p ''$daemonPath
- chown -R ceph:ceph ''$daemonPath
+ if [ ! -d $daemonPath ]; then
+ mkdir -m 755 -p $daemonPath
+ chown -R ceph:ceph $daemonPath
fi
'';
} // optionalAttrs (daemonType == "osd") { path = [ pkgs.getopt ]; }
);
- generateTargetFile = (daemonType:
+
+ makeTarget = (daemonType:
{
"ceph-${daemonType}" = {
description = "Ceph target allowing to start/stop all ceph-${daemonType} services at once";
partOf = [ "ceph.target" ];
+ wantedBy = [ "ceph.target" ];
before = [ "ceph.target" ];
};
}
@@ -82,6 +88,14 @@ in
'';
};
+ mgrModulePath = mkOption {
+ type = types.path;
+ default = "${pkgs.ceph.lib}/lib/ceph/mgr";
+ description = ''
+ Path at which to find ceph-mgr modules.
+ '';
+ };
+
monInitialMembers = mkOption {
type = with types; nullOr commas;
default = null;
@@ -157,6 +171,27 @@ in
A comma-separated list of subnets that will be used as cluster networks in the cluster.
'';
};
+
+ rgwMimeTypesFile = mkOption {
+ type = with types; nullOr path;
+ default = "${pkgs.mime-types}/etc/mime.types";
+ description = ''
+ Path to mime types used by radosgw.
+ '';
+ };
+ };
+
+ extraConfig = mkOption {
+ type = with types; attrsOf str;
+ default = {};
+ example = ''
+ {
+ "ms bind ipv6" = "true";
+ };
+ '';
+ description = ''
+ Extra configuration to add to the global section. Use for setting values that are common for all daemons in the cluster.
+ '';
};
mgr = {
@@ -216,6 +251,7 @@ in
to the id part in ceph i.e. [ "name1" ] would result in osd.name1
'';
};
+
extraConfig = mkOption {
type = with types; attrsOf str;
default = {
@@ -296,9 +332,6 @@ in
{ assertion = cfg.global.fsid != "";
message = "fsid has to be set to a valid uuid for the cluster to function";
}
- { assertion = cfg.mgr.enable == true;
- message = "ceph 12.x requires atleast 1 MGR daemon enabled for the cluster to function";
- }
{ assertion = cfg.mon.enable == true -> cfg.mon.daemons != [];
message = "have to set id of atleast one MON if you're going to enable Monitor";
}
@@ -317,14 +350,12 @@ in
''Not setting up a list of members in monInitialMembers requires that you set the host variable for each mon daemon or else the cluster won't function'';
environment.etc."ceph/ceph.conf".text = let
- # Translate camelCaseOptions to the expected camel case option for ceph.conf
- translatedGlobalConfig = mapAttrs' (name: value: nameValuePair (translateOption name) value) cfg.global;
# Merge the extraConfig set for mgr daemons, as mgr don't have their own section
- globalAndMgrConfig = translatedGlobalConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig;
+ globalSection = expandCamelCaseAttrs (cfg.global // cfg.extraConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig);
# Remove all name-value pairs with null values from the attribute set to avoid making empty sections in the ceph.conf
- globalConfig = mapAttrs' (name: value: nameValuePair (translateOption name) value) (filterAttrs (name: value: value != null) globalAndMgrConfig);
+ globalSection' = filterAttrs (name: value: value != null) globalSection;
totalConfig = {
- "global" = globalConfig;
+ "global" = globalSection';
} // optionalAttrs (cfg.mon.enable && cfg.mon.extraConfig != {}) { "mon" = cfg.mon.extraConfig; }
// optionalAttrs (cfg.mds.enable && cfg.mds.extraConfig != {}) { "mds" = cfg.mds.extraConfig; }
// optionalAttrs (cfg.osd.enable && cfg.osd.extraConfig != {}) { "osd" = cfg.osd.extraConfig; }
@@ -336,8 +367,9 @@ in
name = "ceph";
uid = config.ids.uids.ceph;
description = "Ceph daemon user";
+ group = "ceph";
+ extraGroups = [ "disk" ];
};
-
users.groups = singleton {
name = "ceph";
gid = config.ids.gids.ceph;
@@ -345,22 +377,26 @@ in
systemd.services = let
services = []
- ++ optional cfg.mon.enable (generateDaemonList "mon" cfg.mon.daemons { RestartSec = "10"; })
- ++ optional cfg.mds.enable (generateDaemonList "mds" cfg.mds.daemons { StartLimitBurst = "3"; })
- ++ optional cfg.osd.enable (generateDaemonList "osd" cfg.osd.daemons { StartLimitBurst = "30"; RestartSec = "20s"; })
- ++ optional cfg.rgw.enable (generateDaemonList "rgw" cfg.rgw.daemons { })
- ++ optional cfg.mgr.enable (generateDaemonList "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; });
+ ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons { RestartSec = "10"; })
+ ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons { StartLimitBurst = "3"; })
+ ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons { StartLimitBurst = "30";
+ RestartSec = "20s";
+ PrivateDevices = "no"; # osd needs disk access
+ })
+ ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons { })
+ ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; });
in
mkMerge services;
systemd.targets = let
targets = [
- { "ceph" = { description = "Ceph target allowing to start/stop all ceph service instances at once"; }; }
- ] ++ optional cfg.mon.enable (generateTargetFile "mon")
- ++ optional cfg.mds.enable (generateTargetFile "mds")
- ++ optional cfg.osd.enable (generateTargetFile "osd")
- ++ optional cfg.rgw.enable (generateTargetFile "rgw")
- ++ optional cfg.mgr.enable (generateTargetFile "mgr");
+ { "ceph" = { description = "Ceph target allowing to start/stop all ceph service instances at once";
+ wantedBy = [ "multi-user.target" ]; }; }
+ ] ++ optional cfg.mon.enable (makeTarget "mon")
+ ++ optional cfg.mds.enable (makeTarget "mds")
+ ++ optional cfg.osd.enable (makeTarget "osd")
+ ++ optional cfg.rgw.enable (makeTarget "rgw")
+ ++ optional cfg.mgr.enable (makeTarget "mgr");
in
mkMerge targets;
diff --git a/nixos/tests/ceph.nix b/nixos/tests/ceph.nix
index 7408029c460e..7a6e238d9157 100644
--- a/nixos/tests/ceph.nix
+++ b/nixos/tests/ceph.nix
@@ -1,4 +1,4 @@
-import ./make-test.nix ({pkgs, ...}: rec {
+import ./make-test.nix ({pkgs, lib, ...}: rec {
name = "All-in-one-basic-ceph-cluster";
meta = with pkgs.stdenv.lib.maintainers; {
maintainers = [ lejonet ];
@@ -7,6 +7,7 @@ import ./make-test.nix ({pkgs, ...}: rec {
nodes = {
aio = { pkgs, ... }: {
virtualisation = {
+ memorySize = 1536;
emptyDiskImages = [ 20480 20480 ];
vlans = [ 1 ];
};
@@ -24,9 +25,6 @@ import ./make-test.nix ({pkgs, ...}: rec {
ceph
xfsprogs
];
- nixpkgs.config.packageOverrides = super: {
- ceph = super.ceph.override({ nss = super.nss; libxfs = super.libxfs; libaio = super.libaio; jemalloc = super.jemalloc; });
- };
boot.kernelModules = [ "xfs" ];
@@ -51,6 +49,9 @@ import ./make-test.nix ({pkgs, ...}: rec {
enable = true;
daemons = [ "0" "1" ];
};
+
+ # So that we don't have to battle systemd when bootstraping
+ systemd.targets.ceph.wantedBy = lib.mkForce [];
};
};
@@ -61,24 +62,26 @@ import ./make-test.nix ({pkgs, ...}: rec {
# Create the ceph-related directories
$aio->mustSucceed(
- "mkdir -p /var/lib/ceph/mgr/ceph-aio/",
- "mkdir -p /var/lib/ceph/mon/ceph-aio/",
- "mkdir -p /var/lib/ceph/osd/ceph-{0..1}/",
- "chown ceph:ceph -R /var/lib/ceph/"
+ "mkdir -p /var/lib/ceph/mgr/ceph-aio",
+ "mkdir -p /var/lib/ceph/mon/ceph-aio",
+ "mkdir -p /var/lib/ceph/osd/ceph-{0,1}",
+ "chown ceph:ceph -R /var/lib/ceph/",
+ "mkdir -p /etc/ceph",
+ "chown ceph:ceph -R /etc/ceph"
);
# Bootstrap ceph-mon daemon
$aio->mustSucceed(
- "mkdir -p /var/lib/ceph/bootstrap-osd && chown ceph:ceph /var/lib/ceph/bootstrap-osd",
"sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
- "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
- "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
- "monmaptool --create --add aio 192.168.1.1 --fsid 066ae264-2a5d-4729-8001-6ad265f50b03 /tmp/monmap",
+ "sudo -u ceph ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
+ "sudo -u ceph ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
+ "monmaptool --create --add aio 192.168.1.1 --fsid 066ae264-2a5d-4729-8001-6ad265f50b03 /tmp/monmap",
"sudo -u ceph ceph-mon --mkfs -i aio --monmap /tmp/monmap --keyring /tmp/ceph.mon.keyring",
- "touch /var/lib/ceph/mon/ceph-aio/done",
+ "sudo -u ceph touch /var/lib/ceph/mon/ceph-aio/done",
"systemctl start ceph-mon-aio"
);
$aio->waitForUnit("ceph-mon-aio");
+ $aio->mustSucceed("ceph mon enable-msgr2");
# Can't check ceph status until a mon is up
$aio->succeed("ceph -s | grep 'mon: 1 daemons'");
@@ -90,6 +93,7 @@ import ./make-test.nix ({pkgs, ...}: rec {
);
$aio->waitForUnit("ceph-mgr-aio");
$aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'");
+ $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'");
# Bootstrap both OSDs
$aio->mustSucceed(
@@ -112,8 +116,8 @@ import ./make-test.nix ({pkgs, ...}: rec {
"systemctl start ceph-osd-1"
);
- $aio->waitUntilSucceeds("ceph osd stat | grep '2 osds: 2 up, 2 in'");
- $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'");
+ $aio->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");
+ $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'");
$aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'");
$aio->mustSucceed(
@@ -135,5 +139,23 @@ import ./make-test.nix ({pkgs, ...}: rec {
"ceph osd pool ls | grep 'aio-test'",
"ceph osd pool delete aio-other-test aio-other-test --yes-i-really-really-mean-it"
);
+
+ # As we disable the target in the config, we still want to test that it works as intended
+ $aio->mustSucceed(
+ "systemctl stop ceph-osd-0",
+ "systemctl stop ceph-osd-1",
+ "systemctl stop ceph-mgr-aio",
+ "systemctl stop ceph-mon-aio"
+ );
+ $aio->succeed("systemctl start ceph.target");
+ $aio->waitForUnit("ceph-mon-aio");
+ $aio->waitForUnit("ceph-mgr-aio");
+ $aio->waitForUnit("ceph-osd-0");
+ $aio->waitForUnit("ceph-osd-1");
+ $aio->succeed("ceph -s | grep 'mon: 1 daemons'");
+ $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'");
+ $aio->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");
+ $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active,'");
+ $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'");
'';
})
diff --git a/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch b/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch
new file mode 100644
index 000000000000..9604da87fb50
--- /dev/null
+++ b/pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch
@@ -0,0 +1,70 @@
+diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py
+index a8a3ec07c1..bcc9b86c37 100644
+--- a/src/pybind/mgr/dashboard/module.py
++++ b/src/pybind/mgr/dashboard/module.py
+@@ -25,40 +25,6 @@ except ImportError:
+
+ from .services.sso import load_sso_db
+
+-# The SSL code in CherryPy 3.5.0 is buggy. It was fixed long ago,
+-# but 3.5.0 is still shipping in major linux distributions
+-# (Fedora 27, Ubuntu Xenial), so we must monkey patch it to get SSL working.
+-if cherrypy is not None:
+- v = StrictVersion(cherrypy.__version__)
+- # It was fixed in 3.7.0. Exact lower bound version is probably earlier,
+- # but 3.5.0 is what this monkey patch is tested on.
+- if StrictVersion("3.5.0") <= v < StrictVersion("3.7.0"):
+- from cherrypy.wsgiserver.wsgiserver2 import HTTPConnection,\
+- CP_fileobject
+-
+- def fixed_init(hc_self, server, sock, makefile=CP_fileobject):
+- hc_self.server = server
+- hc_self.socket = sock
+- hc_self.rfile = makefile(sock, "rb", hc_self.rbufsize)
+- hc_self.wfile = makefile(sock, "wb", hc_self.wbufsize)
+- hc_self.requests_seen = 0
+-
+- HTTPConnection.__init__ = fixed_init
+-
+-# When the CherryPy server in 3.2.2 (and later) starts it attempts to verify
+-# that the ports its listening on are in fact bound. When using the any address
+-# "::" it tries both ipv4 and ipv6, and in some environments (e.g. kubernetes)
+-# ipv6 isn't yet configured / supported and CherryPy throws an uncaught
+-# exception.
+-if cherrypy is not None:
+- v = StrictVersion(cherrypy.__version__)
+- # the issue was fixed in 3.2.3. it's present in 3.2.2 (current version on
+- # centos:7) and back to at least 3.0.0.
+- if StrictVersion("3.1.2") <= v < StrictVersion("3.2.3"):
+- # https://github.com/cherrypy/cherrypy/issues/1100
+- from cherrypy.process import servers
+- servers.wait_for_occupied_port = lambda host, port: None
+-
+ if 'COVERAGE_ENABLED' in os.environ:
+ import coverage
+ __cov = coverage.Coverage(config_file="{}/.coveragerc".format(os.path.dirname(__file__)),
+diff --git a/src/pybind/mgr/prometheus/module.py b/src/pybind/mgr/prometheus/module.py
+index b7fecf8d85..dfd4160591 100644
+--- a/src/pybind/mgr/prometheus/module.py
++++ b/src/pybind/mgr/prometheus/module.py
+@@ -18,20 +18,6 @@ from rbd import RBD
+ DEFAULT_ADDR = '::'
+ DEFAULT_PORT = 9283
+
+-# When the CherryPy server in 3.2.2 (and later) starts it attempts to verify
+-# that the ports its listening on are in fact bound. When using the any address
+-# "::" it tries both ipv4 and ipv6, and in some environments (e.g. kubernetes)
+-# ipv6 isn't yet configured / supported and CherryPy throws an uncaught
+-# exception.
+-if cherrypy is not None:
+- v = StrictVersion(cherrypy.__version__)
+- # the issue was fixed in 3.2.3. it's present in 3.2.2 (current version on
+- # centos:7) and back to at least 3.0.0.
+- if StrictVersion("3.1.2") <= v < StrictVersion("3.2.3"):
+- # https://github.com/cherrypy/cherrypy/issues/1100
+- from cherrypy.process import servers
+- servers.wait_for_occupied_port = lambda host, port: None
+-
+ # cherrypy likes to sys.exit on error. don't let it take us down too!
+ def os_exit_noop(*args, **kwargs):
+ pass
diff --git a/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch b/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch
new file mode 100644
index 000000000000..b04082537b53
--- /dev/null
+++ b/pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch
@@ -0,0 +1,11 @@
+--- a/cmake/modules/BuildSPDK.cmake 2018-08-09 09:22:34.950684960 +0200
++++ b/cmake/modules/BuildSPDK.cmake 2018-08-09 09:21:59.986964224 +0200
+@@ -16,7 +16,7 @@
+ # unset $CFLAGS, otherwise it will interfere with how SPDK sets
+ # its include directory.
+ # unset $LDFLAGS, otherwise SPDK will fail to mock some functions.
+- BUILD_COMMAND env -i PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} $(MAKE) EXTRA_CFLAGS="-fPIC"
++ BUILD_COMMAND env PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} $(MAKE) EXTRA_CFLAGS="-fPIC" C_OPT="-mssse3"
+ BUILD_IN_SOURCE 1
+ INSTALL_COMMAND "true")
+ ExternalProject_Get_Property(spdk-ext source_dir)
diff --git a/pkgs/tools/filesystems/ceph/0001-kv-RocksDBStore-API-break-additional.patch b/pkgs/tools/filesystems/ceph/0001-kv-RocksDBStore-API-break-additional.patch
deleted file mode 100644
index 0b6f1d30a4c9..000000000000
--- a/pkgs/tools/filesystems/ceph/0001-kv-RocksDBStore-API-break-additional.patch
+++ /dev/null
@@ -1,58 +0,0 @@
-Seulement dans ceph: ceph.old
-diff -ur ceph.old/src/kv/RocksDBStore.cc ceph/src/kv/RocksDBStore.cc
---- ceph.old/src/kv/RocksDBStore.cc 1980-01-02 00:00:00.000000000 +0100
-+++ ceph/src/kv/RocksDBStore.cc 2018-01-24 14:08:35.017553372 +0100
-@@ -505,7 +505,7 @@
- // considering performance overhead, default is disabled
- if (g_conf->rocksdb_perf) {
- rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
-- rocksdb::perf_context.Reset();
-+ rocksdb::get_perf_context()->Reset();
- }
-
- RocksDBTransactionImpl * _t =
-@@ -532,13 +532,13 @@
- utime_t write_wal_time;
- utime_t write_pre_and_post_process_time;
- write_wal_time.set_from_double(
-- static_cast(rocksdb::perf_context.write_wal_time)/1000000000);
-+ static_cast(rocksdb::get_perf_context()->write_wal_time)/1000000000);
- write_memtable_time.set_from_double(
-- static_cast(rocksdb::perf_context.write_memtable_time)/1000000000);
-+ static_cast(rocksdb::get_perf_context()->write_memtable_time)/1000000000);
- write_delay_time.set_from_double(
-- static_cast(rocksdb::perf_context.write_delay_time)/1000000000);
-+ static_cast(rocksdb::get_perf_context()->write_delay_time)/1000000000);
- write_pre_and_post_process_time.set_from_double(
-- static_cast(rocksdb::perf_context.write_pre_and_post_process_time)/1000000000);
-+ static_cast(rocksdb::get_perf_context()->write_pre_and_post_process_time)/1000000000);
- logger->tinc(l_rocksdb_write_memtable_time, write_memtable_time);
- logger->tinc(l_rocksdb_write_delay_time, write_delay_time);
- logger->tinc(l_rocksdb_write_wal_time, write_wal_time);
-@@ -558,7 +558,7 @@
- // considering performance overhead, default is disabled
- if (g_conf->rocksdb_perf) {
- rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
-- rocksdb::perf_context.Reset();
-+ rocksdb::get_perf_context()->Reset();
- }
-
- RocksDBTransactionImpl * _t =
-@@ -586,13 +586,13 @@
- utime_t write_wal_time;
- utime_t write_pre_and_post_process_time;
- write_wal_time.set_from_double(
-- static_cast(rocksdb::perf_context.write_wal_time)/1000000000);
-+ static_cast(rocksdb::get_perf_context()->write_wal_time)/1000000000);
- write_memtable_time.set_from_double(
-- static_cast(rocksdb::perf_context.write_memtable_time)/1000000000);
-+ static_cast(rocksdb::get_perf_context()->write_memtable_time)/1000000000);
- write_delay_time.set_from_double(
-- static_cast(rocksdb::perf_context.write_delay_time)/1000000000);
-+ static_cast(rocksdb::get_perf_context()->write_delay_time)/1000000000);
- write_pre_and_post_process_time.set_from_double(
-- static_cast(rocksdb::perf_context.write_pre_and_post_process_time)/1000000000);
-+ static_cast(rocksdb::get_perf_context()->write_pre_and_post_process_time)/1000000000);
- logger->tinc(l_rocksdb_write_memtable_time, write_memtable_time);
- logger->tinc(l_rocksdb_write_delay_time, write_delay_time);
- logger->tinc(l_rocksdb_write_wal_time, write_wal_time);
diff --git a/pkgs/tools/filesystems/ceph/0002-fix-absolute-include-path.patch b/pkgs/tools/filesystems/ceph/0002-fix-absolute-include-path.patch
deleted file mode 100644
index 01aa194dddf2..000000000000
--- a/pkgs/tools/filesystems/ceph/0002-fix-absolute-include-path.patch
+++ /dev/null
@@ -1,19 +0,0 @@
-diff -ru ceph/src/key_value_store/kv_flat_btree_async.cc ceph-copy/src/key_value_store/kv_flat_btree_async.cc
---- ceph/src/key_value_store/kv_flat_btree_async.cc 1980-01-02 00:00:00.000000000 +0100
-+++ ceph-copy/src/key_value_store/kv_flat_btree_async.cc 2018-02-13 21:49:59.232860487 +0100
-@@ -15,13 +15,13 @@
- #include "key_value_store/kv_flat_btree_async.h"
- #include "key_value_store/kvs_arg_types.h"
- #include "include/rados/librados.hpp"
--#include "/usr/include/asm-generic/errno.h"
--#include "/usr/include/asm-generic/errno-base.h"
- #include "common/ceph_context.h"
- #include "common/Clock.h"
- #include "include/types.h"
-
-
-+#include
-+#include
- #include
- #include
- #include
diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix
index ef3e36ff6275..efb292687ab4 100644
--- a/pkgs/tools/filesystems/ceph/default.nix
+++ b/pkgs/tools/filesystems/ceph/default.nix
@@ -1,12 +1,198 @@
-{ callPackage, fetchgit, fetchpatch, ... } @ args:
+{ stdenv, runCommand, fetchurl
+, ensureNewerSourcesHook
+, cmake, pkgconfig
+, which, git
+, boost, python3Packages
+, libxml2, zlib, lz4
+, openldap, lttng-ust
+, babeltrace, gperf
+, cunit, snappy
+, rocksdb, makeWrapper
+, leveldb, oathToolkit, removeReferencesTo
-callPackage ./generic.nix (args // rec {
- version = "12.2.7";
+# Optional Dependencies
+, yasm ? null, fcgi ? null, expat ? null
+, curl ? null, fuse ? null
+, libedit ? null, libatomic_ops ? null
+, libs3 ? null
- src = fetchgit {
- url = "https://github.com/ceph/ceph.git";
- rev = "refs/tags/v${version}";
- sha256 = "031nfw2g2fdpxxx39g862phgmdx68hj9r54axazandghfhc1bzrl";
+# Mallocs
+, jemalloc ? null, gperftools ? null
+
+# Crypto Dependencies
+, cryptopp ? null
+, nss ? null, nspr ? null
+
+# Linux Only Dependencies
+, linuxHeaders, utillinux, libuuid, udev, keyutils, rdma-core, rabbitmq-c
+, libaio ? null, libxfs ? null, zfs ? null
+, ...
+}:
+
+# We must have one crypto library
+assert cryptopp != null || (nss != null && nspr != null);
+
+with stdenv; with stdenv.lib;
+let
+ shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null;
+
+ optYasm = shouldUsePkg yasm;
+ optFcgi = shouldUsePkg fcgi;
+ optExpat = shouldUsePkg expat;
+ optCurl = shouldUsePkg curl;
+ optFuse = shouldUsePkg fuse;
+ optLibedit = shouldUsePkg libedit;
+ optLibatomic_ops = shouldUsePkg libatomic_ops;
+ optLibs3 = shouldUsePkg libs3;
+
+ optJemalloc = shouldUsePkg jemalloc;
+ optGperftools = shouldUsePkg gperftools;
+
+ optCryptopp = shouldUsePkg cryptopp;
+ optNss = shouldUsePkg nss;
+ optNspr = shouldUsePkg nspr;
+
+ optLibaio = shouldUsePkg libaio;
+ optLibxfs = shouldUsePkg libxfs;
+ optZfs = shouldUsePkg zfs;
+
+ hasRadosgw = optFcgi != null && optExpat != null && optCurl != null && optLibedit != null;
+
+
+ # Malloc implementation (can be jemalloc, tcmalloc or null)
+ malloc = if optJemalloc != null then optJemalloc else optGperftools;
+
+ # We prefer nss over cryptopp
+ cryptoStr = if optNss != null && optNspr != null then "nss" else
+ if optCryptopp != null then "cryptopp" else "none";
+
+ cryptoLibsMap = {
+ nss = [ optNss optNspr ];
+ cryptopp = [ optCryptopp ];
+ none = [ ];
};
-})
+ ceph-python-env = python3Packages.python.withPackages (ps: [
+ ps.sphinx
+ ps.flask
+ ps.cython
+ ps.setuptools
+ ps.virtualenv
+ # Libraries needed by the python tools
+ ps.Mako
+ ps.cherrypy
+ ps.pecan
+ ps.prettytable
+ ps.pyjwt
+ ps.webob
+ ps.bcrypt
+ ps.six
+ ]);
+
+ version = "14.2.1";
+in rec {
+ ceph = stdenv.mkDerivation {
+ name="ceph-${version}";
+
+ src = fetchurl {
+ url = "http://download.ceph.com/tarballs/ceph-${version}.tar.gz";
+ sha256 = "0qa9p8xp26d45h3jfj1rbwhmqv44f9n1mvccmpzaf2i05v42kmzb";
+ };
+
+ patches = [
+ ./0000-fix-SPDK-build-env.patch
+ ./0000-dont-check-cherrypy-version.patch
+ ];
+
+ nativeBuildInputs = [
+ cmake
+ pkgconfig which git python3Packages.wrapPython makeWrapper
+ (ensureNewerSourcesHook { year = "1980"; })
+ ];
+
+ buildInputs = cryptoLibsMap.${cryptoStr} ++ [
+ boost ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3
+ malloc zlib openldap lttng-ust babeltrace gperf cunit
+ snappy rocksdb lz4 oathToolkit leveldb
+ removeReferencesTo
+ ] ++ optionals stdenv.isLinux [
+ linuxHeaders utillinux libuuid udev keyutils optLibaio optLibxfs optZfs
+ # ceph 14
+ rdma-core rabbitmq-c
+ ] ++ optionals hasRadosgw [
+ optFcgi optExpat optCurl optFuse optLibedit
+ ];
+
+ preConfigure =''
+ substituteInPlace src/common/module.c --replace "/sbin/modinfo" "modinfo"
+ substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe"
+ # Since Boost 1.67 this seems to have changed
+ substituteInPlace CMakeLists.txt --replace "list(APPEND BOOST_COMPONENTS python)" "list(APPEND BOOST_COMPONENTS python37)"
+ substituteInPlace src/CMakeLists.txt --replace "Boost::python " "Boost::python37 "
+
+ # for pybind/rgw to find internal dep
+ export LD_LIBRARY_PATH="$PWD/build/lib:$LD_LIBRARY_PATH"
+ # install target needs to be in PYTHONPATH for "*.pth support" check to succeed
+ export PYTHONPATH=${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/python3.7/site-packages/:$out/lib/python3.7/site-packages/
+
+ patchShebangs src/spdk
+ '';
+
+ cmakeFlags = [
+ "-DWITH_PYTHON3=ON"
+ "-DWITH_SYSTEM_ROCKSDB=OFF"
+
+ "-DWITH_SYSTEM_BOOST=ON"
+ "-DWITH_SYSTEMD=OFF"
+ "-DWITH_TESTS=OFF"
+ # TODO breaks with sandbox, tries to download stuff with npm
+ "-DWITH_MGR_DASHBOARD_FRONTEND=OFF"
+ ];
+
+ preFixup = ''
+ find $lib -type f -exec remove-references-to -t $out '{}' +
+ mv $out/share/ceph/mgr $lib/lib/ceph/
+ '';
+
+ postFixup = ''
+ export PYTHONPATH="${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/"
+ wrapPythonPrograms
+ wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/"
+ wrapProgram $out/bin/ceph-volume --prefix PYTHONPATH ":" "${ceph-python-env}/lib/python3.7/site-packages:$lib/lib/ceph/mgr:$out/lib/python3.7/site-packages/"
+ '';
+
+ enableParallelBuilding = true;
+
+ outputs = [ "out" "lib" "dev" "doc" "man" ];
+
+ meta = {
+ homepage = https://ceph.com/;
+ description = "Distributed storage system";
+ license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
+ maintainers = with maintainers; [ adev ak krav johanot ];
+ platforms = platforms.unix;
+ };
+
+ passthru.version = version;
+ };
+
+ ceph-client = runCommand "ceph-client-${version}" {
+ meta = {
+ homepage = https://ceph.com/;
+ description = "Tools needed to mount Ceph's RADOS Block Devices";
+ license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
+ maintainers = with maintainers; [ adev ak krav ];
+ platforms = platforms.unix;
+ };
+ } ''
+ mkdir -p $out/{bin,etc,lib/python3.7/site-packages}
+ cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin
+ cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin
+ cp -r ${ceph}/bin/rbd-replay* $out/bin
+ cp -r ${ceph}/lib/python3.7/site-packages $out/lib/python3.7/
+ cp -r ${ceph}/etc/bash_completion.d $out/etc
+ # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
+ substituteInPlace $out/bin/ceph --replace ${ceph} $out
+ substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out
+ '';
+}
diff --git a/pkgs/tools/filesystems/ceph/generic.nix b/pkgs/tools/filesystems/ceph/generic.nix
deleted file mode 100644
index e8a4917be99f..000000000000
--- a/pkgs/tools/filesystems/ceph/generic.nix
+++ /dev/null
@@ -1,175 +0,0 @@
-{ stdenv, ensureNewerSourcesHook, cmake, pkgconfig
-, which, git
-, boost, python2Packages
-, libxml2, zlib
-, openldap, lttng-ust
-, babeltrace, gperf
-, cunit, snappy
-, rocksdb, makeWrapper
-
-# Optional Dependencies
-, yasm ? null, fcgi ? null, expat ? null
-, curl ? null, fuse ? null
-, libedit ? null, libatomic_ops ? null, kinetic-cpp-client ? null
-, libs3 ? null
-
-# Mallocs
-, jemalloc ? null, gperftools ? null
-
-# Crypto Dependencies
-, cryptopp ? null
-, nss ? null, nspr ? null
-
-# Linux Only Dependencies
-, linuxHeaders, libuuid, udev, keyutils, libaio ? null, libxfs ? null
-, zfs ? null
-
-# Version specific arguments
-, version, src ? [], buildInputs ? []
-, ...
-}:
-
-# We must have one crypto library
-assert cryptopp != null || (nss != null && nspr != null);
-
-with stdenv;
-with stdenv.lib;
-let
-
- shouldUsePkg = pkg_: let pkg = (builtins.tryEval pkg_).value;
- in if lib.any (lib.meta.platformMatch stdenv.hostPlatform) pkg.meta.platforms
- then pkg else null;
-
- optYasm = shouldUsePkg yasm;
- optFcgi = shouldUsePkg fcgi;
- optExpat = shouldUsePkg expat;
- optCurl = shouldUsePkg curl;
- optFuse = shouldUsePkg fuse;
- optLibedit = shouldUsePkg libedit;
- optLibatomic_ops = shouldUsePkg libatomic_ops;
- optKinetic-cpp-client = shouldUsePkg kinetic-cpp-client;
- optLibs3 = if versionAtLeast version "10.0.0" then null else shouldUsePkg libs3;
-
- optJemalloc = shouldUsePkg jemalloc;
- optGperftools = shouldUsePkg gperftools;
-
- optCryptopp = shouldUsePkg cryptopp;
- optNss = shouldUsePkg nss;
- optNspr = shouldUsePkg nspr;
-
- optLibaio = shouldUsePkg libaio;
- optLibxfs = shouldUsePkg libxfs;
- optZfs = shouldUsePkg zfs;
-
- hasRadosgw = optFcgi != null && optExpat != null && optCurl != null && optLibedit != null;
-
-
- # TODO: Reenable when kinetic support is fixed
- #hasKinetic = versionAtLeast version "9.0.0" && optKinetic-cpp-client != null;
- hasKinetic = false;
-
- # Malloc implementation (can be jemalloc, tcmalloc or null)
- malloc = if optJemalloc != null then optJemalloc else optGperftools;
-
- # We prefer nss over cryptopp
- cryptoStr = if optNss != null && optNspr != null then "nss" else
- if optCryptopp != null then "cryptopp" else "none";
- cryptoLibsMap = {
- nss = [ optNss optNspr ];
- cryptopp = [ optCryptopp ];
- none = [ ];
- };
-
- ceph-python-env = python2Packages.python.withPackages (ps: [
- ps.sphinx
- ps.flask
- ps.cython
- ps.setuptools
- ps.pip
- # Libraries needed by the python tools
- ps.Mako
- ps.pecan
- ps.prettytable
- ps.webob
- ps.cherrypy
- ]);
-
-in
-stdenv.mkDerivation {
- pname = "ceph";
- inherit version;
-
- inherit src;
-
- patches = [
- # ./ceph-patch-cmake-path.patch
- ./0001-kv-RocksDBStore-API-break-additional.patch
- ] ++ optionals stdenv.isLinux [
- ./0002-fix-absolute-include-path.patch
- ];
-
- nativeBuildInputs = [
- cmake
- pkgconfig which git python2Packages.wrapPython makeWrapper
- (ensureNewerSourcesHook { year = "1980"; })
- ];
-
- buildInputs = buildInputs ++ cryptoLibsMap.${cryptoStr} ++ [
- boost ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3
- malloc zlib openldap lttng-ust babeltrace gperf cunit
- snappy rocksdb
- ] ++ optionals stdenv.isLinux [
- linuxHeaders libuuid udev keyutils optLibaio optLibxfs optZfs
- ] ++ optionals hasRadosgw [
- optFcgi optExpat optCurl optFuse optLibedit
- ] ++ optionals hasKinetic [
- optKinetic-cpp-client
- ];
-
-
- preConfigure =''
- # rip off submodule that interfer with system libs
- rm -rf src/boost
- rm -rf src/rocksdb
-
- # require LD_LIBRARY_PATH for cython to find internal dep
- export LD_LIBRARY_PATH="$PWD/build/lib:$LD_LIBRARY_PATH"
-
- # requires setuptools due to embedded in-cmake setup.py usage
- export PYTHONPATH="${python2Packages.setuptools}/lib/python2.7/site-packages/:$PYTHONPATH"
- '';
-
- cmakeFlags = [
- "-DENABLE_GIT_VERSION=OFF"
- "-DWITH_SYSTEM_BOOST=ON"
- "-DWITH_SYSTEM_ROCKSDB=ON"
- "-DWITH_LEVELDB=OFF"
-
- # enforce shared lib
- "-DBUILD_SHARED_LIBS=ON"
-
- # disable cephfs, cmake build broken for now
- "-DWITH_CEPHFS=OFF"
- "-DWITH_LIBCEPHFS=OFF"
- ];
-
- postFixup = ''
- wrapPythonPrograms
- wrapProgram $out/bin/ceph-mgr --set PYTHONPATH $out/${python2Packages.python.sitePackages}
- '';
-
- enableParallelBuilding = true;
-
- outputs = [ "dev" "lib" "out" "doc" ];
-
- meta = {
- homepage = https://ceph.com/;
- description = "Distributed storage system";
- license = licenses.lgpl21;
- maintainers = with maintainers; [ adev ak ];
- platforms = platforms.unix;
- broken = true;
- };
-
- passthru.version = version;
-}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index b5a76fc55413..21f5f14a2dcb 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -2193,9 +2193,11 @@ in
nrg2iso = callPackage ../tools/cd-dvd/nrg2iso { };
libceph = ceph.lib;
- ceph = callPackage ../tools/filesystems/ceph {
- boost = boost166.override { enablePython = true; };
- };
+ inherit (callPackages ../tools/filesystems/ceph {
+ boost = boost167.override { enablePython = true; python = python37; };
+ })
+ ceph
+ ceph-client;
ceph-dev = ceph;
inherit (callPackages ../tools/security/certmgr { })