From 8d7ea96a13ab569d69aa7feaaa99f0d896b4f97a Mon Sep 17 00:00:00 2001 From: Johan Thomsen Date: Fri, 8 Jun 2018 10:16:43 +0200 Subject: [PATCH] nixos/kubernetes: improvements - Added option 'cni.configDir' to allow for having CNI config outside of nix-store Existing behavior (writing verbatim CNI conf-files to nix-store) is still available. - Removed unused option 'apiserver.publicAddress' and changed 'apiserver.address' to 'bindAddress' This conforms better to k8s docs and removes existing --bind-address hardcoding to 0.0.0.0 - Fixed c/p mistake in apiserver systemd unit description - Updated 18.09 release notes to reflect changes to existing options And fixed some typos from previous PR - Make docker images for Kubernetes Dashboard and kube-dns configurable --- nixos/doc/manual/release-notes/rl-1809.xml | 15 +++- nixos/modules/rename.nix | 2 + .../services/cluster/kubernetes/dashboard.nix | 35 ++++++---- .../services/cluster/kubernetes/default.nix | 47 ++++++++----- .../services/cluster/kubernetes/dns.nix | 69 +++++++++++-------- 5 files changed, 105 insertions(+), 63 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-1809.xml b/nixos/doc/manual/release-notes/rl-1809.xml index 72f96f1ca1a2..f57fd75c782d 100644 --- a/nixos/doc/manual/release-notes/rl-1809.xml +++ b/nixos/doc/manual/release-notes/rl-1809.xml @@ -288,11 +288,24 @@ inherit (pkgs.nixos { - Recommented way to access the Kubernetes Dashboard is with HTTPS (TLS) + Recommended way to access the Kubernetes Dashboard is via HTTPS (TLS) Therefore; public service port for the dashboard has changed to 443 (container port 8443) and scheme to https. + + + The option services.kubernetes.apiserver.address + was renamed to services.kubernetes.apiserver.bindAddress. + Note that the default value has changed from 127.0.0.1 to 0.0.0.0. + + + + + The option services.kubernetes.apiserver.publicAddress + was not used and thus has been removed. + + diff --git a/nixos/modules/rename.nix b/nixos/modules/rename.nix index 9b9e9e7109de..2df737452fbc 100644 --- a/nixos/modules/rename.nix +++ b/nixos/modules/rename.nix @@ -32,6 +32,8 @@ with lib; (mkRenamedOptionModule [ "services" "i2pd" "extIp" ] [ "services" "i2pd" "address" ]) (mkRenamedOptionModule [ "services" "kibana" "host" ] [ "services" "kibana" "listenAddress" ]) (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "admissionControl" ] [ "services" "kubernetes" "apiserver" "enableAdmissionPlugins" ]) + (mkRenamedOptionModule [ "services" "kubernetes" "apiserver" "address" ] ["services" "kubernetes" "apiserver" "bindAddress"]) + (mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "publicAddress" ] "") (mkRenamedOptionModule [ "services" "logstash" "address" ] [ "services" "logstash" "listenAddress" ]) (mkRenamedOptionModule [ "services" "mpd" "network" "host" ] [ "services" "mpd" "network" "listenAddress" ]) (mkRenamedOptionModule [ "services" "neo4j" "host" ] [ "services" "neo4j" "listenAddress" ]) diff --git a/nixos/modules/services/cluster/kubernetes/dashboard.nix b/nixos/modules/services/cluster/kubernetes/dashboard.nix index 8c1f35ec651b..9c1f814b683c 100644 --- a/nixos/modules/services/cluster/kubernetes/dashboard.nix +++ b/nixos/modules/services/cluster/kubernetes/dashboard.nix @@ -4,16 +4,6 @@ with lib; let cfg = config.services.kubernetes.addons.dashboard; - - name = "k8s.gcr.io/kubernetes-dashboard-amd64"; - version = "v1.8.3"; - - image = pkgs.dockerTools.pullImage { - imageName = name; - imageDigest = "sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0"; - finalImageTag = version; - sha256 = "18ajcg0q1vignfjk2sm4xj4wzphfz8wah69ps8dklqfvv0164mc8"; - }; in { options.services.kubernetes.addons.dashboard = { enable = mkEnableOption "kubernetes dashboard addon"; @@ -23,10 +13,27 @@ in { type = types.bool; default = elem "RBAC" config.services.kubernetes.apiserver.authorizationMode; }; + + version = mkOption { + description = "Which version of the kubernetes dashboard to deploy"; + type = types.str; + default = "v1.8.3"; + }; + + image = mkOption { + description = "Docker image to seed for the kubernetes dashboard container."; + type = types.attrs; + default = { + imageName = "k8s.gcr.io/kubernetes-dashboard-amd64"; + imageDigest = "sha256:dc4026c1b595435ef5527ca598e1e9c4343076926d7d62b365c44831395adbd0"; + finalImageTag = cfg.version; + sha256 = "18ajcg0q1vignfjk2sm4xj4wzphfz8wah69ps8dklqfvv0164mc8"; + }; + }; }; config = mkIf cfg.enable { - services.kubernetes.kubelet.seedDockerImages = [image]; + services.kubernetes.kubelet.seedDockerImages = [(pkgs.dockerTools.pullImage cfg.image)]; services.kubernetes.addonManager.addons = { kubernetes-dashboard-deployment = { @@ -36,7 +43,7 @@ in { labels = { k8s-addon = "kubernetes-dashboard.addons.k8s.io"; k8s-app = "kubernetes-dashboard"; - version = version; + version = cfg.version; "kubernetes.io/cluster-service" = "true"; "addonmanager.kubernetes.io/mode" = "Reconcile"; }; @@ -52,7 +59,7 @@ in { labels = { k8s-addon = "kubernetes-dashboard.addons.k8s.io"; k8s-app = "kubernetes-dashboard"; - version = version; + version = cfg.version; "kubernetes.io/cluster-service" = "true"; }; annotations = { @@ -63,7 +70,7 @@ in { priorityClassName = "system-cluster-critical"; containers = [{ name = "kubernetes-dashboard"; - image = "${name}:${version}"; + image = with cfg.image; "${imageName}:${finalImageTag}"; ports = [{ containerPort = 8443; protocol = "TCP"; diff --git a/nixos/modules/services/cluster/kubernetes/default.nix b/nixos/modules/services/cluster/kubernetes/default.nix index e624f41601b3..20f2308508c6 100644 --- a/nixos/modules/services/cluster/kubernetes/default.nix +++ b/nixos/modules/services/cluster/kubernetes/default.nix @@ -73,7 +73,9 @@ let mkKubeConfigOptions = prefix: { server = mkOption { description = "${prefix} kube-apiserver server address."; - default = "http://${cfg.apiserver.address}:${toString cfg.apiserver.port}"; + default = "http://${if cfg.apiserver.advertiseAddress != null + then cfg.apiserver.advertiseAddress + else "127.0.0.1"}:${toString cfg.apiserver.port}"; type = types.str; }; @@ -103,12 +105,18 @@ let keyFile = mkDefault cfg.kubeconfig.keyFile; }; - cniConfig = pkgs.buildEnv { - name = "kubernetes-cni-config"; - paths = imap (i: entry: - pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry) - ) cfg.kubelet.cni.config; - }; + cniConfig = + if cfg.kubelet.cni.config != [] && !(isNull cfg.kubelet.cni.configDir) then + throw "Verbatim CNI-config and CNI configDir cannot both be set." + else if !(isNull cfg.kubelet.cni.configDir) then + cfg.kubelet.cni.configDir + else + (pkgs.buildEnv { + name = "kubernetes-cni-config"; + paths = imap (i: entry: + pkgs.writeTextDir "${toString (10+i)}-${entry.type}.conf" (builtins.toJSON entry) + ) cfg.kubelet.cni.config; + }); manifests = pkgs.buildEnv { name = "kubernetes-manifests"; @@ -244,18 +252,13 @@ in { type = types.listOf types.str; }; - address = mkOption { - description = "Kubernetes apiserver listening address."; - default = "127.0.0.1"; - type = types.str; - }; - - publicAddress = mkOption { + bindAddress = mkOption { description = '' - Kubernetes apiserver public listening address used for read only and - secure port. + The IP address on which to listen for the --secure-port port. + The associated interface(s) must be reachable by the rest + of the cluster, and by CLI/web clients. ''; - default = cfg.apiserver.address; + default = "0.0.0.0"; type = types.str; }; @@ -670,6 +673,12 @@ in { }] ''; }; + + configDir = mkOption { + description = "Path to Kubernetes CNI configuration directory."; + type = types.nullOr types.path; + default = null; + }; }; manifests = mkOption { @@ -892,7 +901,7 @@ in { (mkIf cfg.apiserver.enable { systemd.services.kube-apiserver = { - description = "Kubernetes Kubelet Service"; + description = "Kubernetes APIServer Service"; wantedBy = [ "kubernetes.target" ]; after = [ "network.target" "docker.service" ]; serviceConfig = { @@ -906,7 +915,7 @@ in { ${optionalString (cfg.etcd.keyFile != null) "--etcd-keyfile=${cfg.etcd.keyFile}"} \ --insecure-port=${toString cfg.apiserver.port} \ - --bind-address=${toString cfg.apiserver.address} \ + --bind-address=${cfg.apiserver.bindAddress} \ ${optionalString (cfg.apiserver.advertiseAddress != null) "--advertise-address=${cfg.apiserver.advertiseAddress}"} \ --allow-privileged=${boolToString cfg.apiserver.allowPrivileged}\ diff --git a/nixos/modules/services/cluster/kubernetes/dns.nix b/nixos/modules/services/cluster/kubernetes/dns.nix index 9751e5f7cf0a..43bbb50a48d4 100644 --- a/nixos/modules/services/cluster/kubernetes/dns.nix +++ b/nixos/modules/services/cluster/kubernetes/dns.nix @@ -4,28 +4,6 @@ with lib; let version = "1.14.10"; - - k8s-dns-kube-dns = pkgs.dockerTools.pullImage { - imageName = "k8s.gcr.io/k8s-dns-kube-dns-amd64"; - imageDigest = "sha256:b99fc3eee2a9f052f7eb4cc00f15eb12fc405fa41019baa2d6b79847ae7284a8"; - finalImageTag = version; - sha256 = "0x583znk9smqn0fix7ld8sm5jgaxhqhx3fq97b1wkqm7iwhvl3pj"; - }; - - k8s-dns-dnsmasq-nanny = pkgs.dockerTools.pullImage { - imageName = "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64"; - imageDigest = "sha256:bbb2a290a568125b3b996028958eb773f33b5b87a6b37bf38a28f8b62dddb3c8"; - finalImageTag = version; - sha256 = "1fihml7s2mfwgac51cbqpylkwbivc8nyhgi4vb820s83zvl8a6y1"; - }; - - k8s-dns-sidecar = pkgs.dockerTools.pullImage { - imageName = "k8s.gcr.io/k8s-dns-sidecar-amd64"; - imageDigest = "sha256:4f1ab957f87b94a5ec1edc26fae50da2175461f00afecf68940c4aa079bd08a4"; - finalImageTag = version; - sha256 = "08l1bv5jgrhvjzpqpbinrkgvv52snc4fzyd8ya9v18ns2klyz7m0"; - }; - cfg = config.services.kubernetes.addons.dns; in { options.services.kubernetes.addons.dns = { @@ -48,13 +26,46 @@ in { default = "cluster.local"; type = types.str; }; + + kube-dns = mkOption { + description = "Docker image to seed for the kube-dns main container."; + type = types.attrs; + default = { + imageName = "k8s.gcr.io/k8s-dns-kube-dns-amd64"; + imageDigest = "sha256:b99fc3eee2a9f052f7eb4cc00f15eb12fc405fa41019baa2d6b79847ae7284a8"; + finalImageTag = version; + sha256 = "0x583znk9smqn0fix7ld8sm5jgaxhqhx3fq97b1wkqm7iwhvl3pj"; + }; + }; + + dnsmasq-nanny = mkOption { + description = "Docker image to seed for the kube-dns dnsmasq container."; + type = types.attrs; + default = { + imageName = "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64"; + imageDigest = "sha256:bbb2a290a568125b3b996028958eb773f33b5b87a6b37bf38a28f8b62dddb3c8"; + finalImageTag = version; + sha256 = "1fihml7s2mfwgac51cbqpylkwbivc8nyhgi4vb820s83zvl8a6y1"; + }; + }; + + sidecar = mkOption { + description = "Docker image to seed for the kube-dns sidecar container."; + type = types.attrs; + default = { + imageName = "k8s.gcr.io/k8s-dns-sidecar-amd64"; + imageDigest = "sha256:4f1ab957f87b94a5ec1edc26fae50da2175461f00afecf68940c4aa079bd08a4"; + finalImageTag = version; + sha256 = "08l1bv5jgrhvjzpqpbinrkgvv52snc4fzyd8ya9v18ns2klyz7m0"; + }; + }; }; config = mkIf cfg.enable { - services.kubernetes.kubelet.seedDockerImages = [ - k8s-dns-kube-dns - k8s-dns-dnsmasq-nanny - k8s-dns-sidecar + services.kubernetes.kubelet.seedDockerImages = with pkgs.dockerTools; [ + (pullImage cfg.kube-dns) + (pullImage cfg.dnsmasq-nanny) + (pullImage cfg.sidecar) ]; services.kubernetes.addonManager.addons = { @@ -88,7 +99,7 @@ in { containers = [ { name = "kubedns"; - image = "k8s.gcr.io/k8s-dns-kube-dns-amd64:${version}"; + image = with cfg.kube-dns; "${imageName}:${finalImageTag}"; resources = { limits.memory = "170Mi"; requests = { @@ -154,7 +165,7 @@ in { } { name = "dnsmasq"; - image = "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:${version}"; + image = with cfg.dnsmasq-nanny; "${imageName}:${finalImageTag}"; livenessProbe = { httpGet = { path = "/healthcheck/dnsmasq"; @@ -206,7 +217,7 @@ in { } { name = "sidecar"; - image = "k8s.gcr.io/k8s-dns-sidecar-amd64:${version}"; + image = with cfg.sidecar; "${imageName}:${finalImageTag}"; livenessProbe = { httpGet = { path = "/metrics";