From 0d506aa712cf088343dff5863e9e58eb1228c3b0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Mar 2014 14:04:39 +0100 Subject: [PATCH 01/32] Provide a simple way to log into containers On the host, you can run $ socat unix:/var/lib/login.socket -,echo=0,raw to get a login prompt. So this allows logging in even if the container has no SSH access enabled. You can also do $ socat unix:/var/lib/root-shell.socket - to get a plain root shell. (This socket is only accessible by root, obviously.) This makes it easy to execute commands in the container, e.g. $ echo reboot | socat unix:/var/lib/root-shell.socket - --- .../virtualisation/container-login.nix | 56 +++++++++++++++++++ nixos/modules/virtualisation/containers.nix | 1 + 2 files changed, 57 insertions(+) create mode 100644 nixos/modules/virtualisation/container-login.nix diff --git a/nixos/modules/virtualisation/container-login.nix b/nixos/modules/virtualisation/container-login.nix new file mode 100644 index 000000000000..56e772fb797d --- /dev/null +++ b/nixos/modules/virtualisation/container-login.nix @@ -0,0 +1,56 @@ +{ config, pkgs, ... }: + +{ + + config = { + + # Provide a login prompt on /var/lib/login.socket. On the host, + # you can connect to it by running ‘socat + # unix:/var/lib/login.socket -,echo=0,raw’. + systemd.sockets.login = + { description = "Login Socket"; + wantedBy = [ "sockets.target" ]; + socketConfig = + { ListenStream = "/var/lib/login.socket"; + SocketMode = "0600"; # only root can connect, obviously + Accept = true; + }; + }; + + systemd.services."login@" = + { description = "Login %i"; + environment.TERM = "linux"; + serviceConfig = + { Type = "simple"; + StandardInput = "socket"; + ExecStart = "${pkgs.socat}/bin/socat -t0 - exec:${pkgs.shadow}/bin/login,pty,setsid,setpgid,stderr,ctty"; + TimeoutStopSec = 1; # FIXME + }; + }; + + # Provide a non-interactive login root shell on + # /var/lib/root-shell.socket. On the host, you can connect to it + # by running ‘socat unix:/var/lib/root-shell.socket -’. + systemd.sockets.root-shell = + { description = "Root Shell Socket"; + wantedBy = [ "sockets.target" ]; + socketConfig = + { ListenStream = "/var/lib/root-shell.socket"; + SocketMode = "0666"; + Accept = true; + }; + }; + + systemd.services."root-shell@" = + { description = "Root Shell %i"; + serviceConfig = + { Type = "simple"; + StandardInput = "socket"; + ExecStart = "${pkgs.bash}/bin/bash --login"; + TimeoutStopSec = 1; # FIXME + }; + }; + + }; + +} diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index d87284de4fc1..f1fcc18f1f99 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -57,6 +57,7 @@ with pkgs.lib; { boot.isContainer = true; security.initialRootPassword = mkDefault "!"; networking.hostName = mkDefault name; + imports = [ ./container-login.nix ]; }; in [ extraConfig config.config ]; prefix = [ "systemd" "containers" name ]; From f13bd41384afadcd94e316759583359de37d15bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Mar 2014 14:10:25 +0100 Subject: [PATCH 02/32] switch-to-configuration: Restart sockets.target --- nixos/modules/system/activation/switch-to-configuration.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/system/activation/switch-to-configuration.pl b/nixos/modules/system/activation/switch-to-configuration.pl index e0649448c834..3f78ec0d48eb 100644 --- a/nixos/modules/system/activation/switch-to-configuration.pl +++ b/nixos/modules/system/activation/switch-to-configuration.pl @@ -321,7 +321,7 @@ if (scalar @restart > 0) { # that are symlinks to other units. We shouldn't start both at the # same time because we'll get a "Failed to add path to set" error from # systemd. -my @start = unique("default.target", "timers.target", split('\n', read_file($startListFile, err_mode => 'quiet') // "")); +my @start = unique("default.target", "timers.target", "sockets.target", split('\n', read_file($startListFile, err_mode => 'quiet') // "")); print STDERR "starting the following units: ", join(", ", sort(@start)), "\n"; system("@systemd@/bin/systemctl", "start", "--", @start) == 0 or $res = 4; unlink($startListFile); From f9e2af1e8b62809e9af9acc92826c29f63db8043 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Mar 2014 14:16:10 +0100 Subject: [PATCH 03/32] switch-to-configuration: Don't require /etc/NIXOS Check /etc/os-release if /etc/NIXOS doesn't exist. --- nixos/modules/system/activation/switch-to-configuration.pl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nixos/modules/system/activation/switch-to-configuration.pl b/nixos/modules/system/activation/switch-to-configuration.pl index 3f78ec0d48eb..67b2280af3be 100644 --- a/nixos/modules/system/activation/switch-to-configuration.pl +++ b/nixos/modules/system/activation/switch-to-configuration.pl @@ -26,7 +26,10 @@ EOF exit 1; } -die "This is not a NixOS installation (/etc/NIXOS is missing)!\n" unless -f "/etc/NIXOS"; +# This is a NixOS installation if it has /etc/NIXOS or a proper +# /etc/os-release. +die "This is not a NixOS installation!\n" unless + -f "/etc/NIXOS" || (read_file("/etc/os-release", err_mode => 'quiet') // "") =~ /ID=nixos/s; openlog("nixos", "", LOG_USER); From 28b7d67d08a7a76c7508d4b6a8f791c2eba4a1b9 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Mar 2014 15:01:10 +0100 Subject: [PATCH 04/32] httpd: Don't require keys.target This has the unintended side-effect of restarting httpd every time we run switch-to-configuration, even if httpd hasn't changed (because we're doing a "stop keys.target" now). So use a "Wants" dependency instead. --- nixos/modules/services/web-servers/apache-httpd/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/web-servers/apache-httpd/default.nix b/nixos/modules/services/web-servers/apache-httpd/default.nix index a22ef10312d4..949dce96824b 100644 --- a/nixos/modules/services/web-servers/apache-httpd/default.nix +++ b/nixos/modules/services/web-servers/apache-httpd/default.nix @@ -621,7 +621,7 @@ in { description = "Apache HTTPD"; wantedBy = [ "multi-user.target" ]; - requires = [ "keys.target" ]; + wants = [ "keys.target" ]; after = [ "network.target" "fs.target" "postgresql.service" "keys.target" ]; path = From 511b86d22dbf3d4a52ac9d6437f0ee0375468f01 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Mar 2014 15:02:53 +0100 Subject: [PATCH 05/32] Add an option to reload rather than restart changed units --- .../system/activation/switch-to-configuration.pl | 5 ++++- nixos/modules/system/boot/systemd-unit-options.nix | 11 +++++++++++ nixos/modules/system/boot/systemd.nix | 6 +++++- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/nixos/modules/system/activation/switch-to-configuration.pl b/nixos/modules/system/activation/switch-to-configuration.pl index 67b2280af3be..fd2b5b7950d5 100644 --- a/nixos/modules/system/activation/switch-to-configuration.pl +++ b/nixos/modules/system/activation/switch-to-configuration.pl @@ -176,7 +176,10 @@ while (my ($unit, $state) = each %{$activePrev}) { # FIXME: do something? } else { my $unitInfo = parseUnit($newUnitFile); - if (!boolIsTrue($unitInfo->{'X-RestartIfChanged'} // "yes")) { + if (boolIsTrue($unitInfo->{'X-ReloadIfChanged'} // "no")) { + write_file($reloadListFile, { append => 1 }, "$unit\n"); + } + elsif (!boolIsTrue($unitInfo->{'X-RestartIfChanged'} // "yes")) { push @unitsToSkip, $unit; } else { # If this unit is socket-activated, then stop the diff --git a/nixos/modules/system/boot/systemd-unit-options.nix b/nixos/modules/system/boot/systemd-unit-options.nix index 113990814efa..784025f6bab2 100644 --- a/nixos/modules/system/boot/systemd-unit-options.nix +++ b/nixos/modules/system/boot/systemd-unit-options.nix @@ -243,6 +243,17 @@ in rec { ''; }; + reloadIfChanged = mkOption { + type = types.bool; + default = false; + description = '' + Whether the service should be reloaded during a NixOS + configuration switch if its definition has changed. If + enabled, the value of is + ignored. + ''; + }; + stopIfChanged = mkOption { type = types.bool; default = true; diff --git a/nixos/modules/system/boot/systemd.nix b/nixos/modules/system/boot/systemd.nix index 49502b3e6851..b31c631a0a7b 100644 --- a/nixos/modules/system/boot/systemd.nix +++ b/nixos/modules/system/boot/systemd.nix @@ -279,7 +279,11 @@ let [Service] ${let env = cfg.globalEnvironment // def.environment; in concatMapStrings (n: "Environment=\"${n}=${getAttr n env}\"\n") (attrNames env)} - ${optionalString (!def.restartIfChanged) "X-RestartIfChanged=false"} + ${if def.reloadIfChanged then '' + X-ReloadIfChanged=true + '' else if !def.restartIfChanged then '' + X-RestartIfChanged=false + '' else ""} ${optionalString (!def.stopIfChanged) "X-StopIfChanged=false"} ${attrsToSection def.serviceConfig} ''; From ef8e0266a2583eb45444b1faa0512f99d18fca6c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Mar 2014 15:03:29 +0100 Subject: [PATCH 06/32] Don't reboot a container when its configuration changes Instead, just run "switch-to-configuration" inside the container. --- nixos/modules/virtualisation/containers.nix | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index f1fcc18f1f99..034ebe84b42f 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -132,6 +132,14 @@ with pkgs.lib; done fi ''; + + reloadIfChanged = true; + + serviceConfig.ExecReload = + "${pkgs.bash}/bin/bash -c '" + + "echo ${container.path}/bin/switch-to-configuration test " + + "| ${pkgs.socat}/bin/socat unix:${container.root}/var/lib/root-shell.socket -'"; + }) config.systemd.containers; }; From ac215779dd1fdfe58d75bcc22cb6a5b0a58744a4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 17 Mar 2014 15:23:20 +0100 Subject: [PATCH 07/32] Give containers a writable /nix/var/nix/{profiles,gcroots} These are stored on the host in /nix/var/nix/{profiles,gcroots}/per-container/ to ensure that container profiles/roots are not garbage-collected. --- nixos/modules/virtualisation/containers.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 034ebe84b42f..6a4833e1e215 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -110,10 +110,19 @@ with pkgs.lib; if ! [ -e ${container.root}/etc/os-release ]; then touch ${container.root}/etc/os-release fi + + mkdir -p -m 0755 \ + /nix/var/nix/profiles/per-container/${name} \ + /nix/var/nix/gcroots/per-container/${name} ''; serviceConfig.ExecStart = - "${config.systemd.package}/bin/systemd-nspawn -M ${name} -D ${container.root} --bind-ro=/nix ${container.path}/init"; + "${config.systemd.package}/bin/systemd-nspawn" + + " -M ${name} -D ${container.root}" + + " --bind-ro=/nix/store --bind-ro=/nix/var/nix/db --bind-ro=/nix/var/nix/daemon-socket" + + " --bind=/nix/var/nix/profiles/per-container/${name}:/nix/var/nix/profiles" + + " --bind=/nix/var/nix/gcroots/per-container/${name}:/nix/var/nix/gcroots" + + " ${container.path}/init"; preStop = '' From 895bcdd1cb9f98ba032c78d996f3ebc89fd60bc2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Mar 2014 10:49:25 +0100 Subject: [PATCH 08/32] Add support for running a container with a private network interface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For example, the following sets up a container named ‘foo’. The container will have a single network interface eth0, with IP address 10.231.136.2. The host will have an interface c-foo with IP address 10.231.136.1. systemd.containers.foo = { privateNetwork = true; hostAddress = "10.231.136.1"; localAddress = "10.231.136.2"; config = { services.openssh.enable = true; }; }; With ‘privateNetwork = true’, the container has the CAP_NET_ADMIN capability, allowing it to do arbitrary network configuration, such as setting up firewall rules. This is secure because it cannot touch the interfaces of the host. The helper program ‘run-in-netns’ is needed at the moment because ‘ip netns exec’ doesn't quite do the right thing (it remounts /sys without bind-mounting the original /sys/fs/cgroups). --- nixos/modules/services/networking/dhcpcd.nix | 5 +- nixos/modules/virtualisation/containers.nix | 113 +++++++++++++++++-- nixos/modules/virtualisation/run-in-netns.c | 50 ++++++++ 3 files changed, 155 insertions(+), 13 deletions(-) create mode 100644 nixos/modules/virtualisation/run-in-netns.c diff --git a/nixos/modules/services/networking/dhcpcd.nix b/nixos/modules/services/networking/dhcpcd.nix index d4ec96a18f6f..38a21a723d97 100644 --- a/nixos/modules/services/networking/dhcpcd.nix +++ b/nixos/modules/services/networking/dhcpcd.nix @@ -34,8 +34,9 @@ let # Ignore peth* devices; on Xen, they're renamed physical # Ethernet cards used for bridging. Likewise for vif* and tap* - # (Xen) and virbr* and vnet* (libvirt). - denyinterfaces ${toString ignoredInterfaces} peth* vif* tap* tun* virbr* vnet* vboxnet* + # (Xen) and virbr* and vnet* (libvirt) and c-* and ctmp-* (NixOS + # containers). + denyinterfaces ${toString ignoredInterfaces} peth* vif* tap* tun* virbr* vnet* vboxnet* c-* ctmp-* ${config.networking.dhcpcd.extraConfig} ''; diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 6a4833e1e215..49046975d83e 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -2,6 +2,20 @@ with pkgs.lib; +let + + runInNetns = pkgs.stdenv.mkDerivation { + name = "run-in-netns"; + unpackPhase = "true"; + buildPhase = '' + mkdir -p $out/bin + gcc ${./run-in-netns.c} -o $out/bin/run-in-netns + ''; + installPhase = "true"; + }; + +in + { options = { @@ -45,6 +59,39 @@ with pkgs.lib; ''; }; + privateNetwork = mkOption { + type = types.bool; + default = false; + description = '' + Whether to give the container its own private virtual + Ethernet interface. The interface is called + eth0, and is hooked up to the interface + c-container-name + on the host. If this option is not set, then the + container shares the network interfaces of the host, + and can bind to any port on any interface. + ''; + }; + + hostAddress = mkOption { + type = types.nullOr types.string; + default = null; + example = "10.231.136.1"; + description = '' + The IPv4 address assigned to the host interface. + ''; + }; + + localAddress = mkOption { + type = types.nullOr types.string; + default = null; + example = "10.231.136.2"; + description = '' + The IPv4 address assigned to eth0 + in the container. + ''; + }; + }; config = mkMerge @@ -97,32 +144,70 @@ with pkgs.lib; config = { - systemd.services = mapAttrs' (name: container: nameValuePair "container-${name}" - { description = "Container '${name}'"; + systemd.services = mapAttrs' (name: cfg: + let + # FIXME: interface names have a maximum length. + ifaceHost = "c-${name}"; + ifaceCont = "ctmp-${name}"; + ns = "net-${name}"; + in + nameValuePair "container-${name}" { + description = "Container '${name}'"; wantedBy = [ "multi-user.target" ]; - unitConfig.RequiresMountsFor = [ container.root ]; + unitConfig.RequiresMountsFor = [ cfg.root ]; + + path = [ pkgs.iproute ]; preStart = '' - mkdir -p -m 0755 ${container.root}/etc - if ! [ -e ${container.root}/etc/os-release ]; then - touch ${container.root}/etc/os-release + mkdir -p -m 0755 ${cfg.root}/etc + if ! [ -e ${cfg.root}/etc/os-release ]; then + touch ${cfg.root}/etc/os-release fi mkdir -p -m 0755 \ /nix/var/nix/profiles/per-container/${name} \ /nix/var/nix/gcroots/per-container/${name} + '' + + + optionalString cfg.privateNetwork '' + # Cleanup from last time. + ip netns del ${ns} 2> /dev/null || true + ip link del ${ifaceHost} 2> /dev/null || true + ip link del ${ifaceCont} 2> /dev/null || true + + # Create a pair of virtual ethernet devices. On the host, + # we get ‘c- +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char * * argv) +{ + if (argc < 3) { + fprintf(stderr, "%s: missing arguments\n", argv[0]); + return 1; + } + + char nsPath[PATH_MAX]; + + sprintf(nsPath, "/run/netns/%s", argv[1]); + + int fd = open(nsPath, O_RDONLY); + if (fd == -1) { + fprintf(stderr, "%s: opening network namespace: %s\n", argv[0], strerror(errno)); + return 1; + } + + if (setns(fd, CLONE_NEWNET) == -1) { + fprintf(stderr, "%s: setting network namespace: %s\n", argv[0], strerror(errno)); + return 1; + } + + umount2(nsPath, MNT_DETACH); + if (unlink(nsPath) == -1) { + fprintf(stderr, "%s: unlinking network namespace: %s\n", argv[0], strerror(errno)); + return 1; + } + + /* FIXME: Remount /sys so that /sys/class/net reflects the + interfaces visible in the network namespace. This requires + bind-mounting /sys/fs/cgroups etc. */ + + execv(argv[2], argv + 2); + fprintf(stderr, "%s: running command: %s\n", argv[0], strerror(errno)); + return 1; +} From 7b82d1ee27a60157418e241152ef1aa3f909ad7b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Mar 2014 11:04:54 +0100 Subject: [PATCH 09/32] Ensure that the container root can always be accessed via /var/lib/containers --- nixos/modules/virtualisation/containers.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 49046975d83e..ff17fcc1221d 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -172,6 +172,10 @@ in /nix/var/nix/gcroots/per-container/${name} '' + + optionalString (cfg.root != "/var/lib/containers/${name}") '' + ln -sfn "${cfg.root}" "/var/lib/containers/${name}" + '' + + optionalString cfg.privateNetwork '' # Cleanup from last time. ip netns del ${ns} 2> /dev/null || true From 11c4c4ae54bbef647358d2b6d6c3ddf0457f81f7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Mar 2014 11:36:03 +0100 Subject: [PATCH 10/32] =?UTF-8?q?Add=20command=20=E2=80=98nixos-container-?= =?UTF-8?q?shell=E2=80=99=20for=20logging=20into=20a=20container?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- nixos/modules/virtualisation/containers.nix | 54 +++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index ff17fcc1221d..28ee78e3fcce 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -14,6 +14,58 @@ let installPhase = "true"; }; + nixos-container-shell = pkgs.writeScriptBin "nixos-container-shell" + '' + #! ${pkgs.bash}/bin/sh -e + + usage() { + echo "Usage: $0 " >&2 + echo " $0 (-r|--root-shell) " >&2 + } + + args="`getopt --options 'r' -l help -- "$@"`" + eval "set -- $args" + rootShell= + while [ $# -gt 0 ]; do + case "$1" in + (--help) usage; exit 0;; + (-r|--root-shell) rootShell=1;; + (--) shift; break;; + (*) break;; + esac + shift + done + + container="$1" + if [ -z "$container" ]; then + usage + exit 1 + fi + shift + + root="/var/lib/containers/$container" + if ! [ -d "$root" ]; then + echo "$0: container ‘$container’ does not exist" >&2 + exit 1 + fi + + if [ -n "$rootShell" ]; then + socket="$root/var/lib/root-shell.socket" + else + socket="$root/var/lib/login.socket" + fi + if ! [ -S "$socket" ]; then + echo "$0: socket ‘$socket’ does not exist" >&2 + exit 1 + fi + + if [ -n "$rootShell" ]; then + exec ${pkgs.socat}/bin/socat "unix:$socket" - + else + exec ${pkgs.socat}/bin/socat "unix:$socket" -,echo=0,raw + fi + ''; + in { @@ -246,5 +298,7 @@ in ${cfg.localAddress} ${name}.containers '') config.systemd.containers); + environment.systemPackages = optional (config.systemd.containers != {}) nixos-container-shell; + }; } From 5b10ea1f99edf1855d0735330634a463771a5ee2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Mar 2014 11:39:51 +0100 Subject: [PATCH 11/32] Don't run dhcpcd in containers --- nixos/modules/virtualisation/containers.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 28ee78e3fcce..b8388e7f206d 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -156,6 +156,7 @@ in { boot.isContainer = true; security.initialRootPassword = mkDefault "!"; networking.hostName = mkDefault name; + networking.useDHCP = false; imports = [ ./container-login.nix ]; }; in [ extraConfig config.config ]; From 7ee31c7f94d121f5e57779c0c1478aee30d80cb1 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Mar 2014 18:04:38 +0100 Subject: [PATCH 12/32] Fix permissions --- nixos/modules/virtualisation/container-login.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/modules/virtualisation/container-login.nix b/nixos/modules/virtualisation/container-login.nix index 56e772fb797d..09eaf90ae658 100644 --- a/nixos/modules/virtualisation/container-login.nix +++ b/nixos/modules/virtualisation/container-login.nix @@ -12,7 +12,7 @@ wantedBy = [ "sockets.target" ]; socketConfig = { ListenStream = "/var/lib/login.socket"; - SocketMode = "0600"; # only root can connect, obviously + SocketMode = "0666"; Accept = true; }; }; @@ -36,7 +36,7 @@ wantedBy = [ "sockets.target" ]; socketConfig = { ListenStream = "/var/lib/root-shell.socket"; - SocketMode = "0666"; + SocketMode = "0600"; # only root can connect, obviously Accept = true; }; }; From 2ace7edb81509189a15a0b8e8d0ee1886be9e725 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 18 Mar 2014 18:18:35 +0100 Subject: [PATCH 13/32] Rename systemd.containers -> containers That NixOS containers use systemd-nspawn is just an implementation detail (which we could change in the future). --- nixos/modules/virtualisation/containers.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index b8388e7f206d..cdd9e729dc16 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -80,7 +80,7 @@ in ''; }; - systemd.containers = mkOption { + containers = mkOption { type = types.attrsOf (types.submodule ( { config, options, name, ... }: { @@ -160,7 +160,7 @@ in imports = [ ./container-login.nix ]; }; in [ extraConfig config.config ]; - prefix = [ "systemd" "containers" name ]; + prefix = [ "containers" name ]; }).config.system.build.toplevel; }) ]; @@ -291,15 +291,15 @@ in + "echo ${cfg.path}/bin/switch-to-configuration test " + "| ${pkgs.socat}/bin/socat unix:${cfg.root}/var/lib/root-shell.socket -'"; - }) config.systemd.containers; + }) config.containers; # Generate /etc/hosts entries for the containers. networking.extraHosts = concatStrings (mapAttrsToList (name: cfg: optionalString (cfg.localAddress != null) '' ${cfg.localAddress} ${name}.containers - '') config.systemd.containers); + '') config.containers); - environment.systemPackages = optional (config.systemd.containers != {}) nixos-container-shell; + environment.systemPackages = optional (config.containers != {}) nixos-container-shell; }; } From 0cca0f477f168cee994b2c90d20b713f2bb67d85 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Mar 2014 10:15:57 +0100 Subject: [PATCH 14/32] nixos-container-shell -> nixos-container { login | root-shell } --- nixos/modules/virtualisation/containers.nix | 60 +++---------------- .../modules/virtualisation/nixos-container.sh | 54 +++++++++++++++++ 2 files changed, 62 insertions(+), 52 deletions(-) create mode 100644 nixos/modules/virtualisation/nixos-container.sh diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index cdd9e729dc16..e718398815d2 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -14,57 +14,13 @@ let installPhase = "true"; }; - nixos-container-shell = pkgs.writeScriptBin "nixos-container-shell" - '' - #! ${pkgs.bash}/bin/sh -e - - usage() { - echo "Usage: $0 " >&2 - echo " $0 (-r|--root-shell) " >&2 - } - - args="`getopt --options 'r' -l help -- "$@"`" - eval "set -- $args" - rootShell= - while [ $# -gt 0 ]; do - case "$1" in - (--help) usage; exit 0;; - (-r|--root-shell) rootShell=1;; - (--) shift; break;; - (*) break;; - esac - shift - done - - container="$1" - if [ -z "$container" ]; then - usage - exit 1 - fi - shift - - root="/var/lib/containers/$container" - if ! [ -d "$root" ]; then - echo "$0: container ‘$container’ does not exist" >&2 - exit 1 - fi - - if [ -n "$rootShell" ]; then - socket="$root/var/lib/root-shell.socket" - else - socket="$root/var/lib/login.socket" - fi - if ! [ -S "$socket" ]; then - echo "$0: socket ‘$socket’ does not exist" >&2 - exit 1 - fi - - if [ -n "$rootShell" ]; then - exec ${pkgs.socat}/bin/socat "unix:$socket" - - else - exec ${pkgs.socat}/bin/socat "unix:$socket" -,echo=0,raw - fi - ''; + nixos-container = pkgs.substituteAll { + name = "nixos-container"; + dir = "bin"; + isExecutable = true; + src = ./nixos-container.sh; + inherit (pkgs) bash socat; + }; in @@ -299,7 +255,7 @@ in ${cfg.localAddress} ${name}.containers '') config.containers); - environment.systemPackages = optional (config.containers != {}) nixos-container-shell; + environment.systemPackages = optional (config.containers != {}) nixos-container; }; } diff --git a/nixos/modules/virtualisation/nixos-container.sh b/nixos/modules/virtualisation/nixos-container.sh new file mode 100644 index 000000000000..f4e34588d6eb --- /dev/null +++ b/nixos/modules/virtualisation/nixos-container.sh @@ -0,0 +1,54 @@ +#! @bash@/bin/sh -e + +usage() { + echo "Usage: $0 login " >&2 + echo " $0 root-shell " >&2 +} + +args="`getopt --options '' -l help -- "$@"`" +eval "set -- $args" +while [ $# -gt 0 ]; do + case "$1" in + (--help) usage; exit 0;; + (--) shift; break;; + (*) break;; + esac + shift +done + +action="$1" +if [ -z "$action" ]; then usage; exit 1; fi +shift + +getContainerRoot() { + root="/var/lib/containers/$container" + if ! [ -d "$root" ]; then + echo "$0: container ‘$container’ does not exist" >&2 + exit 1 + fi +} + +if [ $action = login ]; then + + container="$1" + if [ -z "$container" ]; then usage; exit 1; fi + shift + + getContainerRoot + + exec @socat@/bin/socat "unix:$root/var/lib/login.socket" -,echo=0,raw + +elif [ $action = root-shell ]; then + + container="$1" + if [ -z "$container" ]; then usage; exit 1; fi + shift + + getContainerRoot + + exec @socat@/bin/socat "unix:$root/var/lib/root-shell.socket" - + +else + echo "$0: unknown action ‘$action’" >&2 + exit 1 +fi From ba88db3cd332e439dd2090b64abb7b9942b5fc94 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 19 Mar 2014 19:55:05 +0100 Subject: [PATCH 15/32] Add support for imperative container management MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The command nixos-container can now create containers. For instance, the following creates and starts a container named ‘database’: $ nixos-container create database The configuration of the container is stored in /var/lib/containers//etc/nixos/configuration.nix. After editing the configuration, you can make the changes take effect by doing $ nixos-container update database The container can also be destroyed: $ nixos-container destroy database Containers are now executed using a template unit, ‘container@.service’, so the unit in this example would be ‘container@database.service’. --- nixos/modules/virtualisation/containers.nix | 164 ++++++++++-------- .../modules/virtualisation/nixos-container.sh | 106 +++++++++-- 2 files changed, 184 insertions(+), 86 deletions(-) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index e718398815d2..573a3d1c5848 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -42,13 +42,6 @@ in { options = { - root = mkOption { - type = types.path; - description = '' - The root directory of the container. - ''; - }; - config = mkOption { description = '' A specification of the desired configuration of this @@ -103,9 +96,7 @@ in }; config = mkMerge - [ { root = mkDefault "/var/lib/containers/${name}"; - } - (mkIf options.config.isDefined { + [ (mkIf options.config.isDefined { path = (import ../../lib/eval-config.nix { modules = let extraConfig = @@ -126,12 +117,10 @@ in example = literalExample '' { webserver = - { root = "/containers/webserver"; - path = "/nix/var/nix/profiles/webserver"; + { path = "/nix/var/nix/profiles/webserver"; }; database = - { root = "/containers/database"; - config = + { config = { config, pkgs, ... }: { services.postgresql.enable = true; services.postgresql.package = pkgs.postgresql92; @@ -153,78 +142,76 @@ in config = { - systemd.services = mapAttrs' (name: cfg: - let - # FIXME: interface names have a maximum length. - ifaceHost = "c-${name}"; - ifaceCont = "ctmp-${name}"; - ns = "net-${name}"; - in - nameValuePair "container-${name}" { - description = "Container '${name}'"; + systemd.services."container@" = + { description = "Container '%I'"; - wantedBy = [ "multi-user.target" ]; - - unitConfig.RequiresMountsFor = [ cfg.root ]; + unitConfig.RequiresMountsFor = [ "/var/lib/containers/%I" ]; path = [ pkgs.iproute ]; - preStart = + environment.INSTANCE = "%I"; + + script = '' - mkdir -p -m 0755 ${cfg.root}/etc - if ! [ -e ${cfg.root}/etc/os-release ]; then - touch ${cfg.root}/etc/os-release + root="/var/lib/containers/$INSTANCE" + mkdir -p -m 0755 "$root/etc" + if ! [ -e "$root/etc/os-release" ]; then + touch "$root/etc/os-release" fi mkdir -p -m 0755 \ - /nix/var/nix/profiles/per-container/${name} \ - /nix/var/nix/gcroots/per-container/${name} - '' + "/nix/var/nix/profiles/per-container/$INSTANCE" \ + "/nix/var/nix/gcroots/per-container/$INSTANCE" - + optionalString (cfg.root != "/var/lib/containers/${name}") '' - ln -sfn "${cfg.root}" "/var/lib/containers/${name}" - '' + SYSTEM_PATH=/nix/var/nix/profiles/system + if [ -f "/etc/containers/$INSTANCE.conf" ]; then + . "/etc/containers/$INSTANCE.conf" + fi - + optionalString cfg.privateNetwork '' # Cleanup from last time. - ip netns del ${ns} 2> /dev/null || true - ip link del ${ifaceHost} 2> /dev/null || true - ip link del ${ifaceCont} 2> /dev/null || true + ifaceHost=c-$INSTANCE + ifaceCont=ctmp-$INSTANCE + ns=net-$INSTANCE + ip netns del $ns 2> /dev/null || true + ip link del $ifaceHost 2> /dev/null || true + ip link del $ifaceCont 2> /dev/null || true - # Create a pair of virtual ethernet devices. On the host, - # we get ‘c-" >&2 + echo "Usage: $0 create [--config ]" >&2 + echo " $0 update " >&2 + echo " $0 destroy " >&2 + echo " $0 login " >&2 echo " $0 root-shell " >&2 + echo " $0 set-root-password " >&2 } -args="`getopt --options '' -l help -- "$@"`" +args="`getopt --options '' -l help -l config: -- "$@"`" eval "set -- $args" +extraConfigFile= while [ $# -gt 0 ]; do case "$1" in (--help) usage; exit 0;; + (--config) shift; extraConfigFile=$1;; (--) shift; break;; (*) break;; esac @@ -28,26 +34,104 @@ getContainerRoot() { fi } -if [ $action = login ]; then +container="$1" +if [ -z "$container" ]; then usage; exit 1; fi +shift - container="$1" - if [ -z "$container" ]; then usage; exit 1; fi - shift +if [ $action = create ]; then + + confFile="/etc/containers/$container.conf" + root="/var/lib/containers/$container" + + if [ -e "$confFile" -o -e "$root/nix" ]; then + echo "$0: container ‘$container’ already exists" >&2 + exit 1 + fi + + profileDir="/nix/var/nix/profiles/per-container/$container" + mkdir -m 0755 -p "$root/etc/nixos" "$profileDir" + + config=" +{ config, pkgs, ... }: + +with pkgs.lib; + +{ boot.isContainer = true; + security.initialRootPassword = mkDefault \"!\"; + networking.hostName = mkDefault \"$container\"; + networking.useDHCP = false; + imports = [ $extraConfigFile ]; +}" + configFile="$root/etc/nixos/configuration.nix" + echo "$config" > "$configFile" + + nix-env -p "$profileDir/system" -I "nixos-config=$configFile" -f '' --set -A system + + # Allocate a new /8 network in the 10.233.* range. + network="$(sed -e 's/.*_ADDRESS=10\.233\.\(.*\)\..*/\1/; t; d' /etc/containers/*.conf | sort -n | tail -n1)" + if [ -z "$network" ]; then network=0; else : $((network++)); fi + + hostAddress="10.233.$network.1" + localAddress="10.233.$network.2" + echo "host IP is $hostAddress, container IP is $localAddress" >&2 + + cat > "$confFile" <&2 + systemctl start "container@$container.service" + +elif [ $action = update ]; then getContainerRoot + configFile="$root/etc/nixos/configuration.nix" + profileDir="/nix/var/nix/profiles/per-container/$container" + + nix-env -p "$profileDir/system" -I "nixos-config=$configFile" -f '' --set -A system + + echo "reloading container@$container.service..." >&2 + systemctl reload "container@$container.service" + +elif [ $action = destroy ]; then + + getContainerRoot + + confFile="/etc/containers/$container.conf" + if [ ! -w "$confFile" ]; then + echo "$0: cannot destroy declarative container (remove it from your configuration.nix instead)" + exit 1 + fi + + if systemctl show "container@$container.service" | grep -q ActiveState=active; then + echo "stopping container@$container.service..." >&2 + systemctl stop "container@$container.service" + fi + + rm -f "$confFile" + +elif [ $action = login ]; then + + getContainerRoot exec @socat@/bin/socat "unix:$root/var/lib/login.socket" -,echo=0,raw elif [ $action = root-shell ]; then - container="$1" - if [ -z "$container" ]; then usage; exit 1; fi - shift - getContainerRoot - exec @socat@/bin/socat "unix:$root/var/lib/root-shell.socket" - +elif [ $action = set-root-password ]; then + + password="$1" + if [ -z "$password" ]; then usage; exit 1; fi + + # FIXME: not very secure. + getContainerRoot + (echo "passwd"; echo "$password"; echo "$password") | @socat@/bin/socat "unix:$root/var/lib/root-shell.socket" - + else echo "$0: unknown action ‘$action’" >&2 exit 1 From 29c469b88db31d56acf02478fffea14f15372b1f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Mar 2014 15:09:38 +0100 Subject: [PATCH 16/32] Allow dashes in container names --- nixos/modules/virtualisation/containers.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 573a3d1c5848..7e45d9f8b22f 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -143,13 +143,13 @@ in config = { systemd.services."container@" = - { description = "Container '%I'"; + { description = "Container '%i'"; - unitConfig.RequiresMountsFor = [ "/var/lib/containers/%I" ]; + unitConfig.RequiresMountsFor = [ "/var/lib/containers/%i" ]; path = [ pkgs.iproute ]; - environment.INSTANCE = "%I"; + environment.INSTANCE = "%i"; script = '' From 6010b0e8868b5a8058302a7b7839cddb21272043 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 20 Mar 2014 15:09:53 +0100 Subject: [PATCH 17/32] nixos-container: NixOps helper functions --- .../modules/virtualisation/nixos-container.sh | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/nixos/modules/virtualisation/nixos-container.sh b/nixos/modules/virtualisation/nixos-container.sh index c72b6fb92814..a3fd9db56d82 100644 --- a/nixos/modules/virtualisation/nixos-container.sh +++ b/nixos/modules/virtualisation/nixos-container.sh @@ -1,21 +1,25 @@ #! @bash@/bin/sh -e usage() { - echo "Usage: $0 create [--config ]" >&2 + echo "Usage: $0 list" >&2 + echo " $0 create [--config ] [--ensure-unique-name]" >&2 echo " $0 update " >&2 echo " $0 destroy " >&2 echo " $0 login " >&2 echo " $0 root-shell " >&2 echo " $0 set-root-password " >&2 + echo " $0 show-ip " >&2 } -args="`getopt --options '' -l help -l config: -- "$@"`" +args="`getopt --options '' -l help -l config: -l ensure-unique-name -- "$@"`" eval "set -- $args" extraConfigFile= +ensureUniqueName= while [ $# -gt 0 ]; do case "$1" in (--help) usage; exit 0;; (--config) shift; extraConfigFile=$1;; + (--ensure-unique-name) ensureUniqueName=1;; (--) shift; break;; (*) break;; esac @@ -34,12 +38,28 @@ getContainerRoot() { fi } +if [ $action = list ]; then + for i in $(cd /etc/containers && echo *.conf); do + echo "$(basename "$i" .conf)" + done + exit 0 +fi + container="$1" if [ -z "$container" ]; then usage; exit 1; fi shift if [ $action = create ]; then + if [ -n "$ensureUniqueName" ]; then + # FIXME: race + nr=0 + while [ -e "/etc/containers/$container-$nr.conf" -o -e "/var/lib/containers/$container-$nr" ]; do + : $((nr++)) + done + container="$container-$nr" + fi + confFile="/etc/containers/$container.conf" root="/var/lib/containers/$container" @@ -61,13 +81,19 @@ with pkgs.lib; networking.hostName = mkDefault \"$container\"; networking.useDHCP = false; imports = [ $extraConfigFile ]; + services.openssh.enable = true; + services.openssh.extraConfig = + '' + UseDNS no + ''; + users.extraUsers.root.openssh.authorizedKeys.keys = [ \"ssh-dss AAAAB3NzaC1kc3MAAACBAOo3foMFsYvc+LEVVTAeXpaxdOFG6O2NE9coxZYN6UtwE477GwkvZ4uKymAekq3TB8I6dDg4QFfE27fIip/rQHJ/Rus+KsxwnTbwPzE0WcZVpkKQsepsoqLkfwMpiPfn5/oxcnJsimwRY/E95aJmmOHdGaYWrc0t4ARa+6teUgdFAAAAFQCSQq2Wil0/X4hDypGGUKlKvYyaWQAAAIAy/0fSDnz1tZOQBGq7q78y406HfWghErrVlrW9g+foJQG5pgXXcdJs9JCIrlaKivUKITDsYnQaCjrZaK8eHnc4ksbkSLfDOxFnR5814ulCftrgEDOv9K1UU3pYketjFMvQCA2U48lR6jG/99CPNXPH55QEFs8H97cIsdLQw9wM4gAAAIEAmzWZlXLzIf3eiHQggXqvw3+C19QvxQITcYHYVTx/XYqZi1VZ/fkY8bNmdcJsWFyOHgEhpEca+xM/SNvH/14rXDmt0wtclLEx/4GVLi59hQCnnKqv7HzJg8RF4v6XTiROBAEEdb4TaFuFn+JCvqPzilTzXTexvZKJECOvfYcY+10= eelco.dolstra@logicblox.com\" ]; }" configFile="$root/etc/nixos/configuration.nix" echo "$config" > "$configFile" nix-env -p "$profileDir/system" -I "nixos-config=$configFile" -f '' --set -A system - # Allocate a new /8 network in the 10.233.* range. + # Allocate a new /8 network in the 10.233.* range. FIXME: race network="$(sed -e 's/.*_ADDRESS=10\.233\.\(.*\)\..*/\1/; t; d' /etc/containers/*.conf | sort -n | tail -n1)" if [ -z "$network" ]; then network=0; else : $((network++)); fi @@ -84,6 +110,11 @@ EOF echo "starting container@$container.service..." >&2 systemctl start "container@$container.service" + # Print generated container name on stdout. + if [ -n "$ensureUniqueName" ]; then + echo "$container" + fi + elif [ $action = update ]; then getContainerRoot @@ -101,7 +132,7 @@ elif [ $action = destroy ]; then getContainerRoot confFile="/etc/containers/$container.conf" - if [ ! -w "$confFile" ]; then + if [ -e "$confFile" -a ! -w "$confFile" ]; then echo "$0: cannot destroy declarative container (remove it from your configuration.nix instead)" exit 1 fi @@ -112,6 +143,7 @@ elif [ $action = destroy ]; then fi rm -f "$confFile" + rm -rf "$root" elif [ $action = login ]; then @@ -132,6 +164,12 @@ elif [ $action = set-root-password ]; then getContainerRoot (echo "passwd"; echo "$password"; echo "$password") | @socat@/bin/socat "unix:$root/var/lib/root-shell.socket" - +elif [ $action = show-ip ]; then + + getContainerRoot + . "/etc/containers/$container.conf" + echo "$LOCAL_ADDRESS" + else echo "$0: unknown action ‘$action’" >&2 exit 1 From 07adfae5519ae12ce1a06d2b57f027c97288d0fb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 24 Mar 2014 09:26:42 +0100 Subject: [PATCH 18/32] Remove hard-coded SSH key --- nixos/modules/virtualisation/nixos-container.sh | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/nixos/modules/virtualisation/nixos-container.sh b/nixos/modules/virtualisation/nixos-container.sh index a3fd9db56d82..47abf96072f3 100644 --- a/nixos/modules/virtualisation/nixos-container.sh +++ b/nixos/modules/virtualisation/nixos-container.sh @@ -2,7 +2,7 @@ usage() { echo "Usage: $0 list" >&2 - echo " $0 create [--config ] [--ensure-unique-name]" >&2 + echo " $0 create [--config ] [--ensure-unique-name]" >&2 echo " $0 update " >&2 echo " $0 destroy " >&2 echo " $0 login " >&2 @@ -13,12 +13,12 @@ usage() { args="`getopt --options '' -l help -l config: -l ensure-unique-name -- "$@"`" eval "set -- $args" -extraConfigFile= +extraConfig= ensureUniqueName= while [ $# -gt 0 ]; do case "$1" in (--help) usage; exit 0;; - (--config) shift; extraConfigFile=$1;; + (--config) shift; extraConfig=$1;; (--ensure-unique-name) ensureUniqueName=1;; (--) shift; break;; (*) break;; @@ -80,13 +80,8 @@ with pkgs.lib; security.initialRootPassword = mkDefault \"!\"; networking.hostName = mkDefault \"$container\"; networking.useDHCP = false; - imports = [ $extraConfigFile ]; - services.openssh.enable = true; - services.openssh.extraConfig = - '' - UseDNS no - ''; - users.extraUsers.root.openssh.authorizedKeys.keys = [ \"ssh-dss AAAAB3NzaC1kc3MAAACBAOo3foMFsYvc+LEVVTAeXpaxdOFG6O2NE9coxZYN6UtwE477GwkvZ4uKymAekq3TB8I6dDg4QFfE27fIip/rQHJ/Rus+KsxwnTbwPzE0WcZVpkKQsepsoqLkfwMpiPfn5/oxcnJsimwRY/E95aJmmOHdGaYWrc0t4ARa+6teUgdFAAAAFQCSQq2Wil0/X4hDypGGUKlKvYyaWQAAAIAy/0fSDnz1tZOQBGq7q78y406HfWghErrVlrW9g+foJQG5pgXXcdJs9JCIrlaKivUKITDsYnQaCjrZaK8eHnc4ksbkSLfDOxFnR5814ulCftrgEDOv9K1UU3pYketjFMvQCA2U48lR6jG/99CPNXPH55QEFs8H97cIsdLQw9wM4gAAAIEAmzWZlXLzIf3eiHQggXqvw3+C19QvxQITcYHYVTx/XYqZi1VZ/fkY8bNmdcJsWFyOHgEhpEca+xM/SNvH/14rXDmt0wtclLEx/4GVLi59hQCnnKqv7HzJg8RF4v6XTiROBAEEdb4TaFuFn+JCvqPzilTzXTexvZKJECOvfYcY+10= eelco.dolstra@logicblox.com\" ]; + imports = [ ]; + $extraConfig }" configFile="$root/etc/nixos/configuration.nix" echo "$config" > "$configFile" From 7ebd856a3801d35872c60b112fce51132b45aff5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 31 Mar 2014 19:16:52 +0200 Subject: [PATCH 19/32] Provide nixos-container unconditionally --- nixos/modules/virtualisation/containers.nix | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 7e45d9f8b22f..16df108c21ec 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -267,9 +267,7 @@ in ${cfg.localAddress} ${name}.containers '') config.containers); - environment.systemPackages = optional (config.containers != {}) nixos-container; - - system.build.foo = nixos-container; + environment.systemPackages = [ nixos-container ]; }; } From bdb658d033df47c97a0cdbe9628fdcf8c086e149 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 31 Mar 2014 19:17:17 +0200 Subject: [PATCH 20/32] nixpkgs-lint: Tweak --- maintainers/scripts/nixpkgs-lint.pl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/maintainers/scripts/nixpkgs-lint.pl b/maintainers/scripts/nixpkgs-lint.pl index d74f5c740f58..7e9ff91ebe06 100755 --- a/maintainers/scripts/nixpkgs-lint.pl +++ b/maintainers/scripts/nixpkgs-lint.pl @@ -31,8 +31,7 @@ GetOptions("package|p=s" => \$filter, "maintainer|m=s" => \$maintainer, "file|f=s" => \$path, "help" => sub { showHelp() } - ) - or die("syntax: $0 ...\n"); + ) or exit 1; # Evaluate Nixpkgs into an XML representation. my $xml = `nix-env -f '$path' -qa '$filter' --xml --meta --drv-path`; From 6da72a4456ea7cc12219c962739f279e5060fd66 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 31 Mar 2014 19:21:36 +0200 Subject: [PATCH 21/32] nixos-container: Rewrite in Perl Also fix race condition when multiple containers are created simultaneously (as NixOps tends to do). --- nixos/modules/virtualisation/containers.nix | 5 +- .../modules/virtualisation/nixos-container.pl | 198 ++++++++++++++++++ .../modules/virtualisation/nixos-container.sh | 171 --------------- 3 files changed, 201 insertions(+), 173 deletions(-) create mode 100644 nixos/modules/virtualisation/nixos-container.pl delete mode 100644 nixos/modules/virtualisation/nixos-container.sh diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 16df108c21ec..9964cd431cd7 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -18,8 +18,9 @@ let name = "nixos-container"; dir = "bin"; isExecutable = true; - src = ./nixos-container.sh; - inherit (pkgs) bash socat; + src = ./nixos-container.pl; + perl = "${pkgs.perl}/bin/perl -I${pkgs.perlPackages.FileSlurp}/lib/perl5/site_perl"; + inherit (pkgs) socat; }; in diff --git a/nixos/modules/virtualisation/nixos-container.pl b/nixos/modules/virtualisation/nixos-container.pl new file mode 100644 index 000000000000..dfc856e8b667 --- /dev/null +++ b/nixos/modules/virtualisation/nixos-container.pl @@ -0,0 +1,198 @@ +#! @perl@ + +use strict; +use File::Path; +use File::Slurp; +use Fcntl ':flock'; +use Getopt::Long qw(:config gnu_getopt); + +my $socat = '@socat@/bin/socat'; + +# Parse the command line. + +sub showHelp { + print < [--config ] [--ensure-unique-name] + nixos-container destroy + nixos-container start + nixos-container stop + nixos-container login + nixos-container root-shell + nixos-container set-root-password + nixos-container show-ip +EOF + exit 0; +} + +my $ensureUniqueName = 0; +my $extraConfig = ""; + +GetOptions( + "help" => sub { showHelp() }, + "ensure-unique-name" => \$ensureUniqueName, + "config=s" => \$extraConfig + ) or exit 1; + +my $action = $ARGV[0] or die "$0: no action specified\n"; + + +# Execute the selected action. + +mkpath("/etc/containers", 0, 0755); +mkpath("/var/lib/containers", 0, 0700); + +if ($action eq "list") { + foreach my $confFile (glob "/etc/containers/*.conf") { + $confFile =~ /\/([^\/]+).conf$/ or next; + print "$1\n"; + } + exit 0; +} + +my $containerName = $ARGV[1] or die "$0: no container name specified\n"; +$containerName =~ /^[a-zA-Z0-9\-]+$/ or die "$0: invalid container name\n"; + +if ($action eq "create") { + # Acquire an exclusive lock to prevent races with other + # invocations of ‘nixos-container create’. + my $lockFN = "/run/lock/nixos-container"; + open(my $lock, '>>', $lockFN) or die "$0: opening $lockFN: $!"; + flock($lock, LOCK_EX) or die "$0: could not lock $lockFN: $!"; + + my $confFile = "/etc/containers/$containerName.conf"; + my $root = "/var/lib/containers/$containerName"; + + # Maybe generate a unique name. + if ($ensureUniqueName) { + my $base = $containerName; + for (my $nr = 0; ; $nr++) { + $containerName = "$base-$nr"; + $confFile = "/etc/containers/$containerName.conf"; + $root = "/var/lib/containers/$containerName"; + last unless -e $confFile || -e $root; + } + } + + die "$0: container ‘$containerName’ already exists\n" if -e $confFile; + + # Get an unused IP address. + my %usedIPs; + foreach my $confFile2 (glob "/etc/containers/*.conf") { + my $s = read_file($confFile2) or die; + $usedIPs{$1} = 1 if $s =~ /^HOST_ADDRESS=([0-9\.]+)$/m; + $usedIPs{$1} = 1 if $s =~ /^LOCAL_ADDRESS=([0-9\.]+)$/m; + } + + my ($ipPrefix, $hostAddress, $localAddress); + for (my $nr = 1; $nr < 255; $nr++) { + $ipPrefix = "10.233.$nr"; + $hostAddress = "$ipPrefix.1"; + $localAddress = "$ipPrefix.2"; + last unless $usedIPs{$hostAddress} || $usedIPs{$localAddress}; + $ipPrefix = undef; + } + + die "$0: out of IP addresses\n" unless defined $ipPrefix; + + my @conf; + push @conf, "PRIVATE_NETWORK=1\n"; + push @conf, "HOST_ADDRESS=$hostAddress\n"; + push @conf, "LOCAL_ADDRESS=$localAddress\n"; + write_file($confFile, \@conf); + + close($lock); + + print STDERR "host IP is $hostAddress, container IP is $localAddress\n"; + + mkpath("$root/etc/nixos", 0, 0755); + + my $nixosConfig = < ]; + $extraConfig +} +EOF + my $nixosConfigFile = "$root/etc/nixos/configuration.nix"; + write_file($nixosConfigFile, $nixosConfig); + + # The per-container directory is restricted to prevent users on + # the host from messing with guest users who happen to have the + # same uid. + my $profileDir = "/nix/var/nix/profiles/per-container"; + mkpath($profileDir, 0, 0700); + $profileDir = "$profileDir/$containerName"; + mkpath($profileDir, 0, 0755); + + system("nix-env", "-p", "$profileDir/system", + "-I", "nixos-config=$nixosConfigFile", "-f", "", + "--set", "-A", "system") == 0 + or die "$0: failed to build initial container configuration\n"; + + print "$containerName\n" if $ensureUniqueName; + exit 0; +} + +my $confFile = "/etc/containers/$containerName.conf"; +die "$0: container ‘$containerName’ does not exist\n" if !-e $confFile; + +sub stopContainer { + system("systemctl", "stop", "container\@$containerName") == 0 + or die "$0: failed to stop container\n"; +} + +if ($action eq "destroy") { + my $root = "/var/lib/containers/$containerName"; + my $profileDir = "/nix/var/nix/profiles/per-container/$containerName"; + + my $status = `systemctl show 'container\@$containerName'`; + stopContainer if $status =~ /ActiveState=active/; + + rmtree($profileDir) if -e $profileDir; + rmtree($root) if -e $root; + unlink($confFile) or die; +} + +elsif ($action eq "start") { + system("systemctl", "start", "container\@$containerName") == 0 + or die "$0: failed to start container\n"; +} + +elsif ($action eq "stop") { + stopContainer; +} + +elsif ($action eq "login") { + exec($socat, "unix:/var/lib/containers/$containerName/var/lib/login.socket", "-,echo=0,raw"); +} + +elsif ($action eq "root-shell") { + exec($socat, "unix:/var/lib/containers/$containerName/var/lib/root-shell.socket", "-"); +} + +elsif ($action eq "set-root-password") { + # FIXME: don't get password from the command line. + my $password = $ARGV[2] or die "$0: no password given\n"; + open(SOCAT, "|-", $socat, "unix:/var/lib/containers/$containerName/var/lib/root-shell.socket", "-"); + print SOCAT "passwd\n"; + print SOCAT "$password\n"; + print SOCAT "$password\n"; + close(SOCAT); +} + +elsif ($action eq "show-ip") { + my $s = read_file($confFile) or die; + $s =~ /^LOCAL_ADDRESS=([0-9\.]+)$/m or die "$0: cannot get IP address\n"; + print "$1\n"; +} + +else { + die "$0: unknown action ‘$action’\n"; +} diff --git a/nixos/modules/virtualisation/nixos-container.sh b/nixos/modules/virtualisation/nixos-container.sh deleted file mode 100644 index 47abf96072f3..000000000000 --- a/nixos/modules/virtualisation/nixos-container.sh +++ /dev/null @@ -1,171 +0,0 @@ -#! @bash@/bin/sh -e - -usage() { - echo "Usage: $0 list" >&2 - echo " $0 create [--config ] [--ensure-unique-name]" >&2 - echo " $0 update " >&2 - echo " $0 destroy " >&2 - echo " $0 login " >&2 - echo " $0 root-shell " >&2 - echo " $0 set-root-password " >&2 - echo " $0 show-ip " >&2 -} - -args="`getopt --options '' -l help -l config: -l ensure-unique-name -- "$@"`" -eval "set -- $args" -extraConfig= -ensureUniqueName= -while [ $# -gt 0 ]; do - case "$1" in - (--help) usage; exit 0;; - (--config) shift; extraConfig=$1;; - (--ensure-unique-name) ensureUniqueName=1;; - (--) shift; break;; - (*) break;; - esac - shift -done - -action="$1" -if [ -z "$action" ]; then usage; exit 1; fi -shift - -getContainerRoot() { - root="/var/lib/containers/$container" - if ! [ -d "$root" ]; then - echo "$0: container ‘$container’ does not exist" >&2 - exit 1 - fi -} - -if [ $action = list ]; then - for i in $(cd /etc/containers && echo *.conf); do - echo "$(basename "$i" .conf)" - done - exit 0 -fi - -container="$1" -if [ -z "$container" ]; then usage; exit 1; fi -shift - -if [ $action = create ]; then - - if [ -n "$ensureUniqueName" ]; then - # FIXME: race - nr=0 - while [ -e "/etc/containers/$container-$nr.conf" -o -e "/var/lib/containers/$container-$nr" ]; do - : $((nr++)) - done - container="$container-$nr" - fi - - confFile="/etc/containers/$container.conf" - root="/var/lib/containers/$container" - - if [ -e "$confFile" -o -e "$root/nix" ]; then - echo "$0: container ‘$container’ already exists" >&2 - exit 1 - fi - - profileDir="/nix/var/nix/profiles/per-container/$container" - mkdir -m 0755 -p "$root/etc/nixos" "$profileDir" - - config=" -{ config, pkgs, ... }: - -with pkgs.lib; - -{ boot.isContainer = true; - security.initialRootPassword = mkDefault \"!\"; - networking.hostName = mkDefault \"$container\"; - networking.useDHCP = false; - imports = [ ]; - $extraConfig -}" - configFile="$root/etc/nixos/configuration.nix" - echo "$config" > "$configFile" - - nix-env -p "$profileDir/system" -I "nixos-config=$configFile" -f '' --set -A system - - # Allocate a new /8 network in the 10.233.* range. FIXME: race - network="$(sed -e 's/.*_ADDRESS=10\.233\.\(.*\)\..*/\1/; t; d' /etc/containers/*.conf | sort -n | tail -n1)" - if [ -z "$network" ]; then network=0; else : $((network++)); fi - - hostAddress="10.233.$network.1" - localAddress="10.233.$network.2" - echo "host IP is $hostAddress, container IP is $localAddress" >&2 - - cat > "$confFile" <&2 - systemctl start "container@$container.service" - - # Print generated container name on stdout. - if [ -n "$ensureUniqueName" ]; then - echo "$container" - fi - -elif [ $action = update ]; then - - getContainerRoot - - configFile="$root/etc/nixos/configuration.nix" - profileDir="/nix/var/nix/profiles/per-container/$container" - - nix-env -p "$profileDir/system" -I "nixos-config=$configFile" -f '' --set -A system - - echo "reloading container@$container.service..." >&2 - systemctl reload "container@$container.service" - -elif [ $action = destroy ]; then - - getContainerRoot - - confFile="/etc/containers/$container.conf" - if [ -e "$confFile" -a ! -w "$confFile" ]; then - echo "$0: cannot destroy declarative container (remove it from your configuration.nix instead)" - exit 1 - fi - - if systemctl show "container@$container.service" | grep -q ActiveState=active; then - echo "stopping container@$container.service..." >&2 - systemctl stop "container@$container.service" - fi - - rm -f "$confFile" - rm -rf "$root" - -elif [ $action = login ]; then - - getContainerRoot - exec @socat@/bin/socat "unix:$root/var/lib/login.socket" -,echo=0,raw - -elif [ $action = root-shell ]; then - - getContainerRoot - exec @socat@/bin/socat "unix:$root/var/lib/root-shell.socket" - - -elif [ $action = set-root-password ]; then - - password="$1" - if [ -z "$password" ]; then usage; exit 1; fi - - # FIXME: not very secure. - getContainerRoot - (echo "passwd"; echo "$password"; echo "$password") | @socat@/bin/socat "unix:$root/var/lib/root-shell.socket" - - -elif [ $action = show-ip ]; then - - getContainerRoot - . "/etc/containers/$container.conf" - echo "$LOCAL_ADDRESS" - -else - echo "$0: unknown action ‘$action’" >&2 - exit 1 -fi From fee81c37398f547faddfff7ee10d945bb4513f09 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Apr 2014 15:42:18 +0200 Subject: [PATCH 22/32] Always enable container logins --- nixos/modules/module-list.nix | 1 + nixos/modules/virtualisation/container-login.nix | 6 ++++-- nixos/modules/virtualisation/containers.nix | 1 - nixos/modules/virtualisation/nixos-container.pl | 1 - 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 1c2fca1f88b5..a7bf69cfc792 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -294,6 +294,7 @@ ./tasks/scsi-link-power-management.nix ./tasks/swraid.nix ./testing/service-runner.nix + ./virtualisation/container-login.nix ./virtualisation/containers.nix ./virtualisation/libvirtd.nix #./virtualisation/nova.nix diff --git a/nixos/modules/virtualisation/container-login.nix b/nixos/modules/virtualisation/container-login.nix index 09eaf90ae658..fb5e333b32ab 100644 --- a/nixos/modules/virtualisation/container-login.nix +++ b/nixos/modules/virtualisation/container-login.nix @@ -1,8 +1,10 @@ -{ config, pkgs, ... }: +{ config, pkgs, lib, ... }: + +with lib; { - config = { + config = mkIf config.boot.isContainer { # Provide a login prompt on /var/lib/login.socket. On the host, # you can connect to it by running ‘socat diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 9964cd431cd7..6c8a6f876c8d 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -105,7 +105,6 @@ in security.initialRootPassword = mkDefault "!"; networking.hostName = mkDefault name; networking.useDHCP = false; - imports = [ ./container-login.nix ]; }; in [ extraConfig config.config ]; prefix = [ "containers" name ]; diff --git a/nixos/modules/virtualisation/nixos-container.pl b/nixos/modules/virtualisation/nixos-container.pl index dfc856e8b667..b08ed0766583 100644 --- a/nixos/modules/virtualisation/nixos-container.pl +++ b/nixos/modules/virtualisation/nixos-container.pl @@ -116,7 +116,6 @@ with pkgs.lib; security.initialRootPassword = mkDefault "!"; networking.hostName = mkDefault "$containerName"; networking.useDHCP = false; - imports = [ ]; $extraConfig } EOF From 269926df0d1e0b03b9697f84e5b9720fa966b293 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Apr 2014 15:43:27 +0200 Subject: [PATCH 23/32] container-login.nix -> container-config.nix --- nixos/modules/module-list.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index a7bf69cfc792..e6694b8c99b8 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -294,7 +294,7 @@ ./tasks/scsi-link-power-management.nix ./tasks/swraid.nix ./testing/service-runner.nix - ./virtualisation/container-login.nix + ./virtualisation/container-config.nix ./virtualisation/containers.nix ./virtualisation/libvirtd.nix #./virtualisation/nova.nix From 1ad9a654be1120a6844c9eb7520188e874178ebe Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Apr 2014 16:02:53 +0200 Subject: [PATCH 24/32] Make starting a container synchronous So now "systemctl start container@foo" will only return after the container has reached multi-user.target. --- ...ntainer-login.nix => container-config.nix} | 14 +++++++++++ nixos/modules/virtualisation/containers.nix | 25 ++++++++++++++++--- 2 files changed, 36 insertions(+), 3 deletions(-) rename nixos/modules/virtualisation/{container-login.nix => container-config.nix} (79%) diff --git a/nixos/modules/virtualisation/container-login.nix b/nixos/modules/virtualisation/container-config.nix similarity index 79% rename from nixos/modules/virtualisation/container-login.nix rename to nixos/modules/virtualisation/container-config.nix index fb5e333b32ab..3d107899e4fe 100644 --- a/nixos/modules/virtualisation/container-login.nix +++ b/nixos/modules/virtualisation/container-config.nix @@ -53,6 +53,20 @@ with lib; }; }; + systemd.services.container-startup-done = + { description = "Container Startup Notification"; + wantedBy = [ "multi-user.target" ]; + after = [ "multi-user.target" ]; + script = + '' + if [ -p /var/lib/startup-done ]; then + echo done > /var/lib/startup-done + fi + ''; + serviceConfig.Type = "oneshot"; + serviceConfig.RemainAfterExit = true; + }; + }; } diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 6c8a6f876c8d..9be79cec3695 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -150,11 +150,21 @@ in path = [ pkgs.iproute ]; environment.INSTANCE = "%i"; + environment.root = "/var/lib/containers/%i"; + + preStart = + '' + mkdir -p -m 0755 $root/var/lib + + # Create a named pipe to get a signal when the container + # has finished booting. + rm -f $root/var/lib/startup-done + mkfifo $root/var/lib/startup-done + ''; script = '' - root="/var/lib/containers/$INSTANCE" - mkdir -p -m 0755 "$root/etc" + mkdir -p -m 0755 "$root/etc" "$root/var/lib" if ! [ -e "$root/etc/os-release" ]; then touch "$root/etc/os-release" fi @@ -209,6 +219,13 @@ in "$SYSTEM_PATH/init" ''; + postStart = + '' + # This blocks until the container-startup-done service + # writes something to this pipe. + read x < $root/var/lib/startup-done + ''; + preStop = '' pid="$(cat /sys/fs/cgroup/systemd/machine/$INSTANCE.nspawn/system/tasks 2> /dev/null)" @@ -238,8 +255,10 @@ in . "/etc/containers/$INSTANCE.conf" fi echo $SYSTEM_PATH/bin/switch-to-configuration test | \ - ${pkgs.socat}/bin/socat unix:/var/lib/containers/$INSTANCE/var/lib/root-shell.socket - + ${pkgs.socat}/bin/socat unix:$root/var/lib/root-shell.socket - ''; + + serviceConfig.SyslogIdentifier = "container %i"; }; # Generate a configuration file in /etc/containers for each From b0b3fa928ad572c67bc3545c2b160bd8aef79d7a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Apr 2014 16:35:11 +0200 Subject: [PATCH 25/32] Disable container support in containers Systemd-nspawn doesn't support nesting, so providing nixos-container inside a container doesn't make sense. --- nixos/modules/virtualisation/containers.nix | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 9be79cec3695..097dd3993eb5 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -140,7 +140,7 @@ in }; - config = { + config = mkIf (!config.boot.isContainer) { systemd.services."container@" = { description = "Container '%i'"; @@ -222,7 +222,8 @@ in postStart = '' # This blocks until the container-startup-done service - # writes something to this pipe. + # writes something to this pipe. FIXME: it also hangs + # until the start timeout expires if systemd-nspawn exits. read x < $root/var/lib/startup-done ''; From 1e4fa227fe434c2042a2c690f62a7072afa36e93 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Apr 2014 16:25:21 +0200 Subject: [PATCH 26/32] nixos-container: Don't destroy declarative containers --- nixos/modules/virtualisation/nixos-container.pl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nixos/modules/virtualisation/nixos-container.pl b/nixos/modules/virtualisation/nixos-container.pl index b08ed0766583..e42a3edd0243 100644 --- a/nixos/modules/virtualisation/nixos-container.pl +++ b/nixos/modules/virtualisation/nixos-container.pl @@ -1,6 +1,7 @@ #! @perl@ use strict; +use POSIX; use File::Path; use File::Slurp; use Fcntl ':flock'; @@ -148,6 +149,9 @@ sub stopContainer { } if ($action eq "destroy") { + die "$0: cannot destroy declarative container (remove it from your configuration.nix instead)\n" + unless POSIX::access($confFile, &POSIX::W_OK); + my $root = "/var/lib/containers/$containerName"; my $profileDir = "/nix/var/nix/profiles/per-container/$containerName"; From 819e7c9fbd57698ecbf3653067c497b5887cfd87 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 3 Apr 2014 16:26:03 +0200 Subject: [PATCH 27/32] Add a test for NixOS containers --- nixos/modules/installer/cd-dvd/channel.nix | 2 +- nixos/tests/containers.nix | 79 ++++++++++++++++++++++ nixos/tests/default.nix | 1 + 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 nixos/tests/containers.nix diff --git a/nixos/modules/installer/cd-dvd/channel.nix b/nixos/modules/installer/cd-dvd/channel.nix index 9aca5b89d258..74428f66dfa1 100644 --- a/nixos/modules/installer/cd-dvd/channel.nix +++ b/nixos/modules/installer/cd-dvd/channel.nix @@ -28,7 +28,7 @@ in { # Provide the NixOS/Nixpkgs sources in /etc/nixos. This is required # for nixos-install. - boot.postBootCommands = + boot.postBootCommands = mkAfter '' if ! [ -e /var/lib/nixos/did-channel-init ]; then echo "unpacking the NixOS/Nixpkgs sources..." diff --git a/nixos/tests/containers.nix b/nixos/tests/containers.nix new file mode 100644 index 000000000000..d72e80b71aff --- /dev/null +++ b/nixos/tests/containers.nix @@ -0,0 +1,79 @@ +# Test for NixOS' container support. + +{ pkgs, ... }: + +{ + + machine = + { config, pkgs, ... }: + { imports = [ ../modules/installer/cd-dvd/channel.nix ]; + virtualisation.writableStore = true; + virtualisation.memorySize = 768; + + containers.webserver = + { privateNetwork = true; + hostAddress = "10.231.136.1"; + localAddress = "10.231.136.2"; + config = + { services.httpd.enable = true; + services.httpd.adminAddr = "foo@example.org"; + }; + }; + + virtualisation.pathsInNixDB = [ pkgs.stdenv ]; + }; + + testScript = + '' + $machine->succeed("nixos-container list") =~ /webserver/; + + # Start the webserver container. + $machine->succeed("nixos-container start webserver"); + + # Since "start" returns after the container has reached + # multi-user.target, we should now be able to access it. + my $ip = $machine->succeed("nixos-container show-ip webserver"); + chomp $ip; + $machine->succeed("ping -c1 $ip"); + $machine->succeed("curl --fail http://$ip/ > /dev/null"); + + # Stop the container. + $machine->succeed("nixos-container stop webserver"); + $machine->fail("curl --fail --connect-timeout 2 http://$ip/ > /dev/null"); + + # Make sure we have a NixOS tree (required by ‘nixos-container create’). + $machine->succeed("nix-env -qa -A nixos.pkgs.hello >&2"); + + # Create some containers imperatively. + my $id1 = $machine->succeed("nixos-container create foo --ensure-unique-name"); + chomp $id1; + $machine->log("created container $id1"); + + my $id2 = $machine->succeed("nixos-container create foo --ensure-unique-name"); + chomp $id2; + $machine->log("created container $id2"); + + die if $id1 eq $id2; + + my $ip1 = $machine->succeed("nixos-container show-ip $id1"); + chomp $ip1; + my $ip2 = $machine->succeed("nixos-container show-ip $id2"); + chomp $ip2; + die if $ip1 eq $ip2; + + # Start one of them. + $machine->succeed("nixos-container start $id1"); + + # Execute commands via the root shell. + $machine->succeed("echo uname | nixos-container root-shell $id1") =~ /Linux/; + $machine->succeed("nixos-container set-root-password $id1 foobar"); + + # Destroy the containers. + $machine->succeed("nixos-container destroy $id1"); + $machine->succeed("nixos-container destroy $id2"); + + # Destroying a declarative container should fail. + $machine->fail("nixos-container destroy webserver"); + ''; + +} diff --git a/nixos/tests/default.nix b/nixos/tests/default.nix index 5b68862a2cd1..20deed2a2492 100644 --- a/nixos/tests/default.nix +++ b/nixos/tests/default.nix @@ -8,6 +8,7 @@ with import ../lib/testing.nix { inherit system minimal; }; { avahi = makeTest (import ./avahi.nix); bittorrent = makeTest (import ./bittorrent.nix); + containers = makeTest (import ./containers.nix); firefox = makeTest (import ./firefox.nix); firewall = makeTest (import ./firewall.nix); installer = makeTests (import ./installer.nix); From 3dca6b98cb6549ef7ba0195a6ac1bbc811f8db66 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Apr 2014 10:53:49 +0200 Subject: [PATCH 28/32] Fix permissions on /var/lib/startup-done --- nixos/modules/virtualisation/containers.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 097dd3993eb5..9d54ddb9948e 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -159,7 +159,7 @@ in # Create a named pipe to get a signal when the container # has finished booting. rm -f $root/var/lib/startup-done - mkfifo $root/var/lib/startup-done + mkfifo -m 0600 $root/var/lib/startup-done ''; script = From da4f180252e9a8f539a019569efc82000ffe5440 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Apr 2014 11:32:50 +0200 Subject: [PATCH 29/32] =?UTF-8?q?Bring=20back=20=E2=80=98nixos-container?= =?UTF-8?q?=20update=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../modules/virtualisation/nixos-container.pl | 71 +++++++++++++------ 1 file changed, 50 insertions(+), 21 deletions(-) diff --git a/nixos/modules/virtualisation/nixos-container.pl b/nixos/modules/virtualisation/nixos-container.pl index e42a3edd0243..b6919852b285 100644 --- a/nixos/modules/virtualisation/nixos-container.pl +++ b/nixos/modules/virtualisation/nixos-container.pl @@ -54,6 +54,25 @@ if ($action eq "list") { my $containerName = $ARGV[1] or die "$0: no container name specified\n"; $containerName =~ /^[a-zA-Z0-9\-]+$/ or die "$0: invalid container name\n"; +sub writeNixOSConfig { + my ($nixosConfigFile) = @_; + + my $nixosConfig = <", + "--set", "-A", "system") == 0 + or die "$0: failed to build container configuration\n"; + + if (isContainerRunning) { + print STDERR "reloading container...\n"; + system("systemctl", "reload", "container\@$containerName") == 0 + or die "$0: failed to reload container\n"; + } +} + elsif ($action eq "login") { - exec($socat, "unix:/var/lib/containers/$containerName/var/lib/login.socket", "-,echo=0,raw"); + exec($socat, "unix:$root/var/lib/login.socket", "-,echo=0,raw"); } elsif ($action eq "root-shell") { - exec($socat, "unix:/var/lib/containers/$containerName/var/lib/root-shell.socket", "-"); + exec($socat, "unix:$root/var/lib/root-shell.socket", "-"); } elsif ($action eq "set-root-password") { # FIXME: don't get password from the command line. my $password = $ARGV[2] or die "$0: no password given\n"; - open(SOCAT, "|-", $socat, "unix:/var/lib/containers/$containerName/var/lib/root-shell.socket", "-"); + open(SOCAT, "|-", $socat, "unix:$root/var/lib/root-shell.socket", "-"); print SOCAT "passwd\n"; print SOCAT "$password\n"; print SOCAT "$password\n"; From ac8c924c0931237461266c2780e744c63880180c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Apr 2014 13:12:34 +0200 Subject: [PATCH 30/32] =?UTF-8?q?nixos-container:=20Add=20=E2=80=98run?= =?UTF-8?q?=E2=80=99=20and=20=E2=80=98root-login=E2=80=99=20commands?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit And remove ‘root-shell’. --- .../virtualisation/container-config.nix | 51 +++++++++++++++---- nixos/modules/virtualisation/containers.nix | 2 +- .../modules/virtualisation/nixos-container.pl | 16 ++++-- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/nixos/modules/virtualisation/container-config.nix b/nixos/modules/virtualisation/container-config.nix index 3d107899e4fe..21e64c8c0957 100644 --- a/nixos/modules/virtualisation/container-config.nix +++ b/nixos/modules/virtualisation/container-config.nix @@ -30,29 +30,60 @@ with lib; }; }; - # Provide a non-interactive login root shell on - # /var/lib/root-shell.socket. On the host, you can connect to it - # by running ‘socat unix:/var/lib/root-shell.socket -’. - systemd.sockets.root-shell = - { description = "Root Shell Socket"; + # Also provide a root login prompt on /var/lib/root-login.socket + # that doesn't ask for a password. This socket can only be used by + # root on the host. + systemd.sockets.root-login = + { description = "Root Login Socket"; wantedBy = [ "sockets.target" ]; socketConfig = - { ListenStream = "/var/lib/root-shell.socket"; - SocketMode = "0600"; # only root can connect, obviously + { ListenStream = "/var/lib/root-login.socket"; + SocketMode = "0600"; Accept = true; }; }; - systemd.services."root-shell@" = - { description = "Root Shell %i"; + systemd.services."root-login@" = + { description = "Root Login %i"; + environment.TERM = "linux"; serviceConfig = { Type = "simple"; StandardInput = "socket"; - ExecStart = "${pkgs.bash}/bin/bash --login"; + ExecStart = "${pkgs.socat}/bin/socat -t0 - \"exec:${pkgs.shadow}/bin/login -f root,pty,setsid,setpgid,stderr,ctty\""; TimeoutStopSec = 1; # FIXME }; }; + # Provide a daemon on /var/lib/run-command.socket that reads a + # command from stdin and executes it. + systemd.sockets.run-command = + { description = "Run Command Socket"; + wantedBy = [ "sockets.target" ]; + socketConfig = + { ListenStream = "/var/lib/run-command.socket"; + SocketMode = "0600"; # only root can connect + Accept = true; + }; + }; + + systemd.services."run-command@" = + { description = "Run Command %i"; + environment.TERM = "linux"; + serviceConfig = + { Type = "simple"; + StandardInput = "socket"; + TimeoutStopSec = 1; # FIXME + }; + script = + '' + #! ${pkgs.stdenv.shell} -e + source /etc/bashrc + read c + eval "command=($c)" + exec "''${command[@]}" + ''; + }; + systemd.services.container-startup-done = { description = "Container Startup Notification"; wantedBy = [ "multi-user.target" ]; diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index 9d54ddb9948e..fbdd3f9034c6 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -256,7 +256,7 @@ in . "/etc/containers/$INSTANCE.conf" fi echo $SYSTEM_PATH/bin/switch-to-configuration test | \ - ${pkgs.socat}/bin/socat unix:$root/var/lib/root-shell.socket - + ${pkgs.socat}/bin/socat unix:$root/var/lib/run-command.socket - ''; serviceConfig.SyslogIdentifier = "container %i"; diff --git a/nixos/modules/virtualisation/nixos-container.pl b/nixos/modules/virtualisation/nixos-container.pl index b6919852b285..d7e8c7339b6d 100644 --- a/nixos/modules/virtualisation/nixos-container.pl +++ b/nixos/modules/virtualisation/nixos-container.pl @@ -19,7 +19,8 @@ Usage: nixos-container list nixos-container start nixos-container stop nixos-container login - nixos-container root-shell + nixos-container root-login + nixos-container run -- args... nixos-container set-root-password nixos-container show-ip EOF @@ -205,14 +206,21 @@ elsif ($action eq "login") { exec($socat, "unix:$root/var/lib/login.socket", "-,echo=0,raw"); } -elsif ($action eq "root-shell") { - exec($socat, "unix:$root/var/lib/root-shell.socket", "-"); +elsif ($action eq "root-login") { + exec($socat, "unix:$root/var/lib/root-login.socket", "-,echo=0,raw"); +} + +elsif ($action eq "run") { + shift @ARGV; shift @ARGV; + open(SOCAT, "|-", $socat, "unix:$root/var/lib/run-command.socket", "-"); + print SOCAT join(' ', map { "'$_'" } @ARGV), "\n"; + close(SOCAT); } elsif ($action eq "set-root-password") { # FIXME: don't get password from the command line. my $password = $ARGV[2] or die "$0: no password given\n"; - open(SOCAT, "|-", $socat, "unix:$root/var/lib/root-shell.socket", "-"); + open(SOCAT, "|-", $socat, "unix:$root/var/lib/run-command.socket", "-"); print SOCAT "passwd\n"; print SOCAT "$password\n"; print SOCAT "$password\n"; From a34bfbab4cac9d7abcab88a47694e1cc32111dba Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Apr 2014 14:23:38 +0200 Subject: [PATCH 31/32] Add option networking.nat.internalInterfaces This allows applying NAT to an interface, rather than an IP range. --- nixos/modules/services/networking/nat.nix | 55 +++++++++++++++++------ 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/nixos/modules/services/networking/nat.nix b/nixos/modules/services/networking/nat.nix index ce28f0188284..d684d8e31222 100644 --- a/nixos/modules/services/networking/nat.nix +++ b/nixos/modules/services/networking/nat.nix @@ -10,6 +10,8 @@ let cfg = config.networking.nat; + dest = if cfg.externalIP == null then "-j MASQUERADE" else "-j SNAT --to-source ${cfg.externalIP}"; + in { @@ -27,14 +29,27 @@ in ''; }; + networking.nat.internalInterfaces = mkOption { + type = types.listOf types.str; + default = []; + example = [ "eth0" ]; + description = + '' + The interfaces for which to perform NAT. Packets coming from + these interface and destined for the external interface will + be rewritten. + ''; + }; + networking.nat.internalIPs = mkOption { type = types.listOf types.str; - example = [ "192.168.1.0/24" ] ; + default = []; + example = [ "192.168.1.0/24" ]; description = '' The IP address ranges for which to perform NAT. Packets - coming from these networks and destined for the external - interface will be rewritten. + coming from these addresses (on any interface) and destined + for the external interface will be rewritten. ''; }; @@ -80,25 +95,37 @@ in preStart = '' + iptables -t nat -F PREROUTING iptables -t nat -F POSTROUTING iptables -t nat -X - '' - + (concatMapStrings (network: - '' - iptables -t nat -A POSTROUTING \ - -s ${network} -o ${cfg.externalInterface} \ - ${if cfg.externalIP == null - then "-j MASQUERADE" - else "-j SNAT --to-source ${cfg.externalIP}"} - '' - ) cfg.internalIPs) + - '' + + # We can't match on incoming interface in POSTROUTING, so + # mark packets coming from the external interfaces. + ${concatMapStrings (iface: '' + iptables -t nat -A PREROUTING \ + -i '${iface}' -j MARK --set-mark 1 + '') cfg.internalInterfaces} + + # NAT the marked packets. + ${optionalString (cfg.internalInterfaces != []) '' + iptables -t nat -A POSTROUTING -m mark --mark 1 \ + -o ${cfg.externalInterface} ${dest} + ''} + + # NAT packets coming from the internal IPs. + ${concatMapStrings (range: '' + iptables -t nat -A POSTROUTING \ + -s '${range}' -o ${cfg.externalInterface} ${dest}} + '') cfg.internalIPs} + echo 1 > /proc/sys/net/ipv4/ip_forward ''; postStop = '' + iptables -t nat -F PREROUTING iptables -t nat -F POSTROUTING + iptables -t nat -X ''; }; }; From 6a7a8a144fae43fae51232703bf742c8bcee8d67 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 10 Apr 2014 14:57:40 +0200 Subject: [PATCH 32/32] Document NixOS containers --- nixos/doc/manual/containers.xml | 242 ++++++++++++++++++++ nixos/doc/manual/manual.xml | 1 + nixos/modules/virtualisation/containers.nix | 2 + 3 files changed, 245 insertions(+) create mode 100644 nixos/doc/manual/containers.xml diff --git a/nixos/doc/manual/containers.xml b/nixos/doc/manual/containers.xml new file mode 100644 index 000000000000..b8f170fc614f --- /dev/null +++ b/nixos/doc/manual/containers.xml @@ -0,0 +1,242 @@ + + +Containers + +NixOS allows you to easily run other NixOS instances as +containers. Containers are a light-weight +approach to virtualisation that runs software in the container at the +same speed as in the host system. NixOS containers share the Nix store +of the host, making container creation very efficient. + +Currently, NixOS containers are not perfectly isolated +from the host system. This means that a user with root access to the +container can do things that affect the host. So you should not give +container root access to untrusted users. + +NixOS containers can be created in two ways: imperatively, using +the command nixos-container, and declaratively, by +specifying them in your configuration.nix. The +declarative approach implies that containers get upgraded along with +your host system when you run nixos-rebuild, which +is often not what you want. By contrast, in the imperative approach, +containers are configured and updated independently from the host +system. + + +
Imperative container management + +We’ll cover imperative container management using +nixos-container first. You create a container with +identifier foo as follows: + + +$ nixos-container create foo + + +This creates the container’s root directory in +/var/lib/containers/foo and a small configuration +file in /etc/containers/foo.conf. It also builds +the container’s initial system configuration and stores it in +/nix/var/nix/profiles/per-container/foo/system. You +can modify the initial configuration of the container on the command +line. For instance, to create a container that has +sshd running, with the given public key for +root: + + +$ nixos-container create foo --config 'services.openssh.enable = true; \ + users.extraUsers.root.openssh.authorizedKeys.keys = ["ssh-dss AAAAB3N…"];' + + + + +Creating a container does not start it. To start the container, +run: + + +$ nixos-container start foo + + +This command will return as soon as the container has booted and has +reached multi-user.target. On the host, the +container runs within a systemd unit called +container@container-name.service. +Thus, if something went wrong, you can get status info using +systemctl: + + +$ systemctl status container@foo + + + + +If the container has started succesfully, you can log in as +root using the root-login operation: + + +$ nixos-container root-login foo +[root@foo:~]# + + +Note that only root on the host can do this (since there is no +authentication). You can also get a regular login prompt using the +login operation, which is available to all users on +the host: + + +$ nixos-container login foo +foo login: alice +Password: *** + + +With nixos-container run, you can execute arbitrary +commands in the container: + + +$ nixos-container run foo -- uname -a +Linux foo 3.4.82 #1-NixOS SMP Thu Mar 20 14:44:05 UTC 2014 x86_64 GNU/Linux + + + + +There are several ways to change the configuration of the +container. First, on the host, you can edit +/var/lib/container/name/etc/nixos/configuration.nix, +and run + + +$ nixos-container update foo + + +This will build and activate the new configuration. You can also +specify a new configuration on the command line: + + +$ nixos-container update foo --config 'services.httpd.enable = true; \ + services.httpd.adminAddr = "foo@example.org";' + +$ curl http://$(nixos-container show-ip foo)/ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">… + + +However, note that this will overwrite the container’s +/etc/nixos/configuration.nix. + +Alternatively, you can change the configuration from within the +container itself by running nixos-rebuild switch +inside the container. Note that the container by default does not have +a copy of the NixOS channel, so you should run nix-channel +--update first. + +Containers can be stopped and started using +nixos-container stop and nixos-container +start, respectively, or by using +systemctl on the container’s service unit. To +destroy a container, including its file system, do + + +$ nixos-container destroy foo + + + + +
+ + +
Declarative container specification + +You can also specify containers and their configuration in the +host’s configuration.nix. For example, the +following specifies that there shall be a container named +database running PostgreSQL: + + +containers.database = + { config = + { config, pkgs, ... }: + { services.postgresql.enable = true; + services.postgresql.package = pkgs.postgresql92; + }; + }; + + +If you run nixos-rebuild switch, the container will +be built and started. If the container was already running, it will be +updated in place, without rebooting. + +By default, declarative containers share the network namespace +of the host, meaning that they can listen on (privileged) +ports. However, they cannot change the network configuration. You can +give a container its own network as follows: + + +containers.database = + { privateNetwork = true; + hostAddress = "192.168.100.10"; + localAddress = "192.168.100.11"; + }; + + +This gives the container a private virtual Ethernet interface with IP +address 192.168.100.11, which is hooked up to a +virtual Ethernet interface on the host with IP address +192.168.100.10. (See the next section for details +on container networking.) + +To disable the container, just remove it from +configuration.nix and run nixos-rebuild +switch. Note that this will not delete the root directory of +the container in /var/lib/containers. + +
+ + +
Networking + +When you create a container using nixos-container +create, it gets it own private IPv4 address in the range +10.233.0.0/16. You can get the container’s IPv4 +address as follows: + + +$ nixos-container show-ip foo +10.233.4.2 + +$ ping -c1 10.233.4.2 +64 bytes from 10.233.4.2: icmp_seq=1 ttl=64 time=0.106 ms + + + + +Networking is implemented using a pair of virtual Ethernet +devices. The network interface in the container is called +eth0, while the matching interface in the host is +called c-container-name +(e.g., c-foo). The container has its own network +namespace and the CAP_NET_ADMIN capability, so it +can perform arbitrary network configuration such as setting up +firewall rules, without affecting or having access to the host’s +network. + +By default, containers cannot talk to the outside network. If +you want that, you should set up Network Address Translation (NAT) +rules on the host to rewrite container traffic to use your external +IP address. This can be accomplished using the following configuration +on the host: + + +networking.nat.enable = true; +networking.nat.internalInterfaces = ["c-+"]; +networking.nat.externalInterface = "eth0"; + +where eth0 should be replaced with the desired +external interface. Note that c-+ is a wildcard +that matches all container interfaces. + +
+ + + + diff --git a/nixos/doc/manual/manual.xml b/nixos/doc/manual/manual.xml index f9775f4f0170..5753a8ff9e74 100644 --- a/nixos/doc/manual/manual.xml +++ b/nixos/doc/manual/manual.xml @@ -54,6 +54,7 @@ + diff --git a/nixos/modules/virtualisation/containers.nix b/nixos/modules/virtualisation/containers.nix index fbdd3f9034c6..c53bd7d3509d 100644 --- a/nixos/modules/virtualisation/containers.nix +++ b/nixos/modules/virtualisation/containers.nix @@ -281,6 +281,8 @@ in ''; }) config.containers; + # FIXME: auto-start containers. + # Generate /etc/hosts entries for the containers. networking.extraHosts = concatStrings (mapAttrsToList (name: cfg: optionalString (cfg.localAddress != null) ''