From 39ba325315fa87e309bd430d71f90e3c1a28d332 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Thu, 31 Oct 2013 03:38:04 -0700 Subject: [PATCH 01/46] Initial checkin --- default.nix | 16 +++++++++ node-packages-generated.nix | 55 +++++++++++++++++++++++++++++ src/bin/nixdocker.js | 69 +++++++++++++++++++++++++++++++++++++ src/docker.nix | 40 +++++++++++++++++++++ src/dockerfile.nix | 26 ++++++++++++++ src/package.json | 10 ++++++ 6 files changed, 216 insertions(+) create mode 100644 default.nix create mode 100644 node-packages-generated.nix create mode 100644 src/bin/nixdocker.js create mode 100644 src/docker.nix create mode 100644 src/dockerfile.nix create mode 100644 src/package.json diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..a81cca3 --- /dev/null +++ b/default.nix @@ -0,0 +1,16 @@ +let + pkgs = import {}; + nodePackages = pkgs.recurseIntoAttrs (import { + inherit pkgs; + inherit (pkgs) stdenv nodejs fetchurl; + neededNatives = [pkgs.python pkgs.utillinux]; + self = nodePackages; + generated = ./node-packages-generated.nix; + }); +in + nodePackages.buildNodePackage { + name = "nixdocker"; + src = [ { outPath = ./src; name = "nixdocker"; } ]; + deps = with nodePackages; [optimist]; + passthru.names = [ "nixdocker" ]; + } \ No newline at end of file diff --git a/node-packages-generated.nix b/node-packages-generated.nix new file mode 100644 index 0000000..f1b4369 --- /dev/null +++ b/node-packages-generated.nix @@ -0,0 +1,55 @@ +{ self, fetchurl, lib }: + +{ + full."minimist"."~0.0.1" = lib.makeOverridable self.buildNodePackage { + name = "minimist-0.0.5"; + src = [ + (fetchurl { + url = "http://registry.npmjs.org/minimist/-/minimist-0.0.5.tgz"; + sha1 = "d7aa327bcecf518f9106ac6b8f003fa3bcea8566"; + }) + ]; + buildInputs = + (self.nativeDeps."minimist"."~0.0.1" or []); + deps = [ + ]; + peerDependencies = [ + ]; + passthru.names = [ "minimist" ]; + }; + full."optimist"."0.6.0" = lib.makeOverridable self.buildNodePackage { + name = "optimist-0.6.0"; + src = [ + (fetchurl { + url = "http://registry.npmjs.org/optimist/-/optimist-0.6.0.tgz"; + sha1 = "69424826f3405f79f142e6fc3d9ae58d4dbb9200"; + }) + ]; + buildInputs = + (self.nativeDeps."optimist"."0.6.0" or []); + deps = [ + self.full."wordwrap"."~0.0.2" + self.full."minimist"."~0.0.1" + ]; + peerDependencies = [ + ]; + passthru.names = [ "optimist" ]; + }; + "optimist" = self.full."optimist"."0.6.0"; + full."wordwrap"."~0.0.2" = lib.makeOverridable self.buildNodePackage { + name = "wordwrap-0.0.2"; + src = [ + (fetchurl { + url = "http://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz"; + sha1 = "b79669bb42ecb409f83d583cad52ca17eaa1643f"; + }) + ]; + buildInputs = + (self.nativeDeps."wordwrap"."~0.0.2" or []); + deps = [ + ]; + peerDependencies = [ + ]; + passthru.names = [ "wordwrap" ]; + }; +} diff --git a/src/bin/nixdocker.js b/src/bin/nixdocker.js new file mode 100644 index 0000000..14de9c3 --- /dev/null +++ b/src/bin/nixdocker.js @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +var optimist = require("optimist"); +var fs = require("fs"); +var path = require("path"); + +var rootPath = path.resolve(fs.realpathSync(process.argv[1]), "../.."); + +console.log("Root path", rootPath); + +var spawn = require('child_process').spawn; +var execFile = require('child_process').execFile; + +function pipeRun(cmd, args, callback) { + var command = spawn(cmd, args); + command.stdout.pipe(process.stdout); + command.stderr.pipe(process.stderr); + + command.on('close', function(code) { + callback(code); + }); +} + +function build(nix, configPath, callback) { + var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath]); + var nixPath; + + nixBuild.stdout.on('data', function(data) { + nixPath = data.toString("ascii"); + }); + nixBuild.stderr.pipe(process.stderr); + + nixBuild.on('close', function(code) { + if (code === 0) { + callback(null, nixPath.trim()); + } else { + callback(code); + } + }); +} + +build(rootPath + "/docker.nix", "configuration.nix", function(err, startPath) { + build(rootPath + "/dockerfile.nix", "configuration.nix", function(err, dockerFilePath) { + execFile("nix-store", ["-qR", startPath], {}, function(err, stdout) { + if (err) { + return console.error(err); + } + var paths = stdout.split("\n"); + var dockerFile = fs.readFileSync(dockerFilePath + "/Dockerfile").toString("ascii"); + var dockerLines = []; + + if(!fs.existsSync("nix_symlink")) { + fs.symlinkSync("/nix", "nix_symlink"); + } + + paths.forEach(function(path) { + if (path) { + dockerLines.push("ADD " + "nix_symlink" + path.substring("/nix".length) + " " + path); + } + }); + dockerLines.push("CMD " + startPath + "/bin/start"); + fs.writeFileSync("Dockerfile", dockerFile.replace("<>", dockerLines.join("\n"))); + pipeRun("docker", ["build", "-t", "test", "."], function(code) { + fs.unlinkSync("nix_symlink"); + process.exit(code); + }); + }); + }); +}); \ No newline at end of file diff --git a/src/docker.nix b/src/docker.nix new file mode 100644 index 0000000..87bb536 --- /dev/null +++ b/src/docker.nix @@ -0,0 +1,40 @@ +{ pkgs ? import {} +, configuration ? import { inherit pkgs; } +}: +let + inherit (pkgs.lib) concatMapStrings getAttr attrNames hasAttr; + stdenv = pkgs.stdenv; + supervisorConfig = pkgs.writeText "supervisord.conf" '' +[supervisord] +logfile=/tmp/supervisord.log + +${concatMapStrings (name: + let + cfg = getAttr name configuration.services; + in + '' + [program:${name}] + command=${cfg.command} + ${if hasAttr "cwd" cfg then + "directory=${cfg.cwd}" + else ""} + '' + ) (attrNames configuration.services) +} +''; + startScript = pkgs.writeText "start" '' + ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${supervisorConfig} -n +''; +in stdenv.mkDerivation { + name = "dockerconfig"; + src = ./.; + + phases = [ "installPhase" ]; + + installPhase = '' + mkdir -p $out/bin + cp ${startScript} $out/bin/start + chmod +x $out/bin/start + ''; +} + \ No newline at end of file diff --git a/src/dockerfile.nix b/src/dockerfile.nix new file mode 100644 index 0000000..583f1f4 --- /dev/null +++ b/src/dockerfile.nix @@ -0,0 +1,26 @@ +{ pkgs ? import {} +, configuration ? import { inherit pkgs; } +}: +let + inherit (pkgs.lib) hasAttr concatMapStrings; + dockerFile = pkgs.writeText "Dockerfile" '' +FROM busybox +<> +${ + if hasAttr "docker" configuration && hasAttr "ports" configuration.docker then + concatMapStrings (port: "EXPOSE ${toString port}\n") configuration.docker.ports + else + "" +} +''; +in pkgs.stdenv.mkDerivation { + name = "dockerfile"; + src = ./.; + + phases = [ "installPhase" ]; + + installPhase = '' + mkdir -p $out + cp ${dockerFile} $out/Dockerfile + ''; +} \ No newline at end of file diff --git a/src/package.json b/src/package.json new file mode 100644 index 0000000..d923f55 --- /dev/null +++ b/src/package.json @@ -0,0 +1,10 @@ +{ + "name" : "nixdocker", + "version" : "0.1.0", + "dependencies" : { + "optimist" : "0.6.0" + }, + "directories": { + "bin": "./bin" + } +} \ No newline at end of file From 509835211f86c8fceefe7cf08cd77ea39dbea5c4 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Thu, 31 Oct 2013 10:15:19 -0700 Subject: [PATCH 02/46] Work --- configuration.nix | 4 ++ default.nix | 6 +-- src/base.nix | 6 +++ src/bin/nix-docker | 126 +++++++++++++++++++++++++++++++++++++++++++ src/bin/nixdocker.js | 69 ------------------------ src/docker.nix | 20 +------ src/dockerfile.nix | 106 ++++++++++++++++++++++++++++++------ src/package.json | 2 +- zedconfig.json | 12 +++++ 9 files changed, 244 insertions(+), 107 deletions(-) create mode 100644 configuration.nix create mode 100644 src/base.nix create mode 100644 src/bin/nix-docker delete mode 100644 src/bin/nixdocker.js create mode 100644 zedconfig.json diff --git a/configuration.nix b/configuration.nix new file mode 100644 index 0000000..9dbe0aa --- /dev/null +++ b/configuration.nix @@ -0,0 +1,4 @@ +{ config, pkgs, ... }: +{ + services.redis.enable = true; +} \ No newline at end of file diff --git a/default.nix b/default.nix index a81cca3..34c70a5 100644 --- a/default.nix +++ b/default.nix @@ -9,8 +9,8 @@ let }); in nodePackages.buildNodePackage { - name = "nixdocker"; - src = [ { outPath = ./src; name = "nixdocker"; } ]; + name = "nix-docker"; + src = [ { outPath = ./src; name = "nix-docker"; } ]; deps = with nodePackages; [optimist]; - passthru.names = [ "nixdocker" ]; + passthru.names = [ "nix-docker" ]; } \ No newline at end of file diff --git a/src/base.nix b/src/base.nix new file mode 100644 index 0000000..654d72e --- /dev/null +++ b/src/base.nix @@ -0,0 +1,6 @@ +{ config, pkgs, ... }: +{ + config = { + fileSystems."/".device = "/dev/disk/by-label/nixos"; + }; +} \ No newline at end of file diff --git a/src/bin/nix-docker b/src/bin/nix-docker new file mode 100644 index 0000000..0d4aa44 --- /dev/null +++ b/src/bin/nix-docker @@ -0,0 +1,126 @@ +#!/usr/bin/env node + +var argv = require("optimist").argv; +var fs = require("fs"); +var path = require("path"); + +var rootPath = path.resolve(fs.realpathSync(process.argv[1]), "../.."); + +var spawn = require('child_process').spawn; +var execFile = require('child_process').execFile; +var configFile = argv.c || "configuration.nix"; +var imageName = argv.t ? argv.t : "nix-docker-build"; + +var tmpNixStore = "nix_store"; + +if(argv.help) { + console.log("Usage: nix-docker [--dockerfile-only] [--mount-build] [-t imagename] [-c configuration.nix] "); + console.log(" --docker-file-only: generate Docker file, but do not build it."); + console.log(" --mount-build: don't add /nix paths to container, but let them be mounted from host at run-time with -v."); + console.log(" -t: name of the image to build."); + console.log(" -c: path to configuration file to build."); + process.exit(0); +} + +function pipeRun(cmd, args, callback) { + var command = spawn(cmd, args); + command.stdout.pipe(process.stdout); + command.stderr.pipe(process.stderr); + + command.on('close', function(code) { + callback(code); + }); +} + +function build(nix, configPath, callback) { + var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath]); + var nixPath; + + nixBuild.stdout.on('data', function(data) { + nixPath = data.toString("ascii"); + }); + nixBuild.stderr.pipe(process.stderr); + + nixBuild.on('close', function(code) { + if (code === 0) { + callback(null, nixPath.trim()); + } else { + callback(code); + } + }); +} + +function cleanup(callback) { + if(!fs.existsSync(tmpNixStore)) { + return callback(); + } + var stats = fs.lstatSync(tmpNixStore); + if(stats.isSymbolicLink()) { + fs.unlinkSync(tmpNixStore); + callback(); + } else { + execFile("chmod", ["-R", "777", tmpNixStore], function() { + execFile("rm", ["-rf", tmpNixStore], callback); + }); + } +} + +function buildImage(dockerFilePath, dockerLines, callback) { + var dockerFile = fs.readFileSync(dockerFilePath + "/Dockerfile").toString("ascii"); + fs.writeFileSync("Dockerfile", dockerFile.replace("<>", dockerLines.join("\n"))); + if(!argv["dockerfile-only"]) { + pipeRun("docker", ["build", "-t", imageName, "."], function(code) { + cleanup(function() { + callback(code); + }); + }); + } +} + +cleanup(function() { + build(rootPath + "/dockerfile.nix", configFile, function(err, dockerFilePath) { + execFile("nix-store", ["-qR", dockerFilePath], {}, function(err, stdout) { + if (err) { + return console.error(err); + } + var paths = stdout.trim().split("\n"); + var dockerLines = []; + + if(argv["big-commit"]) { + fs.mkdirSync(tmpNixStore); + execFile("cp", ["-r", "--no-preserve=ownership"].concat(paths).concat([tmpNixStore]), function(err, stdout, stderr) { + if(err) { + console.log(err); + process.exit(1); + } + console.log(stdout, stderr); + + dockerLines.push("ADD " + tmpNixStore + " /nix/store"); + + console.log("Docker lines", dockerLines); + + buildImage(dockerFilePath, dockerLines, function(code) { + console.log("To run: docker run -t -i " + imageName); + process.exit(code); + }); + }); + } else if(argv["mount-build"]) { + buildImage(dockerFilePath, dockerLines, function(code) { + console.log("To run: docker run -t -i -v /nix/store:/nix/store " + imageName); + process.exit(code); + }); + } else { + fs.symlinkSync("/nix/store", tmpNixStore); + + paths.forEach(function(path) { + dockerLines.push("ADD " + tmpNixStore + path.substring("/nix/store".length) + " " + path); + }); + + buildImage(dockerFilePath, dockerLines, function(code) { + console.log("To run: docker run -t -i " + imageName); + process.exit(code); + }); + } + }); + }); +}); diff --git a/src/bin/nixdocker.js b/src/bin/nixdocker.js deleted file mode 100644 index 14de9c3..0000000 --- a/src/bin/nixdocker.js +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env node - -var optimist = require("optimist"); -var fs = require("fs"); -var path = require("path"); - -var rootPath = path.resolve(fs.realpathSync(process.argv[1]), "../.."); - -console.log("Root path", rootPath); - -var spawn = require('child_process').spawn; -var execFile = require('child_process').execFile; - -function pipeRun(cmd, args, callback) { - var command = spawn(cmd, args); - command.stdout.pipe(process.stdout); - command.stderr.pipe(process.stderr); - - command.on('close', function(code) { - callback(code); - }); -} - -function build(nix, configPath, callback) { - var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath]); - var nixPath; - - nixBuild.stdout.on('data', function(data) { - nixPath = data.toString("ascii"); - }); - nixBuild.stderr.pipe(process.stderr); - - nixBuild.on('close', function(code) { - if (code === 0) { - callback(null, nixPath.trim()); - } else { - callback(code); - } - }); -} - -build(rootPath + "/docker.nix", "configuration.nix", function(err, startPath) { - build(rootPath + "/dockerfile.nix", "configuration.nix", function(err, dockerFilePath) { - execFile("nix-store", ["-qR", startPath], {}, function(err, stdout) { - if (err) { - return console.error(err); - } - var paths = stdout.split("\n"); - var dockerFile = fs.readFileSync(dockerFilePath + "/Dockerfile").toString("ascii"); - var dockerLines = []; - - if(!fs.existsSync("nix_symlink")) { - fs.symlinkSync("/nix", "nix_symlink"); - } - - paths.forEach(function(path) { - if (path) { - dockerLines.push("ADD " + "nix_symlink" + path.substring("/nix".length) + " " + path); - } - }); - dockerLines.push("CMD " + startPath + "/bin/start"); - fs.writeFileSync("Dockerfile", dockerFile.replace("<>", dockerLines.join("\n"))); - pipeRun("docker", ["build", "-t", "test", "."], function(code) { - fs.unlinkSync("nix_symlink"); - process.exit(code); - }); - }); - }); -}); \ No newline at end of file diff --git a/src/docker.nix b/src/docker.nix index 87bb536..ee65917 100644 --- a/src/docker.nix +++ b/src/docker.nix @@ -3,25 +3,7 @@ }: let inherit (pkgs.lib) concatMapStrings getAttr attrNames hasAttr; - stdenv = pkgs.stdenv; - supervisorConfig = pkgs.writeText "supervisord.conf" '' -[supervisord] -logfile=/tmp/supervisord.log - -${concatMapStrings (name: - let - cfg = getAttr name configuration.services; - in - '' - [program:${name}] - command=${cfg.command} - ${if hasAttr "cwd" cfg then - "directory=${cfg.cwd}" - else ""} - '' - ) (attrNames configuration.services) -} -''; + startScript = pkgs.writeText "start" '' ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${supervisorConfig} -n ''; diff --git a/src/dockerfile.nix b/src/dockerfile.nix index 583f1f4..3db1f61 100644 --- a/src/dockerfile.nix +++ b/src/dockerfile.nix @@ -1,26 +1,102 @@ { pkgs ? import {} -, configuration ? import { inherit pkgs; } +, configuration ? }: +with pkgs.lib; let - inherit (pkgs.lib) hasAttr concatMapStrings; + config = import { + modules = [ configuration ./base.nix ]; + }; + + services = removeAttrs config.config.systemd.services [ + "acpid" + "network-setup" + "prepare-kexec" + "systemd-sysctl" + "alsa-store" + "nix-daemon" + "rngd" + "systemd-update-utmp" + "cpufreq" + "nix-gc" + "scsi-link-pm" + "systemd-vconsole-setup" + "cron" + "nscd" + "synergy-client" + "update-locatedb" + "dhcpcd" + "ntpd" + "synergy-server" + "post-resume" + "systemd-modules-load" + "klogd" + "pre-sleep" + "systemd-random-seed" + ]; + + isOneShot = cfg: hasAttr "Type" cfg.serviceConfig && cfg.serviceConfig.Type == "oneshot"; + + runServices = filterAttrs (name: cfg: !(isOneShot cfg)) services; + + oneShotServices = filterAttrs (name: cfg: isOneShot cfg) services; + + configToCommand = cfg: if hasAttr "ExecStart" cfg.serviceConfig then + cfg.serviceConfig.ExecStart + else if hasAttr "script" cfg then + pkgs.writeScript "script" '' + #!/bin/sh + ${cfg.script} + '' + else + ""; + + supervisorConfig = pkgs.writeText "supervisord.conf" '' + [supervisord] + logfile=/tmp/supervisord.log + + ${concatMapStrings (name: + let + cfg = getAttr name runServices; + in + '' + [program:${name}] + command=${configToCommand cfg}'' + ) (attrNames runServices) + } + ''; + + extraUsers = config.config.users.extraUsers; + dockerFile = pkgs.writeText "Dockerfile" '' -FROM busybox +FROM ubuntu <> +# Create users ${ - if hasAttr "docker" configuration && hasAttr "ports" configuration.docker then - concatMapStrings (port: "EXPOSE ${toString port}\n") configuration.docker.ports - else + concatMapStrings (name: "RUN /usr/sbin/useradd ${name} || echo\n") (attrNames extraUsers) +} +# Run one shot services +${ + concatMapStrings (name: "RUN ${configToCommand (getAttr name oneShotServices)}\n") (attrNames oneShotServices) +} + + +${ + # if hasAttr "docker" configuration && hasAttr "ports" configuration.docker then + # concatMapStrings (port: "EXPOSE ${toString port}\n") configuration.docker.ports + # else + # "" "" } +CMD ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${supervisorConfig} -n ''; in pkgs.stdenv.mkDerivation { - name = "dockerfile"; - src = ./.; - - phases = [ "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp ${dockerFile} $out/Dockerfile - ''; + name = "dockerfile"; + src = ./.; + + phases = [ "installPhase" ]; + + installPhase = '' + mkdir -p $out + cp ${dockerFile} $out/Dockerfile + ''; } \ No newline at end of file diff --git a/src/package.json b/src/package.json index d923f55..d349134 100644 --- a/src/package.json +++ b/src/package.json @@ -1,5 +1,5 @@ { - "name" : "nixdocker", + "name" : "nix-docker", "version" : "0.1.0", "dependencies" : { "optimist" : "0.6.0" diff --git a/zedconfig.json b/zedconfig.json new file mode 100644 index 0000000..8f9ab34 --- /dev/null +++ b/zedconfig.json @@ -0,0 +1,12 @@ +{ + "modes": { + "javascript": { + "filenames": ["nix-docker"] + }, + "nix": { + "preferences": { + "tabSize": 2 + } + } + } +} \ No newline at end of file From d9b562701bf17a479d60bb1dc78996a456dbf227 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 1 Nov 2013 08:25:09 -0700 Subject: [PATCH 03/46] Working quite well, but images get HUGE --- base-configuration.nix | 4 ++ configuration.nix | 16 ++++- src/base.nix | 10 ++++ src/bin/nix-docker | 131 ++++++++++++++++++++++------------------- src/docker.nix | 22 ------- src/dockerfile.nix | 129 ++++++++++++++++++---------------------- src/environment.nix | 22 +++++++ src/systemd.nix | 67 +++++++++++++++++++++ src/users.nix | 27 +++++++++ 9 files changed, 274 insertions(+), 154 deletions(-) create mode 100644 base-configuration.nix delete mode 100644 src/docker.nix create mode 100644 src/environment.nix create mode 100644 src/systemd.nix create mode 100644 src/users.nix diff --git a/base-configuration.nix b/base-configuration.nix new file mode 100644 index 0000000..17ef878 --- /dev/null +++ b/base-configuration.nix @@ -0,0 +1,4 @@ +{ config, pkgs, ... }: +{ + +} \ No newline at end of file diff --git a/configuration.nix b/configuration.nix index 9dbe0aa..9fff36a 100644 --- a/configuration.nix +++ b/configuration.nix @@ -1,4 +1,18 @@ { config, pkgs, ... }: { - services.redis.enable = true; + services.redis = { + enable = true; + logfile = "stdout"; + logLevel = "debug"; + port = 1234; + syslog = false; + }; + + #docker.ports = [ 1234 ]; + + users.extraUsers.zef = { + group = "users"; + home = "/home/zef"; + createHome = true; + }; } \ No newline at end of file diff --git a/src/base.nix b/src/base.nix index 654d72e..5ddac72 100644 --- a/src/base.nix +++ b/src/base.nix @@ -1,5 +1,15 @@ { config, pkgs, ... }: +with pkgs.lib; { + options = { + docker = { + ports = mkOption { + default = []; + description = "Ports to expose to the outside world."; + example = [ 80 22 ]; + }; + }; + }; config = { fileSystems."/".device = "/dev/disk/by-label/nixos"; }; diff --git a/src/bin/nix-docker b/src/bin/nix-docker index 0d4aa44..95821cd 100644 --- a/src/bin/nix-docker +++ b/src/bin/nix-docker @@ -9,11 +9,14 @@ var rootPath = path.resolve(fs.realpathSync(process.argv[1]), "../.."); var spawn = require('child_process').spawn; var execFile = require('child_process').execFile; var configFile = argv.c || "configuration.nix"; +var isMountBuild = !! argv["mount-build"]; + +var baseImage = argv.from || "zefhemel/base-nix"; var imageName = argv.t ? argv.t : "nix-docker-build"; var tmpNixStore = "nix_store"; -if(argv.help) { +if (argv.help) { console.log("Usage: nix-docker [--dockerfile-only] [--mount-build] [-t imagename] [-c configuration.nix] "); console.log(" --docker-file-only: generate Docker file, but do not build it."); console.log(" --mount-build: don't add /nix paths to container, but let them be mounted from host at run-time with -v."); @@ -24,6 +27,7 @@ if(argv.help) { function pipeRun(cmd, args, callback) { var command = spawn(cmd, args); + process.stdin.pipe(command.stdin); command.stdout.pipe(process.stdout); command.stderr.pipe(process.stderr); @@ -33,7 +37,7 @@ function pipeRun(cmd, args, callback) { } function build(nix, configPath, callback) { - var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath]); + var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath, "--arg", "mountBuild", "" + isMountBuild, "--argstr", "imageName", imageName, "--argstr", "baseImage", baseImage, "--show-trace"]); var nixPath; nixBuild.stdout.on('data', function(data) { @@ -51,25 +55,19 @@ function build(nix, configPath, callback) { } function cleanup(callback) { - if(!fs.existsSync(tmpNixStore)) { + if (!fs.existsSync(tmpNixStore)) { return callback(); } - var stats = fs.lstatSync(tmpNixStore); - if(stats.isSymbolicLink()) { - fs.unlinkSync(tmpNixStore); - callback(); - } else { - execFile("chmod", ["-R", "777", tmpNixStore], function() { - execFile("rm", ["-rf", tmpNixStore], callback); - }); - } + execFile("chmod", ["-R", "777", tmpNixStore], function() { + execFile("rm", ["-rf", tmpNixStore], callback); + }); } -function buildImage(dockerFilePath, dockerLines, callback) { +function buildImage(dockerFilePath, callback) { var dockerFile = fs.readFileSync(dockerFilePath + "/Dockerfile").toString("ascii"); - fs.writeFileSync("Dockerfile", dockerFile.replace("<>", dockerLines.join("\n"))); - if(!argv["dockerfile-only"]) { - pipeRun("docker", ["build", "-t", imageName, "."], function(code) { + fs.writeFileSync("Dockerfile", dockerFile); + if (!argv["dockerfile-only"]) { + pipeRun("sudo", ["docker", "build", "-rm=true", "-t", imageName, "."], function(code) { cleanup(function() { callback(code); }); @@ -77,50 +75,63 @@ function buildImage(dockerFilePath, dockerLines, callback) { } } +function getAvailableNixPaths(callback) { + var command = spawn("sudo", ["docker", "run", baseImage, "/bin/ls", "/nix/store"]); + process.stdin.pipe(command.stdin); + + var output = ''; + + command.stdout.on("data", function(data) { + output += data.toString("ascii"); + }); + + command.on('close', function() { + callback(null, output.split("\n")); + }); +} + +function copyClosureAndBuild(dockerFilePath, availablePaths) { + execFile("nix-store", ["-qR", dockerFilePath], {}, function(err, stdout) { + if (err) { + return console.error(err); + } + + if (isMountBuild) { + buildImage(dockerFilePath, function(code) { + console.log("To run: sudo docker run -t -i -v /nix/store:/nix/store " + imageName); + process.exit(code); + }); + } else { + var paths = stdout.trim().split("\n"); + paths = paths.filter(function(path) { + return availablePaths.indexOf(path.substring("/nix/store/".length)) === -1; + }); + + console.log("New paths to copy", paths); + + fs.mkdirSync(tmpNixStore); + execFile("cp", ["-r", "--no-preserve=ownership"].concat(paths).concat([tmpNixStore]), function(err, stdout, stderr) { + if (err) { + console.log(err); + process.exit(1); + } + buildImage(dockerFilePath, function(code) { + console.log("To run: sudo docker run -t -i " + imageName); + process.exit(code); + }); + }); + } + }); +} + cleanup(function() { build(rootPath + "/dockerfile.nix", configFile, function(err, dockerFilePath) { - execFile("nix-store", ["-qR", dockerFilePath], {}, function(err, stdout) { - if (err) { - return console.error(err); - } - var paths = stdout.trim().split("\n"); - var dockerLines = []; - - if(argv["big-commit"]) { - fs.mkdirSync(tmpNixStore); - execFile("cp", ["-r", "--no-preserve=ownership"].concat(paths).concat([tmpNixStore]), function(err, stdout, stderr) { - if(err) { - console.log(err); - process.exit(1); - } - console.log(stdout, stderr); - - dockerLines.push("ADD " + tmpNixStore + " /nix/store"); - - console.log("Docker lines", dockerLines); - - buildImage(dockerFilePath, dockerLines, function(code) { - console.log("To run: docker run -t -i " + imageName); - process.exit(code); - }); - }); - } else if(argv["mount-build"]) { - buildImage(dockerFilePath, dockerLines, function(code) { - console.log("To run: docker run -t -i -v /nix/store:/nix/store " + imageName); - process.exit(code); - }); - } else { - fs.symlinkSync("/nix/store", tmpNixStore); - - paths.forEach(function(path) { - dockerLines.push("ADD " + tmpNixStore + path.substring("/nix/store".length) + " " + path); - }); - - buildImage(dockerFilePath, dockerLines, function(code) { - console.log("To run: docker run -t -i " + imageName); - process.exit(code); - }); - } - }); + if (isMountBuild) { + copyClosureAndBuild(dockerFilePath, []); + } else { + getAvailableNixPaths(function(err, availablePaths) { + copyClosureAndBuild(dockerFilePath, availablePaths); + }); + } }); -}); +}); \ No newline at end of file diff --git a/src/docker.nix b/src/docker.nix deleted file mode 100644 index ee65917..0000000 --- a/src/docker.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ pkgs ? import {} -, configuration ? import { inherit pkgs; } -}: -let - inherit (pkgs.lib) concatMapStrings getAttr attrNames hasAttr; - - startScript = pkgs.writeText "start" '' - ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${supervisorConfig} -n -''; -in stdenv.mkDerivation { - name = "dockerconfig"; - src = ./.; - - phases = [ "installPhase" ]; - - installPhase = '' - mkdir -p $out/bin - cp ${startScript} $out/bin/start - chmod +x $out/bin/start - ''; -} - \ No newline at end of file diff --git a/src/dockerfile.nix b/src/dockerfile.nix index 3db1f61..b32505b 100644 --- a/src/dockerfile.nix +++ b/src/dockerfile.nix @@ -1,94 +1,80 @@ { pkgs ? import {} , configuration ? +, mountBuild ? false +, imageName ? "docker-build-nix" +, baseImage ? "ubuntu" }: with pkgs.lib; let +# pkgs = import {} +# config = import { modules = [ ./configuration.nix ./src/base.nix ]; } +# config = import { modules = [ configuration ./base.nix ]; }; - services = removeAttrs config.config.systemd.services [ - "acpid" - "network-setup" - "prepare-kexec" - "systemd-sysctl" - "alsa-store" - "nix-daemon" - "rngd" - "systemd-update-utmp" - "cpufreq" - "nix-gc" - "scsi-link-pm" - "systemd-vconsole-setup" - "cron" - "nscd" - "synergy-client" - "update-locatedb" - "dhcpcd" - "ntpd" - "synergy-server" - "post-resume" - "systemd-modules-load" - "klogd" - "pre-sleep" - "systemd-random-seed" - ]; + localNixPath = pkg: "nix_store/${substring 11 (stringLength pkg.outPath) pkg.outPath}"; - isOneShot = cfg: hasAttr "Type" cfg.serviceConfig && cfg.serviceConfig.Type == "oneshot"; + users = import ./users.nix { inherit pkgs config; }; + systemd = import ./systemd.nix { inherit pkgs config; }; + environment = import ./environment.nix { inherit pkgs config; }; - runServices = filterAttrs (name: cfg: !(isOneShot cfg)) services; + setupScript = '' + cp ${users.groupFile} /etc/group + cp ${users.passwdFile} /etc/passwd + ${environment.updateEtcScript} + ${environment.setupSystemProfile} + ${systemd.oneShotScript} + ''; - oneShotServices = filterAttrs (name: cfg: isOneShot cfg) services; + shellScriptFile = pkgs.writeScript "shell" '' + #!/bin/sh + ${setupScriptFile} + /usr/bin/bash + ''; - configToCommand = cfg: if hasAttr "ExecStart" cfg.serviceConfig then - cfg.serviceConfig.ExecStart - else if hasAttr "script" cfg then - pkgs.writeScript "script" '' - #!/bin/sh - ${cfg.script} - '' - else - ""; + setupScriptFile = pkgs.writeScript "setup" '' + #!/bin/sh -e + ${setupScript} + ''; - supervisorConfig = pkgs.writeText "supervisord.conf" '' - [supervisord] - logfile=/tmp/supervisord.log + runScript = pkgs.writeScript "run" '' + #!/bin/sh + ${if mountBuild then setupScript else ""} + ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${systemd.supervisorConfigFile} -n & + mkdir -p /var/log/supervisord + sleep 2 + touch /var/log/supervisord/test.log + tail -n 100 -f /var/log/supervisord/*.log + ''; - ${concatMapStrings (name: - let - cfg = getAttr name runServices; - in - '' - [program:${name}] - command=${configToCommand cfg}'' - ) (attrNames runServices) + dockerFile = pkgs.writeText "Dockerfile" '' + FROM ${baseImage} + ${if !mountBuild then + '' + ADD nix_store /nix/store + RUN ${setupScriptFile} + '' + else "" + } + RUN ln -sf ${shellScriptFile} /bin/shell + CMD ${runScript} + ${ + concatMapStrings (port: "EXPOSE ${toString port}\n") config.config.docker.ports } ''; - extraUsers = config.config.users.extraUsers; + runContainerScript = pkgs.writeScript "docker-run" '' + #!/usr/bin/env bash - dockerFile = pkgs.writeText "Dockerfile" '' -FROM ubuntu -<> -# Create users -${ - concatMapStrings (name: "RUN /usr/sbin/useradd ${name} || echo\n") (attrNames extraUsers) -} -# Run one shot services -${ - concatMapStrings (name: "RUN ${configToCommand (getAttr name oneShotServices)}\n") (attrNames oneShotServices) -} + OPTIONS="-t -i $*" + if [ "$1" == "-d" ]; then + OPTIONS="$*" + fi + docker run $OPTIONS ${if mountBuild then "-v /nix/store:/nix/store" else ""} ${imageName} + ''; -${ - # if hasAttr "docker" configuration && hasAttr "ports" configuration.docker then - # concatMapStrings (port: "EXPOSE ${toString port}\n") configuration.docker.ports - # else - # "" - "" -} -CMD ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${supervisorConfig} -n -''; in pkgs.stdenv.mkDerivation { name = "dockerfile"; src = ./.; @@ -96,7 +82,8 @@ in pkgs.stdenv.mkDerivation { phases = [ "installPhase" ]; installPhase = '' - mkdir -p $out + mkdir -p $out/bin + cp ${runContainerScript} $out/bin/run-container cp ${dockerFile} $out/Dockerfile ''; } \ No newline at end of file diff --git a/src/environment.nix b/src/environment.nix new file mode 100644 index 0000000..4a06463 --- /dev/null +++ b/src/environment.nix @@ -0,0 +1,22 @@ +{ pkgs +, config }: +with pkgs.lib; +let + etc = config.config.environment.etc; + systemPackages = config.config.environment.systemPackages; +in { + updateEtcScript = concatMapStrings (filename: + let + file = getAttr filename etc; + in + '' + mkdir -p /etc/${dirOf "/${file.target}"} + ln -s ${file.source} ${file.target} + '' + ) (attrNames etc); + + setupSystemProfile = '' + rm -rf /usr + ln -s ${config.config.system.path} /usr + ''; +} \ No newline at end of file diff --git a/src/systemd.nix b/src/systemd.nix new file mode 100644 index 0000000..cacbe80 --- /dev/null +++ b/src/systemd.nix @@ -0,0 +1,67 @@ +{ pkgs +, config }: +with pkgs.lib; +let + services = removeAttrs config.config.systemd.services [ + "acpid" + "network-setup" + "prepare-kexec" + "systemd-sysctl" + "alsa-store" + "nix-daemon" + "rngd" + "systemd-update-utmp" + "cpufreq" + "nix-gc" + "scsi-link-pm" + "systemd-vconsole-setup" + "cron" + "nscd" + "synergy-client" + "update-locatedb" + "dhcpcd" + "ntpd" + "synergy-server" + "post-resume" + "systemd-modules-load" + "klogd" + "pre-sleep" + "systemd-random-seed" + ]; + + isOneShot = cfg: hasAttr "Type" cfg.serviceConfig && cfg.serviceConfig.Type == "oneshot"; + + runServices = filterAttrs (name: cfg: !(isOneShot cfg)) services; + + oneShotServices = filterAttrs (name: cfg: isOneShot cfg) services; + + configToCommand = name: cfg: if hasAttr "ExecStart" cfg.serviceConfig then + cfg.serviceConfig.ExecStart + else if hasAttr "script" cfg then + pkgs.writeScript "${name}-script" '' + #!/bin/sh -e + ${cfg.script} + '' + else + ""; +in { + oneShotScript = concatMapStrings (name: "${configToCommand name (getAttr name oneShotServices)}\n") (attrNames oneShotServices); + supervisorConfigFile = pkgs.writeText "supervisord.conf" '' + [supervisord] + logfile=/var/log/supervisord/supervisord.log + + ${concatMapStrings (name: + let + cfg = getAttr name runServices; + in + '' + [program:${name}] + command=${configToCommand name cfg} + redirect_stderr=true + stdout_logfile=/var/log/supervisord/${name}.log + ${if hasAttr "User" cfg.serviceConfig then "user=${cfg.serviceConfig.User}\n" else ""} + '' + ) (attrNames runServices) + } + ''; +} \ No newline at end of file diff --git a/src/users.nix b/src/users.nix new file mode 100644 index 0000000..8f240b1 --- /dev/null +++ b/src/users.nix @@ -0,0 +1,27 @@ +{ pkgs +, config }: +with pkgs.lib; +let + extraUsers = config.config.users.extraUsers; + extraGroups = config.config.users.extraGroups; +in { + passwdFile = pkgs.writeText "passwd" '' + ${concatMapStrings (name: + let + user = getAttr name extraUsers; + in + if user.createUser then + "${user.name}:x:${toString user.uid}:${toString (getAttr user.group extraGroups).gid}:Description:${user.home}:${user.shell}\n" + else + "" + ) (attrNames extraUsers)} + ''; + groupFile = pkgs.writeText "group" '' + ${concatMapStrings (name: + let + group = getAttr name extraGroups; + in + "${group.name}:x:${toString group.gid}:\n" + ) (attrNames extraGroups)} + ''; +} \ No newline at end of file From 054dee58330f286c9f2b7c7da7aefd6d4b9c18b0 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 1 Nov 2013 08:36:53 -0700 Subject: [PATCH 04/46] Commit --- configuration.nix | 20 ++-- src/all-modules.nix | 4 + src/bin/nix-docker | 5 +- src/dockerfile.nix | 5 +- src/{ => modules}/base.nix | 3 - src/modules/user-groups.nix | 192 ++++++++++++++++++++++++++++++++++++ 6 files changed, 209 insertions(+), 20 deletions(-) create mode 100644 src/all-modules.nix rename src/{ => modules}/base.nix (75%) create mode 100644 src/modules/user-groups.nix diff --git a/configuration.nix b/configuration.nix index 9fff36a..cf2b262 100644 --- a/configuration.nix +++ b/configuration.nix @@ -1,18 +1,10 @@ { config, pkgs, ... }: { - services.redis = { - enable = true; - logfile = "stdout"; - logLevel = "debug"; - port = 1234; - syslog = false; - }; + docker.ports = [ 1234 ]; - #docker.ports = [ 1234 ]; - - users.extraUsers.zef = { - group = "users"; - home = "/home/zef"; - createHome = true; - }; + # users.extraUsers.zef = { + # group = "users"; + # home = "/home/zef"; + # createHome = true; + # }; } \ No newline at end of file diff --git a/src/all-modules.nix b/src/all-modules.nix new file mode 100644 index 0000000..dea70d8 --- /dev/null +++ b/src/all-modules.nix @@ -0,0 +1,4 @@ +[ + ./modules/base.nix + ./modules/user-groups.nix +] \ No newline at end of file diff --git a/src/bin/nix-docker b/src/bin/nix-docker index 95821cd..728d7eb 100644 --- a/src/bin/nix-docker +++ b/src/bin/nix-docker @@ -125,7 +125,10 @@ function copyClosureAndBuild(dockerFilePath, availablePaths) { } cleanup(function() { - build(rootPath + "/dockerfile.nix", configFile, function(err, dockerFilePath) { + build(rootPath + "/dockerfile.nix", configFile, function(code, dockerFilePath) { + if(code) { + process.exit(code); + } if (isMountBuild) { copyClosureAndBuild(dockerFilePath, []); } else { diff --git a/src/dockerfile.nix b/src/dockerfile.nix index b32505b..08e5f6f 100644 --- a/src/dockerfile.nix +++ b/src/dockerfile.nix @@ -9,8 +9,9 @@ let # pkgs = import {} # config = import { modules = [ ./configuration.nix ./src/base.nix ]; } # - config = import { - modules = [ configuration ./base.nix ]; + config = evalModules { + modules = concatLists [ [configuration] (import ./all-modules.nix) ]; + args = { inherit pkgs; }; }; localNixPath = pkg: "nix_store/${substring 11 (stringLength pkg.outPath) pkg.outPath}"; diff --git a/src/base.nix b/src/modules/base.nix similarity index 75% rename from src/base.nix rename to src/modules/base.nix index 5ddac72..402b835 100644 --- a/src/base.nix +++ b/src/modules/base.nix @@ -10,7 +10,4 @@ with pkgs.lib; }; }; }; - config = { - fileSystems."/".device = "/dev/disk/by-label/nixos"; - }; } \ No newline at end of file diff --git a/src/modules/user-groups.nix b/src/modules/user-groups.nix new file mode 100644 index 0000000..df5da8d --- /dev/null +++ b/src/modules/user-groups.nix @@ -0,0 +1,192 @@ +{pkgs, config, ...}: + +with pkgs.lib; + +let + + ids = config.ids; + users = config.users; + + userOpts = { name, config, ... }: { + + options = { + + name = mkOption { + type = types.str; + description = "The name of the user account. If undefined, the name of the attribute set will be used."; + }; + + description = mkOption { + type = types.str; + default = ""; + example = "Alice Q. User"; + description = '' + A short description of the user account, typically the + user's full name. This is actually the “GECOS” or “comment” + field in /etc/passwd. + ''; + }; + + uid = mkOption { + type = with types; uniq (nullOr int); + default = null; + description = "The account UID. If undefined, NixOS will select a free UID."; + }; + + group = mkOption { + type = types.str; + default = "nogroup"; + description = "The user's primary group."; + }; + + extraGroups = mkOption { + type = types.listOf types.str; + default = []; + description = "The user's auxiliary groups."; + }; + + home = mkOption { + type = types.str; + default = "/var/empty"; + description = "The user's home directory."; + }; + + shell = mkOption { + type = types.str; + default = "/run/current-system/sw/sbin/nologin"; + description = "The path to the user's shell."; + }; + + createHome = mkOption { + type = types.bool; + default = false; + description = "If true, the home directory will be created automatically."; + }; + + useDefaultShell = mkOption { + type = types.bool; + default = false; + description = "If true, the user's shell will be set to users.defaultUserShell."; + }; + + password = mkOption { + type = with types; uniq (nullOr str); + default = null; + description = '' + The user's password. If undefined, no password is set for + the user. Warning: do not set confidential information here + because it is world-readable in the Nix store. This option + should only be used for public accounts such as + guest. + ''; + }; + + isSystemUser = mkOption { + type = types.bool; + default = true; + description = "Indicates if the user is a system user or not."; + }; + + createUser = mkOption { + type = types.bool; + default = true; + description = '' + Indicates if the user should be created automatically as a local user. + Set this to false if the user for instance is an LDAP user. NixOS will + then not modify any of the basic properties for the user account. + ''; + }; + + isAlias = mkOption { + type = types.bool; + default = false; + description = "If true, the UID of this user is not required to be unique and can thus alias another user."; + }; + + }; + + config = { + name = mkDefault name; + uid = mkDefault (attrByPath [name] null ids.uids); + shell = mkIf config.useDefaultShell (mkDefault users.defaultUserShell); + }; + + }; + + groupOpts = { name, config, ... }: { + + options = { + + name = mkOption { + type = types.str; + description = "The name of the group. If undefined, the name of the attribute set will be used."; + }; + + gid = mkOption { + type = with types; uniq (nullOr int); + default = null; + description = "The GID of the group. If undefined, NixOS will select a free GID."; + }; + + }; + + config = { + name = mkDefault name; + gid = mkDefault (attrByPath [name] null ids.gids); + }; + + }; + + # Note: the 'X' in front of the password is to distinguish between + # having an empty password, and not having a password. + serializedUser = u: "${u.name}\n${u.description}\n${if u.uid != null then toString u.uid else ""}\n${u.group}\n${toString (concatStringsSep "," u.extraGroups)}\n${u.home}\n${u.shell}\n${toString u.createHome}\n${if u.password != null then "X" + u.password else ""}\n${toString u.isSystemUser}\n${toString u.createUser}\n${toString u.isAlias}\n"; + + usersFile = pkgs.writeText "users" ( + let + p = partition (u: u.isAlias) (attrValues config.users.extraUsers); + in concatStrings (map serializedUser p.wrong ++ map serializedUser p.right)); + +in + +{ + + ###### interface + + options = { + + users.extraUsers = mkOption { + default = {}; + type = types.loaOf types.optionSet; + example = { + alice = { + uid = 1234; + description = "Alice Q. User"; + home = "/home/alice"; + createHome = true; + group = "users"; + extraGroups = ["wheel"]; + shell = "/bin/sh"; + }; + }; + description = '' + Additional user accounts to be created automatically by the system. + This can also be used to set options for root. + ''; + options = [ userOpts ]; + }; + + users.extraGroups = mkOption { + default = {}; + example = + { students.gid = 1001; + hackers = { }; + }; + type = types.loaOf types.optionSet; + description = '' + Additional groups to be created automatically by the system. + ''; + options = [ groupOpts ]; + }; + + }; +} \ No newline at end of file From f0b384659a27b12a7137fe7ed7018eda08f2e61a Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 1 Nov 2013 08:39:18 -0700 Subject: [PATCH 05/46] Yo --- src/environment.nix | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/environment.nix b/src/environment.nix index 4a06463..305977a 100644 --- a/src/environment.nix +++ b/src/environment.nix @@ -9,10 +9,12 @@ in { let file = getAttr filename etc; in - '' - mkdir -p /etc/${dirOf "/${file.target}"} - ln -s ${file.source} ${file.target} - '' + if file.enable then + '' + mkdir -p /etc/${dirOf "/${file.target}"} + ln -s ${file.source} ${file.target} + '' + else "" ) (attrNames etc); setupSystemProfile = '' From c3091f35f00c1f6833ea1fc4d94c2bb22c920981 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Mon, 4 Nov 2013 01:49:30 -0800 Subject: [PATCH 06/46] Apache and redis module, refactoring of code. --- configuration.nix | 25 +- default.nix | 4 +- nix-docker/all-modules.nix | 9 + {src => nix-docker}/bin/nix-docker | 4 +- {src => nix-docker}/dockerfile.nix | 49 +- nix-docker/environment.nix | 13 + nix-docker/modules/config/docker.nix | 30 + nix-docker/modules/config/environment.nix | 20 + .../modules/config}/user-groups.nix | 105 ++- nix-docker/modules/servers/database/redis.nix | 184 +++++ .../modules/servers/http/apache/default.nix | 641 ++++++++++++++++++ .../http/apache/per-server-options.nix | 150 ++++ nix-docker/modules/servers/supervisord.nix | 80 +++ {src => nix-docker}/package.json | 0 {src => nix-docker}/systemd.nix | 0 src/all-modules.nix | 4 - src/environment.nix | 22 - src/modules/base.nix | 13 - src/users.nix | 27 - 19 files changed, 1255 insertions(+), 125 deletions(-) create mode 100644 nix-docker/all-modules.nix rename {src => nix-docker}/bin/nix-docker (96%) rename {src => nix-docker}/dockerfile.nix (51%) create mode 100644 nix-docker/environment.nix create mode 100644 nix-docker/modules/config/docker.nix create mode 100644 nix-docker/modules/config/environment.nix rename {src/modules => nix-docker/modules/config}/user-groups.nix (66%) create mode 100644 nix-docker/modules/servers/database/redis.nix create mode 100644 nix-docker/modules/servers/http/apache/default.nix create mode 100644 nix-docker/modules/servers/http/apache/per-server-options.nix create mode 100644 nix-docker/modules/servers/supervisord.nix rename {src => nix-docker}/package.json (100%) rename {src => nix-docker}/systemd.nix (100%) delete mode 100644 src/all-modules.nix delete mode 100644 src/environment.nix delete mode 100644 src/modules/base.nix delete mode 100644 src/users.nix diff --git a/configuration.nix b/configuration.nix index cf2b262..0cafadb 100644 --- a/configuration.nix +++ b/configuration.nix @@ -1,10 +1,23 @@ { config, pkgs, ... }: { - docker.ports = [ 1234 ]; + docker.ports = [ 1234 80 ]; - # users.extraUsers.zef = { - # group = "users"; - # home = "/home/zef"; - # createHome = true; - # }; + services.redis = { + enable = true; + port = 1234; + logLevel = "debug"; + }; + + services.httpd.enable = true; + services.httpd.port = 80; + services.httpd.documentRoot = ./www; + services.httpd.adminAddr = "zef.hemel@logicblox.com"; + + supervisord.tailLogs = true; + + users.extraUsers.zef = { + group = "users"; + home = "/home/zef"; + createHome = true; + }; } \ No newline at end of file diff --git a/default.nix b/default.nix index 34c70a5..cb3f1cf 100644 --- a/default.nix +++ b/default.nix @@ -10,7 +10,7 @@ let in nodePackages.buildNodePackage { name = "nix-docker"; - src = [ { outPath = ./src; name = "nix-docker"; } ]; + src = [ { outPath = ./nix-docker; name = "nix-docker"; } ]; deps = with nodePackages; [optimist]; - passthru.names = [ "nix-docker" ]; + passthru.names = [ "nix-docker" ]; } \ No newline at end of file diff --git a/nix-docker/all-modules.nix b/nix-docker/all-modules.nix new file mode 100644 index 0000000..69f059f --- /dev/null +++ b/nix-docker/all-modules.nix @@ -0,0 +1,9 @@ +[ + ./modules/config/docker.nix + ./modules/config/user-groups.nix + ./modules/config/environment.nix + ./modules/servers/supervisord.nix + + ./modules/servers/database/redis.nix + ./modules/servers/http/apache/default.nix +] \ No newline at end of file diff --git a/src/bin/nix-docker b/nix-docker/bin/nix-docker similarity index 96% rename from src/bin/nix-docker rename to nix-docker/bin/nix-docker index 728d7eb..dbbbef2 100644 --- a/src/bin/nix-docker +++ b/nix-docker/bin/nix-docker @@ -37,7 +37,7 @@ function pipeRun(cmd, args, callback) { } function build(nix, configPath, callback) { - var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath, "--arg", "mountBuild", "" + isMountBuild, "--argstr", "imageName", imageName, "--argstr", "baseImage", baseImage, "--show-trace"]); + var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath, "--arg", "mountBuild", "" + isMountBuild, "--argstr", "name", imageName, "--argstr", "baseImage", baseImage, "--show-trace"]); var nixPath; nixBuild.stdout.on('data', function(data) { @@ -110,7 +110,7 @@ function copyClosureAndBuild(dockerFilePath, availablePaths) { console.log("New paths to copy", paths); fs.mkdirSync(tmpNixStore); - execFile("cp", ["-r", "--no-preserve=ownership"].concat(paths).concat([tmpNixStore]), function(err, stdout, stderr) { + execFile("cp", ["-r", "--no-preserve=ownership"].concat(paths).concat([tmpNixStore]), function(err) { if (err) { console.log(err); process.exit(1); diff --git a/src/dockerfile.nix b/nix-docker/dockerfile.nix similarity index 51% rename from src/dockerfile.nix rename to nix-docker/dockerfile.nix index 08e5f6f..fcfaec1 100644 --- a/src/dockerfile.nix +++ b/nix-docker/dockerfile.nix @@ -1,14 +1,11 @@ { pkgs ? import {} +, name , configuration ? -, mountBuild ? false -, imageName ? "docker-build-nix" +, mountBuild ? true , baseImage ? "ubuntu" }: with pkgs.lib; let -# pkgs = import {} -# config = import { modules = [ ./configuration.nix ./src/base.nix ]; } -# config = evalModules { modules = concatLists [ [configuration] (import ./all-modules.nix) ]; args = { inherit pkgs; }; @@ -16,50 +13,30 @@ let localNixPath = pkg: "nix_store/${substring 11 (stringLength pkg.outPath) pkg.outPath}"; - users = import ./users.nix { inherit pkgs config; }; systemd = import ./systemd.nix { inherit pkgs config; }; environment = import ./environment.nix { inherit pkgs config; }; - setupScript = '' - cp ${users.groupFile} /etc/group - cp ${users.passwdFile} /etc/passwd - ${environment.updateEtcScript} - ${environment.setupSystemProfile} - ${systemd.oneShotScript} - ''; - shellScriptFile = pkgs.writeScript "shell" '' - #!/bin/sh - ${setupScriptFile} - /usr/bin/bash - ''; - - setupScriptFile = pkgs.writeScript "setup" '' + buildScript = pkgs.writeScript "build" '' #!/bin/sh -e - ${setupScript} + ${config.config.docker.buildScript} ''; - runScript = pkgs.writeScript "run" '' - #!/bin/sh - ${if mountBuild then setupScript else ""} - ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${systemd.supervisorConfigFile} -n & - mkdir -p /var/log/supervisord - sleep 2 - touch /var/log/supervisord/test.log - tail -n 100 -f /var/log/supervisord/*.log + bootScript = pkgs.writeScript "boot" '' + #!/bin/sh -e + ${if mountBuild then config.config.docker.buildScript else ""} + ${config.config.docker.bootScript} ''; dockerFile = pkgs.writeText "Dockerfile" '' FROM ${baseImage} ${if !mountBuild then '' - ADD nix_store /nix/store - RUN ${setupScriptFile} + ADD nix_store /nix/store + RUN ${buildScript} '' - else "" - } - RUN ln -sf ${shellScriptFile} /bin/shell - CMD ${runScript} + else ""} + CMD ${bootScript} ${ concatMapStrings (port: "EXPOSE ${toString port}\n") config.config.docker.ports } @@ -73,7 +50,7 @@ let OPTIONS="$*" fi - docker run $OPTIONS ${if mountBuild then "-v /nix/store:/nix/store" else ""} ${imageName} + docker run $OPTIONS ${if mountBuild then "-v /nix/store:/nix/store" else ""} ${name} ''; in pkgs.stdenv.mkDerivation { diff --git a/nix-docker/environment.nix b/nix-docker/environment.nix new file mode 100644 index 0000000..d3696cc --- /dev/null +++ b/nix-docker/environment.nix @@ -0,0 +1,13 @@ +{ pkgs +, config }: +with pkgs.lib; +let + # etc = config.config.environment.etc; + systemPackages = config.config.environment.systemPackages; + systemEnv = pkgs.buildEnv { name = "system-env"; paths = systemPackages; }; +in { + setupSystemProfile = '' + rm -rf /usr + ln -s ${systemEnv} /usr + ''; +} \ No newline at end of file diff --git a/nix-docker/modules/config/docker.nix b/nix-docker/modules/config/docker.nix new file mode 100644 index 0000000..be40e48 --- /dev/null +++ b/nix-docker/modules/config/docker.nix @@ -0,0 +1,30 @@ +{ config, pkgs, ... }: +with pkgs.lib; +{ + options = { + docker.ports = mkOption { + default = []; + description = "Ports to expose to the outside world."; + example = [ 80 22 ]; + }; + + docker.buildScripts = mkOption { + default = {}; + example = { + setupUsers = "cp passwd /etc/passwd"; + }; + description = "Scripts (as text) to be run during build, executed alphabetically"; + }; + + docker.bootScript = mkOption { + default = ""; + description = "Script (text) to run when container booted."; + }; + + docker.buildScript = mkOption {}; + }; + + config = { + docker.buildScript = concatStrings (attrValues config.docker.buildScripts); + }; +} \ No newline at end of file diff --git a/nix-docker/modules/config/environment.nix b/nix-docker/modules/config/environment.nix new file mode 100644 index 0000000..0916520 --- /dev/null +++ b/nix-docker/modules/config/environment.nix @@ -0,0 +1,20 @@ +{ config, pkgs, ... }: +with pkgs.lib; +{ + options = { + environment.systemPackages = mkOption { + default = []; + description = "Packages to be put in the system profile."; + }; + }; + + config = { + docker.buildScripts.systemEnv = let + systemPackages = config.environment.systemPackages; + systemEnv = pkgs.buildEnv { name = "system-env"; paths = systemPackages; }; + in '' + rm -rf /usr + ln -s ${systemEnv} /usr + ''; + }; +} \ No newline at end of file diff --git a/src/modules/user-groups.nix b/nix-docker/modules/config/user-groups.nix similarity index 66% rename from src/modules/user-groups.nix rename to nix-docker/modules/config/user-groups.nix index df5da8d..e94e240 100644 --- a/src/modules/user-groups.nix +++ b/nix-docker/modules/config/user-groups.nix @@ -4,9 +4,6 @@ with pkgs.lib; let - ids = config.ids; - users = config.users; - userOpts = { name, config, ... }: { options = { @@ -107,8 +104,8 @@ let config = { name = mkDefault name; - uid = mkDefault (attrByPath [name] null ids.uids); - shell = mkIf config.useDefaultShell (mkDefault users.defaultUserShell); + uid = mkDefault null; + shell = mkDefault "/bin/sh"; }; }; @@ -132,20 +129,37 @@ let config = { name = mkDefault name; - gid = mkDefault (attrByPath [name] null ids.gids); + gid = mkDefault null; }; }; - # Note: the 'X' in front of the password is to distinguish between - # having an empty password, and not having a password. - serializedUser = u: "${u.name}\n${u.description}\n${if u.uid != null then toString u.uid else ""}\n${u.group}\n${toString (concatStringsSep "," u.extraGroups)}\n${u.home}\n${u.shell}\n${toString u.createHome}\n${if u.password != null then "X" + u.password else ""}\n${toString u.isSystemUser}\n${toString u.createUser}\n${toString u.isAlias}\n"; + extraUsers = config.users.extraUsers; + extraGroups = config.users.extraGroups; - usersFile = pkgs.writeText "users" ( - let - p = partition (u: u.isAlias) (attrValues config.users.extraUsers); - in concatStrings (map serializedUser p.wrong ++ map serializedUser p.right)); + uidUsers = listToAttrs + (imap (i: name: + let + user = getAttr name extraUsers; + in { + name=name; + value = if user.uid == null then + setAttr user "uid" (builtins.add 1000 i) + else user; + }) + (attrNames extraUsers)); + gidGroups = listToAttrs + (imap (i: name: + let + group = getAttr name extraGroups; + in { + name=name; + value = if group.gid == null then + setAttr group "gid" (builtins.add 1000 i) + else group; + }) + (attrNames extraGroups)); in { @@ -188,5 +202,70 @@ in options = [ groupOpts ]; }; + users.files = mkOption {}; }; + + config = { + + users.extraUsers = { + root = { + uid = 0; + description = "System administrator"; + home = "/root"; + group = "root"; + }; + nobody = { + uid = 1; + description = "Unprivileged account (don't use!)"; + }; + }; + + users.extraGroups = { + root = { gid = 0; }; + wheel = { }; + disk = { }; + kmem = { }; + tty = { }; + floppy = { }; + uucp = { }; + lp = { }; + cdrom = { }; + tape = { }; + audio = { }; + video = { }; + dialout = { }; + nogroup = { }; + users = { }; + utmp = { }; + adm = { }; + }; + + users.files = { + passwdFile = pkgs.writeText "passwd" '' + ${concatMapStrings (name: + let + user = getAttr name uidUsers; + in + if user.createUser then + "${user.name}:x:${toString user.uid}:${toString (getAttr user.group gidGroups).gid}:${user.description}:${user.home}:${user.shell}\n" + else + "" + ) (attrNames uidUsers)} + ''; + groupFile = pkgs.writeText "group" '' + ${concatMapStrings (name: + let + group = getAttr name gidGroups; + in + "${group.name}:x:${toString group.gid}:\n" + ) (attrNames gidGroups)} + ''; + }; + + docker.buildScripts."0-userfiles" = '' + cp ${config.users.files.groupFile} /etc/group + cp ${config.users.files.passwdFile} /etc/passwd + ''; + }; + } \ No newline at end of file diff --git a/nix-docker/modules/servers/database/redis.nix b/nix-docker/modules/servers/database/redis.nix new file mode 100644 index 0000000..fcc4d01 --- /dev/null +++ b/nix-docker/modules/servers/database/redis.nix @@ -0,0 +1,184 @@ +{ config, pkgs, ... }: + +with pkgs.lib; + +let + cfg = config.services.redis; + redisBool = b: if b then "yes" else "no"; + condOption = name: value: if value != null then "${name} ${toString value}" else ""; + + redisConfig = pkgs.writeText "redis.conf" '' + pidfile ${cfg.pidFile} + port ${toString cfg.port} + loglevel ${cfg.logLevel} + logfile ${cfg.logfile} + databases ${toString cfg.databases} + ${concatMapStrings (d: "save ${toString (builtins.elemAt d 0)} ${toString (builtins.elemAt d 1)}\n") cfg.save} + dbfilename ${cfg.dbFilename} + dir ${toString cfg.dbpath} + ${if cfg.slaveOf != null then "slaveof ${cfg.slaveOf.ip} ${toString cfg.slaveOf.port}" else ""} + ${condOption "masterauth" cfg.masterAuth} + ${condOption "requirepass" cfg.requirePass} + appendOnly ${redisBool cfg.appendOnly} + appendfsync ${cfg.appendFsync} + slowlog-log-slower-than ${toString cfg.slowLogLogSlowerThan} + slowlog-max-len ${toString cfg.slowLogMaxLen} + ${cfg.extraConfig} + ''; +in +{ + + ###### interface + + options = { + + services.redis = { + + enable = mkOption { + default = false; + description = "Whether to enable the Redis server."; + }; + + package = mkOption { + default = pkgs.redis; + description = "Which Redis derivation to use."; + }; + + user = mkOption { + default = "redis"; + description = "User account under which Redis runs"; + }; + + pidFile = mkOption { + default = "/var/lib/redis/redis.pid"; + description = ""; + }; + + port = mkOption { + default = 6379; + description = "The port for Redis to listen to"; + type = with types; int; + }; + + logLevel = mkOption { + default = "notice"; # debug, verbose, notice, warning + example = "debug"; + description = "Specify the server verbosity level, options: debug, verbose, notice, warning"; + type = with types; string; + }; + + logfile = mkOption { + default = "stdout"; + description = "Specify the log file name. Also 'stdout' can be used to force Redis to log on the standard output."; + example = "/var/log/redis.log"; + type = with types; string; + }; + + databases = mkOption { + default = 16; + description = "Set the number of databases."; + type = with types; int; + }; + + save = mkOption { + default = [ [900 1] [300 10] [60 10000] ]; + description = "The schedule in which data is persisted to disk, represented as a list of lists where the first element represent the amount of seconds and the second the number of changes."; + example = [ [900 1] [300 10] [60 10000] ]; + }; + + dbFilename = mkOption { + default = "dump.rdb"; + description = "The filename where to dump the DB"; + type = with types; string; + }; + + dbpath = mkOption { + default = "/var/lib/redis"; + description = "The DB will be written inside this directory, with the filename specified using the 'dbFilename' configuration"; + type = with types; string; + }; + + slaveOf = mkOption { + default = null; # { ip, port } + description = "An attribute set with two attributes: ip and port to which this redis instance acts as a slave"; + example = { ip = "192.168.1.100"; port = 6379; }; + }; + + masterAuth = mkOption { + default = null; + description = ''If the master is password protected (using the requirePass configuration) + it is possible to tell the slave to authenticate before starting the replication synchronization + process, otherwise the master will refuse the slave request. + (STORED PLAIN TEXT, WORLD-READABLE IN NIX STORE)''; + }; + + requirePass = mkOption { + default = null; + description = "Password for database (STORED PLAIN TEXT, WORLD-READABLE IN NIX STORE)"; + example = "letmein!"; + }; + + appendOnly = mkOption { + default = false; + description = "By default data is only periodically persisted to disk, enable this option to use an append-only file for improved persistence."; + type = with types; bool; + }; + + appendOnlyFilename = mkOption { + default = "appendonly.aof"; + description = "Filename for the append-only file (stored inside of dbpath)"; + type = with types; string; + }; + + appendFsync = mkOption { + default = "everysec"; # no, always, everysec + description = "How often to fsync the append-only log, options: no, always, everysec"; + type = with types; string; + }; + + slowLogLogSlowerThan = mkOption { + default = 10000; + description = "Log queries whose execution take longer than X in milliseconds"; + example = 1000; + type = with types; int; + }; + + slowLogMaxLen = mkOption { + default = 128; + description = "Maximum number of items to keep in slow log"; + type = with types; int; + }; + + extraConfig = mkOption { + default = ""; + description = "Extra configuration options for redis.conf"; + type = with types; string; + }; + }; + + }; + + + ###### implementation + + config = mkIf config.services.redis.enable { + + users.extraUsers.redis = + { name = cfg.user; + description = "Redis database user"; + }; + + environment.systemPackages = [ cfg.package ]; + + docker.buildScripts.redisInit = '' + if ! test -e ${cfg.dbpath}; then + install -d -m0700 -o ${cfg.user} ${cfg.dbpath} + fi + ''; + + supervisord.services.redis = + { user = cfg.user; + command = "${cfg.package}/bin/redis-server ${redisConfig}"; + }; + }; +} diff --git a/nix-docker/modules/servers/http/apache/default.nix b/nix-docker/modules/servers/http/apache/default.nix new file mode 100644 index 0000000..ba2dc39 --- /dev/null +++ b/nix-docker/modules/servers/http/apache/default.nix @@ -0,0 +1,641 @@ +{ config, pkgs, ... }: + +with pkgs.lib; + +let + + mainCfg = config.services.httpd; + + httpd = mainCfg.package; + + version24 = !versionOlder httpd.version "2.4"; + + httpdConf = mainCfg.configFile; + + php = pkgs.php.override { apacheHttpd = httpd; }; + + getPort = cfg: if cfg.port != 0 then cfg.port else if cfg.enableSSL then 443 else 80; + + extraModules = attrByPath ["extraModules"] [] mainCfg; + extraForeignModules = filter builtins.isAttrs extraModules; + extraApacheModules = filter (x: !(builtins.isAttrs x)) extraModules; # I'd prefer using builtins.isString here, but doesn't exist yet + + + makeServerInfo = cfg: { + # Canonical name must not include a trailing slash. + canonicalName = + (if cfg.enableSSL then "https" else "http") + "://" + + cfg.hostName + + (if getPort cfg != (if cfg.enableSSL then 443 else 80) then ":${toString (getPort cfg)}" else ""); + + # Admin address: inherit from the main server if not specified for + # a virtual host. + adminAddr = if cfg.adminAddr != null then cfg.adminAddr else mainCfg.adminAddr; + + vhostConfig = cfg; + serverConfig = mainCfg; + fullConfig = config; # machine config + }; + + + allHosts = [mainCfg] ++ mainCfg.virtualHosts; + + + callSubservices = serverInfo: defs: + let f = svc: + let + svcFunction = + if svc ? function then svc.function + else import (toString "${toString ./.}/${if svc ? serviceType then svc.serviceType else svc.serviceName}.nix"); + config = (evalModules + { modules = [ { options = res.options; config = svc.config or svc; } ]; + check = false; + }).config; + defaults = { + extraConfig = ""; + extraModules = []; + extraModulesPre = []; + extraPath = []; + extraServerPath = []; + globalEnvVars = []; + robotsEntries = ""; + startupScript = ""; + enablePHP = false; + phpOptions = ""; + options = {}; + }; + res = defaults // svcFunction { inherit config pkgs serverInfo php; }; + in res; + in map f defs; + + + # !!! callSubservices is expensive + subservicesFor = cfg: callSubservices (makeServerInfo cfg) cfg.extraSubservices; + + mainSubservices = subservicesFor mainCfg; + + allSubservices = mainSubservices ++ concatMap subservicesFor mainCfg.virtualHosts; + + + # !!! should be in lib + writeTextInDir = name: text: + pkgs.runCommand name {inherit text;} "ensureDir $out; echo -n \"$text\" > $out/$name"; + + + enableSSL = any (vhost: vhost.enableSSL) allHosts; + + + # Names of modules from ${httpd}/modules that we want to load. + apacheModules = + [ # HTTP authentication mechanisms: basic and digest. + "auth_basic" "auth_digest" + + # Authentication: is the user who he claims to be? + "authn_file" "authn_dbm" "authn_anon" + (if version24 then "authn_core" else "authn_alias") + + # Authorization: is the user allowed access? + "authz_user" "authz_groupfile" "authz_host" + + # Other modules. + "ext_filter" "include" "log_config" "env" "mime_magic" + "cern_meta" "expires" "headers" "usertrack" /* "unique_id" */ "setenvif" + "mime" "dav" "status" "autoindex" "asis" "info" "dav_fs" + "vhost_alias" "negotiation" "dir" "imagemap" "actions" "speling" + "userdir" "alias" "rewrite" "proxy" "proxy_http" + ] + ++ optionals version24 [ + "mpm_${mainCfg.multiProcessingModule}" + "authz_core" + "unixd" + ] + ++ (if mainCfg.multiProcessingModule == "prefork" then [ "cgi" ] else [ "cgid" ]) + ++ optional enableSSL "ssl" + ++ extraApacheModules; + + + allDenied = if version24 then '' + Require all denied + '' else '' + Order deny,allow + Deny from all + ''; + + allGranted = if version24 then '' + Require all granted + '' else '' + Order allow,deny + Allow from all + ''; + + + loggingConf = '' + ErrorLog ${mainCfg.logDir}/error_log + + LogLevel notice + + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + LogFormat "%{Referer}i -> %U" referer + LogFormat "%{User-agent}i" agent + + CustomLog ${mainCfg.logDir}/access_log ${mainCfg.logFormat} + ''; + + + browserHacks = '' + BrowserMatch "Mozilla/2" nokeepalive + BrowserMatch "MSIE 4\.0b2;" nokeepalive downgrade-1.0 force-response-1.0 + BrowserMatch "RealPlayer 4\.0" force-response-1.0 + BrowserMatch "Java/1\.0" force-response-1.0 + BrowserMatch "JDK/1\.0" force-response-1.0 + BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully + BrowserMatch "^WebDrive" redirect-carefully + BrowserMatch "^WebDAVFS/1.[012]" redirect-carefully + BrowserMatch "^gnome-vfs" redirect-carefully + ''; + + + sslConf = '' + SSLSessionCache shm:${mainCfg.stateDir}/ssl_scache(512000) + + SSLMutex posixsem + + SSLRandomSeed startup builtin + SSLRandomSeed connect builtin + ''; + + + mimeConf = '' + TypesConfig ${httpd}/conf/mime.types + + AddType application/x-x509-ca-cert .crt + AddType application/x-pkcs7-crl .crl + AddType application/x-httpd-php .php .phtml + + + MIMEMagicFile ${httpd}/conf/magic + + + AddEncoding x-compress Z + AddEncoding x-gzip gz tgz + ''; + + + perServerConf = isMainServer: cfg: let + + serverInfo = makeServerInfo cfg; + + subservices = callSubservices serverInfo cfg.extraSubservices; + + documentRoot = if cfg.documentRoot != null then cfg.documentRoot else + pkgs.runCommand "empty" {} "ensureDir $out"; + + documentRootConf = '' + DocumentRoot "${documentRoot}" + + + Options Indexes FollowSymLinks + AllowOverride None + ${allGranted} + + ''; + + robotsTxt = pkgs.writeText "robots.txt" '' + ${# If this is a vhost, the include the entries for the main server as well. + if isMainServer then "" + else concatMapStrings (svc: svc.robotsEntries) mainSubservices} + ${concatMapStrings (svc: svc.robotsEntries) subservices} + ''; + + robotsConf = '' + Alias /robots.txt ${robotsTxt} + ''; + + in '' + ServerName ${serverInfo.canonicalName} + + ${concatMapStrings (alias: "ServerAlias ${alias}\n") cfg.serverAliases} + + ${if cfg.sslServerCert != null then '' + SSLCertificateFile ${cfg.sslServerCert} + SSLCertificateKeyFile ${cfg.sslServerKey} + '' else ""} + + ${if cfg.enableSSL then '' + SSLEngine on + '' else if enableSSL then /* i.e., SSL is enabled for some host, but not this one */ + '' + SSLEngine off + '' else ""} + + ${if isMainServer || cfg.adminAddr != null then '' + ServerAdmin ${cfg.adminAddr} + '' else ""} + + ${if !isMainServer && mainCfg.logPerVirtualHost then '' + ErrorLog ${mainCfg.logDir}/error_log-${cfg.hostName} + CustomLog ${mainCfg.logDir}/access_log-${cfg.hostName} ${cfg.logFormat} + '' else ""} + + ${robotsConf} + + ${if isMainServer || cfg.documentRoot != null then documentRootConf else ""} + + ${if cfg.enableUserDir then '' + + UserDir public_html + UserDir disabled root + + + AllowOverride FileInfo AuthConfig Limit Indexes + Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec + + ${allGranted} + + + ${allDenied} + + + + '' else ""} + + ${if cfg.globalRedirect != null then '' + RedirectPermanent / ${cfg.globalRedirect} + '' else ""} + + ${ + let makeFileConf = elem: '' + Alias ${elem.urlPath} ${elem.file} + ''; + in concatMapStrings makeFileConf cfg.servedFiles + } + + ${ + let makeDirConf = elem: '' + Alias ${elem.urlPath} ${elem.dir}/ + + Options +Indexes + ${allGranted} + AllowOverride All + + ''; + in concatMapStrings makeDirConf cfg.servedDirs + } + + ${concatMapStrings (svc: svc.extraConfig) subservices} + + ${cfg.extraConfig} + ''; + + + confFile = pkgs.writeText "httpd.conf" '' + + ServerRoot ${httpd} + + ${optionalString version24 '' + DefaultRuntimeDir ${mainCfg.stateDir}/runtime + ''} + + PidFile ${mainCfg.stateDir}/httpd.pid + + ${optionalString (mainCfg.multiProcessingModule != "prefork") '' + # mod_cgid requires this. + ScriptSock ${mainCfg.stateDir}/cgisock + ''} + + + MaxClients ${toString mainCfg.maxClients} + MaxRequestsPerChild ${toString mainCfg.maxRequestsPerChild} + + + ${let + ports = map getPort allHosts; + uniquePorts = uniqList {inputList = ports;}; + in concatMapStrings (port: "Listen ${toString port}\n") uniquePorts + } + + User ${mainCfg.user} + Group ${mainCfg.group} + + ${let + load = {name, path}: "LoadModule ${name}_module ${path}\n"; + allModules = + concatMap (svc: svc.extraModulesPre) allSubservices + ++ map (name: {inherit name; path = "${httpd}/modules/mod_${name}.so";}) apacheModules + ++ optional enablePHP { name = "php5"; path = "${php}/modules/libphp5.so"; } + ++ concatMap (svc: svc.extraModules) allSubservices + ++ extraForeignModules; + in concatMapStrings load allModules + } + + AddHandler type-map var + + + ${allDenied} + + + ${mimeConf} + ${loggingConf} + ${browserHacks} + + Include ${httpd}/conf/extra/httpd-default.conf + Include ${httpd}/conf/extra/httpd-autoindex.conf + Include ${httpd}/conf/extra/httpd-multilang-errordoc.conf + Include ${httpd}/conf/extra/httpd-languages.conf + + ${if enableSSL then sslConf else ""} + + # Fascist default - deny access to everything. + + Options FollowSymLinks + AllowOverride None + ${allDenied} + + + # But do allow access to files in the store so that we don't have + # to generate clauses for every generated file that we + # want to serve. + + ${allGranted} + + + # Generate directives for the main server. + ${perServerConf true mainCfg} + + # Always enable virtual hosts; it doesn't seem to hurt. + ${let + ports = map getPort allHosts; + uniquePorts = uniqList {inputList = ports;}; + directives = concatMapStrings (port: "NameVirtualHost *:${toString port}\n") uniquePorts; + in optionalString (!version24) directives + } + + ${let + makeVirtualHost = vhost: '' + + ${perServerConf false vhost} + + ''; + in concatMapStrings makeVirtualHost mainCfg.virtualHosts + } + ''; + + + enablePHP = any (svc: svc.enablePHP) allSubservices; + + + # Generate the PHP configuration file. Should probably be factored + # out into a separate module. + phpIni = pkgs.runCommand "php.ini" + { options = concatStringsSep "\n" + ([ mainCfg.phpOptions ] ++ (map (svc: svc.phpOptions) allSubservices)); + } + '' + cat ${php}/etc/php-recommended.ini > $out + echo "$options" >> $out + ''; + +in + + +{ + + ###### interface + + options = { + + services.httpd = { + + enable = mkOption { + type = types.bool; + default = false; + description = "Whether to enable the Apache HTTP Server."; + }; + + package = mkOption { + type = types.path; + default = pkgs.apacheHttpd.override { mpm = mainCfg.multiProcessingModule; }; + example = "pkgs.apacheHttpd_2_4"; + description = '' + Overridable attribute of the Apache HTTP Server package to use. + ''; + }; + + configFile = mkOption { + type = types.path; + default = confFile; + example = literalExample ''pkgs.writeText "httpd.conf" "# my custom config file ...";''; + description = '' + Override the configuration file used by Apache. By default, + NixOS generates one automatically. + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = '' + Cnfiguration lines appended to the generated Apache + configuration file. Note that this mechanism may not work + when is overridden. + ''; + }; + + extraModules = mkOption { + type = types.listOf types.unspecified; + default = []; + example = literalExample ''[ "proxy_connect" { name = "php5"; path = "''${php}/modules/libphp5.so"; } ]''; + description = '' + Additional Apache modules to be used. These can be + specified as a string in the case of modules distributed + with Apache, or as an attribute set specifying the + name and path of the + module. + ''; + }; + + logPerVirtualHost = mkOption { + type = types.bool; + default = false; + description = '' + If enabled, each virtual host gets its own + access_log and + error_log, namely suffixed by the + of the virtual host. + ''; + }; + + user = mkOption { + type = types.str; + default = "wwwrun"; + description = '' + User account under which httpd runs. The account is created + automatically if it doesn't exist. + ''; + }; + + group = mkOption { + type = types.str; + default = "wwwrun"; + description = '' + Group under which httpd runs. The account is created + automatically if it doesn't exist. + ''; + }; + + logDir = mkOption { + type = types.path; + default = "/var/log/httpd"; + description = '' + Directory for Apache's log files. It is created automatically. + ''; + }; + + stateDir = mkOption { + type = types.path; + default = "/run/httpd"; + description = '' + Directory for Apache's transient runtime state (such as PID + files). It is created automatically. Note that the default, + /run/httpd, is deleted at boot time. + ''; + }; + + virtualHosts = mkOption { + type = types.listOf (types.submodule ( + { options = import ./per-server-options.nix { + inherit pkgs; + forMainServer = false; + }; + })); + default = []; + example = [ + { hostName = "foo"; + documentRoot = "/data/webroot-foo"; + } + { hostName = "bar"; + documentRoot = "/data/webroot-bar"; + } + ]; + description = '' + Specification of the virtual hosts served by Apache. Each + element should be an attribute set specifying the + configuration of the virtual host. The available options + are the non-global options permissible for the main host. + ''; + }; + + phpOptions = mkOption { + type = types.lines; + default = ""; + example = + '' + date.timezone = "CET" + ''; + description = + "Options appended to the PHP configuration file php.ini."; + }; + + multiProcessingModule = mkOption { + type = types.str; + default = "prefork"; + example = "worker"; + description = + '' + Multi-processing module to be used by Apache. Available + modules are prefork (the default; + handles each request in a separate child process), + worker (hybrid approach that starts a + number of child processes each running a number of + threads) and event (a recent variant of + worker that handles persistent + connections more efficiently). + ''; + }; + + maxClients = mkOption { + type = types.int; + default = 150; + example = 8; + description = "Maximum number of httpd processes (prefork)"; + }; + + maxRequestsPerChild = mkOption { + type = types.int; + default = 0; + example = 500; + description = + "Maximum number of httpd requests answered per httpd child (prefork), 0 means unlimited"; + }; + } + + # Include the options shared between the main server and virtual hosts. + // (import ./per-server-options.nix { + inherit pkgs; + forMainServer = true; + }); + + }; + + + ###### implementation + + config = mkIf config.services.httpd.enable { + + users.extraUsers.wwwrun = + { name = "wwwrun"; + group = "wwwrun"; + description = "Apache httpd user"; + }; + + users.extraGroups.wwwrun = {}; + + environment.systemPackages = [httpd] ++ concatMap (svc: svc.extraPath) allSubservices; + + services.httpd.phpOptions = + '' + ; Needed for PHP's mail() function. + sendmail_path = sendmail -t -i + + ; Apparently PHP doesn't use $TZ. + date.timezone = "${config.time.timeZone}" + ''; + + docker.buildScripts.apache = + '' + mkdir -m 0750 -p ${mainCfg.stateDir} + chown root.${mainCfg.group} ${mainCfg.stateDir} + ${optionalString version24 '' + mkdir -m 0750 -p "${mainCfg.stateDir}/runtime" + chown root.${mainCfg.group} "${mainCfg.stateDir}/runtime" + ''} + mkdir -m 0700 -p ${mainCfg.logDir} + + ${optionalString (mainCfg.documentRoot != null) + '' + # Create the document root directory if does not exists yet + mkdir -p ${mainCfg.documentRoot} + '' + } + + # Get rid of old semaphores. These tend to accumulate across + # server restarts, eventually preventing it from restarting + # successfully. + for i in $(${pkgs.utillinux}/bin/ipcs -s | grep ' ${mainCfg.user} ' | cut -f2 -d ' '); do + ${pkgs.utillinux}/bin/ipcrm -s $i + done + + # Run the startup hooks for the subservices. + for i in ${toString (map (svn: svn.startupScript) allSubservices)}; do + echo Running Apache startup hook $i... + $i + done + ''; + + supervisord.services.httpd = + { + command = "${httpd}/bin/httpd -f ${httpdConf} -DFOREGROUND"; + }; + }; +} diff --git a/nix-docker/modules/servers/http/apache/per-server-options.nix b/nix-docker/modules/servers/http/apache/per-server-options.nix new file mode 100644 index 0000000..53f34e2 --- /dev/null +++ b/nix-docker/modules/servers/http/apache/per-server-options.nix @@ -0,0 +1,150 @@ +# This file defines the options that can be used both for the Apache +# main server configuration, and for the virtual hosts. (The latter +# has additional options that affect the web server as a whole, like +# the user/group to run under.) + +{ forMainServer, pkgs }: + +with pkgs.lib; + +{ + + hostName = mkOption { + type = types.str; + default = "localhost"; + description = "Canonical hostname for the server."; + }; + + serverAliases = mkOption { + type = types.listOf types.str; + default = []; + example = ["www.example.org" "www.example.org:8080" "example.org"]; + description = '' + Additional names of virtual hosts served by this virtual host configuration. + ''; + }; + + port = mkOption { + type = types.int; + default = 0; + description = '' + Port for the server. 0 means use the default port: 80 for http + and 443 for https (i.e. when enableSSL is set). + ''; + }; + + enableSSL = mkOption { + type = types.bool; + default = false; + description = "Whether to enable SSL (https) support."; + }; + + # Note: sslServerCert and sslServerKey can be left empty, but this + # only makes sense for virtual hosts (they will inherit from the + # main server). + + sslServerCert = mkOption { + type = types.nullOr types.path; + default = null; + example = "/var/host.cert"; + description = "Path to server SSL certificate."; + }; + + sslServerKey = mkOption { + type = types.path; + example = "/var/host.key"; + description = "Path to server SSL certificate key."; + }; + + adminAddr = mkOption ({ + type = types.nullOr types.str; + example = "admin@example.org"; + description = "E-mail address of the server administrator."; + } // (if forMainServer then {} else {default = null;})); + + documentRoot = mkOption { + type = types.nullOr types.path; + default = null; + example = "/data/webserver/docs"; + description = '' + The path of Apache's document root directory. If left undefined, + an empty directory in the Nix store will be used as root. + ''; + }; + + servedDirs = mkOption { + type = types.listOf types.attrs; + default = []; + example = [ + { urlPath = "/nix"; + dir = "/home/eelco/Dev/nix-homepage"; + } + ]; + description = '' + This option provides a simple way to serve static directories. + ''; + }; + + servedFiles = mkOption { + type = types.listOf types.attrs; + default = []; + example = [ + { urlPath = "/foo/bar.png"; + dir = "/home/eelco/some-file.png"; + } + ]; + description = '' + This option provides a simple way to serve individual, static files. + ''; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + example = '' + + Options FollowSymlinks + AllowOverride All + + ''; + description = '' + These lines go to httpd.conf verbatim. They will go after + directories and directory aliases defined by default. + ''; + }; + + extraSubservices = mkOption { + type = types.listOf types.unspecified; + default = []; + description = "Extra subservices to enable in the webserver."; + }; + + enableUserDir = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable serving ~/public_html as + /~username. + ''; + }; + + globalRedirect = mkOption { + type = types.nullOr types.str; + default = null; + example = http://newserver.example.org/; + description = '' + If set, all requests for this host are redirected permanently to + the given URL. + ''; + }; + + logFormat = mkOption { + type = types.str; + default = "common"; + example = "combined"; + description = " + Log format for Apache's log files. Possible values are: combined, common, referer, agent. + "; + }; + +} diff --git a/nix-docker/modules/servers/supervisord.nix b/nix-docker/modules/servers/supervisord.nix new file mode 100644 index 0000000..89bc3ee --- /dev/null +++ b/nix-docker/modules/servers/supervisord.nix @@ -0,0 +1,80 @@ +{ config, pkgs, ... }: +with pkgs.lib; +let + serviceOpts = { name, config, ...}: { + options = { + command = mkOption { + description = "The command to execute"; + }; + directory = mkOption { + default = "/"; + description = "Current directory when running the command"; + }; + user = mkOption { + default = "root"; + description = "The user to run the command as"; + }; + }; + }; + services = config.supervisord.services; +in { + options = { + supervisord = { + enable = mkOption { + default = true; + type = types.bool; + }; + + services = mkOption { + default = {}; + type = types.loaOf types.optionSet; + description = '' + Supervisord services to start + ''; + options = [ serviceOpts ]; + }; + + tailLogs = mkOption { + default = false; + type = types.bool; + description = '' + Whether or not to tail all logs to standard out. + ''; + }; + + configFile = mkOption {}; + }; + }; + + config = mkIf config.supervisord.enable { + supervisord.configFile = pkgs.writeText "supervisord.conf" '' + [supervisord] + logfile=/var/log/supervisord/supervisord.log + + ${concatMapStrings (name: + let + cfg = getAttr name services; + in + '' + [program:${name}] + command=${cfg.command} + directory=${cfg.directory} + redirect_stderr=true + stdout_logfile=/var/log/supervisord/${name}.log + user=${cfg.user} + '' + ) (attrNames services) + } + ''; + + docker.bootScript = '' + mkdir -p /var/log/supervisord + ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${config.supervisord.configFile} ${if config.supervisord.tailLogs then '' + + sleep 2 + touch /var/log/supervisord/test.log + tail -n 100 -f /var/log/supervisord/*.log + '' else "-n"} + ''; + }; +} \ No newline at end of file diff --git a/src/package.json b/nix-docker/package.json similarity index 100% rename from src/package.json rename to nix-docker/package.json diff --git a/src/systemd.nix b/nix-docker/systemd.nix similarity index 100% rename from src/systemd.nix rename to nix-docker/systemd.nix diff --git a/src/all-modules.nix b/src/all-modules.nix deleted file mode 100644 index dea70d8..0000000 --- a/src/all-modules.nix +++ /dev/null @@ -1,4 +0,0 @@ -[ - ./modules/base.nix - ./modules/user-groups.nix -] \ No newline at end of file diff --git a/src/environment.nix b/src/environment.nix deleted file mode 100644 index 4a06463..0000000 --- a/src/environment.nix +++ /dev/null @@ -1,22 +0,0 @@ -{ pkgs -, config }: -with pkgs.lib; -let - etc = config.config.environment.etc; - systemPackages = config.config.environment.systemPackages; -in { - updateEtcScript = concatMapStrings (filename: - let - file = getAttr filename etc; - in - '' - mkdir -p /etc/${dirOf "/${file.target}"} - ln -s ${file.source} ${file.target} - '' - ) (attrNames etc); - - setupSystemProfile = '' - rm -rf /usr - ln -s ${config.config.system.path} /usr - ''; -} \ No newline at end of file diff --git a/src/modules/base.nix b/src/modules/base.nix deleted file mode 100644 index 402b835..0000000 --- a/src/modules/base.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ config, pkgs, ... }: -with pkgs.lib; -{ - options = { - docker = { - ports = mkOption { - default = []; - description = "Ports to expose to the outside world."; - example = [ 80 22 ]; - }; - }; - }; -} \ No newline at end of file diff --git a/src/users.nix b/src/users.nix deleted file mode 100644 index 8f240b1..0000000 --- a/src/users.nix +++ /dev/null @@ -1,27 +0,0 @@ -{ pkgs -, config }: -with pkgs.lib; -let - extraUsers = config.config.users.extraUsers; - extraGroups = config.config.users.extraGroups; -in { - passwdFile = pkgs.writeText "passwd" '' - ${concatMapStrings (name: - let - user = getAttr name extraUsers; - in - if user.createUser then - "${user.name}:x:${toString user.uid}:${toString (getAttr user.group extraGroups).gid}:Description:${user.home}:${user.shell}\n" - else - "" - ) (attrNames extraUsers)} - ''; - groupFile = pkgs.writeText "group" '' - ${concatMapStrings (name: - let - group = getAttr name extraGroups; - in - "${group.name}:x:${toString group.gid}:\n" - ) (attrNames extraGroups)} - ''; -} \ No newline at end of file From 20c4115a21757b5c533a3d159844e4c768de3d99 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Mon, 4 Nov 2013 07:21:26 -0800 Subject: [PATCH 07/46] This is getting really awesome. --- base-configuration.nix | 2 +- configuration.nix | 8 +- nix-docker/all-modules.nix | 11 +- nix-docker/bin/nix-docker | 33 +- nix-docker/dockerfile.nix | 25 +- nix-docker/environment.nix | 13 - nix-docker/modules/config/docker.nix | 13 + nix-docker/modules/config/environment.nix | 32 +- nix-docker/modules/config/user-groups.nix | 1 + nix-docker/modules/servers/database/redis.nix | 184 ---------- nix-docker/modules/servers/openssh.nix | 333 ++++++++++++++++++ nix-docker/modules/servers/supervisord.nix | 9 +- nix-docker/systemd.nix | 67 ---- 13 files changed, 439 insertions(+), 292 deletions(-) delete mode 100644 nix-docker/environment.nix delete mode 100644 nix-docker/modules/servers/database/redis.nix create mode 100644 nix-docker/modules/servers/openssh.nix delete mode 100644 nix-docker/systemd.nix diff --git a/base-configuration.nix b/base-configuration.nix index 17ef878..63204d9 100644 --- a/base-configuration.nix +++ b/base-configuration.nix @@ -1,4 +1,4 @@ { config, pkgs, ... }: { - + services.mysql.enable = true; } \ No newline at end of file diff --git a/configuration.nix b/configuration.nix index 0cafadb..6d56a41 100644 --- a/configuration.nix +++ b/configuration.nix @@ -1,6 +1,7 @@ { config, pkgs, ... }: { docker.ports = [ 1234 80 ]; + docker.verbose = true; services.redis = { enable = true; @@ -13,11 +14,16 @@ services.httpd.documentRoot = ./www; services.httpd.adminAddr = "zef.hemel@logicblox.com"; + services.mysql.enable = true; + services.openssh.enable = true; + supervisord.tailLogs = true; users.extraUsers.zef = { group = "users"; - home = "/home/zef"; + home = "/"; + shell = "/bin/bash"; createHome = true; + openssh.authorizedKeys.keys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCjWpdyDIsS09lWlOsMG9OMTHB/N/afVU12BwKcyjjhbezPdFEgHK4cZBN7m1bvoFKl832BdB+ZjeRH4UGBcUpvrFu1vE7Lf/0vZDU7qzzWQE9V+tfSPwDiXPf9QnCYeZmYPDHUHDUEse9LKBZbt6UKF1tuTD8ussV5jvEFBaesDhCqD1TJ4b4O877cdx9+VTOuDSEDm32jQ2az27d1b/5DoEKBe5cJSC3PhObAQ7OAYrVVBFX9ffKpaSvV6yqo+rhCmXP9DjNgBwMtElreoXL3h5Xbw2AiER5oHNUAEA2XGpnOVOr7ZZUAbMC0/0dq387jQZCqe7gIDZCqjDpGhUa9 zefhemel@gmail.com" ]; }; } \ No newline at end of file diff --git a/nix-docker/all-modules.nix b/nix-docker/all-modules.nix index 69f059f..a4060ca 100644 --- a/nix-docker/all-modules.nix +++ b/nix-docker/all-modules.nix @@ -4,6 +4,15 @@ ./modules/config/environment.nix ./modules/servers/supervisord.nix - ./modules/servers/database/redis.nix + ./modules/shim/systemd.nix + + + + + + + + # These modules needed some patching to work well ./modules/servers/http/apache/default.nix + ./modules/servers/openssh.nix ] \ No newline at end of file diff --git a/nix-docker/bin/nix-docker b/nix-docker/bin/nix-docker index dbbbef2..e3ed3c3 100644 --- a/nix-docker/bin/nix-docker +++ b/nix-docker/bin/nix-docker @@ -9,17 +9,16 @@ var rootPath = path.resolve(fs.realpathSync(process.argv[1]), "../.."); var spawn = require('child_process').spawn; var execFile = require('child_process').execFile; var configFile = argv.c || "configuration.nix"; -var isMountBuild = !! argv["mount-build"]; +var fullBuild = !!argv.b; -var baseImage = argv.from || "zefhemel/base-nix"; +var baseImage = argv.from = "zefhemel/base-nix"; var imageName = argv.t ? argv.t : "nix-docker-build"; var tmpNixStore = "nix_store"; if (argv.help) { - console.log("Usage: nix-docker [--dockerfile-only] [--mount-build] [-t imagename] [-c configuration.nix] "); - console.log(" --docker-file-only: generate Docker file, but do not build it."); - console.log(" --mount-build: don't add /nix paths to container, but let them be mounted from host at run-time with -v."); + console.log("Usage: nix-docker [-b] [-t imagename] [-c configuration.nix] "); + console.log(" -b: build full (portable) docker image."); console.log(" -t: name of the image to build."); console.log(" -c: path to configuration file to build."); process.exit(0); @@ -37,7 +36,7 @@ function pipeRun(cmd, args, callback) { } function build(nix, configPath, callback) { - var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath, "--arg", "mountBuild", "" + isMountBuild, "--argstr", "name", imageName, "--argstr", "baseImage", baseImage, "--show-trace"]); + var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath, "--arg", "mountBuild", "" + !fullBuild, "--argstr", "name", imageName, "--argstr", "baseImage", baseImage, "--show-trace"]); var nixPath; nixBuild.stdout.on('data', function(data) { @@ -55,6 +54,9 @@ function build(nix, configPath, callback) { } function cleanup(callback) { + if(!fullBuild) { + return callback(); + } if (!fs.existsSync(tmpNixStore)) { return callback(); } @@ -96,12 +98,7 @@ function copyClosureAndBuild(dockerFilePath, availablePaths) { return console.error(err); } - if (isMountBuild) { - buildImage(dockerFilePath, function(code) { - console.log("To run: sudo docker run -t -i -v /nix/store:/nix/store " + imageName); - process.exit(code); - }); - } else { + if (fullBuild) { var paths = stdout.trim().split("\n"); paths = paths.filter(function(path) { return availablePaths.indexOf(path.substring("/nix/store/".length)) === -1; @@ -120,6 +117,11 @@ function copyClosureAndBuild(dockerFilePath, availablePaths) { process.exit(code); }); }); + } else { + buildImage(dockerFilePath, function(code) { + console.log("To run: sudo docker run -t -i -v /nix/store:/nix/store " + imageName); + process.exit(code); + }); } }); } @@ -129,12 +131,13 @@ cleanup(function() { if(code) { process.exit(code); } - if (isMountBuild) { - copyClosureAndBuild(dockerFilePath, []); - } else { + if (fullBuild) { getAvailableNixPaths(function(err, availablePaths) { copyClosureAndBuild(dockerFilePath, availablePaths); }); + } else { + console.log("Result in", dockerFilePath, "test with sudo ./result/sbin/docker-run"); + console.log("To deploy: nix-copy-closure " + dockerFilePath + " machine && ssh root@machine" + dockerFilePath + "/sbin/docker-run") } }); }); \ No newline at end of file diff --git a/nix-docker/dockerfile.nix b/nix-docker/dockerfile.nix index fcfaec1..eb9e184 100644 --- a/nix-docker/dockerfile.nix +++ b/nix-docker/dockerfile.nix @@ -16,20 +16,22 @@ let systemd = import ./systemd.nix { inherit pkgs config; }; environment = import ./environment.nix { inherit pkgs config; }; + verboseFlag = if config.config.docker.verbose then "v" else ""; + buildScript = pkgs.writeScript "build" '' - #!/bin/sh -e + #!/bin/sh -e${verboseFlag} ${config.config.docker.buildScript} ''; bootScript = pkgs.writeScript "boot" '' - #!/bin/sh -e + #!/bin/sh -e${verboseFlag} ${if mountBuild then config.config.docker.buildScript else ""} ${config.config.docker.bootScript} ''; dockerFile = pkgs.writeText "Dockerfile" '' - FROM ${baseImage} + FROM ${if mountBuild then "busybox" else baseImage} ${if !mountBuild then '' ADD nix_store /nix/store @@ -42,26 +44,35 @@ let } ''; + imageHash = substring 11 8 dockerFile.outPath; + runContainerScript = pkgs.writeScript "docker-run" '' #!/usr/bin/env bash + if [ "" == "$(docker images | grep -E "${name}\s*${imageHash}")" ]; then + docker build -t ${name}:${imageHash} $(dirname $0)/.. + fi + OPTIONS="-t -i $*" if [ "$1" == "-d" ]; then OPTIONS="$*" fi - docker run $OPTIONS ${if mountBuild then "-v /nix/store:/nix/store" else ""} ${name} + docker run $OPTIONS ${if mountBuild then "-v /nix/store:/nix/store" else ""} ${name}:${imageHash} ''; in pkgs.stdenv.mkDerivation { - name = "dockerfile"; + name = replaceChars ["/"] ["-"] name; src = ./.; phases = [ "installPhase" ]; installPhase = '' - mkdir -p $out/bin - cp ${runContainerScript} $out/bin/run-container + mkdir -p $out + ${if mountBuild then '' + mkdir -p $out/sbin + cp ${runContainerScript} $out/sbin/docker-run + '' else ""} cp ${dockerFile} $out/Dockerfile ''; } \ No newline at end of file diff --git a/nix-docker/environment.nix b/nix-docker/environment.nix deleted file mode 100644 index d3696cc..0000000 --- a/nix-docker/environment.nix +++ /dev/null @@ -1,13 +0,0 @@ -{ pkgs -, config }: -with pkgs.lib; -let - # etc = config.config.environment.etc; - systemPackages = config.config.environment.systemPackages; - systemEnv = pkgs.buildEnv { name = "system-env"; paths = systemPackages; }; -in { - setupSystemProfile = '' - rm -rf /usr - ln -s ${systemEnv} /usr - ''; -} \ No newline at end of file diff --git a/nix-docker/modules/config/docker.nix b/nix-docker/modules/config/docker.nix index be40e48..73e8ff8 100644 --- a/nix-docker/modules/config/docker.nix +++ b/nix-docker/modules/config/docker.nix @@ -22,9 +22,22 @@ with pkgs.lib; }; docker.buildScript = mkOption {}; + + docker.verbose = mkOption { + default = false; + type = types.bool; + }; + + networking = mkOption {}; + security = mkOption {}; + #system.nssModules.path = mkOption {}; + #services.samba = mkOption{}; }; config = { docker.buildScript = concatStrings (attrValues config.docker.buildScripts); + networking.enableIPv6 = false; + #system.nssModules.path = ""; + #services.samba.syncPasswordsByPam = false; }; } \ No newline at end of file diff --git a/nix-docker/modules/config/environment.nix b/nix-docker/modules/config/environment.nix index 0916520..ac705eb 100644 --- a/nix-docker/modules/config/environment.nix +++ b/nix-docker/modules/config/environment.nix @@ -1,20 +1,48 @@ { config, pkgs, ... }: with pkgs.lib; -{ +let + etc2 = filter (f: f.enable) (attrValues config.environment.etc); + + etc = pkgs.stdenv.mkDerivation { + name = "etc"; + + builder = ; + + preferLocalBuild = true; + + /* !!! Use toXML. */ + sources = map (x: x.source) etc2; + targets = map (x: x.target) etc2; + modes = map (x: x.mode) etc2; + }; +in { options = { environment.systemPackages = mkOption { default = []; description = "Packages to be put in the system profile."; }; + + system.activationScripts.etc = mkOption {}; # Ignore + system.build.etc = mkOption {}; # Ignore + }; config = { - docker.buildScripts.systemEnv = let + docker.buildScripts."0-systemEnv" = let systemPackages = config.environment.systemPackages; systemEnv = pkgs.buildEnv { name = "system-env"; paths = systemPackages; }; in '' rm -rf /usr + chmod 777 /tmp ln -s ${systemEnv} /usr + ln -s /usr/bin/bash /bin/bash + ''; + + environment.systemPackages = with pkgs; [ coreutils bash ]; + + docker.buildScripts."0-etc" = '' + echo "setting up /etc..." + ${pkgs.perl}/bin/perl ${} ${etc}/etc ''; }; } \ No newline at end of file diff --git a/nix-docker/modules/config/user-groups.nix b/nix-docker/modules/config/user-groups.nix index e94e240..fdcc79f 100644 --- a/nix-docker/modules/config/user-groups.nix +++ b/nix-docker/modules/config/user-groups.nix @@ -218,6 +218,7 @@ in uid = 1; description = "Unprivileged account (don't use!)"; }; + ldap = {}; }; users.extraGroups = { diff --git a/nix-docker/modules/servers/database/redis.nix b/nix-docker/modules/servers/database/redis.nix deleted file mode 100644 index fcc4d01..0000000 --- a/nix-docker/modules/servers/database/redis.nix +++ /dev/null @@ -1,184 +0,0 @@ -{ config, pkgs, ... }: - -with pkgs.lib; - -let - cfg = config.services.redis; - redisBool = b: if b then "yes" else "no"; - condOption = name: value: if value != null then "${name} ${toString value}" else ""; - - redisConfig = pkgs.writeText "redis.conf" '' - pidfile ${cfg.pidFile} - port ${toString cfg.port} - loglevel ${cfg.logLevel} - logfile ${cfg.logfile} - databases ${toString cfg.databases} - ${concatMapStrings (d: "save ${toString (builtins.elemAt d 0)} ${toString (builtins.elemAt d 1)}\n") cfg.save} - dbfilename ${cfg.dbFilename} - dir ${toString cfg.dbpath} - ${if cfg.slaveOf != null then "slaveof ${cfg.slaveOf.ip} ${toString cfg.slaveOf.port}" else ""} - ${condOption "masterauth" cfg.masterAuth} - ${condOption "requirepass" cfg.requirePass} - appendOnly ${redisBool cfg.appendOnly} - appendfsync ${cfg.appendFsync} - slowlog-log-slower-than ${toString cfg.slowLogLogSlowerThan} - slowlog-max-len ${toString cfg.slowLogMaxLen} - ${cfg.extraConfig} - ''; -in -{ - - ###### interface - - options = { - - services.redis = { - - enable = mkOption { - default = false; - description = "Whether to enable the Redis server."; - }; - - package = mkOption { - default = pkgs.redis; - description = "Which Redis derivation to use."; - }; - - user = mkOption { - default = "redis"; - description = "User account under which Redis runs"; - }; - - pidFile = mkOption { - default = "/var/lib/redis/redis.pid"; - description = ""; - }; - - port = mkOption { - default = 6379; - description = "The port for Redis to listen to"; - type = with types; int; - }; - - logLevel = mkOption { - default = "notice"; # debug, verbose, notice, warning - example = "debug"; - description = "Specify the server verbosity level, options: debug, verbose, notice, warning"; - type = with types; string; - }; - - logfile = mkOption { - default = "stdout"; - description = "Specify the log file name. Also 'stdout' can be used to force Redis to log on the standard output."; - example = "/var/log/redis.log"; - type = with types; string; - }; - - databases = mkOption { - default = 16; - description = "Set the number of databases."; - type = with types; int; - }; - - save = mkOption { - default = [ [900 1] [300 10] [60 10000] ]; - description = "The schedule in which data is persisted to disk, represented as a list of lists where the first element represent the amount of seconds and the second the number of changes."; - example = [ [900 1] [300 10] [60 10000] ]; - }; - - dbFilename = mkOption { - default = "dump.rdb"; - description = "The filename where to dump the DB"; - type = with types; string; - }; - - dbpath = mkOption { - default = "/var/lib/redis"; - description = "The DB will be written inside this directory, with the filename specified using the 'dbFilename' configuration"; - type = with types; string; - }; - - slaveOf = mkOption { - default = null; # { ip, port } - description = "An attribute set with two attributes: ip and port to which this redis instance acts as a slave"; - example = { ip = "192.168.1.100"; port = 6379; }; - }; - - masterAuth = mkOption { - default = null; - description = ''If the master is password protected (using the requirePass configuration) - it is possible to tell the slave to authenticate before starting the replication synchronization - process, otherwise the master will refuse the slave request. - (STORED PLAIN TEXT, WORLD-READABLE IN NIX STORE)''; - }; - - requirePass = mkOption { - default = null; - description = "Password for database (STORED PLAIN TEXT, WORLD-READABLE IN NIX STORE)"; - example = "letmein!"; - }; - - appendOnly = mkOption { - default = false; - description = "By default data is only periodically persisted to disk, enable this option to use an append-only file for improved persistence."; - type = with types; bool; - }; - - appendOnlyFilename = mkOption { - default = "appendonly.aof"; - description = "Filename for the append-only file (stored inside of dbpath)"; - type = with types; string; - }; - - appendFsync = mkOption { - default = "everysec"; # no, always, everysec - description = "How often to fsync the append-only log, options: no, always, everysec"; - type = with types; string; - }; - - slowLogLogSlowerThan = mkOption { - default = 10000; - description = "Log queries whose execution take longer than X in milliseconds"; - example = 1000; - type = with types; int; - }; - - slowLogMaxLen = mkOption { - default = 128; - description = "Maximum number of items to keep in slow log"; - type = with types; int; - }; - - extraConfig = mkOption { - default = ""; - description = "Extra configuration options for redis.conf"; - type = with types; string; - }; - }; - - }; - - - ###### implementation - - config = mkIf config.services.redis.enable { - - users.extraUsers.redis = - { name = cfg.user; - description = "Redis database user"; - }; - - environment.systemPackages = [ cfg.package ]; - - docker.buildScripts.redisInit = '' - if ! test -e ${cfg.dbpath}; then - install -d -m0700 -o ${cfg.user} ${cfg.dbpath} - fi - ''; - - supervisord.services.redis = - { user = cfg.user; - command = "${cfg.package}/bin/redis-server ${redisConfig}"; - }; - }; -} diff --git a/nix-docker/modules/servers/openssh.nix b/nix-docker/modules/servers/openssh.nix new file mode 100644 index 0000000..a661915 --- /dev/null +++ b/nix-docker/modules/servers/openssh.nix @@ -0,0 +1,333 @@ +{ config, pkgs, ... }: + +with pkgs.lib; + +let + + cfg = config.services.openssh; + cfgc = config.programs.ssh; + + nssModulesPath = config.system.nssModules.path; + + permitRootLoginCheck = v: + v == "yes" || + v == "without-password" || + v == "forced-commands-only" || + v == "no"; + + knownHosts = map (h: getAttr h cfg.knownHosts) (attrNames cfg.knownHosts); + + knownHostsFile = pkgs.writeText "ssh_known_hosts" ( + flip concatMapStrings knownHosts (h: + "${concatStringsSep "," h.hostNames} ${builtins.readFile h.publicKeyFile}" + ) + ); + + userOptions = { + + openssh.authorizedKeys = { + keys = mkOption { + type = types.listOf types.str; + default = []; + description = '' + A list of verbatim OpenSSH public keys that should be added to the + user's authorized keys. The keys are added to a file that the SSH + daemon reads in addition to the the user's authorized_keys file. + You can combine the keys and + keyFiles options. + ''; + }; + + keyFiles = mkOption { + type = types.listOf types.str; + default = []; + description = '' + A list of files each containing one OpenSSH public key that should be + added to the user's authorized keys. The contents of the files are + read at build time and added to a file that the SSH daemon reads in + addition to the the user's authorized_keys file. You can combine the + keyFiles and keys options. + ''; + }; + }; + + }; + + authKeysFiles = let + mkAuthKeyFile = u: { + target = "ssh/authorized_keys.d/${u.name}"; + mode = "0444"; + source = pkgs.writeText "${u.name}-authorized_keys" '' + ${concatStringsSep "\n" u.openssh.authorizedKeys.keys} + ${concatMapStrings (f: builtins.readFile f + "\n") u.openssh.authorizedKeys.keyFiles} + ''; + }; + usersWithKeys = attrValues (flip filterAttrs config.users.extraUsers (n: u: + length u.openssh.authorizedKeys.keys != 0 || length u.openssh.authorizedKeys.keyFiles != 0 + )); + in map mkAuthKeyFile usersWithKeys; + +in + +{ + + ###### interface + + options = { + + services.openssh = { + + enable = mkOption { + type = types.bool; + default = false; + description = '' + Whether to enable the OpenSSH secure shell daemon, which + allows secure remote logins. + ''; + }; + + forwardX11 = mkOption { + type = types.bool; + default = cfgc.setXAuthLocation; + description = '' + Whether to allow X11 connections to be forwarded. + ''; + }; + + allowSFTP = mkOption { + type = types.bool; + default = true; + description = '' + Whether to enable the SFTP subsystem in the SSH daemon. This + enables the use of commands such as sftp and + sshfs. + ''; + }; + + permitRootLogin = mkOption { + default = "without-password"; + type = types.addCheck types.str permitRootLoginCheck; + description = '' + Whether the root user can login using ssh. Valid values are + yes, without-password, + forced-commands-only or + no. + ''; + }; + + gatewayPorts = mkOption { + type = types.str; + default = "no"; + description = '' + Specifies whether remote hosts are allowed to connect to + ports forwarded for the client. See + sshd_config + 5. + ''; + }; + + ports = mkOption { + type = types.listOf types.int; + default = [22]; + description = '' + Specifies on which ports the SSH daemon listens. + ''; + }; + + passwordAuthentication = mkOption { + type = types.bool; + default = true; + description = '' + Specifies whether password authentication is allowed. + ''; + }; + + challengeResponseAuthentication = mkOption { + type = types.bool; + default = true; + description = '' + Specifies whether challenge/response authentication is allowed. + ''; + }; + + hostKeys = mkOption { + type = types.listOf types.attrs; + default = + [ { path = "/etc/ssh/ssh_host_dsa_key"; + type = "dsa"; + bits = 1024; + } + { path = "/etc/ssh/ssh_host_ecdsa_key"; + type = "ecdsa"; + bits = 521; + } + ]; + description = '' + NixOS can automatically generate SSH host keys. This option + specifies the path, type and size of each key. See + ssh-keygen + 1 for supported types + and sizes. + ''; + }; + + authorizedKeysFiles = mkOption { + type = types.listOf types.str; + default = []; + description = "Files from with authorized keys are read."; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = "Verbatim contents of sshd_config."; + }; + + knownHosts = mkOption { + default = {}; + type = types.loaOf types.optionSet; + description = '' + The set of system-wide known SSH hosts. + ''; + example = [ + { + hostNames = [ "myhost" "myhost.mydomain.com" "10.10.1.4" ]; + publicKeyFile = literalExample "./pubkeys/myhost_ssh_host_dsa_key.pub"; + } + { + hostNames = [ "myhost2" ]; + publicKeyFile = literalExample "./pubkeys/myhost2_ssh_host_dsa_key.pub"; + } + ]; + options = { + hostNames = mkOption { + type = types.listOf types.string; + default = []; + description = '' + A list of host names and/or IP numbers used for accessing + the host's ssh service. + ''; + }; + publicKeyFile = mkOption { + description = '' + The path to the public key file for the host. The public + key file is read at build time and saved in the Nix store. + You can fetch a public key file from a running SSH server + with the ssh-keyscan command. + ''; + }; + }; + }; + + }; + + users.extraUsers = mkOption { + options = [ userOptions ]; + }; + + }; + + + ###### implementation + + config = mkIf cfg.enable { + + users.extraUsers = singleton + { name = "sshd"; + uid = config.ids.uids.sshd; + description = "SSH privilege separation user"; + home = "/var/empty"; + }; + + environment.etc = authKeysFiles ++ [ + { source = "${pkgs.openssh}/etc/ssh/moduli"; + target = "ssh/moduli"; + } + { source = knownHostsFile; + target = "ssh/ssh_known_hosts"; + } + ]; + + systemd.services.sshd = + { description = "SSH Daemon"; + + wantedBy = [ "multi-user.target" ]; + + stopIfChanged = false; + + path = [ pkgs.openssh pkgs.gawk ]; + + environment.LD_LIBRARY_PATH = ""; + environment.LOCALE_ARCHIVE = "/run/current-system/sw/lib/locale/locale-archive"; + + preStart = + '' + mkdir -m 0755 -p /etc/ssh + + ${flip concatMapStrings cfg.hostKeys (k: '' + if ! [ -f "${k.path}" ]; then + ssh-keygen -t "${k.type}" -b "${toString k.bits}" -f "${k.path}" -N "" + fi + '')} + ''; + + serviceConfig = + { ExecStart = + "${pkgs.openssh}/sbin/sshd " + + "-f ${pkgs.writeText "sshd_config" cfg.extraConfig} -D"; + Restart = "always"; + KillMode = "process"; + PIDFile = "/run/sshd.pid"; + }; + }; + + services.openssh.authorizedKeysFiles = + [ ".ssh/authorized_keys" ".ssh/authorized_keys2" "/etc/ssh/authorized_keys.d/%u" ]; + + services.openssh.extraConfig = + '' + PidFile /run/sshd.pid + + Protocol 2 + + UsePAM no + + AddressFamily ${if config.networking.enableIPv6 then "any" else "inet"} + ${concatMapStrings (port: '' + Port ${toString port} + '') cfg.ports} + + ${optionalString cfgc.setXAuthLocation '' + XAuthLocation ${pkgs.xorg.xauth}/bin/xauth + ''} + + ${if cfg.forwardX11 then '' + X11Forwarding yes + '' else '' + X11Forwarding no + ''} + + ${optionalString cfg.allowSFTP '' + Subsystem sftp ${pkgs.openssh}/libexec/sftp-server + ''} + + PermitRootLogin ${cfg.permitRootLogin} + GatewayPorts ${cfg.gatewayPorts} + PasswordAuthentication ${if cfg.passwordAuthentication then "yes" else "no"} + ChallengeResponseAuthentication ${if cfg.challengeResponseAuthentication then "yes" else "no"} + + PrintMotd no # handled by pam_motd + + AuthorizedKeysFile ${toString cfg.authorizedKeysFiles} + + ${flip concatMapStrings cfg.hostKeys (k: '' + HostKey ${k.path} + '')} + ''; + + assertions = [{ assertion = if cfg.forwardX11 then cfgc.setXAuthLocation else true; + message = "cannot enable X11 forwarding without setting xauth location";}]; + + }; + +} diff --git a/nix-docker/modules/servers/supervisord.nix b/nix-docker/modules/servers/supervisord.nix index 89bc3ee..9888204 100644 --- a/nix-docker/modules/servers/supervisord.nix +++ b/nix-docker/modules/servers/supervisord.nix @@ -14,6 +14,12 @@ let default = "root"; description = "The user to run the command as"; }; + environment = mkOption { + default = {}; + example = { + PATH = "/some/path"; + }; + }; }; }; services = config.supervisord.services; @@ -58,6 +64,7 @@ in { '' [program:${name}] command=${cfg.command} + environment=${concatMapStrings (name: "${name}=\"${toString (getAttr name cfg.environment)}\",") (attrNames cfg.environment)} directory=${cfg.directory} redirect_stderr=true stdout_logfile=/var/log/supervisord/${name}.log @@ -70,7 +77,7 @@ in { docker.bootScript = '' mkdir -p /var/log/supervisord ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${config.supervisord.configFile} ${if config.supervisord.tailLogs then '' - + sleep 2 touch /var/log/supervisord/test.log tail -n 100 -f /var/log/supervisord/*.log diff --git a/nix-docker/systemd.nix b/nix-docker/systemd.nix deleted file mode 100644 index cacbe80..0000000 --- a/nix-docker/systemd.nix +++ /dev/null @@ -1,67 +0,0 @@ -{ pkgs -, config }: -with pkgs.lib; -let - services = removeAttrs config.config.systemd.services [ - "acpid" - "network-setup" - "prepare-kexec" - "systemd-sysctl" - "alsa-store" - "nix-daemon" - "rngd" - "systemd-update-utmp" - "cpufreq" - "nix-gc" - "scsi-link-pm" - "systemd-vconsole-setup" - "cron" - "nscd" - "synergy-client" - "update-locatedb" - "dhcpcd" - "ntpd" - "synergy-server" - "post-resume" - "systemd-modules-load" - "klogd" - "pre-sleep" - "systemd-random-seed" - ]; - - isOneShot = cfg: hasAttr "Type" cfg.serviceConfig && cfg.serviceConfig.Type == "oneshot"; - - runServices = filterAttrs (name: cfg: !(isOneShot cfg)) services; - - oneShotServices = filterAttrs (name: cfg: isOneShot cfg) services; - - configToCommand = name: cfg: if hasAttr "ExecStart" cfg.serviceConfig then - cfg.serviceConfig.ExecStart - else if hasAttr "script" cfg then - pkgs.writeScript "${name}-script" '' - #!/bin/sh -e - ${cfg.script} - '' - else - ""; -in { - oneShotScript = concatMapStrings (name: "${configToCommand name (getAttr name oneShotServices)}\n") (attrNames oneShotServices); - supervisorConfigFile = pkgs.writeText "supervisord.conf" '' - [supervisord] - logfile=/var/log/supervisord/supervisord.log - - ${concatMapStrings (name: - let - cfg = getAttr name runServices; - in - '' - [program:${name}] - command=${configToCommand name cfg} - redirect_stderr=true - stdout_logfile=/var/log/supervisord/${name}.log - ${if hasAttr "User" cfg.serviceConfig then "user=${cfg.serviceConfig.User}\n" else ""} - '' - ) (attrNames runServices) - } - ''; -} \ No newline at end of file From a5e01e7c18f25ec45422bb6f27b33e4bdf5a88ac Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Mon, 4 Nov 2013 08:32:40 -0800 Subject: [PATCH 08/46] Added volume support, forced running nix-docker as root when building stand-alone images --- configuration.nix | 4 +- nix-docker/bin/nix-docker | 52 ++++++++++++----------- nix-docker/{dockerfile.nix => docker.nix} | 7 ++- nix-docker/modules/config/docker.nix | 11 +++-- nix-docker/modules/config/environment.nix | 2 +- nix-docker/modules/shim/systemd.nix | 51 ++++++++++++++++++++++ 6 files changed, 93 insertions(+), 34 deletions(-) rename nix-docker/{dockerfile.nix => docker.nix} (92%) create mode 100644 nix-docker/modules/shim/systemd.nix diff --git a/configuration.nix b/configuration.nix index 6d56a41..8ff3c53 100644 --- a/configuration.nix +++ b/configuration.nix @@ -1,7 +1,7 @@ { config, pkgs, ... }: { - docker.ports = [ 1234 80 ]; - docker.verbose = true; + docker.ports = [ 1234 80 22 ]; + docker.volumes = [ "/data" ]; services.redis = { enable = true; diff --git a/nix-docker/bin/nix-docker b/nix-docker/bin/nix-docker index e3ed3c3..a8e9bd8 100644 --- a/nix-docker/bin/nix-docker +++ b/nix-docker/bin/nix-docker @@ -11,10 +11,10 @@ var execFile = require('child_process').execFile; var configFile = argv.c || "configuration.nix"; var fullBuild = !!argv.b; -var baseImage = argv.from = "zefhemel/base-nix"; -var imageName = argv.t ? argv.t : "nix-docker-build"; +var baseImage = argv.from || "zefhemel/base-nix"; +var imageName = argv.t || "nix-docker-build"; -var tmpNixStore = "nix_store"; +var nixClosureCopy = "nix-closure"; if (argv.help) { console.log("Usage: nix-docker [-b] [-t imagename] [-c configuration.nix] "); @@ -36,7 +36,7 @@ function pipeRun(cmd, args, callback) { } function build(nix, configPath, callback) { - var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath, "--arg", "mountBuild", "" + !fullBuild, "--argstr", "name", imageName, "--argstr", "baseImage", baseImage, "--show-trace"]); + var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath, "--arg", "mountBuild", "" + !fullBuild, "--argstr", "name", imageName, "--argstr", "baseImage", baseImage, "--show-trace"].concat(fullBuild ? ["--no-out-link"] : [])); var nixPath; nixBuild.stdout.on('data', function(data) { @@ -57,28 +57,24 @@ function cleanup(callback) { if(!fullBuild) { return callback(); } - if (!fs.existsSync(tmpNixStore)) { + if (!fs.existsSync(nixClosureCopy)) { return callback(); } - execFile("chmod", ["-R", "777", tmpNixStore], function() { - execFile("rm", ["-rf", tmpNixStore], callback); - }); + pipeRun("rm", ["-rf", nixClosureCopy], callback); } function buildImage(dockerFilePath, callback) { var dockerFile = fs.readFileSync(dockerFilePath + "/Dockerfile").toString("ascii"); fs.writeFileSync("Dockerfile", dockerFile); - if (!argv["dockerfile-only"]) { - pipeRun("sudo", ["docker", "build", "-rm=true", "-t", imageName, "."], function(code) { - cleanup(function() { - callback(code); - }); + pipeRun("docker", ["build", "-rm=true", "-t", imageName, "."], function(code) { + cleanup(function() { + callback(code); }); - } + }); } function getAvailableNixPaths(callback) { - var command = spawn("sudo", ["docker", "run", baseImage, "/bin/ls", "/nix/store"]); + var command = spawn("docker", ["run", baseImage, "/bin/ls", "/nix/store"]); process.stdin.pipe(command.stdin); var output = ''; @@ -106,15 +102,17 @@ function copyClosureAndBuild(dockerFilePath, availablePaths) { console.log("New paths to copy", paths); - fs.mkdirSync(tmpNixStore); - execFile("cp", ["-r", "--no-preserve=ownership"].concat(paths).concat([tmpNixStore]), function(err) { - if (err) { - console.log(err); - process.exit(1); - } - buildImage(dockerFilePath, function(code) { - console.log("To run: sudo docker run -t -i " + imageName); - process.exit(code); + fs.mkdirSync(nixClosureCopy); + execFile("cp", ["-r"].concat(paths).concat([nixClosureCopy]), function(err) { + pipeRun("chown", ["-R", "root:root", nixClosureCopy], function() { + if (err) { + console.log(err); + process.exit(1); + } + buildImage(dockerFilePath, function(code) { + console.log("To run: sudo docker run -t -i " + imageName); + process.exit(code); + }); }); }); } else { @@ -126,8 +124,12 @@ function copyClosureAndBuild(dockerFilePath, availablePaths) { }); } +if(fullBuild && process.env.USER !== "root") { + console.error("When doing a full build you need to run nix-docker as root. Rerun this command with 'sudo -E nix-docker ...'"); + process.exit(1); +} cleanup(function() { - build(rootPath + "/dockerfile.nix", configFile, function(code, dockerFilePath) { + build(rootPath + "/docker.nix", configFile, function(code, dockerFilePath) { if(code) { process.exit(code); } diff --git a/nix-docker/dockerfile.nix b/nix-docker/docker.nix similarity index 92% rename from nix-docker/dockerfile.nix rename to nix-docker/docker.nix index eb9e184..3b806fb 100644 --- a/nix-docker/dockerfile.nix +++ b/nix-docker/docker.nix @@ -2,7 +2,7 @@ , name , configuration ? , mountBuild ? true -, baseImage ? "ubuntu" +, baseImage ? "busybox" }: with pkgs.lib; let @@ -34,7 +34,7 @@ let FROM ${if mountBuild then "busybox" else baseImage} ${if !mountBuild then '' - ADD nix_store /nix/store + ADD nix-closure /nix/store RUN ${buildScript} '' else ""} @@ -42,6 +42,9 @@ let ${ concatMapStrings (port: "EXPOSE ${toString port}\n") config.config.docker.ports } + ${ + concatMapStrings (port: "VOLUME ${toString port}\n") config.config.docker.volumes + } ''; imageHash = substring 11 8 dockerFile.outPath; diff --git a/nix-docker/modules/config/docker.nix b/nix-docker/modules/config/docker.nix index 73e8ff8..2573fc1 100644 --- a/nix-docker/modules/config/docker.nix +++ b/nix-docker/modules/config/docker.nix @@ -8,6 +8,12 @@ with pkgs.lib; example = [ 80 22 ]; }; + docker.volumes = mkOption { + default = []; + description = "Volumes to create for container."; + example = [ "/var/lib" "/var/log" ]; + }; + docker.buildScripts = mkOption { default = {}; example = { @@ -28,16 +34,13 @@ with pkgs.lib; type = types.bool; }; + # HACK: Let's ignore these for now networking = mkOption {}; security = mkOption {}; - #system.nssModules.path = mkOption {}; - #services.samba = mkOption{}; }; config = { docker.buildScript = concatStrings (attrValues config.docker.buildScripts); networking.enableIPv6 = false; - #system.nssModules.path = ""; - #services.samba.syncPasswordsByPam = false; }; } \ No newline at end of file diff --git a/nix-docker/modules/config/environment.nix b/nix-docker/modules/config/environment.nix index ac705eb..115fc8d 100644 --- a/nix-docker/modules/config/environment.nix +++ b/nix-docker/modules/config/environment.nix @@ -35,7 +35,7 @@ in { rm -rf /usr chmod 777 /tmp ln -s ${systemEnv} /usr - ln -s /usr/bin/bash /bin/bash + ln -sf /usr/bin/bash /bin/bash ''; environment.systemPackages = with pkgs; [ coreutils bash ]; diff --git a/nix-docker/modules/shim/systemd.nix b/nix-docker/modules/shim/systemd.nix new file mode 100644 index 0000000..c268ca0 --- /dev/null +++ b/nix-docker/modules/shim/systemd.nix @@ -0,0 +1,51 @@ +{ pkgs, config, ... }: +with pkgs.lib; +let + services = config.systemd.services; + + isOneShot = cfg: hasAttr "Type" cfg.serviceConfig && cfg.serviceConfig.Type == "oneshot"; + + runServices = filterAttrs (name: cfg: !(isOneShot cfg)) services; + + oneShotServices = filterAttrs (name: cfg: isOneShot cfg) services; + + configToCommand = name: cfg: '' + #!/bin/sh -e + ${if hasAttr "preStart" cfg then cfg.preStart else ""} + ${if hasAttr "ExecStart" cfg.serviceConfig then + cfg.serviceConfig.ExecStart + else if hasAttr "script" cfg then + cfg.script + else + "" + } + ${if hasAttr "postStart" cfg then cfg.postStart else ""} + ''; + +in { + + options = { + systemd.services = mkOption { }; # TODO make more specific + }; + + config = { + docker.buildScripts."1-systemd-oneshot" = concatMapStrings (name: "${configToCommand name (getAttr name oneShotServices)}\n") (attrNames oneShotServices); + + supervisord.services = listToAttrs (map (name: + let + cfg = getAttr name runServices; + in + { + name = name; + value = { + command = pkgs.writeScript "${name}-run" (configToCommand name cfg); + user = if hasAttr "User" cfg.serviceConfig then cfg.serviceConfig.User else "root"; + environment = (if hasAttr "environment" cfg then cfg.environment else {}) // + (if hasAttr "path" cfg then + { PATH = concatStringsSep ":" (map (prg: "${prg}/bin") cfg.path); } + else {}); + }; + } + ) (attrNames runServices)); + }; +} \ No newline at end of file From 5c991c5c9a3d0079590a7028a40425307382bfd1 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Mon, 4 Nov 2013 08:43:02 -0800 Subject: [PATCH 09/46] Nix and Docker ubuntu install scripts --- scripts/README.md | 1 + scripts/install-docker.sh | 13 ++++++ scripts/install-nix.sh | 96 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 scripts/README.md create mode 100644 scripts/install-docker.sh create mode 100644 scripts/install-nix.sh diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..0d1a420 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1 @@ +These are scripts to automatically install Docker and Nix and has been tested to work on Ubuntu 13.04. diff --git a/scripts/install-docker.sh b/scripts/install-docker.sh new file mode 100644 index 0000000..38bcb87 --- /dev/null +++ b/scripts/install-docker.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +sudo sh -c "wget -qO- https://get.docker.io/gpg | apt-key add -" + +# Add the Docker repository to your apt sources list. +sudo sh -c "echo deb http://get.docker.io/ubuntu docker main\ +> /etc/apt/sources.list.d/docker.list" + +# update +sudo apt-get update + +# install +sudo apt-get install -y lxc-docker \ No newline at end of file diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh new file mode 100644 index 0000000..1ecd125 --- /dev/null +++ b/scripts/install-nix.sh @@ -0,0 +1,96 @@ +#!/bin/sh + +set -e + +# Install the binary tarball... +cd / +wget -O - http://hydra.nixos.org/job/nix/trunk/binaryTarball.x86_64-linux/latest/download | tar xvj +/usr/bin/nix-finish-install +rm /usr/bin/nix-finish-install + +# Setup multiuser + +# Allow all users to create profiles +mkdir -p /nix/var/nix/profiles/per-user +chmod 1777 /nix/var/nix/profiles/per-user + +# Add build users +# 9 is the exit code when the group already exists +groupadd -r nixbld || [ "$?" -eq 9 ] +for n in 1 2 3 4 5 6 7 8 9 10; do + useradd -c "Nix build user $n" -d /var/empty -g nixbld -G nixbld \ + -M -N -r -s `which nologin` nixbld$n || [ "$?" -eq 9 ] +done +chown root:nixbld /nix/store +chmod 1775 /nix/store +mkdir -p /etc/nix +grep -w build-users-group /etc/nix/nix.conf 2>/dev/null || echo "build-users-group = nixbld" >> /etc/nix/nix.conf +grep -w binary-caches /etc/nix/nix.conf 2>/dev/null || echo "binary-caches = http://cache.nixos.org" >> /etc/nix/nix.conf +grep -w trusted-binary-caches /etc/nix/nix.conf 2>/dev/null || echo "trusted-binary-caches = http://hydra.nixos.org http://builder.logicblox.com http://cache.nixos.org" >> /etc/nix/nix.conf + +# Use a multiuser-compatible profile script +unlink /etc/profile.d/nix.sh +cat > /etc/profile.d/nix.sh <&2 + if [ -n "\$OWNS_STORE" ]; then + _NIX_PROFILE_LINK=/nix/var/nix/nix/profiles/default + else + mkdir -p "/nix/var/nix/profiles/per-user/\$LOGNAME" + _NIX_PROFILE_LINK="/nix/var/nix/profiles/per-user/\$LOGNAME/profile" + fi + ln -s "\$_NIX_PROFILE_LINK" "\$NIX_LINK" + fi + + # Subscribe the root user to the Nixpkgs channel by default. + if [ -n "\$OWNS_STORE" ] && [ ! -e "\$HOME/.nix-channels" ]; then + echo "http://nixos.org/channels/nixpkgs-unstable nixpkgs" > "\$HOME/.nix-channels" + fi + + # Set up nix-defexpr + NIX_DEFEXPR="\$HOME/.nix-defexpr" + if ! [ -e "\$NIX_DEFEXPR" ]; then + echo "creating \$NIX_DEFEXPR" >&2 + mkdir -p "\$NIX_DEFEXPR" + _NIX_CHANNEL_LINK=/nix/var/nix/profiles/per-user/root/channels/nixpkgs + ln -s "\$_NIX_CHANNEL_LINK" "\$NIX_DEFEXPR" + fi + + if [ -z "\$OWNS_STORE" ]; then + export NIX_REMOTE=daemon + export PATH="/nix/var/nix/profiles/default/bin:\$PATH" + fi + export PATH="\$NIX_LINK/bin:\$PATH" + + # Set up NIX_PATH + export NIX_PATH="\${NIX_PATH:+\$NIX_PATH:}/nix/var/nix/profiles/per-user/\$LOGNAME/channels" + unset OWNS_STORE +fi +EOF +cat >> /etc/environment < /etc/init/nix-daemon.conf < Date: Tue, 5 Nov 2013 07:26:41 -0800 Subject: [PATCH 10/46] Lots of cool nix-docker stuff --- README.md | 165 ++++++++++++++++++++ build-base.sh | 2 + default.nix | 19 +-- nix-docker/all-modules.nix | 1 + nix-docker/bin/nix-docker | 199 ++++++++----------------- nix-docker/modules/shim/systemd.nix | 4 +- node-packages-generated.nix | 55 ------- samples/Dockerfile | 8 + samples/apache-config.nix | 14 ++ configuration.nix => samples/fancy.nix | 2 +- samples/result | 1 + samples/ssh-config.nix | 19 +++ samples/www/index.html | 1 + scripts/install-nix.sh | 8 +- zedconfig.json | 9 +- 15 files changed, 293 insertions(+), 214 deletions(-) create mode 100644 README.md create mode 100755 build-base.sh mode change 100644 => 100755 nix-docker/bin/nix-docker delete mode 100644 node-packages-generated.nix create mode 100644 samples/Dockerfile create mode 100644 samples/apache-config.nix rename configuration.nix => samples/fancy.nix (94%) create mode 120000 samples/result create mode 100644 samples/ssh-config.nix create mode 100644 samples/www/index.html diff --git a/README.md b/README.md new file mode 100644 index 0000000..cc8d644 --- /dev/null +++ b/README.md @@ -0,0 +1,165 @@ +nix-docker +========== + +The problem +----------- + +[Docker is great](http://www.infoq.com/articles/docker-containers). It's a nice, +pragmatic solution to deploying applications in a portable way. There is one +thing I do not like about Docker, or two: + +(1) How container images are provisioned +(2) How images are distributed + +We can argue about (2) and I'm sure it it will improve. Docker images are distributed +via a Docker registry. The main one lives at http://index.docker.io, and if your +application can be openly available to everyone, this works great. If you need +to keep applications in-house, you have to host your own registry server, and +support for that (e.g. with authentication) is still limited. + +But my main gripe is (1). Docker images are provisioned using a +[Dockerfile](http://docs.docker.io/en/latest/use/builder/), which basically +a simple imperative script that builds up an image from a base image step by +step. Typical Dockerfile lines look like this: + + RUN apt-get update + RUN apt-get upgrade -y + RUN apt-get install -y openssh-server python curl + +Every such command that you run is committed, resulting in an AuFS layer. This +can be helpful, because Docker can now do basic caching. For instance, if +your build fails at the third line, it can use the image resulting from +the first two lines and start from there. There's a number problems with this +approach: + +1. Can those first two lines really be cached, or may their result be dependent + on the time of being run? Answer: yes, running these lines tomorrow may yield + different results than running them today, but Docker will naively assume + they will always result in the same thing. +2. AuFS can not handle an unlimited number of layers, if your Dockerfile has too + many commands the build will simply fail. So, you better make them count. +3. There's very little support for reuse. There's no include files or configuration + language. All you can do is create a hierarchy of images, where you create + a base image with software common to all other images, and then you use `FROM` + to base future image on. And you're still limited by (2), the layers all add + up. + +What Nix brings to the table +---------------------------- + +[Nix](http://nixos.org/nix/) is a relatively new package manager for Unix systems, +it's not specific to Linux, it works on Mac as well, for instance. In short, it's +package management done right: + +* Packages and sytems are built using the Nix functional language, which is a + full-blown functional language designed specifically for deploying simple and + complex systems. +* Completele dependency closures are automatically derived, so do not have to + be constructed by hand (as is the case with dpkg and RPM). +* Rather than scattering files all over the disk (configuration files in `/etc`, + binaries in `/usr/bin` or `/usr/local/bin` or is it `/bin`), components are + stored in isolation in the Nix store (`/nix/store`) not interfering with each + other. +* Nix has excellent support for modularization. + +While Nix itself is "just" a package manager, there are many tools built on top +of it, including a full-blown Linux distribution called [NixOS](http://nixos.org/nixos/). + +NixOS enables you to declaratively specify your entire system configuration using +the Nix language. Based on this configuration, it can build the entire system, which +can be deployed locally or remotely via [NixOps](https://github.com/NixOS/nixops). + +In the context of Docker, the problem with NixOS is that it's truly a full-blown +OS. All services run using [systemd](http://www.freedesktop.org/wiki/Software/systemd/), +kernels are deployed and a bunch of utility processes are running at all times. + +I tried to see if it's feasible to deploy a system configuration built for NixOS +in a Docker container, but couldn't get it to run because systemd wouldn't run, +beside that, configurations quickly get large, many hundreds of megabytes. So +much for light-weight containers, huh. + +Nevertheless, NixOS configurations are kind of nice and clean and would make sense +for Docker as well. For instance, here's how to run a simple Apache server +serving static files from `./www` directory: + + { config, pkgs, ... }: + { + services.httpd = { + enable = true; + documentRoot = ./www; + adminAddr = "zef.hemel@logicblox.com"; + }; + } + +That's all it takes. The `./www` there refers to the path `./www` local to the +system configuration file, by the way. When the system configuration is built, +the contents of `./www` is automatically copied into the Nix store and becomes +part of the dependencies of the system configuration. + +Wouldn't it be cool to deploy NixOS configs in docker containers? That's what +nix-docker attempts to offer. + +Installation +------------ +To use nix-docker you need Nix installed. If you're using Ubuntu, the easiest way +to do so is by running the `script/install-nix.sh` script as root: + + curl https://raw.github.com/zefhemel/nix-docker/master/scripts/install-nix.sh | sudo sh + +After this script runs successfully, log out and back in. + +Next, install nix-docker itself: + + git clone https://github.com/zefhemel/nix-docker.git + cd nix-docker + nix-env -f . -i nix-docker + +You can now run nix-docker in its default mode (more on this later). To be able +to produce full Docker images, you also need +[Docker itself installed](http://docs.docker.io/en/latest/installation/ubuntulinux/). + +Usage +----- + +`nix-docker` can run in two modes: + +1. full Docker image building mode (by passing in `-b`) +2. using the host's Nix store mode + +The first option is the most portable. `nix-docker` will produce regular Docker +images that you can push to a Docker registry and deploy anywhere where Docker +runs. The second option builds a very minimal Docker images on-demand containing +only some meta data (like `EXPOSE` and `VOLUME`, `RUN` commands in a Dockerfile) +and is used by mounting in the host's Nix store into the container via +`-v /nix/store:/nix/store`. There's two reasons you may want to do the latter: + +1. Build times are _much_ faster, since Nix build are fully incremental, only + things that have not been build before will be built. +2. You don't polute your `/var/lib/docker` with a lot of copies of your software. + +To build a stand-alone Docker image: + + nix-docker -b -t my-image configuration.nix + +This will build the configuration specified in `configuration.nix`, have a look +in the `samples/` directory for examples. It will produce a docker image named +`my-image` which you can then run anywhere. Use `username/my-image` to be able +to push them to the Docker index. + +To build a host-mounted package: + + nix-docker -t my-image configuration.nix + +This will produce a Nix package (symlinked in the current directory in `result`) +containing a script you can use to spawn the container using Docker, e.g.: + + sudo -e ./result/sbin/docker-run + +to run the container in the foreground, or: + + sudo -e ./result/sbin/docker-run -d + +to daemonize it. What the `docker-run` script will do is check if there's already +a docker image available with the current image name and tag based on the Nix +build hash. If not, it will quickly build it first (these images take up barely +any space on disk). Then, it will boot up the container. diff --git a/build-base.sh b/build-base.sh new file mode 100755 index 0000000..ae14dac --- /dev/null +++ b/build-base.sh @@ -0,0 +1,2 @@ +#!/bin/sh +sudo -E nix-docker/bin/nix-docker base-configuration.nix -t zefhemel/base-nix --from busybox -b diff --git a/default.nix b/default.nix index cb3f1cf..28988df 100644 --- a/default.nix +++ b/default.nix @@ -1,16 +1,11 @@ let pkgs = import {}; - nodePackages = pkgs.recurseIntoAttrs (import { - inherit pkgs; - inherit (pkgs) stdenv nodejs fetchurl; - neededNatives = [pkgs.python pkgs.utillinux]; - self = nodePackages; - generated = ./node-packages-generated.nix; - }); in - nodePackages.buildNodePackage { - name = "nix-docker"; - src = [ { outPath = ./nix-docker; name = "nix-docker"; } ]; - deps = with nodePackages; [optimist]; - passthru.names = [ "nix-docker" ]; + pkgs.stdenv.mkDerivation { + name = "nix-docker-0.1"; + src = ./nix-docker; + installPhase = '' + mkdir -p $out + cp -R * $out/ + ''; } \ No newline at end of file diff --git a/nix-docker/all-modules.nix b/nix-docker/all-modules.nix index a4060ca..6562dc6 100644 --- a/nix-docker/all-modules.nix +++ b/nix-docker/all-modules.nix @@ -11,6 +11,7 @@ + # These modules needed some patching to work well ./modules/servers/http/apache/default.nix diff --git a/nix-docker/bin/nix-docker b/nix-docker/bin/nix-docker old mode 100644 new mode 100755 index a8e9bd8..97fda93 --- a/nix-docker/bin/nix-docker +++ b/nix-docker/bin/nix-docker @@ -1,145 +1,76 @@ -#!/usr/bin/env node +#!/usr/bin/python -var argv = require("optimist").argv; -var fs = require("fs"); -var path = require("path"); +import os.path +import subprocess +import argparse +import sys +import shutil -var rootPath = path.resolve(fs.realpathSync(process.argv[1]), "../.."); +parser = argparse.ArgumentParser(description='Build Docker images with Nix', prog='nix-docker') +parser.add_argument('configuration', nargs='?', help='configuration.nix file to build', default="configuration.nix") +parser.add_argument('-b', action='store_true', help="Build a docker image") +parser.add_argument('-t', help='Image name', default="nix-docker-build") +parser.add_argument('--from', help="Image to use as a base image", default="zefhemel/base-nix") -var spawn = require('child_process').spawn; -var execFile = require('child_process').execFile; -var configFile = argv.c || "configuration.nix"; -var fullBuild = !!argv.b; +args = parser.parse_args() -var baseImage = argv.from || "zefhemel/base-nix"; -var imageName = argv.t || "nix-docker-build"; +base_image = getattr(args, "from") +configuration_file = args.configuration +image_name = args.t +full_build = args.b +nix_closure_temp = "nix-closure" -var nixClosureCopy = "nix-closure"; +root_path = os.path.normpath(os.path.realpath(sys.argv[0]) + "/../..") -if (argv.help) { - console.log("Usage: nix-docker [-b] [-t imagename] [-c configuration.nix] "); - console.log(" -b: build full (portable) docker image."); - console.log(" -t: name of the image to build."); - console.log(" -c: path to configuration file to build."); - process.exit(0); -} +def build(): + print "Building Nix closure for the image..." + if args.b: + mountBuild = "false" + else: + mountBuild = "true" + cmd = ["nix-build", "%s/docker.nix" % root_path, '-I', 'configuration=' + configuration_file, "--arg", "mountBuild", mountBuild, "--argstr", "name", image_name, "--argstr", "baseImage", base_image, "--show-trace"] + if full_build: + cmd.append("--no-out-link") + nix_path = subprocess.check_output(cmd) + return nix_path.strip() -function pipeRun(cmd, args, callback) { - var command = spawn(cmd, args); - process.stdin.pipe(command.stdin); - command.stdout.pipe(process.stdout); - command.stderr.pipe(process.stderr); +def get_available_nix_paths(): + try: + paths_string = subprocess.check_output(["docker", "run", base_image, "/bin/ls", "/nix/store"]) + return paths_string.strip().split("\n") + except: + return [] - command.on('close', function(code) { - callback(code); - }); -} +def cleanup(): + subprocess.check_call(["rm", "-rf", nix_closure_temp]) -function build(nix, configPath, callback) { - var nixBuild = spawn('nix-build', [nix, '-I', 'configuration=' + configPath, "--arg", "mountBuild", "" + !fullBuild, "--argstr", "name", imageName, "--argstr", "baseImage", baseImage, "--show-trace"].concat(fullBuild ? ["--no-out-link"] : [])); - var nixPath; +def copy_closure_and_build(package_nix_path, available_paths): + paths = subprocess.check_output(["nix-store", "-qR", package_nix_path]).strip().split("\n") + print "Available", available_paths + paths_to_copy = filter(lambda path: not path[len("/nix/store/"):] in available_paths, paths) + print "New Nix store paths to copy to image:", paths_to_copy + os.mkdir(nix_closure_temp) + cmd = ["cp", "-r"] + cmd.extend(paths_to_copy) + cmd.append(nix_closure_temp) + shutil.copyfile("%s/Dockerfile" % package_nix_path, "Dockerfile") + subprocess.check_call(cmd) + subprocess.check_call(["chown", "-R", "root:root", nix_closure_temp]) + subprocess.check_call(["docker", "build", "-rm=true", "-t", image_name, "."]) + cleanup() - nixBuild.stdout.on('data', function(data) { - nixPath = data.toString("ascii"); - }); - nixBuild.stderr.pipe(process.stderr); +if __name__ == '__main__': + if full_build and os.getenv("USER") != "root": + print "When doing a full build you need to run nix-docker as root. Rerun this command with 'sudo -E nix-docker ...'" + sys.exit(1) - nixBuild.on('close', function(code) { - if (code === 0) { - callback(null, nixPath.trim()); - } else { - callback(code); - } - }); -} - -function cleanup(callback) { - if(!fullBuild) { - return callback(); - } - if (!fs.existsSync(nixClosureCopy)) { - return callback(); - } - pipeRun("rm", ["-rf", nixClosureCopy], callback); -} - -function buildImage(dockerFilePath, callback) { - var dockerFile = fs.readFileSync(dockerFilePath + "/Dockerfile").toString("ascii"); - fs.writeFileSync("Dockerfile", dockerFile); - pipeRun("docker", ["build", "-rm=true", "-t", imageName, "."], function(code) { - cleanup(function() { - callback(code); - }); - }); -} - -function getAvailableNixPaths(callback) { - var command = spawn("docker", ["run", baseImage, "/bin/ls", "/nix/store"]); - process.stdin.pipe(command.stdin); - - var output = ''; - - command.stdout.on("data", function(data) { - output += data.toString("ascii"); - }); - - command.on('close', function() { - callback(null, output.split("\n")); - }); -} - -function copyClosureAndBuild(dockerFilePath, availablePaths) { - execFile("nix-store", ["-qR", dockerFilePath], {}, function(err, stdout) { - if (err) { - return console.error(err); - } - - if (fullBuild) { - var paths = stdout.trim().split("\n"); - paths = paths.filter(function(path) { - return availablePaths.indexOf(path.substring("/nix/store/".length)) === -1; - }); - - console.log("New paths to copy", paths); - - fs.mkdirSync(nixClosureCopy); - execFile("cp", ["-r"].concat(paths).concat([nixClosureCopy]), function(err) { - pipeRun("chown", ["-R", "root:root", nixClosureCopy], function() { - if (err) { - console.log(err); - process.exit(1); - } - buildImage(dockerFilePath, function(code) { - console.log("To run: sudo docker run -t -i " + imageName); - process.exit(code); - }); - }); - }); - } else { - buildImage(dockerFilePath, function(code) { - console.log("To run: sudo docker run -t -i -v /nix/store:/nix/store " + imageName); - process.exit(code); - }); - } - }); -} - -if(fullBuild && process.env.USER !== "root") { - console.error("When doing a full build you need to run nix-docker as root. Rerun this command with 'sudo -E nix-docker ...'"); - process.exit(1); -} -cleanup(function() { - build(rootPath + "/docker.nix", configFile, function(code, dockerFilePath) { - if(code) { - process.exit(code); - } - if (fullBuild) { - getAvailableNixPaths(function(err, availablePaths) { - copyClosureAndBuild(dockerFilePath, availablePaths); - }); - } else { - console.log("Result in", dockerFilePath, "test with sudo ./result/sbin/docker-run"); - console.log("To deploy: nix-copy-closure " + dockerFilePath + " machine && ssh root@machine" + dockerFilePath + "/sbin/docker-run") - } - }); -}); \ No newline at end of file + package_nix_path = build() + if full_build: + cleanup() + available_paths = get_available_nix_paths() + copy_closure_and_build(package_nix_path, available_paths) + print "To run: sudo docker run -t -i", image_name + else: + print "Result in", package_nix_path, "test with sudo ./result/sbin/docker-run" + print "To deploy: nix-copy-closure -s root@", package_nix_path + print "To run: ssh root@", package_nix_path + "/sbin/docker-run -d" \ No newline at end of file diff --git a/nix-docker/modules/shim/systemd.nix b/nix-docker/modules/shim/systemd.nix index c268ca0..8c7a6e9 100644 --- a/nix-docker/modules/shim/systemd.nix +++ b/nix-docker/modules/shim/systemd.nix @@ -25,7 +25,9 @@ let in { options = { - systemd.services = mkOption { }; # TODO make more specific + systemd.services = mkOption { + default = {}; + }; # TODO make more specific }; config = { diff --git a/node-packages-generated.nix b/node-packages-generated.nix deleted file mode 100644 index f1b4369..0000000 --- a/node-packages-generated.nix +++ /dev/null @@ -1,55 +0,0 @@ -{ self, fetchurl, lib }: - -{ - full."minimist"."~0.0.1" = lib.makeOverridable self.buildNodePackage { - name = "minimist-0.0.5"; - src = [ - (fetchurl { - url = "http://registry.npmjs.org/minimist/-/minimist-0.0.5.tgz"; - sha1 = "d7aa327bcecf518f9106ac6b8f003fa3bcea8566"; - }) - ]; - buildInputs = - (self.nativeDeps."minimist"."~0.0.1" or []); - deps = [ - ]; - peerDependencies = [ - ]; - passthru.names = [ "minimist" ]; - }; - full."optimist"."0.6.0" = lib.makeOverridable self.buildNodePackage { - name = "optimist-0.6.0"; - src = [ - (fetchurl { - url = "http://registry.npmjs.org/optimist/-/optimist-0.6.0.tgz"; - sha1 = "69424826f3405f79f142e6fc3d9ae58d4dbb9200"; - }) - ]; - buildInputs = - (self.nativeDeps."optimist"."0.6.0" or []); - deps = [ - self.full."wordwrap"."~0.0.2" - self.full."minimist"."~0.0.1" - ]; - peerDependencies = [ - ]; - passthru.names = [ "optimist" ]; - }; - "optimist" = self.full."optimist"."0.6.0"; - full."wordwrap"."~0.0.2" = lib.makeOverridable self.buildNodePackage { - name = "wordwrap-0.0.2"; - src = [ - (fetchurl { - url = "http://registry.npmjs.org/wordwrap/-/wordwrap-0.0.2.tgz"; - sha1 = "b79669bb42ecb409f83d583cad52ca17eaa1643f"; - }) - ]; - buildInputs = - (self.nativeDeps."wordwrap"."~0.0.2" or []); - deps = [ - ]; - peerDependencies = [ - ]; - passthru.names = [ "wordwrap" ]; - }; -} diff --git a/samples/Dockerfile b/samples/Dockerfile new file mode 100644 index 0000000..370cfa9 --- /dev/null +++ b/samples/Dockerfile @@ -0,0 +1,8 @@ +FROM zefhemel/base-nix +ADD nix-closure /nix/store +RUN /nix/store/da7mjvimgmjpx7z0hra2yc15mganrvpq-build + +CMD /nix/store/jdyh9vgya6wfz3rkiqg75l370b2d10gd-boot +EXPOSE 22 + + diff --git a/samples/apache-config.nix b/samples/apache-config.nix new file mode 100644 index 0000000..b5d3d33 --- /dev/null +++ b/samples/apache-config.nix @@ -0,0 +1,14 @@ +{ config, pkgs, ... }: +{ + # Expose the apache port (80) + docker.ports = [ + config.services.httpd.port + ]; + + services.httpd = { + enable = true; + port = 80; + documentRoot = ./www; + adminAddr = "zef.hemel@logicblox.com"; + }; +} \ No newline at end of file diff --git a/configuration.nix b/samples/fancy.nix similarity index 94% rename from configuration.nix rename to samples/fancy.nix index 8ff3c53..73101e4 100644 --- a/configuration.nix +++ b/samples/fancy.nix @@ -1,7 +1,7 @@ +# Boots up all kinds of services: redis, apache, ssh, mysql { config, pkgs, ... }: { docker.ports = [ 1234 80 22 ]; - docker.volumes = [ "/data" ]; services.redis = { enable = true; diff --git a/samples/result b/samples/result new file mode 120000 index 0000000..0947640 --- /dev/null +++ b/samples/result @@ -0,0 +1 @@ +/nix/store/c5crki76l41vlr9b9sdmslwa3sms1pgh-nix-docker-build \ No newline at end of file diff --git a/samples/ssh-config.nix b/samples/ssh-config.nix new file mode 100644 index 0000000..497a337 --- /dev/null +++ b/samples/ssh-config.nix @@ -0,0 +1,19 @@ +# Runs an SSH server in a Docker container +# Creates a user "you" that you can login with +{ config, pkgs, ... }: +{ + docker.ports = [ 22 ]; + + services.openssh.enable = true; + + users.extraUsers.you = { + group = "users"; + home = "/"; + shell = "/bin/bash"; + createHome = true; + openssh.authorizedKeys.keys = [ + # Replace with your own SSH key (e.g. from ~/.ssh/id_rsa.pub) + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCjWpdyDIsS09lWlOsMG9OMTHB/N/afVU12BwKcyjjhbezPdFEgHK4cZBN7m1bvoFKl832BdB+ZjeRH4UGBcUpvrFu1vE7Lf/0vZDU7qzzWQE9V+tfSPwDiXPf9QnCYeZmYPDHUHDUEse9LKBZbt6UKF1tuTD8ussV5jvEFBaesDhCqD1TJ4b4O877cdx9+VTOuDSEDm32jQ2az27d1b/5DoEKBe5cJSC3PhObAQ7OAYrVVBFX9ffKpaSvV6yqo+rhCmXP9DjNgBwMtElreoXL3h5Xbw2AiER5oHNUAEA2XGpnOVOr7ZZUAbMC0/0dq387jQZCqe7gIDZCqjDpGhUa9" + ]; + }; +} \ No newline at end of file diff --git a/samples/www/index.html b/samples/www/index.html new file mode 100644 index 0000000..b64ccf4 --- /dev/null +++ b/samples/www/index.html @@ -0,0 +1 @@ +Hello world 2 diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh index 1ecd125..0bee307 100644 --- a/scripts/install-nix.sh +++ b/scripts/install-nix.sh @@ -75,9 +75,9 @@ if test -n "\$HOME"; then unset OWNS_STORE fi EOF -cat >> /etc/environment < /etc/init/nix-daemon.conf < Date: Tue, 5 Nov 2013 08:07:51 -0800 Subject: [PATCH 11/46] Updated README --- README.md | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index cc8d644..dfef0d2 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,33 @@ part of the dependencies of the system configuration. Wouldn't it be cool to deploy NixOS configs in docker containers? That's what nix-docker attempts to offer. +How it works +------------ + +`nix-docker` can be run in two modes: + +1. A "traditional" Docker image building mode +2. A thin image mode which mounts the hosts's Nix store into the container + +The first mode will build the system configuration, copy it into a temporary +directory and the `ADD` it into the image in a generated Dockerfile. The result +is a standard, portable Docker image that you can push to a repository. + +To reduce the per-image size, it's possible to use a base image, `nix-docker` is +clever enought to check what `/nix/store` paths are available in the base image +already and not to copy those again for the new image thereby greatly reducing +image sizes. + +The second option builds a very minimal Docker images on-demand containing +only some meta data (like `EXPOSE` and `VOLUME`, `RUN` commands in a Dockerfile) +and is used by mounting in the host's Nix store into the container via +`-v /nix/store:/nix/store`. There's two reasons you may want to use this mode +(even just during development): + +1. Build times are _much_ faster, since Nix builds are fully incremental, only + things that have not been build before will be built. +2. You don't polute your `/var/lib/docker` with a lot of copies of your software. + Installation ------------ To use nix-docker you need Nix installed. If you're using Ubuntu, the easiest way @@ -121,22 +148,6 @@ to produce full Docker images, you also need Usage ----- -`nix-docker` can run in two modes: - -1. full Docker image building mode (by passing in `-b`) -2. using the host's Nix store mode - -The first option is the most portable. `nix-docker` will produce regular Docker -images that you can push to a Docker registry and deploy anywhere where Docker -runs. The second option builds a very minimal Docker images on-demand containing -only some meta data (like `EXPOSE` and `VOLUME`, `RUN` commands in a Dockerfile) -and is used by mounting in the host's Nix store into the container via -`-v /nix/store:/nix/store`. There's two reasons you may want to do the latter: - -1. Build times are _much_ faster, since Nix build are fully incremental, only - things that have not been build before will be built. -2. You don't polute your `/var/lib/docker` with a lot of copies of your software. - To build a stand-alone Docker image: nix-docker -b -t my-image configuration.nix @@ -153,13 +164,13 @@ To build a host-mounted package: This will produce a Nix package (symlinked in the current directory in `result`) containing a script you can use to spawn the container using Docker, e.g.: - sudo -e ./result/sbin/docker-run + sudo -E ./result/sbin/docker-run to run the container in the foreground, or: - sudo -e ./result/sbin/docker-run -d + sudo -E ./result/sbin/docker-run -d to daemonize it. What the `docker-run` script will do is check if there's already a docker image available with the current image name and tag based on the Nix build hash. If not, it will quickly build it first (these images take up barely -any space on disk). Then, it will boot up the container. +any space on disk). Then, it will boot up the container. \ No newline at end of file From 8dd0fe8124067d1ec129772d691e3c2281f61278 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Tue, 5 Nov 2013 08:10:57 -0800 Subject: [PATCH 12/46] Updated README --- README.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index dfef0d2..604aa68 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,11 @@ The problem ----------- [Docker is great](http://www.infoq.com/articles/docker-containers). It's a nice, -pragmatic solution to deploying applications in a portable way. There is one -thing I do not like about Docker, or two: +pragmatic solution to deploying applications in a portable way. There are two +things I don't like about Docker: -(1) How container images are provisioned -(2) How images are distributed +1. How container images are provisioned +2. (This is more minor:) how images are distributed We can argue about (2) and I'm sure it it will improve. Docker images are distributed via a Docker registry. The main one lives at http://index.docker.io, and if your @@ -173,4 +173,13 @@ to run the container in the foreground, or: to daemonize it. What the `docker-run` script will do is check if there's already a docker image available with the current image name and tag based on the Nix build hash. If not, it will quickly build it first (these images take up barely -any space on disk). Then, it will boot up the container. \ No newline at end of file +any space on disk). Then, it will boot up the container. + +Distributing host-mounted packages is done by first copying the Nix closure +resulting from the build to the target machine (when you do the build it +will give you example commands to run): + + nix-copy-closure root@targetmachine /nix/store/.... + +Then, you can spawn the container remotely with the script path provided +in the output of the build command. \ No newline at end of file From 430ac838a64b8f97fe0223124a6f50cee3b29df8 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Tue, 5 Nov 2013 19:36:26 +0100 Subject: [PATCH 13/46] Update install-docker.sh --- scripts/install-docker.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/install-docker.sh b/scripts/install-docker.sh index 38bcb87..4cd8634 100644 --- a/scripts/install-docker.sh +++ b/scripts/install-docker.sh @@ -1,5 +1,9 @@ #!/bin/sh +sudo apt-get update + +sudo apt-get install linux-image-extra-`uname -r` + sudo sh -c "wget -qO- https://get.docker.io/gpg | apt-key add -" # Add the Docker repository to your apt sources list. @@ -10,4 +14,4 @@ sudo sh -c "echo deb http://get.docker.io/ubuntu docker main\ sudo apt-get update # install -sudo apt-get install -y lxc-docker \ No newline at end of file +sudo apt-get install -y lxc-docker From 58a8e536ae785ec64040785739797aa775f4d7cc Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Tue, 5 Nov 2013 19:36:43 +0100 Subject: [PATCH 14/46] Update install-docker.sh --- scripts/install-docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-docker.sh b/scripts/install-docker.sh index 4cd8634..3b561d1 100644 --- a/scripts/install-docker.sh +++ b/scripts/install-docker.sh @@ -2,7 +2,7 @@ sudo apt-get update -sudo apt-get install linux-image-extra-`uname -r` +sudo apt-get install -y linux-image-extra-`uname -r` sudo sh -c "wget -qO- https://get.docker.io/gpg | apt-key add -" From d4973712aae285e85fab74f25d318e32af5b44df Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 6 Nov 2013 10:58:39 +0100 Subject: [PATCH 15/46] Update install-nix.sh --- scripts/install-nix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh index 0bee307..7aecf29 100644 --- a/scripts/install-nix.sh +++ b/scripts/install-nix.sh @@ -4,7 +4,7 @@ set -e # Install the binary tarball... cd / -wget -O - http://hydra.nixos.org/job/nix/trunk/binaryTarball.x86_64-linux/latest/download | tar xvj +wget -O - http://hydra.nixos.org/job/nix/trunk/binaryTarball.x86_64-linux/latest/download | tar xj /usr/bin/nix-finish-install rm /usr/bin/nix-finish-install From 6ee827512c6db3128bbfcabce2f25484ad683618 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 6 Nov 2013 11:08:33 +0100 Subject: [PATCH 16/46] Fixes --- nix-docker/bin/nix-docker | 4 +++- nix-docker/docker.nix | 2 +- samples/Dockerfile | 6 +++--- zedconfig.json | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/nix-docker/bin/nix-docker b/nix-docker/bin/nix-docker index 97fda93..b0ebff3 100755 --- a/nix-docker/bin/nix-docker +++ b/nix-docker/bin/nix-docker @@ -11,6 +11,7 @@ parser.add_argument('configuration', nargs='?', help='configuration.nix file to parser.add_argument('-b', action='store_true', help="Build a docker image") parser.add_argument('-t', help='Image name', default="nix-docker-build") parser.add_argument('--from', help="Image to use as a base image", default="zefhemel/base-nix") +parser.add_argument('-f', action='store_true', help="Force nix-docker into something it does not what to do.") args = parser.parse_args() @@ -19,6 +20,7 @@ configuration_file = args.configuration image_name = args.t full_build = args.b nix_closure_temp = "nix-closure" +force = args.f root_path = os.path.normpath(os.path.realpath(sys.argv[0]) + "/../..") @@ -60,7 +62,7 @@ def copy_closure_and_build(package_nix_path, available_paths): cleanup() if __name__ == '__main__': - if full_build and os.getenv("USER") != "root": + if full_build and os.getenv("USER") != "root" and not force: print "When doing a full build you need to run nix-docker as root. Rerun this command with 'sudo -E nix-docker ...'" sys.exit(1) diff --git a/nix-docker/docker.nix b/nix-docker/docker.nix index 3b806fb..90413a7 100644 --- a/nix-docker/docker.nix +++ b/nix-docker/docker.nix @@ -1,4 +1,4 @@ -{ pkgs ? import {} +{ pkgs ? import { system = "x86_64-linux"; } , name , configuration ? , mountBuild ? true diff --git a/samples/Dockerfile b/samples/Dockerfile index 370cfa9..dba5e8e 100644 --- a/samples/Dockerfile +++ b/samples/Dockerfile @@ -1,8 +1,8 @@ FROM zefhemel/base-nix ADD nix-closure /nix/store -RUN /nix/store/da7mjvimgmjpx7z0hra2yc15mganrvpq-build +RUN /nix/store/l38wlw0rmyp971d2dx720dc8yij2yvb4-build -CMD /nix/store/jdyh9vgya6wfz3rkiqg75l370b2d10gd-boot -EXPOSE 22 +CMD /nix/store/bgrkm48m6kxhh1glvvmg75870a61fgqw-boot +EXPOSE 80 diff --git a/zedconfig.json b/zedconfig.json index edce644..e8d8c1c 100644 --- a/zedconfig.json +++ b/zedconfig.json @@ -2,6 +2,6 @@ "modes": { "python": { "filenames": ["nix-docker"] - } + } } } \ No newline at end of file From 4725124022b683b3bc5a33e5fb141b3b8e3fc5f0 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 6 Nov 2013 11:20:49 +0100 Subject: [PATCH 17/46] Update install-nix.sh --- scripts/install-nix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh index 7aecf29..3ed2ea8 100644 --- a/scripts/install-nix.sh +++ b/scripts/install-nix.sh @@ -42,7 +42,7 @@ if test -n "\$HOME"; then if ! [ -L "\$NIX_LINK" ]; then echo "creating \$NIX_LINK" >&2 if [ -n "\$OWNS_STORE" ]; then - _NIX_PROFILE_LINK=/nix/var/nix/nix/profiles/default + _NIX_PROFILE_LINK=/nix/var/nix/profiles/default else mkdir -p "/nix/var/nix/profiles/per-user/\$LOGNAME" _NIX_PROFILE_LINK="/nix/var/nix/profiles/per-user/\$LOGNAME/profile" From b05786d55245d83d6a493428b54f8891299643f1 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 6 Nov 2013 11:25:30 +0100 Subject: [PATCH 18/46] Update install-nix.sh --- scripts/install-nix.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh index 3ed2ea8..b788af1 100644 --- a/scripts/install-nix.sh +++ b/scripts/install-nix.sh @@ -8,7 +8,7 @@ wget -O - http://hydra.nixos.org/job/nix/trunk/binaryTarball.x86_64-linux/latest /usr/bin/nix-finish-install rm /usr/bin/nix-finish-install -# Setup multiuser +# Setup multiuserbu # Allow all users to create profiles mkdir -p /nix/var/nix/profiles/per-user @@ -26,7 +26,7 @@ chmod 1775 /nix/store mkdir -p /etc/nix grep -w build-users-group /etc/nix/nix.conf 2>/dev/null || echo "build-users-group = nixbld" >> /etc/nix/nix.conf grep -w binary-caches /etc/nix/nix.conf 2>/dev/null || echo "binary-caches = http://cache.nixos.org" >> /etc/nix/nix.conf -grep -w trusted-binary-caches /etc/nix/nix.conf 2>/dev/null || echo "trusted-binary-caches = http://hydra.nixos.org http://builder.logicblox.com http://cache.nixos.org" >> /etc/nix/nix.conf +grep -w trusted-binary-caches /etc/nix/nix.conf 2>/dev/null || echo "trusted-binary-caches = http://hydra.nixos.org http://cache.nixos.org" >> /etc/nix/nix.conf # Use a multiuser-compatible profile script unlink /etc/profile.d/nix.sh From 51f50680bd7186c94eb1e5bee32bea8a5110cff9 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 6 Nov 2013 12:08:08 +0100 Subject: [PATCH 19/46] Update install-nix.sh --- scripts/install-nix.sh | 50 ++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh index b788af1..7f21b2c 100644 --- a/scripts/install-nix.sh +++ b/scripts/install-nix.sh @@ -8,6 +8,9 @@ wget -O - http://hydra.nixos.org/job/nix/trunk/binaryTarball.x86_64-linux/latest /usr/bin/nix-finish-install rm /usr/bin/nix-finish-install +# Hack +chmod 777 /nix/var/nix/profiles + # Setup multiuserbu # Allow all users to create profiles @@ -31,53 +34,52 @@ grep -w trusted-binary-caches /etc/nix/nix.conf 2>/dev/null || echo "trusted-bin # Use a multiuser-compatible profile script unlink /etc/profile.d/nix.sh cat > /etc/profile.d/nix.sh <&2 - if [ -n "\$OWNS_STORE" ]; then - _NIX_PROFILE_LINK=/nix/var/nix/profiles/default - else - mkdir -p "/nix/var/nix/profiles/per-user/\$LOGNAME" - _NIX_PROFILE_LINK="/nix/var/nix/profiles/per-user/\$LOGNAME/profile" - fi - ln -s "\$_NIX_PROFILE_LINK" "\$NIX_LINK" + if ! [ -L "$NIX_LINK" ]; then + echo "creating $NIX_LINK" >&2 + mkdir -p "/nix/var/nix/profiles/per-user/$LOGNAME" + _NIX_PROFILE_LINK="/nix/var/nix/profiles/per-user/$LOGNAME/profile" + ln -s /nix/var/nix/profiles/default $_NIX_PROFILE_LINK + ln -s "$_NIX_PROFILE_LINK" "$NIX_LINK" fi # Subscribe the root user to the Nixpkgs channel by default. - if [ -n "\$OWNS_STORE" ] && [ ! -e "\$HOME/.nix-channels" ]; then - echo "http://nixos.org/channels/nixpkgs-unstable nixpkgs" > "\$HOME/.nix-channels" + if [ ! -e "$HOME/.nix-channels" ]; then + echo "http://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" fi # Set up nix-defexpr - NIX_DEFEXPR="\$HOME/.nix-defexpr" - if ! [ -e "\$NIX_DEFEXPR" ]; then - echo "creating \$NIX_DEFEXPR" >&2 - mkdir -p "\$NIX_DEFEXPR" - _NIX_CHANNEL_LINK=/nix/var/nix/profiles/per-user/root/channels/nixpkgs - ln -s "\$_NIX_CHANNEL_LINK" "\$NIX_DEFEXPR" + NIX_DEFEXPR="$HOME/.nix-defexpr" + if ! [ -e "$NIX_DEFEXPR" ]; then + #echo "creating $NIX_DEFEXPR" >&2 + #mkdir -p "$NIX_DEFEXPR" + #_NIX_CHANNEL_LINK=/nix/var/nix/profiles/per-user/$LOGNAME/channels/nixpkgs + #ln -s "$_NIX_CHANNEL_LINK" "$NIX_DEFEXPR" + /nix/var/nix/profiles/default/bin/nix-channel --update fi - if [ -z "\$OWNS_STORE" ]; then + if [ -z "$OWNS_STORE" ]; then export NIX_REMOTE=daemon - export PATH="/nix/var/nix/profiles/default/bin:\$PATH" + export PATH="/nix/var/nix/profiles/default/bin:$PATH" fi - export PATH="\$NIX_LINK/bin:\$PATH" + export PATH="$NIX_LINK/bin:$PATH" # Set up NIX_PATH - export NIX_PATH="\${NIX_PATH:+\$NIX_PATH:}/nix/var/nix/profiles/per-user/\$LOGNAME/channels" + export NIX_PATH="${NIX_PATH:+$NIX_PATH:}/nix/var/nix/profiles/per-user/$LOGNAME/channels" unset OWNS_STORE fi EOF -# Add default nix profile to global path to make nix-copy-closure work +# Add default nix profile to global path and enable it during sudo sed -i 's/"$/:\/nix\/var\/nix\/profiles\/default\/bin"/' /etc/environment +sed -i 's/secure_path="/secure_path="\/nix\/var\/nix\/profiles\/default\/bin:/' /etc/sudoers # Install upstart job cat > /etc/init/nix-daemon.conf < Date: Wed, 6 Nov 2013 12:18:37 +0100 Subject: [PATCH 20/46] Update install-nix.sh --- scripts/install-nix.sh | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh index 7f21b2c..7efa664 100644 --- a/scripts/install-nix.sh +++ b/scripts/install-nix.sh @@ -34,45 +34,45 @@ grep -w trusted-binary-caches /etc/nix/nix.conf 2>/dev/null || echo "trusted-bin # Use a multiuser-compatible profile script unlink /etc/profile.d/nix.sh cat > /etc/profile.d/nix.sh <&2 - mkdir -p "/nix/var/nix/profiles/per-user/$LOGNAME" - _NIX_PROFILE_LINK="/nix/var/nix/profiles/per-user/$LOGNAME/profile" + if ! [ -L "$\NIX_LINK" ]; then + echo "creating $\NIX_LINK" >&2 + mkdir -p "/nix/var/nix/profiles/per-user/\$LOGNAME" + _NIX_PROFILE_LINK="/nix/var/nix/profiles/per-user/\$LOGNAME/profile" ln -s /nix/var/nix/profiles/default $_NIX_PROFILE_LINK - ln -s "$_NIX_PROFILE_LINK" "$NIX_LINK" + ln -s "\$_NIX_PROFILE_LINK" "\$NIX_LINK" fi # Subscribe the root user to the Nixpkgs channel by default. - if [ ! -e "$HOME/.nix-channels" ]; then - echo "http://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$HOME/.nix-channels" + if [ ! -e "\$HOME/.nix-channels" ]; then + echo "http://nixos.org/channels/nixpkgs-unstable nixpkgs" > "\$HOME/.nix-channels" fi # Set up nix-defexpr - NIX_DEFEXPR="$HOME/.nix-defexpr" - if ! [ -e "$NIX_DEFEXPR" ]; then - #echo "creating $NIX_DEFEXPR" >&2 - #mkdir -p "$NIX_DEFEXPR" - #_NIX_CHANNEL_LINK=/nix/var/nix/profiles/per-user/$LOGNAME/channels/nixpkgs - #ln -s "$_NIX_CHANNEL_LINK" "$NIX_DEFEXPR" - /nix/var/nix/profiles/default/bin/nix-channel --update + NIX_DEFEXPR="\$HOME/.nix-defexpr" + if ! [ -e "\$NIX_DEFEXPR" ]; then + echo "creating \$NIX_DEFEXPR" >&2 + mkdir -p "\$NIX_DEFEXPR" + _NIX_CHANNEL_LINK=/nix/var/nix/profiles/per-user/root/channels + ln -s "\$_NIX_CHANNEL_LINK" "\$NIX_DEFEXPR/channels" + #/nix/var/nix/profiles/default/bin/nix-channel --update fi - if [ -z "$OWNS_STORE" ]; then + if [ -z "\$OWNS_STORE" ]; then export NIX_REMOTE=daemon - export PATH="/nix/var/nix/profiles/default/bin:$PATH" + export PATH="/nix/var/nix/profiles/default/bin:\$PATH" fi - export PATH="$NIX_LINK/bin:$PATH" + export PATH="\$NIX_LINK/bin:$PATH" # Set up NIX_PATH - export NIX_PATH="${NIX_PATH:+$NIX_PATH:}/nix/var/nix/profiles/per-user/$LOGNAME/channels" + export NIX_PATH="${NIX_PATH:+\$NIX_PATH:}/nix/var/nix/profiles/per-user/\$LOGNAME/channels" unset OWNS_STORE fi EOF From 38a5c6e602f411e9fe2a534f3cb5fc37115bb405 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 6 Nov 2013 12:24:35 +0100 Subject: [PATCH 21/46] Update install-nix.sh --- scripts/install-nix.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh index 7efa664..47c5f23 100644 --- a/scripts/install-nix.sh +++ b/scripts/install-nix.sh @@ -43,10 +43,10 @@ if test -n "\$HOME"; then # Set the default profile. if ! [ -L "$\NIX_LINK" ]; then - echo "creating $\NIX_LINK" >&2 + echo "creating \$NIX_LINK" >&2 mkdir -p "/nix/var/nix/profiles/per-user/\$LOGNAME" _NIX_PROFILE_LINK="/nix/var/nix/profiles/per-user/\$LOGNAME/profile" - ln -s /nix/var/nix/profiles/default $_NIX_PROFILE_LINK + ln -s /nix/var/nix/profiles/default \$_NIX_PROFILE_LINK ln -s "\$_NIX_PROFILE_LINK" "\$NIX_LINK" fi @@ -69,7 +69,7 @@ if test -n "\$HOME"; then export NIX_REMOTE=daemon export PATH="/nix/var/nix/profiles/default/bin:\$PATH" fi - export PATH="\$NIX_LINK/bin:$PATH" + export PATH="\$NIX_LINK/bin:\$PATH" # Set up NIX_PATH export NIX_PATH="${NIX_PATH:+\$NIX_PATH:}/nix/var/nix/profiles/per-user/\$LOGNAME/channels" From 5f79543586136604c07316dde2dfecfeacfae363 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 6 Nov 2013 17:51:37 +0100 Subject: [PATCH 22/46] Work --- README.md | 19 ++++++++----------- Vagrantfile | 20 ++++++++++++++++++++ build-base.sh | 2 +- default.nix | 1 + nix-docker/bin/nix-docker | 19 +++++++++++++------ samples/result | 1 - scripts/install-docker.sh | 12 ++++++------ scripts/install-nix.sh | 13 +++++++++---- zedconfig.json | 3 +++ 9 files changed, 61 insertions(+), 29 deletions(-) create mode 100644 Vagrantfile delete mode 120000 samples/result diff --git a/README.md b/README.md index 604aa68..899cf43 100644 --- a/README.md +++ b/README.md @@ -128,22 +128,19 @@ and is used by mounting in the host's Nix store into the container via Installation ------------ -To use nix-docker you need Nix installed. If you're using Ubuntu, the easiest way -to do so is by running the `script/install-nix.sh` script as root: +To use nix-docker you need [Nix](http://nixos.org/nix) installed as well as +[Docker](http://www.docker.io). The easy way to do this is to use [Vagrant](http://vagrantup.com). - curl https://raw.github.com/zefhemel/nix-docker/master/scripts/install-nix.sh | sudo sh - -After this script runs successfully, log out and back in. - -Next, install nix-docker itself: +When you have Vagrant installed: git clone https://github.com/zefhemel/nix-docker.git cd nix-docker - nix-env -f . -i nix-docker + vagrant up + vagrant ssh -You can now run nix-docker in its default mode (more on this later). To be able -to produce full Docker images, you also need -[Docker itself installed](http://docs.docker.io/en/latest/installation/ubuntulinux/). +If all went well, you're now in a VM that has both Docker and Nix installed +and `nix-docker` in its path. You can now cd into the nix-docker/samples directory +to try to build some of the examples. Usage ----- diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..63be66b --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,20 @@ +Vagrant.configure("2") do |config| + config.vm.box = "raring64" + config.vm.box_url = "http://goo.gl/ceHWg" + + config.vm.provision :shell, :path => "scripts/install-docker.sh" + config.vm.provision :shell, :path => "scripts/install-nix.sh" + config.vm.provision "shell", + inline: < /etc/profile.d/path.sh +chmod 777 /var/run/docker.sock +eos + + config.vm.network "private_network", ip: "192.168.22.22" + + config.vm.synced_folder ".", "/home/vagrant/nix-docker" + + config.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", "1024"] + end +end diff --git a/build-base.sh b/build-base.sh index ae14dac..df5a4a8 100755 --- a/build-base.sh +++ b/build-base.sh @@ -1,2 +1,2 @@ #!/bin/sh -sudo -E nix-docker/bin/nix-docker base-configuration.nix -t zefhemel/base-nix --from busybox -b +sudo -E nix-docker -b -t zefhemel/base-nix --from busybox base-configuration.nix \ No newline at end of file diff --git a/default.nix b/default.nix index 28988df..959c4f2 100644 --- a/default.nix +++ b/default.nix @@ -4,6 +4,7 @@ in pkgs.stdenv.mkDerivation { name = "nix-docker-0.1"; src = ./nix-docker; + buildInputs = [ pkgs.python27 ]; installPhase = '' mkdir -p $out cp -R * $out/ diff --git a/nix-docker/bin/nix-docker b/nix-docker/bin/nix-docker index b0ebff3..a3bf7af 100755 --- a/nix-docker/bin/nix-docker +++ b/nix-docker/bin/nix-docker @@ -5,6 +5,7 @@ import subprocess import argparse import sys import shutil +import tempfile parser = argparse.ArgumentParser(description='Build Docker images with Nix', prog='nix-docker') parser.add_argument('configuration', nargs='?', help='configuration.nix file to build', default="configuration.nix") @@ -19,7 +20,10 @@ base_image = getattr(args, "from") configuration_file = args.configuration image_name = args.t full_build = args.b -nix_closure_temp = "nix-closure" + +temp_dir = tempfile.mkdtemp() + +nix_closure_temp = "%s/nix-closure" % temp_dir force = args.f root_path = os.path.normpath(os.path.realpath(sys.argv[0]) + "/../..") @@ -30,7 +34,7 @@ def build(): mountBuild = "false" else: mountBuild = "true" - cmd = ["nix-build", "%s/docker.nix" % root_path, '-I', 'configuration=' + configuration_file, "--arg", "mountBuild", mountBuild, "--argstr", "name", image_name, "--argstr", "baseImage", base_image, "--show-trace"] + cmd = ["nix-build", "%s/docker.nix" % root_path, '-I', 'configuration=' + configuration_file, "--arg", "mountBuild", mountBuild, "--argstr", "name", image_name, "--argstr", "baseImage", base_image] if full_build: cmd.append("--no-out-link") nix_path = subprocess.check_output(cmd) @@ -44,7 +48,7 @@ def get_available_nix_paths(): return [] def cleanup(): - subprocess.check_call(["rm", "-rf", nix_closure_temp]) + subprocess.check_call(["rm", "-rf", temp_dir]) def copy_closure_and_build(package_nix_path, available_paths): paths = subprocess.check_output(["nix-store", "-qR", package_nix_path]).strip().split("\n") @@ -55,10 +59,10 @@ def copy_closure_and_build(package_nix_path, available_paths): cmd = ["cp", "-r"] cmd.extend(paths_to_copy) cmd.append(nix_closure_temp) - shutil.copyfile("%s/Dockerfile" % package_nix_path, "Dockerfile") + shutil.copyfile("%s/Dockerfile" % package_nix_path, "%s/Dockerfile" % temp_dir) subprocess.check_call(cmd) subprocess.check_call(["chown", "-R", "root:root", nix_closure_temp]) - subprocess.check_call(["docker", "build", "-rm=true", "-t", image_name, "."]) + subprocess.check_call(["docker", "build", "-rm=true", "-t", image_name, temp_dir]) cleanup() if __name__ == '__main__': @@ -66,9 +70,12 @@ if __name__ == '__main__': print "When doing a full build you need to run nix-docker as root. Rerun this command with 'sudo -E nix-docker ...'" sys.exit(1) + if not os.path.exists(configuration_file): + print "Could not find configuration file: %s" % configuration_file + sys.exit(1) + package_nix_path = build() if full_build: - cleanup() available_paths = get_available_nix_paths() copy_closure_and_build(package_nix_path, available_paths) print "To run: sudo docker run -t -i", image_name diff --git a/samples/result b/samples/result deleted file mode 120000 index 0947640..0000000 --- a/samples/result +++ /dev/null @@ -1 +0,0 @@ -/nix/store/c5crki76l41vlr9b9sdmslwa3sms1pgh-nix-docker-build \ No newline at end of file diff --git a/scripts/install-docker.sh b/scripts/install-docker.sh index 3b561d1..3b057d1 100644 --- a/scripts/install-docker.sh +++ b/scripts/install-docker.sh @@ -1,17 +1,17 @@ #!/bin/sh -sudo apt-get update +apt-get update -sudo apt-get install -y linux-image-extra-`uname -r` +apt-get install -y linux-image-extra-`uname -r` -sudo sh -c "wget -qO- https://get.docker.io/gpg | apt-key add -" +sh -c "wget -qO- https://get.docker.io/gpg | apt-key add -" # Add the Docker repository to your apt sources list. -sudo sh -c "echo deb http://get.docker.io/ubuntu docker main\ +sh -c "echo deb http://get.docker.io/ubuntu docker main\ > /etc/apt/sources.list.d/docker.list" # update -sudo apt-get update +apt-get update # install -sudo apt-get install -y lxc-docker +apt-get install -y lxc-docker diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh index 47c5f23..1fe98f2 100644 --- a/scripts/install-nix.sh +++ b/scripts/install-nix.sh @@ -2,9 +2,14 @@ set -e +# Clean up old Nix stuff +#rm -f /etc/profile.d/nix.sh +#rm -rf /nix + # Install the binary tarball... +apt-get install -y curl cd / -wget -O - http://hydra.nixos.org/job/nix/trunk/binaryTarball.x86_64-linux/latest/download | tar xj +curl -L http://hydra.nixos.org/job/nix/trunk/binaryTarball.x86_64-linux/latest/download | tar xj /usr/bin/nix-finish-install rm /usr/bin/nix-finish-install @@ -42,11 +47,11 @@ if test -n "\$HOME"; then fi # Set the default profile. - if ! [ -L "$\NIX_LINK" ]; then + if ! [ -L "\$NIX_LINK" ]; then echo "creating \$NIX_LINK" >&2 mkdir -p "/nix/var/nix/profiles/per-user/\$LOGNAME" _NIX_PROFILE_LINK="/nix/var/nix/profiles/per-user/\$LOGNAME/profile" - ln -s /nix/var/nix/profiles/default \$_NIX_PROFILE_LINK + ln -s /nix/var/nix/profiles/default \$_NIX_PROFILE_LINK ln -s "\$_NIX_PROFILE_LINK" "\$NIX_LINK" fi @@ -72,7 +77,7 @@ if test -n "\$HOME"; then export PATH="\$NIX_LINK/bin:\$PATH" # Set up NIX_PATH - export NIX_PATH="${NIX_PATH:+\$NIX_PATH:}/nix/var/nix/profiles/per-user/\$LOGNAME/channels" + export NIX_PATH="\$NIX_DEFEXPR/channels" unset OWNS_STORE fi EOF diff --git a/zedconfig.json b/zedconfig.json index e8d8c1c..cb15520 100644 --- a/zedconfig.json +++ b/zedconfig.json @@ -2,6 +2,9 @@ "modes": { "python": { "filenames": ["nix-docker"] + }, + "ruby": { + "filenames": ["Vagrantfile"] } } } \ No newline at end of file From 1a65dc2dfe351c8f2eba467c5dcb1881ac92fa24 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 6 Nov 2013 17:54:34 +0100 Subject: [PATCH 23/46] Update readme --- README.md | 4 +++- samples/Dockerfile | 8 -------- 2 files changed, 3 insertions(+), 9 deletions(-) delete mode 100644 samples/Dockerfile diff --git a/README.md b/README.md index 899cf43..a9ae988 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,9 @@ When you have Vagrant installed: If all went well, you're now in a VM that has both Docker and Nix installed and `nix-docker` in its path. You can now cd into the nix-docker/samples directory -to try to build some of the examples. +to try to build some of the examples. Note that the `~/nix-docker` directory +is mounted from your host machine, so you can edit your files with your +favorite editor and have them available within the VM. Usage ----- diff --git a/samples/Dockerfile b/samples/Dockerfile deleted file mode 100644 index dba5e8e..0000000 --- a/samples/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM zefhemel/base-nix -ADD nix-closure /nix/store -RUN /nix/store/l38wlw0rmyp971d2dx720dc8yij2yvb4-build - -CMD /nix/store/bgrkm48m6kxhh1glvvmg75870a61fgqw-boot -EXPOSE 80 - - From 591ac4f57b86ca9c3464749bea28dca27d5b10c9 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 6 Nov 2013 17:58:06 +0100 Subject: [PATCH 24/46] README --- README.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a9ae988..8a48f96 100644 --- a/README.md +++ b/README.md @@ -126,10 +126,9 @@ and is used by mounting in the host's Nix store into the container via things that have not been build before will be built. 2. You don't polute your `/var/lib/docker` with a lot of copies of your software. -Installation ------------- -To use nix-docker you need [Nix](http://nixos.org/nix) installed as well as -[Docker](http://www.docker.io). The easy way to do this is to use [Vagrant](http://vagrantup.com). +Installation with Vagrant +------------------------- +The easy way to do this is to use [Vagrant](http://vagrantup.com). When you have Vagrant installed: @@ -144,6 +143,17 @@ to try to build some of the examples. Note that the `~/nix-docker` directory is mounted from your host machine, so you can edit your files with your favorite editor and have them available within the VM. +Installation +------------ + +To use nix-docker you need [Nix](http://nixos.org/nix) installed as well as +[Docker](http://www.docker.io). Realistically, your best way to do this on +an Ubuntu (12.04 or 13.04) box. Once these are installed, installing `nix-docker` +is as simple as: + + git clone https://github.com/zefhemel/nix-docker.git + nix-env -f nix-docker/default.nix -i nix-docker + Usage ----- From ac638b60efd2221111aceb90bdd67125ef021266 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 6 Nov 2013 18:02:46 +0100 Subject: [PATCH 25/46] Vagrant file tweaks --- Vagrantfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 63be66b..5846e01 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -4,8 +4,7 @@ Vagrant.configure("2") do |config| config.vm.provision :shell, :path => "scripts/install-docker.sh" config.vm.provision :shell, :path => "scripts/install-nix.sh" - config.vm.provision "shell", - inline: < /etc/profile.d/path.sh chmod 777 /var/run/docker.sock eos From d4ebe5d9395315e3ed83b2122e30b0ea4328c35b Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Thu, 7 Nov 2013 11:01:50 +0100 Subject: [PATCH 26/46] Updated Vagrantfile --- Vagrantfile | 28 ++++++++++++++++++++++------ scripts/install-docker.sh | 4 ++++ scripts/install-nix.sh | 10 ++++++---- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 5846e01..daa5fca 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -2,18 +2,34 @@ Vagrant.configure("2") do |config| config.vm.box = "raring64" config.vm.box_url = "http://goo.gl/ceHWg" - config.vm.provision :shell, :path => "scripts/install-docker.sh" - config.vm.provision :shell, :path => "scripts/install-nix.sh" + config.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", "1024"] + + # We'll attach an extra 50GB disk for all nix and docker data + file_to_disk = "disk.vmdk" + vb.customize ['createhd', '--filename', file_to_disk, '--size', 50 * 1024] + vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', file_to_disk] + end + config.vm.provision :shell, inline: < /etc/profile.d/path.sh -chmod 777 /var/run/docker.sock + +if [ ! -d /data ]; then + mkfs.ext4 -F /dev/sdb + mkdir -p /nix + echo "/dev/sdb /nix ext4 defaults 0 2" >> /etc/fstab + mount /nix + mkdir -p /nix/docker-lib + ln -s /nix/docker-lib /var/lib/docker +fi eos + config.vm.provision :shell, :path => "scripts/install-docker.sh" + config.vm.provision :shell, inline: "chmod 777 /var/run/docker.sock" + config.vm.provision :shell, :path => "scripts/install-nix.sh" + config.vm.network "private_network", ip: "192.168.22.22" config.vm.synced_folder ".", "/home/vagrant/nix-docker" - config.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--memory", "1024"] - end end diff --git a/scripts/install-docker.sh b/scripts/install-docker.sh index 3b057d1..4cf9e6f 100644 --- a/scripts/install-docker.sh +++ b/scripts/install-docker.sh @@ -1,5 +1,9 @@ #!/bin/sh +if [ "$(which docker)" != "" ]; then + exit 0 +fi + apt-get update apt-get install -y linux-image-extra-`uname -r` diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh index 1fe98f2..4f432a8 100644 --- a/scripts/install-nix.sh +++ b/scripts/install-nix.sh @@ -2,12 +2,14 @@ set -e -# Clean up old Nix stuff -#rm -f /etc/profile.d/nix.sh -#rm -rf /nix +# Check if Nix is already installed + +if [ -d "/nix/store" ]; then + exit 0 +fi # Install the binary tarball... -apt-get install -y curl +apt-get install -y curl vim cd / curl -L http://hydra.nixos.org/job/nix/trunk/binaryTarball.x86_64-linux/latest/download | tar xj /usr/bin/nix-finish-install From 2438f7701a30b9d27fbf4039306a81343c3b9933 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Thu, 7 Nov 2013 16:28:53 +0100 Subject: [PATCH 27/46] Tweaks and README --- README.md | 146 ++-------------------- nix-docker/bin/nix-docker | 6 +- nix-docker/modules/config/user-groups.nix | 24 ++-- 3 files changed, 24 insertions(+), 152 deletions(-) diff --git a/README.md b/README.md index 8a48f96..1faffee 100644 --- a/README.md +++ b/README.md @@ -1,130 +1,10 @@ nix-docker ========== -The problem ------------ +Use [NixOS](http://nixos.org/nixos) configurations to provision [Docker](http://docker.io) +containers. -[Docker is great](http://www.infoq.com/articles/docker-containers). It's a nice, -pragmatic solution to deploying applications in a portable way. There are two -things I don't like about Docker: - -1. How container images are provisioned -2. (This is more minor:) how images are distributed - -We can argue about (2) and I'm sure it it will improve. Docker images are distributed -via a Docker registry. The main one lives at http://index.docker.io, and if your -application can be openly available to everyone, this works great. If you need -to keep applications in-house, you have to host your own registry server, and -support for that (e.g. with authentication) is still limited. - -But my main gripe is (1). Docker images are provisioned using a -[Dockerfile](http://docs.docker.io/en/latest/use/builder/), which basically -a simple imperative script that builds up an image from a base image step by -step. Typical Dockerfile lines look like this: - - RUN apt-get update - RUN apt-get upgrade -y - RUN apt-get install -y openssh-server python curl - -Every such command that you run is committed, resulting in an AuFS layer. This -can be helpful, because Docker can now do basic caching. For instance, if -your build fails at the third line, it can use the image resulting from -the first two lines and start from there. There's a number problems with this -approach: - -1. Can those first two lines really be cached, or may their result be dependent - on the time of being run? Answer: yes, running these lines tomorrow may yield - different results than running them today, but Docker will naively assume - they will always result in the same thing. -2. AuFS can not handle an unlimited number of layers, if your Dockerfile has too - many commands the build will simply fail. So, you better make them count. -3. There's very little support for reuse. There's no include files or configuration - language. All you can do is create a hierarchy of images, where you create - a base image with software common to all other images, and then you use `FROM` - to base future image on. And you're still limited by (2), the layers all add - up. - -What Nix brings to the table ----------------------------- - -[Nix](http://nixos.org/nix/) is a relatively new package manager for Unix systems, -it's not specific to Linux, it works on Mac as well, for instance. In short, it's -package management done right: - -* Packages and sytems are built using the Nix functional language, which is a - full-blown functional language designed specifically for deploying simple and - complex systems. -* Completele dependency closures are automatically derived, so do not have to - be constructed by hand (as is the case with dpkg and RPM). -* Rather than scattering files all over the disk (configuration files in `/etc`, - binaries in `/usr/bin` or `/usr/local/bin` or is it `/bin`), components are - stored in isolation in the Nix store (`/nix/store`) not interfering with each - other. -* Nix has excellent support for modularization. - -While Nix itself is "just" a package manager, there are many tools built on top -of it, including a full-blown Linux distribution called [NixOS](http://nixos.org/nixos/). - -NixOS enables you to declaratively specify your entire system configuration using -the Nix language. Based on this configuration, it can build the entire system, which -can be deployed locally or remotely via [NixOps](https://github.com/NixOS/nixops). - -In the context of Docker, the problem with NixOS is that it's truly a full-blown -OS. All services run using [systemd](http://www.freedesktop.org/wiki/Software/systemd/), -kernels are deployed and a bunch of utility processes are running at all times. - -I tried to see if it's feasible to deploy a system configuration built for NixOS -in a Docker container, but couldn't get it to run because systemd wouldn't run, -beside that, configurations quickly get large, many hundreds of megabytes. So -much for light-weight containers, huh. - -Nevertheless, NixOS configurations are kind of nice and clean and would make sense -for Docker as well. For instance, here's how to run a simple Apache server -serving static files from `./www` directory: - - { config, pkgs, ... }: - { - services.httpd = { - enable = true; - documentRoot = ./www; - adminAddr = "zef.hemel@logicblox.com"; - }; - } - -That's all it takes. The `./www` there refers to the path `./www` local to the -system configuration file, by the way. When the system configuration is built, -the contents of `./www` is automatically copied into the Nix store and becomes -part of the dependencies of the system configuration. - -Wouldn't it be cool to deploy NixOS configs in docker containers? That's what -nix-docker attempts to offer. - -How it works ------------- - -`nix-docker` can be run in two modes: - -1. A "traditional" Docker image building mode -2. A thin image mode which mounts the hosts's Nix store into the container - -The first mode will build the system configuration, copy it into a temporary -directory and the `ADD` it into the image in a generated Dockerfile. The result -is a standard, portable Docker image that you can push to a repository. - -To reduce the per-image size, it's possible to use a base image, `nix-docker` is -clever enought to check what `/nix/store` paths are available in the base image -already and not to copy those again for the new image thereby greatly reducing -image sizes. - -The second option builds a very minimal Docker images on-demand containing -only some meta data (like `EXPOSE` and `VOLUME`, `RUN` commands in a Dockerfile) -and is used by mounting in the host's Nix store into the container via -`-v /nix/store:/nix/store`. There's two reasons you may want to use this mode -(even just during development): - -1. Build times are _much_ faster, since Nix builds are fully incremental, only - things that have not been build before will be built. -2. You don't polute your `/var/lib/docker` with a lot of copies of your software. +[Read about the what and why in this blog post](http://zef.me/6049/nix-docker) Installation with Vagrant ------------------------- @@ -138,18 +18,18 @@ When you have Vagrant installed: vagrant ssh If all went well, you're now in a VM that has both Docker and Nix installed -and `nix-docker` in its path. You can now cd into the nix-docker/samples directory -to try to build some of the examples. Note that the `~/nix-docker` directory -is mounted from your host machine, so you can edit your files with your -favorite editor and have them available within the VM. +and `nix-docker` in its path. You can now cd into the nix-docker/samples +directory to try to build some of the examples. Note that the `~/nix-docker` +directory is mounted from your host machine, so you can edit your files with +your favorite editor and have them available within the VM. Installation ------------ To use nix-docker you need [Nix](http://nixos.org/nix) installed as well as [Docker](http://www.docker.io). Realistically, your best way to do this on -an Ubuntu (12.04 or 13.04) box. Once these are installed, installing `nix-docker` -is as simple as: +an Ubuntu (12.04 or 13.04) box. Once these are installed, installing +`nix-docker` is as simple as: git clone https://github.com/zefhemel/nix-docker.git nix-env -f nix-docker/default.nix -i nix-docker @@ -179,10 +59,10 @@ to run the container in the foreground, or: sudo -E ./result/sbin/docker-run -d -to daemonize it. What the `docker-run` script will do is check if there's already -a docker image available with the current image name and tag based on the Nix -build hash. If not, it will quickly build it first (these images take up barely -any space on disk). Then, it will boot up the container. +to daemonize it. What the `docker-run` script will do is check if there's +already a docker image available with the current image name and tag based on +the Nix build hash. If not, it will quickly build it first (these images take up +barely any space on disk). Then, it will boot up the container. Distributing host-mounted packages is done by first copying the Nix closure resulting from the build to the target machine (when you do the build it diff --git a/nix-docker/bin/nix-docker b/nix-docker/bin/nix-docker index a3bf7af..2660bc3 100755 --- a/nix-docker/bin/nix-docker +++ b/nix-docker/bin/nix-docker @@ -11,7 +11,7 @@ parser = argparse.ArgumentParser(description='Build Docker images with Nix', pro parser.add_argument('configuration', nargs='?', help='configuration.nix file to build', default="configuration.nix") parser.add_argument('-b', action='store_true', help="Build a docker image") parser.add_argument('-t', help='Image name', default="nix-docker-build") -parser.add_argument('--from', help="Image to use as a base image", default="zefhemel/base-nix") +parser.add_argument('--from', help="Image to use as a base image", default="busybox") parser.add_argument('-f', action='store_true', help="Force nix-docker into something it does not what to do.") args = parser.parse_args() @@ -54,7 +54,9 @@ def copy_closure_and_build(package_nix_path, available_paths): paths = subprocess.check_output(["nix-store", "-qR", package_nix_path]).strip().split("\n") print "Available", available_paths paths_to_copy = filter(lambda path: not path[len("/nix/store/"):] in available_paths, paths) - print "New Nix store paths to copy to image:", paths_to_copy + print "New Nix store paths to copy to image:" + for path in paths_to_copy: + print " ", path os.mkdir(nix_closure_temp) cmd = ["cp", "-r"] cmd.extend(paths_to_copy) diff --git a/nix-docker/modules/config/user-groups.nix b/nix-docker/modules/config/user-groups.nix index fdcc79f..7bbedd4 100644 --- a/nix-docker/modules/config/user-groups.nix +++ b/nix-docker/modules/config/user-groups.nix @@ -201,8 +201,6 @@ in ''; options = [ groupOpts ]; }; - - users.files = mkOption {}; }; config = { @@ -241,9 +239,8 @@ in adm = { }; }; - users.files = { - passwdFile = pkgs.writeText "passwd" '' - ${concatMapStrings (name: + environment.etc.passwd.text = + concatMapStrings (name: let user = getAttr name uidUsers; in @@ -251,22 +248,15 @@ in "${user.name}:x:${toString user.uid}:${toString (getAttr user.group gidGroups).gid}:${user.description}:${user.home}:${user.shell}\n" else "" - ) (attrNames uidUsers)} - ''; - groupFile = pkgs.writeText "group" '' - ${concatMapStrings (name: + ) (attrNames uidUsers); + + environment.etc.group.text = + concatMapStrings (name: let group = getAttr name gidGroups; in "${group.name}:x:${toString group.gid}:\n" - ) (attrNames gidGroups)} - ''; - }; - - docker.buildScripts."0-userfiles" = '' - cp ${config.users.files.groupFile} /etc/group - cp ${config.users.files.passwdFile} /etc/passwd - ''; + ) (attrNames gidGroups); }; } \ No newline at end of file From c6f209f144b2052b9a9a222e61e0491bdd53e519 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 8 Nov 2013 11:26:23 +0100 Subject: [PATCH 28/46] Various tweaks --- nix-docker/docker.nix | 1 + nix-docker/modules/config/environment.nix | 9 +- nix-docker/modules/config/user-groups.nix | 10 ++ nix-docker/modules/servers/supervisord.nix | 5 + samples/wordpress.nix | 124 +++++++++++++++++++++ scripts/install-nix.sh | 2 +- 6 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 samples/wordpress.nix diff --git a/nix-docker/docker.nix b/nix-docker/docker.nix index 90413a7..249399b 100644 --- a/nix-docker/docker.nix +++ b/nix-docker/docker.nix @@ -26,6 +26,7 @@ let bootScript = pkgs.writeScript "boot" '' #!/bin/sh -e${verboseFlag} + umask ${config.config.environment.umask} ${if mountBuild then config.config.docker.buildScript else ""} ${config.config.docker.bootScript} ''; diff --git a/nix-docker/modules/config/environment.nix b/nix-docker/modules/config/environment.nix index 115fc8d..fd3ce6d 100644 --- a/nix-docker/modules/config/environment.nix +++ b/nix-docker/modules/config/environment.nix @@ -22,9 +22,14 @@ in { description = "Packages to be put in the system profile."; }; + environment.umask = mkOption { + default = "002"; + type = with types; string; + }; + + # HACK HACK system.activationScripts.etc = mkOption {}; # Ignore system.build.etc = mkOption {}; # Ignore - }; config = { @@ -38,7 +43,7 @@ in { ln -sf /usr/bin/bash /bin/bash ''; - environment.systemPackages = with pkgs; [ coreutils bash ]; + environment.systemPackages = with pkgs; [ coreutils bash ]; docker.buildScripts."0-etc" = '' echo "setting up /etc..." diff --git a/nix-docker/modules/config/user-groups.nix b/nix-docker/modules/config/user-groups.nix index 7bbedd4..89f75a3 100644 --- a/nix-docker/modules/config/user-groups.nix +++ b/nix-docker/modules/config/user-groups.nix @@ -257,6 +257,16 @@ in in "${group.name}:x:${toString group.gid}:\n" ) (attrNames gidGroups); + + docker.buildScripts."1-create-homes" = concatMapStrings (name: + let + user = getAttr name uidUsers; + in + if user.createHome then + "mkdir -p ${user.home}; chown ${user.name}:${user.group} ${user.home}\n" + else + "" + ) (attrNames uidUsers); }; } \ No newline at end of file diff --git a/nix-docker/modules/servers/supervisord.nix b/nix-docker/modules/servers/supervisord.nix index 9888204..01a6f3c 100644 --- a/nix-docker/modules/servers/supervisord.nix +++ b/nix-docker/modules/servers/supervisord.nix @@ -20,6 +20,10 @@ let PATH = "/some/path"; }; }; + startsecs = mkOption { + default = 1; + example = 0; + }; }; }; services = config.supervisord.services; @@ -69,6 +73,7 @@ in { redirect_stderr=true stdout_logfile=/var/log/supervisord/${name}.log user=${cfg.user} + startsecs=${toString cfg.startsecs} '' ) (attrNames services) } diff --git a/samples/wordpress.nix b/samples/wordpress.nix new file mode 100644 index 0000000..be872f2 --- /dev/null +++ b/samples/wordpress.nix @@ -0,0 +1,124 @@ +{ config, pkgs, ... }: +# We'll start with defining some local variables +let + # Settings used to configure the wordpress MySQL database + mysqlHost = "localhost"; + mysqlDb = "wordpress"; + mysqlUser = "wordpress"; + mysqlPassword = "wordpress"; + + # Our bare-bones wp-config.php file using the above settings + wordpressConfig = pkgs.writeText "wp-config.php" '' + + RewriteEngine On + RewriteBase / + RewriteCond %{REQUEST_FILENAME} !-f + RewriteCond %{REQUEST_FILENAME} !-d + RewriteRule . /index.php [L] + + ''; + + # For shits and giggles, let's package the responsive theme + responsiveTheme = pkgs.stdenv.mkDerivation rec { + name = "responsive-theme"; + # Download the theme from the wordpress site + src = pkgs.fetchurl { + url = http://wordpress.org/themes/download/responsive.1.9.3.9.zip; + sha256 = "0397a8nd8q384z2i4lw4a1ij835walp3b5bfdmr76mdl27vbkvmd"; + }; + # We need unzip to build this package + buildInputs = [ pkgs.unzip ]; + # Installing simply means copying all files to the output directory + installPhase = "mkdir -p $out; cp -R * $out/"; + }; + + # The wordpress package itself + wordpress = pkgs.stdenv.mkDerivation { + name = "wordpress"; + # Fetch directly from the wordpress site, want to upgrade? + # Just change the version URL and update the hash + src = pkgs.fetchurl { + url = http://wordpress.org/wordpress-3.7.1.tar.gz; + sha256 = "1m2dlr54fqf5m4kgqc5hrrisrsia0r5j1r2xv1f12gmzb63swsvg"; + }; + installPhase = '' + mkdir -p $out + # Copy all the wordpress files we downloaded + cp -R * $out/ + # We'll symlink the wordpress config + ln -s ${wordpressConfig} $out/wp-config.php + # As well as our custom .htaccess + ln -s ${htaccess} $out/.htaccess + # And the responsive theme + ln -s ${responsiveTheme} $out/wp-content/themes/responsive + # You can add plugins the same way + ''; + }; +in { + # Expose just port 80 + docker.ports = [ 80 ]; + + # And let's store valuable data in a volume + docker.volumes = [ "/data" ]; + + # Apache configuration + services.httpd = { + enable = true; + adminAddr = "zef@zef.me"; + + # We'll set the wordpress package as our document root + documentRoot = wordpress; + + # And enable the PHP5 apache module + extraModules = [ { name = "php5"; path = "${pkgs.php}/modules/libphp5.so"; } ]; + + # And some extra config to make things work nicely + extraConfig = '' + + DirectoryIndex index.php + Allow from * + Options FollowSymLinks + AllowOverride All + + ''; + }; + + # Disable these when not using "localhost" as database name + services.mysql.enable = true; + # Let's store our data in the volume, so it'll survive restarts + services.mysql.dataDir = "/data/mysql"; + + # This service runs evey time you start and ensures that the wordpress + # MySQL database is created, if not it'll create it and setup grant rights + # if there's a database already, it'll exit immediately + supervisord.services.initWordpress = { + command = pkgs.writeScript "init-wordpress.sh" '' + #!/bin/sh + + if [ ! -d /data/mysql/${mysqlDb} ]; then + # Wait until MySQL is up + while [ ! -e /var/run/mysql/mysqld.pid ]; do + sleep 1 + done + mysql -e 'CREATE DATABASE ${mysqlDb};' + mysql -e 'GRANT ALL ON ${mysqlDb}.* TO ${mysqlUser}@localhost IDENTIFIED BY "${mysqlPassword}";' + fi + ''; + # This script can exit immediately, no worries + startsecs = 0; + }; +} \ No newline at end of file diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh index 4f432a8..387e887 100644 --- a/scripts/install-nix.sh +++ b/scripts/install-nix.sh @@ -9,7 +9,7 @@ if [ -d "/nix/store" ]; then fi # Install the binary tarball... -apt-get install -y curl vim +apt-get install -y curl cd / curl -L http://hydra.nixos.org/job/nix/trunk/binaryTarball.x86_64-linux/latest/download | tar xj /usr/bin/nix-finish-install From d786d3499a378917c4b6ab065d814abeb6acceaf Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 8 Nov 2013 11:27:04 +0100 Subject: [PATCH 29/46] Various tweaks --- samples/wordpress.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/wordpress.nix b/samples/wordpress.nix index be872f2..321b050 100644 --- a/samples/wordpress.nix +++ b/samples/wordpress.nix @@ -68,6 +68,7 @@ let # You can add plugins the same way ''; }; +# This is where the body of our configuration starts in { # Expose just port 80 docker.ports = [ 80 ]; From 89d1a662295168793454aa241c0da43aae0a3b6f Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 8 Nov 2013 11:38:34 +0100 Subject: [PATCH 30/46] Moved uploads to /data too --- samples/wordpress.nix | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/samples/wordpress.nix b/samples/wordpress.nix index 321b050..487a02a 100644 --- a/samples/wordpress.nix +++ b/samples/wordpress.nix @@ -7,6 +7,9 @@ let mysqlUser = "wordpress"; mysqlPassword = "wordpress"; + mysqlDataPath = "/data/mysql"; + wordpressUploads = "/data/uploads"; + # Our bare-bones wp-config.php file using the above settings wordpressConfig = pkgs.writeText "wp-config.php" '' Date: Fri, 8 Nov 2013 11:56:33 +0100 Subject: [PATCH 31/46] Sample tweaks --- samples/wordpress.nix | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/samples/wordpress.nix b/samples/wordpress.nix index 487a02a..76673fe 100644 --- a/samples/wordpress.nix +++ b/samples/wordpress.nix @@ -2,12 +2,12 @@ # We'll start with defining some local variables let # Settings used to configure the wordpress MySQL database - mysqlHost = "localhost"; - mysqlDb = "wordpress"; - mysqlUser = "wordpress"; - mysqlPassword = "wordpress"; + mysqlHost = "localhost"; + mysqlDb = "wordpress"; + mysqlUser = "wordpress"; + mysqlPassword = "wordpress"; - mysqlDataPath = "/data/mysql"; + mysqlDataPath = "/data/mysql"; wordpressUploads = "/data/uploads"; # Our bare-bones wp-config.php file using the above settings @@ -36,7 +36,7 @@ let ''; # For shits and giggles, let's package the responsive theme - responsiveTheme = pkgs.stdenv.mkDerivation rec { + responsiveTheme = pkgs.stdenv.mkDerivation { name = "responsive-theme"; # Download the theme from the wordpress site src = pkgs.fetchurl { From 3d820187a993b1c85472fcb34471990f006e9ef2 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 8 Nov 2013 12:00:56 +0100 Subject: [PATCH 32/46] Store logs in /data too --- samples/wordpress.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/samples/wordpress.nix b/samples/wordpress.nix index 76673fe..eeeda26 100644 --- a/samples/wordpress.nix +++ b/samples/wordpress.nix @@ -9,6 +9,7 @@ let mysqlDataPath = "/data/mysql"; wordpressUploads = "/data/uploads"; + apacheLogs = "/data/log"; # Our bare-bones wp-config.php file using the above settings wordpressConfig = pkgs.writeText "wp-config.php" '' @@ -89,6 +90,9 @@ in { # We'll set the wordpress package as our document root documentRoot = wordpress; + # Let's store our logs in the volume as well + logDir = apacheLogs; + # And enable the PHP5 apache module extraModules = [ { name = "php5"; path = "${pkgs.php}/modules/libphp5.so"; } ]; From 3f234958bc68b76c9bbf56f76ee6c1b29e553958 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 8 Nov 2013 12:03:23 +0100 Subject: [PATCH 33/46] Store logs in /data too --- samples/wordpress.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/wordpress.nix b/samples/wordpress.nix index eeeda26..d4cca15 100644 --- a/samples/wordpress.nix +++ b/samples/wordpress.nix @@ -7,6 +7,7 @@ let mysqlUser = "wordpress"; mysqlPassword = "wordpress"; + # Data paths mysqlDataPath = "/data/mysql"; wordpressUploads = "/data/uploads"; apacheLogs = "/data/log"; From 99b6f91e19261d3077c3e84a597626567025e681 Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Mon, 17 Feb 2014 00:31:11 +0100 Subject: [PATCH 34/46] refractor the code for another purpose --- README.md | 74 -- Vagrantfile | 35 - base-configuration.nix | 4 - build-base.sh | 2 - default.nix | 52 +- environment.nix | 23 + foo.nix | 8 + nix-docker/all-modules.nix | 19 - nix-docker/bin/nix-docker | 87 --- nix-docker/docker.nix | 82 --- nix-docker/modules/config/docker.nix | 46 -- nix-docker/modules/config/environment.nix | 53 -- nix-docker/modules/config/user-groups.nix | 272 -------- .../modules/servers/http/apache/default.nix | 641 ------------------ .../http/apache/per-server-options.nix | 150 ---- nix-docker/modules/servers/openssh.nix | 333 --------- nix-docker/package.json | 10 - samples/apache-config.nix | 14 - samples/fancy.nix | 29 - samples/ssh-config.nix | 19 - samples/wordpress.nix | 138 ---- samples/www/index.html | 1 - scripts/README.md | 1 - scripts/install-docker.sh | 21 - scripts/install-nix.sh | 105 --- .../supervisord.nix => supervisord.nix | 21 +- .../modules/shim/systemd.nix => systemd.nix | 22 +- user.nix | 17 + zedconfig.json | 10 - 29 files changed, 114 insertions(+), 2175 deletions(-) delete mode 100644 README.md delete mode 100644 Vagrantfile delete mode 100644 base-configuration.nix delete mode 100755 build-base.sh create mode 100644 environment.nix create mode 100644 foo.nix delete mode 100644 nix-docker/all-modules.nix delete mode 100755 nix-docker/bin/nix-docker delete mode 100644 nix-docker/docker.nix delete mode 100644 nix-docker/modules/config/docker.nix delete mode 100644 nix-docker/modules/config/environment.nix delete mode 100644 nix-docker/modules/config/user-groups.nix delete mode 100644 nix-docker/modules/servers/http/apache/default.nix delete mode 100644 nix-docker/modules/servers/http/apache/per-server-options.nix delete mode 100644 nix-docker/modules/servers/openssh.nix delete mode 100644 nix-docker/package.json delete mode 100644 samples/apache-config.nix delete mode 100644 samples/fancy.nix delete mode 100644 samples/ssh-config.nix delete mode 100644 samples/wordpress.nix delete mode 100644 samples/www/index.html delete mode 100644 scripts/README.md delete mode 100644 scripts/install-docker.sh delete mode 100644 scripts/install-nix.sh rename nix-docker/modules/servers/supervisord.nix => supervisord.nix (80%) rename nix-docker/modules/shim/systemd.nix => systemd.nix (62%) create mode 100644 user.nix delete mode 100644 zedconfig.json diff --git a/README.md b/README.md deleted file mode 100644 index 1faffee..0000000 --- a/README.md +++ /dev/null @@ -1,74 +0,0 @@ -nix-docker -========== - -Use [NixOS](http://nixos.org/nixos) configurations to provision [Docker](http://docker.io) -containers. - -[Read about the what and why in this blog post](http://zef.me/6049/nix-docker) - -Installation with Vagrant -------------------------- -The easy way to do this is to use [Vagrant](http://vagrantup.com). - -When you have Vagrant installed: - - git clone https://github.com/zefhemel/nix-docker.git - cd nix-docker - vagrant up - vagrant ssh - -If all went well, you're now in a VM that has both Docker and Nix installed -and `nix-docker` in its path. You can now cd into the nix-docker/samples -directory to try to build some of the examples. Note that the `~/nix-docker` -directory is mounted from your host machine, so you can edit your files with -your favorite editor and have them available within the VM. - -Installation ------------- - -To use nix-docker you need [Nix](http://nixos.org/nix) installed as well as -[Docker](http://www.docker.io). Realistically, your best way to do this on -an Ubuntu (12.04 or 13.04) box. Once these are installed, installing -`nix-docker` is as simple as: - - git clone https://github.com/zefhemel/nix-docker.git - nix-env -f nix-docker/default.nix -i nix-docker - -Usage ------ - -To build a stand-alone Docker image: - - nix-docker -b -t my-image configuration.nix - -This will build the configuration specified in `configuration.nix`, have a look -in the `samples/` directory for examples. It will produce a docker image named -`my-image` which you can then run anywhere. Use `username/my-image` to be able -to push them to the Docker index. - -To build a host-mounted package: - - nix-docker -t my-image configuration.nix - -This will produce a Nix package (symlinked in the current directory in `result`) -containing a script you can use to spawn the container using Docker, e.g.: - - sudo -E ./result/sbin/docker-run - -to run the container in the foreground, or: - - sudo -E ./result/sbin/docker-run -d - -to daemonize it. What the `docker-run` script will do is check if there's -already a docker image available with the current image name and tag based on -the Nix build hash. If not, it will quickly build it first (these images take up -barely any space on disk). Then, it will boot up the container. - -Distributing host-mounted packages is done by first copying the Nix closure -resulting from the build to the target machine (when you do the build it -will give you example commands to run): - - nix-copy-closure root@targetmachine /nix/store/.... - -Then, you can spawn the container remotely with the script path provided -in the output of the build command. \ No newline at end of file diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index daa5fca..0000000 --- a/Vagrantfile +++ /dev/null @@ -1,35 +0,0 @@ -Vagrant.configure("2") do |config| - config.vm.box = "raring64" - config.vm.box_url = "http://goo.gl/ceHWg" - - config.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--memory", "1024"] - - # We'll attach an extra 50GB disk for all nix and docker data - file_to_disk = "disk.vmdk" - vb.customize ['createhd', '--filename', file_to_disk, '--size', 50 * 1024] - vb.customize ['storageattach', :id, '--storagectl', 'SATA Controller', '--port', 1, '--device', 0, '--type', 'hdd', '--medium', file_to_disk] - end - - config.vm.provision :shell, inline: < /etc/profile.d/path.sh - -if [ ! -d /data ]; then - mkfs.ext4 -F /dev/sdb - mkdir -p /nix - echo "/dev/sdb /nix ext4 defaults 0 2" >> /etc/fstab - mount /nix - mkdir -p /nix/docker-lib - ln -s /nix/docker-lib /var/lib/docker -fi -eos - - config.vm.provision :shell, :path => "scripts/install-docker.sh" - config.vm.provision :shell, inline: "chmod 777 /var/run/docker.sock" - config.vm.provision :shell, :path => "scripts/install-nix.sh" - - config.vm.network "private_network", ip: "192.168.22.22" - - config.vm.synced_folder ".", "/home/vagrant/nix-docker" - -end diff --git a/base-configuration.nix b/base-configuration.nix deleted file mode 100644 index 63204d9..0000000 --- a/base-configuration.nix +++ /dev/null @@ -1,4 +0,0 @@ -{ config, pkgs, ... }: -{ - services.mysql.enable = true; -} \ No newline at end of file diff --git a/build-base.sh b/build-base.sh deleted file mode 100755 index df5a4a8..0000000 --- a/build-base.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -sudo -E nix-docker -b -t zefhemel/base-nix --from busybox base-configuration.nix \ No newline at end of file diff --git a/default.nix b/default.nix index 959c4f2..65bb454 100644 --- a/default.nix +++ b/default.nix @@ -1,12 +1,42 @@ +{ pkgs ? import { system = "x86_64-linux"; } +, name +, configuration ? +, baseImage ? "busybox" +}: +with pkgs.lib; let - pkgs = import {}; -in - pkgs.stdenv.mkDerivation { - name = "nix-docker-0.1"; - src = ./nix-docker; - buildInputs = [ pkgs.python27 ]; - installPhase = '' - mkdir -p $out - cp -R * $out/ - ''; - } \ No newline at end of file + moduleList = [ + ./user.nix ./supervisord.nix ./systemd.nix ./environment.nix + + + + + + + + + ]; + + config = (evalModules { + modules = [configuration] ++ moduleList; + args = { inherit pkgs; }; + }).config; + + systemd = import ./systemd.nix { inherit pkgs config; }; + + startScript = pkgs.writeScript "build" '' + #!/bin/sh + ${config.userNix.startScript} + ''; + +in pkgs.stdenv.mkDerivation { + inherit name; + src = ./.; + + phases = [ "installPhase" ]; + + installPhase = '' + mkdir -p $out/etc/start + ln -s ${startScript} $out/etc/start + ''; +} diff --git a/environment.nix b/environment.nix new file mode 100644 index 0000000..fd3422b --- /dev/null +++ b/environment.nix @@ -0,0 +1,23 @@ +{ config, pkgs, ... }: +with pkgs.lib; +{ + options = { + environment.systemPackages = mkOption { + default = []; + description = "Packages to be put in the system profile."; + }; + + environment.umask = mkOption { + default = "002"; + type = with types; string; + }; + + # HACK HACK + system.activationScripts.etc = mkOption {}; # Ignore + system.build.etc = mkOption {}; # Ignore + }; + + config = { + environment.systemPackages = with pkgs; [ coreutils ]; + }; +} diff --git a/foo.nix b/foo.nix new file mode 100644 index 0000000..c4f0607 --- /dev/null +++ b/foo.nix @@ -0,0 +1,8 @@ +{ config, pkgs, ... }: +{ + config = { + services.postgresql.enable = true; + services.postgresql.package = pkgs.postgresql92; + services.postgresql.dataDir = "/tmp/postgres"; + }; +} diff --git a/nix-docker/all-modules.nix b/nix-docker/all-modules.nix deleted file mode 100644 index 6562dc6..0000000 --- a/nix-docker/all-modules.nix +++ /dev/null @@ -1,19 +0,0 @@ -[ - ./modules/config/docker.nix - ./modules/config/user-groups.nix - ./modules/config/environment.nix - ./modules/servers/supervisord.nix - - ./modules/shim/systemd.nix - - - - - - - - - # These modules needed some patching to work well - ./modules/servers/http/apache/default.nix - ./modules/servers/openssh.nix -] \ No newline at end of file diff --git a/nix-docker/bin/nix-docker b/nix-docker/bin/nix-docker deleted file mode 100755 index 2660bc3..0000000 --- a/nix-docker/bin/nix-docker +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/python - -import os.path -import subprocess -import argparse -import sys -import shutil -import tempfile - -parser = argparse.ArgumentParser(description='Build Docker images with Nix', prog='nix-docker') -parser.add_argument('configuration', nargs='?', help='configuration.nix file to build', default="configuration.nix") -parser.add_argument('-b', action='store_true', help="Build a docker image") -parser.add_argument('-t', help='Image name', default="nix-docker-build") -parser.add_argument('--from', help="Image to use as a base image", default="busybox") -parser.add_argument('-f', action='store_true', help="Force nix-docker into something it does not what to do.") - -args = parser.parse_args() - -base_image = getattr(args, "from") -configuration_file = args.configuration -image_name = args.t -full_build = args.b - -temp_dir = tempfile.mkdtemp() - -nix_closure_temp = "%s/nix-closure" % temp_dir -force = args.f - -root_path = os.path.normpath(os.path.realpath(sys.argv[0]) + "/../..") - -def build(): - print "Building Nix closure for the image..." - if args.b: - mountBuild = "false" - else: - mountBuild = "true" - cmd = ["nix-build", "%s/docker.nix" % root_path, '-I', 'configuration=' + configuration_file, "--arg", "mountBuild", mountBuild, "--argstr", "name", image_name, "--argstr", "baseImage", base_image] - if full_build: - cmd.append("--no-out-link") - nix_path = subprocess.check_output(cmd) - return nix_path.strip() - -def get_available_nix_paths(): - try: - paths_string = subprocess.check_output(["docker", "run", base_image, "/bin/ls", "/nix/store"]) - return paths_string.strip().split("\n") - except: - return [] - -def cleanup(): - subprocess.check_call(["rm", "-rf", temp_dir]) - -def copy_closure_and_build(package_nix_path, available_paths): - paths = subprocess.check_output(["nix-store", "-qR", package_nix_path]).strip().split("\n") - print "Available", available_paths - paths_to_copy = filter(lambda path: not path[len("/nix/store/"):] in available_paths, paths) - print "New Nix store paths to copy to image:" - for path in paths_to_copy: - print " ", path - os.mkdir(nix_closure_temp) - cmd = ["cp", "-r"] - cmd.extend(paths_to_copy) - cmd.append(nix_closure_temp) - shutil.copyfile("%s/Dockerfile" % package_nix_path, "%s/Dockerfile" % temp_dir) - subprocess.check_call(cmd) - subprocess.check_call(["chown", "-R", "root:root", nix_closure_temp]) - subprocess.check_call(["docker", "build", "-rm=true", "-t", image_name, temp_dir]) - cleanup() - -if __name__ == '__main__': - if full_build and os.getenv("USER") != "root" and not force: - print "When doing a full build you need to run nix-docker as root. Rerun this command with 'sudo -E nix-docker ...'" - sys.exit(1) - - if not os.path.exists(configuration_file): - print "Could not find configuration file: %s" % configuration_file - sys.exit(1) - - package_nix_path = build() - if full_build: - available_paths = get_available_nix_paths() - copy_closure_and_build(package_nix_path, available_paths) - print "To run: sudo docker run -t -i", image_name - else: - print "Result in", package_nix_path, "test with sudo ./result/sbin/docker-run" - print "To deploy: nix-copy-closure -s root@", package_nix_path - print "To run: ssh root@", package_nix_path + "/sbin/docker-run -d" \ No newline at end of file diff --git a/nix-docker/docker.nix b/nix-docker/docker.nix deleted file mode 100644 index 249399b..0000000 --- a/nix-docker/docker.nix +++ /dev/null @@ -1,82 +0,0 @@ -{ pkgs ? import { system = "x86_64-linux"; } -, name -, configuration ? -, mountBuild ? true -, baseImage ? "busybox" -}: -with pkgs.lib; -let - config = evalModules { - modules = concatLists [ [configuration] (import ./all-modules.nix) ]; - args = { inherit pkgs; }; - }; - - localNixPath = pkg: "nix_store/${substring 11 (stringLength pkg.outPath) pkg.outPath}"; - - systemd = import ./systemd.nix { inherit pkgs config; }; - environment = import ./environment.nix { inherit pkgs config; }; - - verboseFlag = if config.config.docker.verbose then "v" else ""; - - - buildScript = pkgs.writeScript "build" '' - #!/bin/sh -e${verboseFlag} - ${config.config.docker.buildScript} - ''; - - bootScript = pkgs.writeScript "boot" '' - #!/bin/sh -e${verboseFlag} - umask ${config.config.environment.umask} - ${if mountBuild then config.config.docker.buildScript else ""} - ${config.config.docker.bootScript} - ''; - - dockerFile = pkgs.writeText "Dockerfile" '' - FROM ${if mountBuild then "busybox" else baseImage} - ${if !mountBuild then - '' - ADD nix-closure /nix/store - RUN ${buildScript} - '' - else ""} - CMD ${bootScript} - ${ - concatMapStrings (port: "EXPOSE ${toString port}\n") config.config.docker.ports - } - ${ - concatMapStrings (port: "VOLUME ${toString port}\n") config.config.docker.volumes - } - ''; - - imageHash = substring 11 8 dockerFile.outPath; - - runContainerScript = pkgs.writeScript "docker-run" '' - #!/usr/bin/env bash - - if [ "" == "$(docker images | grep -E "${name}\s*${imageHash}")" ]; then - docker build -t ${name}:${imageHash} $(dirname $0)/.. - fi - - OPTIONS="-t -i $*" - if [ "$1" == "-d" ]; then - OPTIONS="$*" - fi - - docker run $OPTIONS ${if mountBuild then "-v /nix/store:/nix/store" else ""} ${name}:${imageHash} - ''; - -in pkgs.stdenv.mkDerivation { - name = replaceChars ["/"] ["-"] name; - src = ./.; - - phases = [ "installPhase" ]; - - installPhase = '' - mkdir -p $out - ${if mountBuild then '' - mkdir -p $out/sbin - cp ${runContainerScript} $out/sbin/docker-run - '' else ""} - cp ${dockerFile} $out/Dockerfile - ''; -} \ No newline at end of file diff --git a/nix-docker/modules/config/docker.nix b/nix-docker/modules/config/docker.nix deleted file mode 100644 index 2573fc1..0000000 --- a/nix-docker/modules/config/docker.nix +++ /dev/null @@ -1,46 +0,0 @@ -{ config, pkgs, ... }: -with pkgs.lib; -{ - options = { - docker.ports = mkOption { - default = []; - description = "Ports to expose to the outside world."; - example = [ 80 22 ]; - }; - - docker.volumes = mkOption { - default = []; - description = "Volumes to create for container."; - example = [ "/var/lib" "/var/log" ]; - }; - - docker.buildScripts = mkOption { - default = {}; - example = { - setupUsers = "cp passwd /etc/passwd"; - }; - description = "Scripts (as text) to be run during build, executed alphabetically"; - }; - - docker.bootScript = mkOption { - default = ""; - description = "Script (text) to run when container booted."; - }; - - docker.buildScript = mkOption {}; - - docker.verbose = mkOption { - default = false; - type = types.bool; - }; - - # HACK: Let's ignore these for now - networking = mkOption {}; - security = mkOption {}; - }; - - config = { - docker.buildScript = concatStrings (attrValues config.docker.buildScripts); - networking.enableIPv6 = false; - }; -} \ No newline at end of file diff --git a/nix-docker/modules/config/environment.nix b/nix-docker/modules/config/environment.nix deleted file mode 100644 index fd3ce6d..0000000 --- a/nix-docker/modules/config/environment.nix +++ /dev/null @@ -1,53 +0,0 @@ -{ config, pkgs, ... }: -with pkgs.lib; -let - etc2 = filter (f: f.enable) (attrValues config.environment.etc); - - etc = pkgs.stdenv.mkDerivation { - name = "etc"; - - builder = ; - - preferLocalBuild = true; - - /* !!! Use toXML. */ - sources = map (x: x.source) etc2; - targets = map (x: x.target) etc2; - modes = map (x: x.mode) etc2; - }; -in { - options = { - environment.systemPackages = mkOption { - default = []; - description = "Packages to be put in the system profile."; - }; - - environment.umask = mkOption { - default = "002"; - type = with types; string; - }; - - # HACK HACK - system.activationScripts.etc = mkOption {}; # Ignore - system.build.etc = mkOption {}; # Ignore - }; - - config = { - docker.buildScripts."0-systemEnv" = let - systemPackages = config.environment.systemPackages; - systemEnv = pkgs.buildEnv { name = "system-env"; paths = systemPackages; }; - in '' - rm -rf /usr - chmod 777 /tmp - ln -s ${systemEnv} /usr - ln -sf /usr/bin/bash /bin/bash - ''; - - environment.systemPackages = with pkgs; [ coreutils bash ]; - - docker.buildScripts."0-etc" = '' - echo "setting up /etc..." - ${pkgs.perl}/bin/perl ${} ${etc}/etc - ''; - }; -} \ No newline at end of file diff --git a/nix-docker/modules/config/user-groups.nix b/nix-docker/modules/config/user-groups.nix deleted file mode 100644 index 89f75a3..0000000 --- a/nix-docker/modules/config/user-groups.nix +++ /dev/null @@ -1,272 +0,0 @@ -{pkgs, config, ...}: - -with pkgs.lib; - -let - - userOpts = { name, config, ... }: { - - options = { - - name = mkOption { - type = types.str; - description = "The name of the user account. If undefined, the name of the attribute set will be used."; - }; - - description = mkOption { - type = types.str; - default = ""; - example = "Alice Q. User"; - description = '' - A short description of the user account, typically the - user's full name. This is actually the “GECOS” or “comment” - field in /etc/passwd. - ''; - }; - - uid = mkOption { - type = with types; uniq (nullOr int); - default = null; - description = "The account UID. If undefined, NixOS will select a free UID."; - }; - - group = mkOption { - type = types.str; - default = "nogroup"; - description = "The user's primary group."; - }; - - extraGroups = mkOption { - type = types.listOf types.str; - default = []; - description = "The user's auxiliary groups."; - }; - - home = mkOption { - type = types.str; - default = "/var/empty"; - description = "The user's home directory."; - }; - - shell = mkOption { - type = types.str; - default = "/run/current-system/sw/sbin/nologin"; - description = "The path to the user's shell."; - }; - - createHome = mkOption { - type = types.bool; - default = false; - description = "If true, the home directory will be created automatically."; - }; - - useDefaultShell = mkOption { - type = types.bool; - default = false; - description = "If true, the user's shell will be set to users.defaultUserShell."; - }; - - password = mkOption { - type = with types; uniq (nullOr str); - default = null; - description = '' - The user's password. If undefined, no password is set for - the user. Warning: do not set confidential information here - because it is world-readable in the Nix store. This option - should only be used for public accounts such as - guest. - ''; - }; - - isSystemUser = mkOption { - type = types.bool; - default = true; - description = "Indicates if the user is a system user or not."; - }; - - createUser = mkOption { - type = types.bool; - default = true; - description = '' - Indicates if the user should be created automatically as a local user. - Set this to false if the user for instance is an LDAP user. NixOS will - then not modify any of the basic properties for the user account. - ''; - }; - - isAlias = mkOption { - type = types.bool; - default = false; - description = "If true, the UID of this user is not required to be unique and can thus alias another user."; - }; - - }; - - config = { - name = mkDefault name; - uid = mkDefault null; - shell = mkDefault "/bin/sh"; - }; - - }; - - groupOpts = { name, config, ... }: { - - options = { - - name = mkOption { - type = types.str; - description = "The name of the group. If undefined, the name of the attribute set will be used."; - }; - - gid = mkOption { - type = with types; uniq (nullOr int); - default = null; - description = "The GID of the group. If undefined, NixOS will select a free GID."; - }; - - }; - - config = { - name = mkDefault name; - gid = mkDefault null; - }; - - }; - - extraUsers = config.users.extraUsers; - extraGroups = config.users.extraGroups; - - uidUsers = listToAttrs - (imap (i: name: - let - user = getAttr name extraUsers; - in { - name=name; - value = if user.uid == null then - setAttr user "uid" (builtins.add 1000 i) - else user; - }) - (attrNames extraUsers)); - - gidGroups = listToAttrs - (imap (i: name: - let - group = getAttr name extraGroups; - in { - name=name; - value = if group.gid == null then - setAttr group "gid" (builtins.add 1000 i) - else group; - }) - (attrNames extraGroups)); -in - -{ - - ###### interface - - options = { - - users.extraUsers = mkOption { - default = {}; - type = types.loaOf types.optionSet; - example = { - alice = { - uid = 1234; - description = "Alice Q. User"; - home = "/home/alice"; - createHome = true; - group = "users"; - extraGroups = ["wheel"]; - shell = "/bin/sh"; - }; - }; - description = '' - Additional user accounts to be created automatically by the system. - This can also be used to set options for root. - ''; - options = [ userOpts ]; - }; - - users.extraGroups = mkOption { - default = {}; - example = - { students.gid = 1001; - hackers = { }; - }; - type = types.loaOf types.optionSet; - description = '' - Additional groups to be created automatically by the system. - ''; - options = [ groupOpts ]; - }; - }; - - config = { - - users.extraUsers = { - root = { - uid = 0; - description = "System administrator"; - home = "/root"; - group = "root"; - }; - nobody = { - uid = 1; - description = "Unprivileged account (don't use!)"; - }; - ldap = {}; - }; - - users.extraGroups = { - root = { gid = 0; }; - wheel = { }; - disk = { }; - kmem = { }; - tty = { }; - floppy = { }; - uucp = { }; - lp = { }; - cdrom = { }; - tape = { }; - audio = { }; - video = { }; - dialout = { }; - nogroup = { }; - users = { }; - utmp = { }; - adm = { }; - }; - - environment.etc.passwd.text = - concatMapStrings (name: - let - user = getAttr name uidUsers; - in - if user.createUser then - "${user.name}:x:${toString user.uid}:${toString (getAttr user.group gidGroups).gid}:${user.description}:${user.home}:${user.shell}\n" - else - "" - ) (attrNames uidUsers); - - environment.etc.group.text = - concatMapStrings (name: - let - group = getAttr name gidGroups; - in - "${group.name}:x:${toString group.gid}:\n" - ) (attrNames gidGroups); - - docker.buildScripts."1-create-homes" = concatMapStrings (name: - let - user = getAttr name uidUsers; - in - if user.createHome then - "mkdir -p ${user.home}; chown ${user.name}:${user.group} ${user.home}\n" - else - "" - ) (attrNames uidUsers); - }; - -} \ No newline at end of file diff --git a/nix-docker/modules/servers/http/apache/default.nix b/nix-docker/modules/servers/http/apache/default.nix deleted file mode 100644 index ba2dc39..0000000 --- a/nix-docker/modules/servers/http/apache/default.nix +++ /dev/null @@ -1,641 +0,0 @@ -{ config, pkgs, ... }: - -with pkgs.lib; - -let - - mainCfg = config.services.httpd; - - httpd = mainCfg.package; - - version24 = !versionOlder httpd.version "2.4"; - - httpdConf = mainCfg.configFile; - - php = pkgs.php.override { apacheHttpd = httpd; }; - - getPort = cfg: if cfg.port != 0 then cfg.port else if cfg.enableSSL then 443 else 80; - - extraModules = attrByPath ["extraModules"] [] mainCfg; - extraForeignModules = filter builtins.isAttrs extraModules; - extraApacheModules = filter (x: !(builtins.isAttrs x)) extraModules; # I'd prefer using builtins.isString here, but doesn't exist yet - - - makeServerInfo = cfg: { - # Canonical name must not include a trailing slash. - canonicalName = - (if cfg.enableSSL then "https" else "http") + "://" + - cfg.hostName + - (if getPort cfg != (if cfg.enableSSL then 443 else 80) then ":${toString (getPort cfg)}" else ""); - - # Admin address: inherit from the main server if not specified for - # a virtual host. - adminAddr = if cfg.adminAddr != null then cfg.adminAddr else mainCfg.adminAddr; - - vhostConfig = cfg; - serverConfig = mainCfg; - fullConfig = config; # machine config - }; - - - allHosts = [mainCfg] ++ mainCfg.virtualHosts; - - - callSubservices = serverInfo: defs: - let f = svc: - let - svcFunction = - if svc ? function then svc.function - else import (toString "${toString ./.}/${if svc ? serviceType then svc.serviceType else svc.serviceName}.nix"); - config = (evalModules - { modules = [ { options = res.options; config = svc.config or svc; } ]; - check = false; - }).config; - defaults = { - extraConfig = ""; - extraModules = []; - extraModulesPre = []; - extraPath = []; - extraServerPath = []; - globalEnvVars = []; - robotsEntries = ""; - startupScript = ""; - enablePHP = false; - phpOptions = ""; - options = {}; - }; - res = defaults // svcFunction { inherit config pkgs serverInfo php; }; - in res; - in map f defs; - - - # !!! callSubservices is expensive - subservicesFor = cfg: callSubservices (makeServerInfo cfg) cfg.extraSubservices; - - mainSubservices = subservicesFor mainCfg; - - allSubservices = mainSubservices ++ concatMap subservicesFor mainCfg.virtualHosts; - - - # !!! should be in lib - writeTextInDir = name: text: - pkgs.runCommand name {inherit text;} "ensureDir $out; echo -n \"$text\" > $out/$name"; - - - enableSSL = any (vhost: vhost.enableSSL) allHosts; - - - # Names of modules from ${httpd}/modules that we want to load. - apacheModules = - [ # HTTP authentication mechanisms: basic and digest. - "auth_basic" "auth_digest" - - # Authentication: is the user who he claims to be? - "authn_file" "authn_dbm" "authn_anon" - (if version24 then "authn_core" else "authn_alias") - - # Authorization: is the user allowed access? - "authz_user" "authz_groupfile" "authz_host" - - # Other modules. - "ext_filter" "include" "log_config" "env" "mime_magic" - "cern_meta" "expires" "headers" "usertrack" /* "unique_id" */ "setenvif" - "mime" "dav" "status" "autoindex" "asis" "info" "dav_fs" - "vhost_alias" "negotiation" "dir" "imagemap" "actions" "speling" - "userdir" "alias" "rewrite" "proxy" "proxy_http" - ] - ++ optionals version24 [ - "mpm_${mainCfg.multiProcessingModule}" - "authz_core" - "unixd" - ] - ++ (if mainCfg.multiProcessingModule == "prefork" then [ "cgi" ] else [ "cgid" ]) - ++ optional enableSSL "ssl" - ++ extraApacheModules; - - - allDenied = if version24 then '' - Require all denied - '' else '' - Order deny,allow - Deny from all - ''; - - allGranted = if version24 then '' - Require all granted - '' else '' - Order allow,deny - Allow from all - ''; - - - loggingConf = '' - ErrorLog ${mainCfg.logDir}/error_log - - LogLevel notice - - LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined - LogFormat "%h %l %u %t \"%r\" %>s %b" common - LogFormat "%{Referer}i -> %U" referer - LogFormat "%{User-agent}i" agent - - CustomLog ${mainCfg.logDir}/access_log ${mainCfg.logFormat} - ''; - - - browserHacks = '' - BrowserMatch "Mozilla/2" nokeepalive - BrowserMatch "MSIE 4\.0b2;" nokeepalive downgrade-1.0 force-response-1.0 - BrowserMatch "RealPlayer 4\.0" force-response-1.0 - BrowserMatch "Java/1\.0" force-response-1.0 - BrowserMatch "JDK/1\.0" force-response-1.0 - BrowserMatch "Microsoft Data Access Internet Publishing Provider" redirect-carefully - BrowserMatch "^WebDrive" redirect-carefully - BrowserMatch "^WebDAVFS/1.[012]" redirect-carefully - BrowserMatch "^gnome-vfs" redirect-carefully - ''; - - - sslConf = '' - SSLSessionCache shm:${mainCfg.stateDir}/ssl_scache(512000) - - SSLMutex posixsem - - SSLRandomSeed startup builtin - SSLRandomSeed connect builtin - ''; - - - mimeConf = '' - TypesConfig ${httpd}/conf/mime.types - - AddType application/x-x509-ca-cert .crt - AddType application/x-pkcs7-crl .crl - AddType application/x-httpd-php .php .phtml - - - MIMEMagicFile ${httpd}/conf/magic - - - AddEncoding x-compress Z - AddEncoding x-gzip gz tgz - ''; - - - perServerConf = isMainServer: cfg: let - - serverInfo = makeServerInfo cfg; - - subservices = callSubservices serverInfo cfg.extraSubservices; - - documentRoot = if cfg.documentRoot != null then cfg.documentRoot else - pkgs.runCommand "empty" {} "ensureDir $out"; - - documentRootConf = '' - DocumentRoot "${documentRoot}" - - - Options Indexes FollowSymLinks - AllowOverride None - ${allGranted} - - ''; - - robotsTxt = pkgs.writeText "robots.txt" '' - ${# If this is a vhost, the include the entries for the main server as well. - if isMainServer then "" - else concatMapStrings (svc: svc.robotsEntries) mainSubservices} - ${concatMapStrings (svc: svc.robotsEntries) subservices} - ''; - - robotsConf = '' - Alias /robots.txt ${robotsTxt} - ''; - - in '' - ServerName ${serverInfo.canonicalName} - - ${concatMapStrings (alias: "ServerAlias ${alias}\n") cfg.serverAliases} - - ${if cfg.sslServerCert != null then '' - SSLCertificateFile ${cfg.sslServerCert} - SSLCertificateKeyFile ${cfg.sslServerKey} - '' else ""} - - ${if cfg.enableSSL then '' - SSLEngine on - '' else if enableSSL then /* i.e., SSL is enabled for some host, but not this one */ - '' - SSLEngine off - '' else ""} - - ${if isMainServer || cfg.adminAddr != null then '' - ServerAdmin ${cfg.adminAddr} - '' else ""} - - ${if !isMainServer && mainCfg.logPerVirtualHost then '' - ErrorLog ${mainCfg.logDir}/error_log-${cfg.hostName} - CustomLog ${mainCfg.logDir}/access_log-${cfg.hostName} ${cfg.logFormat} - '' else ""} - - ${robotsConf} - - ${if isMainServer || cfg.documentRoot != null then documentRootConf else ""} - - ${if cfg.enableUserDir then '' - - UserDir public_html - UserDir disabled root - - - AllowOverride FileInfo AuthConfig Limit Indexes - Options MultiViews Indexes SymLinksIfOwnerMatch IncludesNoExec - - ${allGranted} - - - ${allDenied} - - - - '' else ""} - - ${if cfg.globalRedirect != null then '' - RedirectPermanent / ${cfg.globalRedirect} - '' else ""} - - ${ - let makeFileConf = elem: '' - Alias ${elem.urlPath} ${elem.file} - ''; - in concatMapStrings makeFileConf cfg.servedFiles - } - - ${ - let makeDirConf = elem: '' - Alias ${elem.urlPath} ${elem.dir}/ - - Options +Indexes - ${allGranted} - AllowOverride All - - ''; - in concatMapStrings makeDirConf cfg.servedDirs - } - - ${concatMapStrings (svc: svc.extraConfig) subservices} - - ${cfg.extraConfig} - ''; - - - confFile = pkgs.writeText "httpd.conf" '' - - ServerRoot ${httpd} - - ${optionalString version24 '' - DefaultRuntimeDir ${mainCfg.stateDir}/runtime - ''} - - PidFile ${mainCfg.stateDir}/httpd.pid - - ${optionalString (mainCfg.multiProcessingModule != "prefork") '' - # mod_cgid requires this. - ScriptSock ${mainCfg.stateDir}/cgisock - ''} - - - MaxClients ${toString mainCfg.maxClients} - MaxRequestsPerChild ${toString mainCfg.maxRequestsPerChild} - - - ${let - ports = map getPort allHosts; - uniquePorts = uniqList {inputList = ports;}; - in concatMapStrings (port: "Listen ${toString port}\n") uniquePorts - } - - User ${mainCfg.user} - Group ${mainCfg.group} - - ${let - load = {name, path}: "LoadModule ${name}_module ${path}\n"; - allModules = - concatMap (svc: svc.extraModulesPre) allSubservices - ++ map (name: {inherit name; path = "${httpd}/modules/mod_${name}.so";}) apacheModules - ++ optional enablePHP { name = "php5"; path = "${php}/modules/libphp5.so"; } - ++ concatMap (svc: svc.extraModules) allSubservices - ++ extraForeignModules; - in concatMapStrings load allModules - } - - AddHandler type-map var - - - ${allDenied} - - - ${mimeConf} - ${loggingConf} - ${browserHacks} - - Include ${httpd}/conf/extra/httpd-default.conf - Include ${httpd}/conf/extra/httpd-autoindex.conf - Include ${httpd}/conf/extra/httpd-multilang-errordoc.conf - Include ${httpd}/conf/extra/httpd-languages.conf - - ${if enableSSL then sslConf else ""} - - # Fascist default - deny access to everything. - - Options FollowSymLinks - AllowOverride None - ${allDenied} - - - # But do allow access to files in the store so that we don't have - # to generate clauses for every generated file that we - # want to serve. - - ${allGranted} - - - # Generate directives for the main server. - ${perServerConf true mainCfg} - - # Always enable virtual hosts; it doesn't seem to hurt. - ${let - ports = map getPort allHosts; - uniquePorts = uniqList {inputList = ports;}; - directives = concatMapStrings (port: "NameVirtualHost *:${toString port}\n") uniquePorts; - in optionalString (!version24) directives - } - - ${let - makeVirtualHost = vhost: '' - - ${perServerConf false vhost} - - ''; - in concatMapStrings makeVirtualHost mainCfg.virtualHosts - } - ''; - - - enablePHP = any (svc: svc.enablePHP) allSubservices; - - - # Generate the PHP configuration file. Should probably be factored - # out into a separate module. - phpIni = pkgs.runCommand "php.ini" - { options = concatStringsSep "\n" - ([ mainCfg.phpOptions ] ++ (map (svc: svc.phpOptions) allSubservices)); - } - '' - cat ${php}/etc/php-recommended.ini > $out - echo "$options" >> $out - ''; - -in - - -{ - - ###### interface - - options = { - - services.httpd = { - - enable = mkOption { - type = types.bool; - default = false; - description = "Whether to enable the Apache HTTP Server."; - }; - - package = mkOption { - type = types.path; - default = pkgs.apacheHttpd.override { mpm = mainCfg.multiProcessingModule; }; - example = "pkgs.apacheHttpd_2_4"; - description = '' - Overridable attribute of the Apache HTTP Server package to use. - ''; - }; - - configFile = mkOption { - type = types.path; - default = confFile; - example = literalExample ''pkgs.writeText "httpd.conf" "# my custom config file ...";''; - description = '' - Override the configuration file used by Apache. By default, - NixOS generates one automatically. - ''; - }; - - extraConfig = mkOption { - type = types.lines; - default = ""; - description = '' - Cnfiguration lines appended to the generated Apache - configuration file. Note that this mechanism may not work - when is overridden. - ''; - }; - - extraModules = mkOption { - type = types.listOf types.unspecified; - default = []; - example = literalExample ''[ "proxy_connect" { name = "php5"; path = "''${php}/modules/libphp5.so"; } ]''; - description = '' - Additional Apache modules to be used. These can be - specified as a string in the case of modules distributed - with Apache, or as an attribute set specifying the - name and path of the - module. - ''; - }; - - logPerVirtualHost = mkOption { - type = types.bool; - default = false; - description = '' - If enabled, each virtual host gets its own - access_log and - error_log, namely suffixed by the - of the virtual host. - ''; - }; - - user = mkOption { - type = types.str; - default = "wwwrun"; - description = '' - User account under which httpd runs. The account is created - automatically if it doesn't exist. - ''; - }; - - group = mkOption { - type = types.str; - default = "wwwrun"; - description = '' - Group under which httpd runs. The account is created - automatically if it doesn't exist. - ''; - }; - - logDir = mkOption { - type = types.path; - default = "/var/log/httpd"; - description = '' - Directory for Apache's log files. It is created automatically. - ''; - }; - - stateDir = mkOption { - type = types.path; - default = "/run/httpd"; - description = '' - Directory for Apache's transient runtime state (such as PID - files). It is created automatically. Note that the default, - /run/httpd, is deleted at boot time. - ''; - }; - - virtualHosts = mkOption { - type = types.listOf (types.submodule ( - { options = import ./per-server-options.nix { - inherit pkgs; - forMainServer = false; - }; - })); - default = []; - example = [ - { hostName = "foo"; - documentRoot = "/data/webroot-foo"; - } - { hostName = "bar"; - documentRoot = "/data/webroot-bar"; - } - ]; - description = '' - Specification of the virtual hosts served by Apache. Each - element should be an attribute set specifying the - configuration of the virtual host. The available options - are the non-global options permissible for the main host. - ''; - }; - - phpOptions = mkOption { - type = types.lines; - default = ""; - example = - '' - date.timezone = "CET" - ''; - description = - "Options appended to the PHP configuration file php.ini."; - }; - - multiProcessingModule = mkOption { - type = types.str; - default = "prefork"; - example = "worker"; - description = - '' - Multi-processing module to be used by Apache. Available - modules are prefork (the default; - handles each request in a separate child process), - worker (hybrid approach that starts a - number of child processes each running a number of - threads) and event (a recent variant of - worker that handles persistent - connections more efficiently). - ''; - }; - - maxClients = mkOption { - type = types.int; - default = 150; - example = 8; - description = "Maximum number of httpd processes (prefork)"; - }; - - maxRequestsPerChild = mkOption { - type = types.int; - default = 0; - example = 500; - description = - "Maximum number of httpd requests answered per httpd child (prefork), 0 means unlimited"; - }; - } - - # Include the options shared between the main server and virtual hosts. - // (import ./per-server-options.nix { - inherit pkgs; - forMainServer = true; - }); - - }; - - - ###### implementation - - config = mkIf config.services.httpd.enable { - - users.extraUsers.wwwrun = - { name = "wwwrun"; - group = "wwwrun"; - description = "Apache httpd user"; - }; - - users.extraGroups.wwwrun = {}; - - environment.systemPackages = [httpd] ++ concatMap (svc: svc.extraPath) allSubservices; - - services.httpd.phpOptions = - '' - ; Needed for PHP's mail() function. - sendmail_path = sendmail -t -i - - ; Apparently PHP doesn't use $TZ. - date.timezone = "${config.time.timeZone}" - ''; - - docker.buildScripts.apache = - '' - mkdir -m 0750 -p ${mainCfg.stateDir} - chown root.${mainCfg.group} ${mainCfg.stateDir} - ${optionalString version24 '' - mkdir -m 0750 -p "${mainCfg.stateDir}/runtime" - chown root.${mainCfg.group} "${mainCfg.stateDir}/runtime" - ''} - mkdir -m 0700 -p ${mainCfg.logDir} - - ${optionalString (mainCfg.documentRoot != null) - '' - # Create the document root directory if does not exists yet - mkdir -p ${mainCfg.documentRoot} - '' - } - - # Get rid of old semaphores. These tend to accumulate across - # server restarts, eventually preventing it from restarting - # successfully. - for i in $(${pkgs.utillinux}/bin/ipcs -s | grep ' ${mainCfg.user} ' | cut -f2 -d ' '); do - ${pkgs.utillinux}/bin/ipcrm -s $i - done - - # Run the startup hooks for the subservices. - for i in ${toString (map (svn: svn.startupScript) allSubservices)}; do - echo Running Apache startup hook $i... - $i - done - ''; - - supervisord.services.httpd = - { - command = "${httpd}/bin/httpd -f ${httpdConf} -DFOREGROUND"; - }; - }; -} diff --git a/nix-docker/modules/servers/http/apache/per-server-options.nix b/nix-docker/modules/servers/http/apache/per-server-options.nix deleted file mode 100644 index 53f34e2..0000000 --- a/nix-docker/modules/servers/http/apache/per-server-options.nix +++ /dev/null @@ -1,150 +0,0 @@ -# This file defines the options that can be used both for the Apache -# main server configuration, and for the virtual hosts. (The latter -# has additional options that affect the web server as a whole, like -# the user/group to run under.) - -{ forMainServer, pkgs }: - -with pkgs.lib; - -{ - - hostName = mkOption { - type = types.str; - default = "localhost"; - description = "Canonical hostname for the server."; - }; - - serverAliases = mkOption { - type = types.listOf types.str; - default = []; - example = ["www.example.org" "www.example.org:8080" "example.org"]; - description = '' - Additional names of virtual hosts served by this virtual host configuration. - ''; - }; - - port = mkOption { - type = types.int; - default = 0; - description = '' - Port for the server. 0 means use the default port: 80 for http - and 443 for https (i.e. when enableSSL is set). - ''; - }; - - enableSSL = mkOption { - type = types.bool; - default = false; - description = "Whether to enable SSL (https) support."; - }; - - # Note: sslServerCert and sslServerKey can be left empty, but this - # only makes sense for virtual hosts (they will inherit from the - # main server). - - sslServerCert = mkOption { - type = types.nullOr types.path; - default = null; - example = "/var/host.cert"; - description = "Path to server SSL certificate."; - }; - - sslServerKey = mkOption { - type = types.path; - example = "/var/host.key"; - description = "Path to server SSL certificate key."; - }; - - adminAddr = mkOption ({ - type = types.nullOr types.str; - example = "admin@example.org"; - description = "E-mail address of the server administrator."; - } // (if forMainServer then {} else {default = null;})); - - documentRoot = mkOption { - type = types.nullOr types.path; - default = null; - example = "/data/webserver/docs"; - description = '' - The path of Apache's document root directory. If left undefined, - an empty directory in the Nix store will be used as root. - ''; - }; - - servedDirs = mkOption { - type = types.listOf types.attrs; - default = []; - example = [ - { urlPath = "/nix"; - dir = "/home/eelco/Dev/nix-homepage"; - } - ]; - description = '' - This option provides a simple way to serve static directories. - ''; - }; - - servedFiles = mkOption { - type = types.listOf types.attrs; - default = []; - example = [ - { urlPath = "/foo/bar.png"; - dir = "/home/eelco/some-file.png"; - } - ]; - description = '' - This option provides a simple way to serve individual, static files. - ''; - }; - - extraConfig = mkOption { - type = types.lines; - default = ""; - example = '' - - Options FollowSymlinks - AllowOverride All - - ''; - description = '' - These lines go to httpd.conf verbatim. They will go after - directories and directory aliases defined by default. - ''; - }; - - extraSubservices = mkOption { - type = types.listOf types.unspecified; - default = []; - description = "Extra subservices to enable in the webserver."; - }; - - enableUserDir = mkOption { - type = types.bool; - default = false; - description = '' - Whether to enable serving ~/public_html as - /~username. - ''; - }; - - globalRedirect = mkOption { - type = types.nullOr types.str; - default = null; - example = http://newserver.example.org/; - description = '' - If set, all requests for this host are redirected permanently to - the given URL. - ''; - }; - - logFormat = mkOption { - type = types.str; - default = "common"; - example = "combined"; - description = " - Log format for Apache's log files. Possible values are: combined, common, referer, agent. - "; - }; - -} diff --git a/nix-docker/modules/servers/openssh.nix b/nix-docker/modules/servers/openssh.nix deleted file mode 100644 index a661915..0000000 --- a/nix-docker/modules/servers/openssh.nix +++ /dev/null @@ -1,333 +0,0 @@ -{ config, pkgs, ... }: - -with pkgs.lib; - -let - - cfg = config.services.openssh; - cfgc = config.programs.ssh; - - nssModulesPath = config.system.nssModules.path; - - permitRootLoginCheck = v: - v == "yes" || - v == "without-password" || - v == "forced-commands-only" || - v == "no"; - - knownHosts = map (h: getAttr h cfg.knownHosts) (attrNames cfg.knownHosts); - - knownHostsFile = pkgs.writeText "ssh_known_hosts" ( - flip concatMapStrings knownHosts (h: - "${concatStringsSep "," h.hostNames} ${builtins.readFile h.publicKeyFile}" - ) - ); - - userOptions = { - - openssh.authorizedKeys = { - keys = mkOption { - type = types.listOf types.str; - default = []; - description = '' - A list of verbatim OpenSSH public keys that should be added to the - user's authorized keys. The keys are added to a file that the SSH - daemon reads in addition to the the user's authorized_keys file. - You can combine the keys and - keyFiles options. - ''; - }; - - keyFiles = mkOption { - type = types.listOf types.str; - default = []; - description = '' - A list of files each containing one OpenSSH public key that should be - added to the user's authorized keys. The contents of the files are - read at build time and added to a file that the SSH daemon reads in - addition to the the user's authorized_keys file. You can combine the - keyFiles and keys options. - ''; - }; - }; - - }; - - authKeysFiles = let - mkAuthKeyFile = u: { - target = "ssh/authorized_keys.d/${u.name}"; - mode = "0444"; - source = pkgs.writeText "${u.name}-authorized_keys" '' - ${concatStringsSep "\n" u.openssh.authorizedKeys.keys} - ${concatMapStrings (f: builtins.readFile f + "\n") u.openssh.authorizedKeys.keyFiles} - ''; - }; - usersWithKeys = attrValues (flip filterAttrs config.users.extraUsers (n: u: - length u.openssh.authorizedKeys.keys != 0 || length u.openssh.authorizedKeys.keyFiles != 0 - )); - in map mkAuthKeyFile usersWithKeys; - -in - -{ - - ###### interface - - options = { - - services.openssh = { - - enable = mkOption { - type = types.bool; - default = false; - description = '' - Whether to enable the OpenSSH secure shell daemon, which - allows secure remote logins. - ''; - }; - - forwardX11 = mkOption { - type = types.bool; - default = cfgc.setXAuthLocation; - description = '' - Whether to allow X11 connections to be forwarded. - ''; - }; - - allowSFTP = mkOption { - type = types.bool; - default = true; - description = '' - Whether to enable the SFTP subsystem in the SSH daemon. This - enables the use of commands such as sftp and - sshfs. - ''; - }; - - permitRootLogin = mkOption { - default = "without-password"; - type = types.addCheck types.str permitRootLoginCheck; - description = '' - Whether the root user can login using ssh. Valid values are - yes, without-password, - forced-commands-only or - no. - ''; - }; - - gatewayPorts = mkOption { - type = types.str; - default = "no"; - description = '' - Specifies whether remote hosts are allowed to connect to - ports forwarded for the client. See - sshd_config - 5. - ''; - }; - - ports = mkOption { - type = types.listOf types.int; - default = [22]; - description = '' - Specifies on which ports the SSH daemon listens. - ''; - }; - - passwordAuthentication = mkOption { - type = types.bool; - default = true; - description = '' - Specifies whether password authentication is allowed. - ''; - }; - - challengeResponseAuthentication = mkOption { - type = types.bool; - default = true; - description = '' - Specifies whether challenge/response authentication is allowed. - ''; - }; - - hostKeys = mkOption { - type = types.listOf types.attrs; - default = - [ { path = "/etc/ssh/ssh_host_dsa_key"; - type = "dsa"; - bits = 1024; - } - { path = "/etc/ssh/ssh_host_ecdsa_key"; - type = "ecdsa"; - bits = 521; - } - ]; - description = '' - NixOS can automatically generate SSH host keys. This option - specifies the path, type and size of each key. See - ssh-keygen - 1 for supported types - and sizes. - ''; - }; - - authorizedKeysFiles = mkOption { - type = types.listOf types.str; - default = []; - description = "Files from with authorized keys are read."; - }; - - extraConfig = mkOption { - type = types.lines; - default = ""; - description = "Verbatim contents of sshd_config."; - }; - - knownHosts = mkOption { - default = {}; - type = types.loaOf types.optionSet; - description = '' - The set of system-wide known SSH hosts. - ''; - example = [ - { - hostNames = [ "myhost" "myhost.mydomain.com" "10.10.1.4" ]; - publicKeyFile = literalExample "./pubkeys/myhost_ssh_host_dsa_key.pub"; - } - { - hostNames = [ "myhost2" ]; - publicKeyFile = literalExample "./pubkeys/myhost2_ssh_host_dsa_key.pub"; - } - ]; - options = { - hostNames = mkOption { - type = types.listOf types.string; - default = []; - description = '' - A list of host names and/or IP numbers used for accessing - the host's ssh service. - ''; - }; - publicKeyFile = mkOption { - description = '' - The path to the public key file for the host. The public - key file is read at build time and saved in the Nix store. - You can fetch a public key file from a running SSH server - with the ssh-keyscan command. - ''; - }; - }; - }; - - }; - - users.extraUsers = mkOption { - options = [ userOptions ]; - }; - - }; - - - ###### implementation - - config = mkIf cfg.enable { - - users.extraUsers = singleton - { name = "sshd"; - uid = config.ids.uids.sshd; - description = "SSH privilege separation user"; - home = "/var/empty"; - }; - - environment.etc = authKeysFiles ++ [ - { source = "${pkgs.openssh}/etc/ssh/moduli"; - target = "ssh/moduli"; - } - { source = knownHostsFile; - target = "ssh/ssh_known_hosts"; - } - ]; - - systemd.services.sshd = - { description = "SSH Daemon"; - - wantedBy = [ "multi-user.target" ]; - - stopIfChanged = false; - - path = [ pkgs.openssh pkgs.gawk ]; - - environment.LD_LIBRARY_PATH = ""; - environment.LOCALE_ARCHIVE = "/run/current-system/sw/lib/locale/locale-archive"; - - preStart = - '' - mkdir -m 0755 -p /etc/ssh - - ${flip concatMapStrings cfg.hostKeys (k: '' - if ! [ -f "${k.path}" ]; then - ssh-keygen -t "${k.type}" -b "${toString k.bits}" -f "${k.path}" -N "" - fi - '')} - ''; - - serviceConfig = - { ExecStart = - "${pkgs.openssh}/sbin/sshd " + - "-f ${pkgs.writeText "sshd_config" cfg.extraConfig} -D"; - Restart = "always"; - KillMode = "process"; - PIDFile = "/run/sshd.pid"; - }; - }; - - services.openssh.authorizedKeysFiles = - [ ".ssh/authorized_keys" ".ssh/authorized_keys2" "/etc/ssh/authorized_keys.d/%u" ]; - - services.openssh.extraConfig = - '' - PidFile /run/sshd.pid - - Protocol 2 - - UsePAM no - - AddressFamily ${if config.networking.enableIPv6 then "any" else "inet"} - ${concatMapStrings (port: '' - Port ${toString port} - '') cfg.ports} - - ${optionalString cfgc.setXAuthLocation '' - XAuthLocation ${pkgs.xorg.xauth}/bin/xauth - ''} - - ${if cfg.forwardX11 then '' - X11Forwarding yes - '' else '' - X11Forwarding no - ''} - - ${optionalString cfg.allowSFTP '' - Subsystem sftp ${pkgs.openssh}/libexec/sftp-server - ''} - - PermitRootLogin ${cfg.permitRootLogin} - GatewayPorts ${cfg.gatewayPorts} - PasswordAuthentication ${if cfg.passwordAuthentication then "yes" else "no"} - ChallengeResponseAuthentication ${if cfg.challengeResponseAuthentication then "yes" else "no"} - - PrintMotd no # handled by pam_motd - - AuthorizedKeysFile ${toString cfg.authorizedKeysFiles} - - ${flip concatMapStrings cfg.hostKeys (k: '' - HostKey ${k.path} - '')} - ''; - - assertions = [{ assertion = if cfg.forwardX11 then cfgc.setXAuthLocation else true; - message = "cannot enable X11 forwarding without setting xauth location";}]; - - }; - -} diff --git a/nix-docker/package.json b/nix-docker/package.json deleted file mode 100644 index d349134..0000000 --- a/nix-docker/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name" : "nix-docker", - "version" : "0.1.0", - "dependencies" : { - "optimist" : "0.6.0" - }, - "directories": { - "bin": "./bin" - } -} \ No newline at end of file diff --git a/samples/apache-config.nix b/samples/apache-config.nix deleted file mode 100644 index b5d3d33..0000000 --- a/samples/apache-config.nix +++ /dev/null @@ -1,14 +0,0 @@ -{ config, pkgs, ... }: -{ - # Expose the apache port (80) - docker.ports = [ - config.services.httpd.port - ]; - - services.httpd = { - enable = true; - port = 80; - documentRoot = ./www; - adminAddr = "zef.hemel@logicblox.com"; - }; -} \ No newline at end of file diff --git a/samples/fancy.nix b/samples/fancy.nix deleted file mode 100644 index 73101e4..0000000 --- a/samples/fancy.nix +++ /dev/null @@ -1,29 +0,0 @@ -# Boots up all kinds of services: redis, apache, ssh, mysql -{ config, pkgs, ... }: -{ - docker.ports = [ 1234 80 22 ]; - - services.redis = { - enable = true; - port = 1234; - logLevel = "debug"; - }; - - services.httpd.enable = true; - services.httpd.port = 80; - services.httpd.documentRoot = ./www; - services.httpd.adminAddr = "zef.hemel@logicblox.com"; - - services.mysql.enable = true; - services.openssh.enable = true; - - supervisord.tailLogs = true; - - users.extraUsers.zef = { - group = "users"; - home = "/"; - shell = "/bin/bash"; - createHome = true; - openssh.authorizedKeys.keys = [ "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCjWpdyDIsS09lWlOsMG9OMTHB/N/afVU12BwKcyjjhbezPdFEgHK4cZBN7m1bvoFKl832BdB+ZjeRH4UGBcUpvrFu1vE7Lf/0vZDU7qzzWQE9V+tfSPwDiXPf9QnCYeZmYPDHUHDUEse9LKBZbt6UKF1tuTD8ussV5jvEFBaesDhCqD1TJ4b4O877cdx9+VTOuDSEDm32jQ2az27d1b/5DoEKBe5cJSC3PhObAQ7OAYrVVBFX9ffKpaSvV6yqo+rhCmXP9DjNgBwMtElreoXL3h5Xbw2AiER5oHNUAEA2XGpnOVOr7ZZUAbMC0/0dq387jQZCqe7gIDZCqjDpGhUa9 zefhemel@gmail.com" ]; - }; -} \ No newline at end of file diff --git a/samples/ssh-config.nix b/samples/ssh-config.nix deleted file mode 100644 index 497a337..0000000 --- a/samples/ssh-config.nix +++ /dev/null @@ -1,19 +0,0 @@ -# Runs an SSH server in a Docker container -# Creates a user "you" that you can login with -{ config, pkgs, ... }: -{ - docker.ports = [ 22 ]; - - services.openssh.enable = true; - - users.extraUsers.you = { - group = "users"; - home = "/"; - shell = "/bin/bash"; - createHome = true; - openssh.authorizedKeys.keys = [ - # Replace with your own SSH key (e.g. from ~/.ssh/id_rsa.pub) - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCjWpdyDIsS09lWlOsMG9OMTHB/N/afVU12BwKcyjjhbezPdFEgHK4cZBN7m1bvoFKl832BdB+ZjeRH4UGBcUpvrFu1vE7Lf/0vZDU7qzzWQE9V+tfSPwDiXPf9QnCYeZmYPDHUHDUEse9LKBZbt6UKF1tuTD8ussV5jvEFBaesDhCqD1TJ4b4O877cdx9+VTOuDSEDm32jQ2az27d1b/5DoEKBe5cJSC3PhObAQ7OAYrVVBFX9ffKpaSvV6yqo+rhCmXP9DjNgBwMtElreoXL3h5Xbw2AiER5oHNUAEA2XGpnOVOr7ZZUAbMC0/0dq387jQZCqe7gIDZCqjDpGhUa9" - ]; - }; -} \ No newline at end of file diff --git a/samples/wordpress.nix b/samples/wordpress.nix deleted file mode 100644 index d4cca15..0000000 --- a/samples/wordpress.nix +++ /dev/null @@ -1,138 +0,0 @@ -{ config, pkgs, ... }: -# We'll start with defining some local variables -let - # Settings used to configure the wordpress MySQL database - mysqlHost = "localhost"; - mysqlDb = "wordpress"; - mysqlUser = "wordpress"; - mysqlPassword = "wordpress"; - - # Data paths - mysqlDataPath = "/data/mysql"; - wordpressUploads = "/data/uploads"; - apacheLogs = "/data/log"; - - # Our bare-bones wp-config.php file using the above settings - wordpressConfig = pkgs.writeText "wp-config.php" '' - - RewriteEngine On - RewriteBase / - RewriteCond %{REQUEST_FILENAME} !-f - RewriteCond %{REQUEST_FILENAME} !-d - RewriteRule . /index.php [L] - - ''; - - # For shits and giggles, let's package the responsive theme - responsiveTheme = pkgs.stdenv.mkDerivation { - name = "responsive-theme"; - # Download the theme from the wordpress site - src = pkgs.fetchurl { - url = http://wordpress.org/themes/download/responsive.1.9.3.9.zip; - sha256 = "0397a8nd8q384z2i4lw4a1ij835walp3b5bfdmr76mdl27vbkvmd"; - }; - # We need unzip to build this package - buildInputs = [ pkgs.unzip ]; - # Installing simply means copying all files to the output directory - installPhase = "mkdir -p $out; cp -R * $out/"; - }; - - # The wordpress package itself - wordpress = pkgs.stdenv.mkDerivation { - name = "wordpress"; - # Fetch directly from the wordpress site, want to upgrade? - # Just change the version URL and update the hash - src = pkgs.fetchurl { - url = http://wordpress.org/wordpress-3.7.1.tar.gz; - sha256 = "1m2dlr54fqf5m4kgqc5hrrisrsia0r5j1r2xv1f12gmzb63swsvg"; - }; - installPhase = '' - mkdir -p $out - # Copy all the wordpress files we downloaded - cp -R * $out/ - # We'll symlink the wordpress config - ln -s ${wordpressConfig} $out/wp-config.php - # As well as our custom .htaccess - ln -s ${htaccess} $out/.htaccess - # And the uploads directory - ln -s ${wordpressUploads} $out/wp-content/uploads - # And the responsive theme - ln -s ${responsiveTheme} $out/wp-content/themes/responsive - # You can add plugins the same way - ''; - }; -# This is where the body of our configuration starts -in { - # Expose just port 80 - docker.ports = [ 80 ]; - - # And let's store valuable data in a volume - docker.volumes = [ "/data" ]; - - # Apache configuration - services.httpd = { - enable = true; - adminAddr = "zef@zef.me"; - - # We'll set the wordpress package as our document root - documentRoot = wordpress; - - # Let's store our logs in the volume as well - logDir = apacheLogs; - - # And enable the PHP5 apache module - extraModules = [ { name = "php5"; path = "${pkgs.php}/modules/libphp5.so"; } ]; - - # And some extra config to make things work nicely - extraConfig = '' - - DirectoryIndex index.php - Allow from * - Options FollowSymLinks - AllowOverride All - - ''; - }; - - # Disable these when not using "localhost" as database name - services.mysql.enable = true; - # Let's store our data in the volume, so it'll survive restarts - services.mysql.dataDir = mysqlDataPath; - - # This service runs evey time you start and ensures two things: - # 1. That our uploads directory exists and is owned by Apache - # 2. that the MySQL database exists - supervisord.services.initApp = { - command = pkgs.writeScript "init-wordpress.sh" '' - #!/bin/sh - - mkdir -p ${wordpressUploads} - chown ${config.services.httpd.user} ${wordpressUploads} - - if [ ! -d /data/mysql/${mysqlDb} ]; then - # Wait until MySQL is up - while [ ! -e /var/run/mysql/mysqld.pid ]; do - sleep 1 - done - mysql -e 'CREATE DATABASE ${mysqlDb};' - mysql -e 'GRANT ALL ON ${mysqlDb}.* TO ${mysqlUser}@localhost IDENTIFIED BY "${mysqlPassword}";' - fi - ''; - # This script can exit immediately, no worries - startsecs = 0; - }; -} \ No newline at end of file diff --git a/samples/www/index.html b/samples/www/index.html deleted file mode 100644 index b64ccf4..0000000 --- a/samples/www/index.html +++ /dev/null @@ -1 +0,0 @@ -Hello world 2 diff --git a/scripts/README.md b/scripts/README.md deleted file mode 100644 index 0d1a420..0000000 --- a/scripts/README.md +++ /dev/null @@ -1 +0,0 @@ -These are scripts to automatically install Docker and Nix and has been tested to work on Ubuntu 13.04. diff --git a/scripts/install-docker.sh b/scripts/install-docker.sh deleted file mode 100644 index 4cf9e6f..0000000 --- a/scripts/install-docker.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/sh - -if [ "$(which docker)" != "" ]; then - exit 0 -fi - -apt-get update - -apt-get install -y linux-image-extra-`uname -r` - -sh -c "wget -qO- https://get.docker.io/gpg | apt-key add -" - -# Add the Docker repository to your apt sources list. -sh -c "echo deb http://get.docker.io/ubuntu docker main\ -> /etc/apt/sources.list.d/docker.list" - -# update -apt-get update - -# install -apt-get install -y lxc-docker diff --git a/scripts/install-nix.sh b/scripts/install-nix.sh deleted file mode 100644 index 387e887..0000000 --- a/scripts/install-nix.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/sh - -set -e - -# Check if Nix is already installed - -if [ -d "/nix/store" ]; then - exit 0 -fi - -# Install the binary tarball... -apt-get install -y curl -cd / -curl -L http://hydra.nixos.org/job/nix/trunk/binaryTarball.x86_64-linux/latest/download | tar xj -/usr/bin/nix-finish-install -rm /usr/bin/nix-finish-install - -# Hack -chmod 777 /nix/var/nix/profiles - -# Setup multiuserbu - -# Allow all users to create profiles -mkdir -p /nix/var/nix/profiles/per-user -chmod 1777 /nix/var/nix/profiles/per-user - -# Add build users -# 9 is the exit code when the group already exists -groupadd -r nixbld || [ "$?" -eq 9 ] -for n in 1 2 3 4 5 6 7 8 9 10; do - useradd -c "Nix build user $n" -d /var/empty -g nixbld -G nixbld \ - -M -N -r -s `which nologin` nixbld$n || [ "$?" -eq 9 ] -done -chown root:nixbld /nix/store -chmod 1775 /nix/store -mkdir -p /etc/nix -grep -w build-users-group /etc/nix/nix.conf 2>/dev/null || echo "build-users-group = nixbld" >> /etc/nix/nix.conf -grep -w binary-caches /etc/nix/nix.conf 2>/dev/null || echo "binary-caches = http://cache.nixos.org" >> /etc/nix/nix.conf -grep -w trusted-binary-caches /etc/nix/nix.conf 2>/dev/null || echo "trusted-binary-caches = http://hydra.nixos.org http://cache.nixos.org" >> /etc/nix/nix.conf - -# Use a multiuser-compatible profile script -unlink /etc/profile.d/nix.sh -cat > /etc/profile.d/nix.sh <&2 - mkdir -p "/nix/var/nix/profiles/per-user/\$LOGNAME" - _NIX_PROFILE_LINK="/nix/var/nix/profiles/per-user/\$LOGNAME/profile" - ln -s /nix/var/nix/profiles/default \$_NIX_PROFILE_LINK - ln -s "\$_NIX_PROFILE_LINK" "\$NIX_LINK" - fi - - # Subscribe the root user to the Nixpkgs channel by default. - if [ ! -e "\$HOME/.nix-channels" ]; then - echo "http://nixos.org/channels/nixpkgs-unstable nixpkgs" > "\$HOME/.nix-channels" - fi - - # Set up nix-defexpr - NIX_DEFEXPR="\$HOME/.nix-defexpr" - if ! [ -e "\$NIX_DEFEXPR" ]; then - echo "creating \$NIX_DEFEXPR" >&2 - mkdir -p "\$NIX_DEFEXPR" - _NIX_CHANNEL_LINK=/nix/var/nix/profiles/per-user/root/channels - ln -s "\$_NIX_CHANNEL_LINK" "\$NIX_DEFEXPR/channels" - #/nix/var/nix/profiles/default/bin/nix-channel --update - fi - - if [ -z "\$OWNS_STORE" ]; then - export NIX_REMOTE=daemon - export PATH="/nix/var/nix/profiles/default/bin:\$PATH" - fi - export PATH="\$NIX_LINK/bin:\$PATH" - - # Set up NIX_PATH - export NIX_PATH="\$NIX_DEFEXPR/channels" - unset OWNS_STORE -fi -EOF - -# Add default nix profile to global path and enable it during sudo -sed -i 's/"$/:\/nix\/var\/nix\/profiles\/default\/bin"/' /etc/environment -sed -i 's/secure_path="/secure_path="\/nix\/var\/nix\/profiles\/default\/bin:/' /etc/sudoers - -# Install upstart job -cat > /etc/init/nix-daemon.conf < Date: Mon, 17 Feb 2014 23:15:41 +0100 Subject: [PATCH 35/46] Fix killing of processes, split control scripts --- default.nix | 32 ++++++++++++++++++++++++++++---- foo.nix | 2 ++ supervisord.nix | 30 ++++++++++++++---------------- systemd.nix | 3 +++ 4 files changed, 47 insertions(+), 20 deletions(-) diff --git a/default.nix b/default.nix index 65bb454..ab26f30 100644 --- a/default.nix +++ b/default.nix @@ -14,6 +14,7 @@ let + ]; @@ -24,19 +25,42 @@ let systemd = import ./systemd.nix { inherit pkgs config; }; - startScript = pkgs.writeScript "build" '' + startServices = pkgs.writeScript "startServices" '' #!/bin/sh + export STATEDIR="${"\$"}{STATEDIR-$(pwd)/var}" + + mkdir -p $STATEDIR/{run,log} + + # Run start scripts first ${config.userNix.startScript} + + # Run supervisord + ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${config.supervisord.configFile} -j $STATEDIR/run/supervisord.pid -d $STATEDIR -q $STATEDIR/log/ -l $STATEDIR/supervisord.log ''; + stopServices = pkgs.writeScript "stopServices" '' + #!/bin/sh + ${pkgs.pythonPackages.supervisor}/bin/supervisorctl -c ${config.supervisord.configFile} shutdown + ''; + + controlServices = pkgs.writeScript "controlServices" '' + #!/bin/sh + ${pkgs.pythonPackages.supervisor}/bin/supervisorctl -c ${config.supervisord.configFile} + ''; + + in pkgs.stdenv.mkDerivation { - inherit name; + name = "${name}-services"; src = ./.; phases = [ "installPhase" ]; installPhase = '' - mkdir -p $out/etc/start - ln -s ${startScript} $out/etc/start + ensureDir $out/bin/ + ln -s ${startServices} $out/bin/${name}-start-services + ln -s ${stopServices} $out/bin/${name}-stop-services + ln -s ${controlServices} $out/bin/${name}-control-services ''; + + passthru.config = config; } diff --git a/foo.nix b/foo.nix index c4f0607..8833c31 100644 --- a/foo.nix +++ b/foo.nix @@ -4,5 +4,7 @@ services.postgresql.enable = true; services.postgresql.package = pkgs.postgresql92; services.postgresql.dataDir = "/tmp/postgres"; + services.elasticsearch.enable = true; + services.elasticsearch.dataDir = "/tmp/elasticsearch"; }; } diff --git a/supervisord.nix b/supervisord.nix index c0cf2b9..327a38d 100644 --- a/supervisord.nix +++ b/supervisord.nix @@ -16,6 +16,9 @@ let PATH = "/some/path"; }; }; + stopsignal = mkOption { + default = "TERM"; + }; startsecs = mkOption { default = 1; example = 0; @@ -55,7 +58,15 @@ in { config = mkIf config.supervisord.enable { supervisord.configFile = pkgs.writeText "supervisord.conf" '' [supervisord] - logfile=/tmp/supervisor/var/log/supervisord/supervisord.log + + [supervisorctl] + serverurl = http://localhost:64125 + + [inet_http_server] + port = 127.0.0.1:64125 + + [rpcinterface:supervisor] + supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface ${concatMapStrings (name: let @@ -67,25 +78,12 @@ in { environment=${concatMapStrings (name: "${name}=\"${toString (getAttr name cfg.environment)}\",") (attrNames cfg.environment)} directory=${cfg.directory} redirect_stderr=true - stdout_logfile=/tmp/supervisor/var/log/supervisord/${name}.log startsecs=${toString cfg.startsecs} + stopsignal=${cfg.stopsignal} + stopasgroup=true '' ) (attrNames services) } ''; - - userNix.startScript = let - systemPackages = config.environment.systemPackages; - systemEnv = pkgs.buildEnv { name = "system-env"; paths = systemPackages; }; - in '' - mkdir -p /tmp/supervisor/var/log/supervisord - export PATH="${systemEnv}/bin:${systemEnv}/sbin" - ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${config.supervisord.configFile} ${if config.supervisord.tailLogs then '' - - sleep 2 - touch $(pwd)/var/log/supervisord/test.log - tail -n 100 -f /var/log/supervisord/*.log - '' else "-n"} - ''; }; } diff --git a/systemd.nix b/systemd.nix index 34b4f66..21fcaa2 100644 --- a/systemd.nix +++ b/systemd.nix @@ -54,6 +54,9 @@ in { { PATH = "%(ENV_PATH)s:" + concatStringsSep ":" (map (prg: "${prg}/bin") cfg.path); } else { PATH="%(ENV_PATH)s"; }); + stopsignal = if hasAttr "KillSignal" cfg.serviceConfig then + substring 3 (stringLength cfg.serviceConfig.KillSignal) cfg.serviceConfig.KillSignal + else "TERM"; }; } ) (attrNames runServices)); From 9391516e62314a509b9166948978d9534f31ff33 Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 7 Mar 2014 17:54:32 +0100 Subject: [PATCH 36/46] Change the way how configuration is passed from single config to list --- default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/default.nix b/default.nix index ab26f30..ad36301 100644 --- a/default.nix +++ b/default.nix @@ -1,6 +1,6 @@ { pkgs ? import { system = "x86_64-linux"; } , name -, configuration ? +, configuration ? [] , baseImage ? "busybox" }: with pkgs.lib; @@ -19,7 +19,7 @@ let ]; config = (evalModules { - modules = [configuration] ++ moduleList; + modules = configuration ++ moduleList; args = { inherit pkgs; }; }).config; From d7fe7d18fe8e39e5560dd49ac49590d4f717c51f Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 7 Mar 2014 17:57:45 +0100 Subject: [PATCH 37/46] fix the way how environment variables, especially PATH is handled --- default.nix | 1 + supervisord.nix | 12 +++++++++++- systemd.nix | 7 ++----- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/default.nix b/default.nix index ad36301..aeb00b9 100644 --- a/default.nix +++ b/default.nix @@ -28,6 +28,7 @@ let startServices = pkgs.writeScript "startServices" '' #!/bin/sh export STATEDIR="${"\$"}{STATEDIR-$(pwd)/var}" + export PATH="${pkgs.coreutils}/bin" mkdir -p $STATEDIR/{run,log} diff --git a/supervisord.nix b/supervisord.nix index 327a38d..fb31666 100644 --- a/supervisord.nix +++ b/supervisord.nix @@ -16,6 +16,10 @@ let PATH = "/some/path"; }; }; + path = mkOption { + default = []; + description = "Current directory when running the command"; + }; stopsignal = mkOption { default = "TERM"; }; @@ -75,7 +79,13 @@ in { '' [program:${name}] command=${cfg.command} - environment=${concatMapStrings (name: "${name}=\"${toString (getAttr name cfg.environment)}\",") (attrNames cfg.environment)} + environment=${concatStrings + (mapAttrsToList (name: value: "${name}=\"${value}\",") ( + cfg.environment // { PATH = concatStringsSep ":" + [("%(ENV_PATH)s") (cfg.path) (maybeAttr "PATH" "" cfg.environment)]; + } + ) + )} directory=${cfg.directory} redirect_stderr=true startsecs=${toString cfg.startsecs} diff --git a/systemd.nix b/systemd.nix index 21fcaa2..b102dea 100644 --- a/systemd.nix +++ b/systemd.nix @@ -49,11 +49,8 @@ in { name = name; value = { command = pkgs.writeScript "${name}-run" (configToCommand name cfg); - environment = (if hasAttr "environment" cfg then cfg.environment else {}) // - (if hasAttr "path" cfg then - { PATH = "%(ENV_PATH)s:" + concatStringsSep ":" (map (prg: "${prg}/bin") cfg.path); } - else { - PATH="%(ENV_PATH)s"; }); + environment = cfg.environment; + path = cfg.path; stopsignal = if hasAttr "KillSignal" cfg.serviceConfig then substring 3 (stringLength cfg.serviceConfig.KillSignal) cfg.serviceConfig.KillSignal else "TERM"; From d1e715c411517d6c296b7d797f2345f2dc29a594 Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 7 Mar 2014 17:59:44 +0100 Subject: [PATCH 38/46] set systemd.services.options to original system option interface --- systemd.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/systemd.nix b/systemd.nix index b102dea..c8488f2 100644 --- a/systemd.nix +++ b/systemd.nix @@ -1,5 +1,6 @@ { pkgs, config, ... }: -with pkgs.lib; + with pkgs.lib; + with import { inherit config pkgs; }; let services = config.systemd.services; @@ -35,6 +36,8 @@ in { options = { systemd.services = mkOption { default = {}; + type = types.attrsOf types.optionSet; + options = [ serviceOptions ]; }; # TODO make more specific }; From 1b4195d95d14bf98c4bbd2c475a38abd9a7204e2 Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 7 Mar 2014 18:00:16 +0100 Subject: [PATCH 39/46] systemd: only create enabled services --- systemd.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemd.nix b/systemd.nix index c8488f2..ac2173d 100644 --- a/systemd.nix +++ b/systemd.nix @@ -59,6 +59,6 @@ in { else "TERM"; }; } - ) (attrNames runServices)); + ) (attrNames (filterAttrs (n: v: v.enable) runServices))); }; } From 6ba793340580084e4cd318eff328283f083f43a4 Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 7 Mar 2014 18:01:00 +0100 Subject: [PATCH 40/46] systemd: fix string comparisson in command filter --- systemd.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/systemd.nix b/systemd.nix index ac2173d..fe3fd5e 100644 --- a/systemd.nix +++ b/systemd.nix @@ -13,7 +13,7 @@ let filterCommand = cmd: let filtered = substring 1 (stringLength cmd -2) cmd; - splitted = pkgs.lib.splitString " " filtered; + splitted = pkgs.lib.splitString (" " + substring 0 0 filtered) filtered; in if eqStrings (substring 0 1 cmd) "@" then traceVal (head splitted) + concatStringsSep " " (drop 2 splitted) else cmd; From ac43fc3f8ee0d729eb3f4c1645bab52a24d0c86b Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 7 Mar 2014 18:01:25 +0100 Subject: [PATCH 41/46] default: syntax fixup --- default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index aeb00b9..cef5fc2 100644 --- a/default.nix +++ b/default.nix @@ -3,7 +3,8 @@ , configuration ? [] , baseImage ? "busybox" }: -with pkgs.lib; + with pkgs.lib; + let moduleList = [ ./user.nix ./supervisord.nix ./systemd.nix ./environment.nix From cb77a8cb411520a1a24ab9717bcc398eee92fda9 Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 7 Mar 2014 18:01:46 +0100 Subject: [PATCH 42/46] default: add several additional modules --- default.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/default.nix b/default.nix index cef5fc2..aa0a97d 100644 --- a/default.nix +++ b/default.nix @@ -16,7 +16,10 @@ let - + + + + ]; config = (evalModules { From bfcaf380a4df869988ed332d976dcad550630d1c Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 7 Mar 2014 18:02:24 +0100 Subject: [PATCH 43/46] default: fix supervisor log path --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/default.nix b/default.nix index aa0a97d..1446810 100644 --- a/default.nix +++ b/default.nix @@ -40,7 +40,7 @@ let ${config.userNix.startScript} # Run supervisord - ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${config.supervisord.configFile} -j $STATEDIR/run/supervisord.pid -d $STATEDIR -q $STATEDIR/log/ -l $STATEDIR/supervisord.log + ${pkgs.pythonPackages.supervisor}/bin/supervisord -c ${config.supervisord.configFile} -j $STATEDIR/run/supervisord.pid -d $STATEDIR -q $STATEDIR/log/ -l $STATEDIR/log/supervisord.log ''; stopServices = pkgs.writeScript "stopServices" '' From 3b862ceec31b09d5894a42fc901cf59b35cc93c7 Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 7 Mar 2014 18:03:33 +0100 Subject: [PATCH 44/46] default: create environment as output where we additionally add environment.systemPackages --- default.nix | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/default.nix b/default.nix index 1446810..8c0d7f9 100644 --- a/default.nix +++ b/default.nix @@ -53,19 +53,23 @@ let ${pkgs.pythonPackages.supervisor}/bin/supervisorctl -c ${config.supervisord.configFile} ''; + servicesControl = pkgs.stdenv.mkDerivation { + name = "${name}-servicesControl"; + src = ./.; -in pkgs.stdenv.mkDerivation { + phases = [ "installPhase" ]; + + installPhase = '' + ensureDir $out/bin/ + ln -s ${startServices} $out/bin/${name}-start-services + ln -s ${stopServices} $out/bin/${name}-stop-services + ln -s ${controlServices} $out/bin/${name}-control-services + ''; + + passthru.config = config; + }; + +in pkgs.buildEnv { name = "${name}-services"; - src = ./.; - - phases = [ "installPhase" ]; - - installPhase = '' - ensureDir $out/bin/ - ln -s ${startServices} $out/bin/${name}-start-services - ln -s ${stopServices} $out/bin/${name}-stop-services - ln -s ${controlServices} $out/bin/${name}-control-services - ''; - - passthru.config = config; + paths = [ servicesControl ] ++ config.environment.systemPackages; } From aba8854afc8ee072fa5e80c8739e457cd63c3f2b Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 14 Mar 2014 16:37:23 +0000 Subject: [PATCH 45/46] Add support for forking services --- supervisord.nix | 5 ++++- systemd.nix | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/supervisord.nix b/supervisord.nix index fb31666..9958371 100644 --- a/supervisord.nix +++ b/supervisord.nix @@ -27,6 +27,9 @@ let default = 1; example = 0; }; + pidfile = mkOption { + default = null; + }; }; }; services = config.supervisord.services; @@ -78,7 +81,7 @@ in { in '' [program:${name}] - command=${cfg.command} + command=${if cfg.pidfile == null then cfg.command else "${pkgs.pythonPackages.supervisor}/bin/pidproxy ${cfg.pidfile} ${cfg.command}"} environment=${concatStrings (mapAttrsToList (name: value: "${name}=\"${value}\",") ( cfg.environment // { PATH = concatStringsSep ":" diff --git a/systemd.nix b/systemd.nix index fe3fd5e..184ad87 100644 --- a/systemd.nix +++ b/systemd.nix @@ -57,6 +57,7 @@ in { stopsignal = if hasAttr "KillSignal" cfg.serviceConfig then substring 3 (stringLength cfg.serviceConfig.KillSignal) cfg.serviceConfig.KillSignal else "TERM"; + pidfile = if hasAttr "PIDFile" cfg.serviceConfig then cfg.serviceConfig.PIDFile else null; }; } ) (attrNames (filterAttrs (n: v: v.enable) runServices))); From bbc5bf5bcf4466c7acd3ef978ae77daa8cae9bac Mon Sep 17 00:00:00 2001 From: Jaka Hudoklin Date: Fri, 14 Mar 2014 16:41:35 +0000 Subject: [PATCH 46/46] Remove example --- foo.nix | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 foo.nix diff --git a/foo.nix b/foo.nix deleted file mode 100644 index 8833c31..0000000 --- a/foo.nix +++ /dev/null @@ -1,10 +0,0 @@ -{ config, pkgs, ... }: -{ - config = { - services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql92; - services.postgresql.dataDir = "/tmp/postgres"; - services.elasticsearch.enable = true; - services.elasticsearch.dataDir = "/tmp/elasticsearch"; - }; -}