Get cross-compilation ready for release. (#1263)

* Add cross-compilation for `lmdb`.
* Got built caching working in CI with `cachix`.
* Cache cross compilation dependencies and toolchains.
* Do release builds in CI.
* Upload release builds to `bootstrap.urbit.org` on successful build.
* Lots of optimization work for CI.
* Boot from a solid pill in CI and load arvo with `-A`.
* Increase `vere` HTTP timeout to 15m.

.gitignore

@@ -3,9 +3,9 @@
 /result-*
 /work
 /zod
+/bus
 /fakezod*
 tags
 TAGS
-bin/darwin
-bin/linux32
-bin/linux64
+cross/
+release/

.travis.yml

@@ -10,7 +10,24 @@ before_install:
 script:
   - cachix authtoken "$CACHIX_AUTH_TOKEN" >/dev/null
   - cachix use urbit2
-  - make
-  - make test
-  - sh/update-brass-pill # To verify that this works.
+
   - ./sh/cachix
+
+  - make
+  - make release
+
+  - sh/ci-tests
+
+deploy:
+  - skip_cleanup: true
+    provider: gcs
+    access_key_id: GOOGTADOPP55X5ZTH3IKAXQW
+    secret_access_key:
+      secure: rSPif0VHX3Q3QpWM9iUt/Z9sicCY8ABuwVFPeT3YUnAAmSXM46PIHKieoGs79kx9IelFyQsM8xS0XWyt4S/haR7VaTJY+zHJjf0AnA1sr5ZIV70t3K0SXzq4+E1ffOZRiC0VmHatPz10wjaIpHxpjcROqQV4M1dBCq2H/rpccIE=
+    bucket: bootstrap.urbit.org
+    local-dir: release/
+    acl: public-read
+    on:
+      condition: "-d release/"
+      repo: urbit/urbit
+      all_branches: true

Makefile

@@ -1,8 +1,4 @@
-PKGS = minima extrema rusthello prog
-
-################################################################################
-
-.PHONY: build build-all install release test clean
+.PHONY: build build-all install cross release test clean
 
 build:
 	nix-build -A urbit -A herb --no-out-link
@@ -13,10 +9,11 @@ build-all:
 install:
 	nix-env -f . -iA urbit -iA urbit-debug -iA herb
 
+cross:
+	sh/cross
+
 release:
-	sh/release urbit linux32
-	sh/release urbit linux64
-	sh/release urbit darwin
+	sh/release
 
 test:
 	sh/test

bin/brass.pill

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e1b2d19d2b71a3c407358ff69b14de0029bd6a7c486b819f59274d615652917
-size 5210146
+oid sha256:a79ceaa44b0c056dc34b15e759757fe3dc470be01530d96a4bc6f44098f73220
+size 5503628

bin/solid.pill

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3487ad86985d61dcd3106e7114167d0d6c071ffab645c6c9bfe131ec8bd6dd3c
+size 8133778

nix/cachix/local.nix

@@ -0,0 +1,23 @@
+# All the non-release builds that should be cached in `cachix`.
+
+let
+
+  pkgs = import ../pkgs {};
+  deps = import ../deps {};
+
+  # Cache the result of cloning source repos.
+  repos = {
+    argon2-src     = deps.argon2.src;
+    ed25519-src    = deps.ed25519.src;
+    h2o-src        = deps.h2o.src;
+    murmur3-src    = deps.murmur3.src;
+    scrypt-src     = deps.scrypt.src;
+    secp256k1-src  = deps.secp256k1.src;
+    sni-src        = deps.sni.src;
+    softfloat3-src = deps.softfloat3.src;
+    uv-src         = deps.uv.src;
+  };
+
+in
+
+  deps // pkgs // repos

nix/cachix/release.nix

@@ -0,0 +1,11 @@
+let
+
+  util         = import ./util.nix;
+  nixcrpkgs    = import ../nixcrpkgs.nix;
+  release      = import ../release.nix;
+  all_releases = util.flattenSetPrefix release;
+  crosstools   = { inherit (nixcrpkgs.native) pkgconf; };
+
+in
+
+  crosstools // all_releases

nix/cachix/util.nix

@@ -0,0 +1,26 @@
+# Some utility functions:
+
+rec {
+
+  # The inverse of builtins.listToAttrs
+  attrsToList = o:
+    map (a: { name=a; value=builtins.getAttr a o; })
+      (builtins.attrNames o);
+
+  # ∀o,x,y. produce o' such that o'.y == o.x.y (assuming no conflicts)
+  flattenSet = o:
+    builtins.foldl' (acc: v: acc // v) {}
+      (builtins.attrValues o);
+
+  prefixSetAttrs = prefix: o:
+    builtins.listToAttrs
+      (map ({name, value}: { name=prefix + name; value=value; })
+        (attrsToList o));
+
+  # ∀o,x,y. produce o' such that o'.x-y == o.x.y
+  flattenSetPrefix = o:
+    (builtins.foldl' (acc: o: acc // o) {}
+      (map ({name, value}: prefixSetAttrs name value)
+        (attrsToList o)));
+
+}

nix/dep-repos.nix

@@ -1,17 +0,0 @@
-# This is a hack to give us access to the fetching of dependency repos
-# as nix builds. We use this to cache those Cachix for faster rebuilds
-# in CI.
-
-with (import ./deps {});
-
-{
-  argon2-src     = argon2.src;
-  ed25519-src    = ed25519.src;
-  h2o-src        = h2o.src;
-  murmur3-src    = murmur3.src;
-  scrypt-src     = scrypt.src;
-  secp256k1-src  = secp256k1.src;
-  sni-src        = sni.src;
-  softfloat3-src = softfloat3.src;
-  uv-src         = uv.src;
-}

nix/nixcrpkgs/macos/default.nix

@@ -184,7 +184,7 @@ let
     global_license_set = { };
 
     # Make it easy to build or refer to the build tools.
-    inherit clang tapi ld ranlib ar sdk toolchain;
+    inherit clang tapi ld ranlib ar sdk toolchain strip;
 
     make_derivation = import ../make_derivation.nix crossenv;
   };

nix/nixcrpkgs/pkgs.nix

@@ -20,6 +20,10 @@ rec {
     inherit crossenv;
   };
 
+  lmdb = import ./pkgs/lmdb {
+    inherit crossenv;
+  };
+
   ncurses = import ./pkgs/ncurses {
     inherit crossenv;
   };

nix/nixcrpkgs/pkgs/lmdb/builder.sh

@@ -0,0 +1,16 @@
+source $setup
+
+cp -r $src ./src
+chmod -R u+w ./src
+cd src/libraries/liblmdb
+
+sed -i 's/liblmdb.a liblmdb..SOEXT.$/liblmdb.a/' Makefile
+sed -i "s/gcc/$host-gcc/"                        Makefile
+sed -i "s/ar/$host-ar/"                          Makefile
+sed -i 's/^CC.*/CC = '"$host-gcc/"               Makefile
+
+cat Makefile
+
+make CFLAGS+="-fPIC"
+
+make DESTDIR="$out" prefix=/ install

nix/nixcrpkgs/pkgs/lmdb/default.nix

@@ -0,0 +1,14 @@
+{ crossenv }:
+
+crossenv.make_derivation rec {
+  name = "lmdb-${version}";
+  version = "0.9.23";
+  builder = ./builder.sh;
+
+  src = crossenv.nixpkgs.fetchFromGitHub {
+    owner = "LMDB";
+    repo = "lmdb";
+    rev = "LMDB_${version}";
+    sha256 = "0ag7l5180ajvm73y59m7sn3p52xm8m972d08cshxhpwgwa4v35k6";
+  };
+}

nix/ops/boot-ship.nix

@@ -0,0 +1,18 @@
+{ pkgs ? import ../nixpkgs.nix
+, debug ? false
+, ship ? "zod"
+, pill ? ../../bin/solid.pill
+}:
+
+let
+
+  deps  = import ../deps { inherit pkgs; };
+  tlon  = import ../pkgs { inherit pkgs; };
+  arvo  = tlon.arvo;
+  urbit = tlon.urbit;
+
+in
+
+import ./fakeship {
+  inherit pkgs tlon deps arvo pill ship debug;
+}

nix/ops/brass/builder.sh

@@ -2,26 +2,21 @@ source $stdenv/setup
 
 set -ex
 
-cp -r $FAKEZOD ./zod
-chmod -R u+rw ./zod
+cp -r $PIER ./pier
+chmod -R u+rw ./pier
 
-$URBIT -d ./zod
+$URBIT -d ./pier
 
 cleanup () {
-  if [ -e ./zod/.vere.lock ]
-  then kill $(< ./zod/.vere.lock) || true;
+  if [ -e ./pier/.vere.lock ]
+  then kill $(< ./pier/.vere.lock) || true;
   fi
 }
 
 trap cleanup EXIT
 
-herb ./zod -p hood -d '+hood/autoload |'
-herb ./zod -p hood -d "+hood/mount %"
-
-rm -r ./zod/home
-cp -r $ARVO ./zod/home
-
-herb ./zod -p hood       -d "+hood/commit %home"
-herb ./zod -P brass.pill -d '+brass'
+herb ./pier -P brass.pill -d '+brass'
 
 mv brass.pill $out
+
+set +x

nix/ops/brass/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, tlon, deps, fakezod, arvo, debug }:
+{ pkgs, tlon, deps, pier, arvo, debug }:
 
 let
 
@@ -13,7 +13,7 @@ pkgs.stdenv.mkDerivation rec {
   builder     = ./builder.sh;
   buildInputs = [ tlon.herb pkgs.coreutils ];
 
-  URBIT   = urbitExe;
-  FAKEZOD = fakezod;
-  ARVO    = arvo;
+  URBIT = urbitExe;
+  PIER  = pier;
+  ARVO  = arvo;
 }

nix/ops/default.nix

@@ -7,40 +7,38 @@ let
   arvo  = tlon.arvo;
   urbit = tlon.urbit;
 
+  bootbrass = ../../bin/brass.pill;
+  bootsolid = ../../bin/solid.pill;
+
+  zod = import ./fakeship {
+    inherit pkgs tlon deps arvo debug;
+    pill = bootsolid;
+    ship = "zod";
+  };
+
+  bus = import ./fakeship {
+    inherit pkgs tlon deps arvo debug;
+    pill = bootsolid;
+    ship = "bus";
+  };
+
 in
 
 rec {
 
-  bootzod = import ./fakeship {
-    inherit pkgs tlon deps debug;
-    brass = ../../bin/brass.pill;
-    ship = "zod";
-  };
-
-  bootbus = import ./fakeship {
-    inherit pkgs tlon deps debug;
-    brass = ../../bin/brass.pill;
-    ship = "bus";
-  };
-
   test = import ./test {
-    inherit pkgs tlon deps arvo debug;
-    ship = bootzod;
+    inherit pkgs tlon deps debug;
+    ship = bus;
   };
 
   solid = import ./solid {
     inherit arvo pkgs tlon deps debug;
-    fakezod = bootzod;
+    pier = zod;
   };
 
   brass = import ./brass {
     inherit arvo pkgs tlon deps debug;
-    fakezod = bootzod;
-  };
-
-  fakezod = import ./fakeship {
-    inherit pkgs tlon deps brass debug;
-    ship = "zod";
+    pier = zod;
   };
 
 }

nix/ops/fakeship/builder.sh

@@ -2,7 +2,7 @@ source $stdenv/setup
 
 set -ex
 
-$URBIT -d -F $SHIP -B "$PILL" $out
+$URBIT -d -F $SHIP -A "$ARVO" -B "$PILL" $out
 
 check () {
   [ 3 -eq "$(herb $out -d 3)" ]
@@ -17,3 +17,5 @@ else
     kill $(< $out/.vere.lock) || true
     exit 1
 fi
+
+set +x

nix/ops/fakeship/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, tlon, deps, brass, ship, debug }:
+{ pkgs, tlon, deps, pill, ship, arvo, debug }:
 
 let
 
@@ -13,6 +13,7 @@ pkgs.stdenv.mkDerivation {
   builder     = ./builder.sh;
   buildInputs = [ tlon.herb ];
   URBIT       = urbitExe;
-  PILL        = brass;
+  ARVO        = arvo;
+  PILL        = pill;
   SHIP        = ship;
 }

nix/ops/solid/builder.sh

@@ -2,30 +2,21 @@ source $stdenv/setup
 
 set -ex
 
-cp -r $FAKEZOD ./zod
-chmod -R u+rw ./zod
+cp -r $PIER ./pier
+chmod -R u+rw ./pier
 
-$URBIT -d ./zod
+$URBIT -d ./pier
 
 cleanup () {
-  if [ -e ./zod/.vere.lock ]
-  then kill $(< ./zod/.vere.lock) || true;
+  if [ -e ./pier/.vere.lock ]
+  then kill $(< ./pier/.vere.lock) || true;
   fi
 }
 
 trap cleanup EXIT
 
-herb ./zod -p hood -d '+hood/autoload |'
-herb ./zod -p hood -d "+hood/mount %"
+herb ./pier -P solid.pill -d '+solid, =dub &'
 
-rm -r ./zod/home
-cp -r $ARVO ./zod/home
+mv solid.pill $out
 
-# cp -r $ARVO/sys            ./zod/home/
-# cp    $ARVO/gen/solid.hoon ./zod/home/gen/
-# cp    $ARVO/lib/pill.hoon  ./zod/home/lib/
-
-herb ./zod -p hood       -d "+hood/commit %home"
-herb ./zod -P urbit.pill -d '+solid, =dub &'
-
-mv urbit.pill $out
+set +x

nix/ops/solid/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, tlon, deps, fakezod, arvo, debug }:
+{ pkgs, tlon, deps, pier, arvo, debug }:
 
 let
 
@@ -13,7 +13,7 @@ pkgs.stdenv.mkDerivation rec {
   builder     = ./builder.sh;
   buildInputs = [ tlon.herb ];
 
-  URBIT   = urbitExe;
-  FAKEZOD = fakezod;
-  ARVO    = arvo;
+  URBIT = urbitExe;
+  PIER  = pier;
+  ARVO  = arvo;
 }

nix/ops/test/builder.sh

@@ -20,14 +20,6 @@ shutdown () {
 
 trap shutdown EXIT
 
-herb ./ship -p hood -d '+hood/autoload |'
-herb ./ship -p hood -d '+hood/mount %'
-
-rm -r ./ship/home
-cp -r $ARVO ./ship/home
-
-herb ./ship -p hood -d '+hood/commit %home'
-
 # Start the test app
 herb ./ship -p hood -d '+hood/start %test'
 
@@ -64,3 +56,5 @@ mkdir $out
 cp test-renders-output   $out/renders
 cp test-cores-output     $out/cores
 cp test-generator-output $out/generator
+
+set +x

nix/ops/test/default.nix

@@ -1,4 +1,4 @@
-{ pkgs, tlon, deps, ship, arvo, debug }:
+{ pkgs, tlon, deps, ship, debug }:
 
 let
 
@@ -15,5 +15,4 @@ pkgs.stdenv.mkDerivation rec {
 
   URBIT = urbitExe;
   SHIP  = ship;
-  ARVO  = arvo;
 }

nix/pkgs/urbit/default.nix

@@ -29,7 +29,7 @@ pkgs.stdenv.mkDerivation {
   # See https://github.com/NixOS/nixpkgs/issues/18995
   hardeningDisable = if debug then [ "all" ] else [];
 
-  CFLAGS           = if debug then "-O1 -g" else "-O3";
+  CFLAGS           = if debug then "-O3 -g -Werror" else "-O3 -Werror";
   MEMORY_DEBUG     = debug;
   CPU_DEBUG        = debug;
   EVENT_TIME_DEBUG = false;

nix/release.nix

@@ -10,7 +10,6 @@ let
       deps = crossdeps env;
     };
 
-  linux32 = release "linux32" nixcrpkgs.linux32;
   linux64 = release "linux64" nixcrpkgs.linux64;
   darwin  = release "darwin"  nixcrpkgs.mac;
 
@@ -21,31 +20,21 @@ let
     import ./pkgs/urbit/release.nix env
       { ent = ent env; debug = false; name = "urbit"; };
 
-  urbit-debug = env:
-    import ./pkgs/urbit/release.nix env
-      { ent = ent env; debug = true; name = "urbit-debug"; };
+  builds-for-platform = plat:
+    plat.deps // {
+      inherit (plat.env) curl libgmp libsigsegv ncurses openssl zlib lmdb;
+      inherit (plat.env) cmake_toolchain;
+      ent         = ent         plat;
+      urbit       = urbit       plat;
+    };
+
+  darwin_extra = {
+    inherit (darwin.env) ranlib ld sdk ar toolchain tapi strip;
+  };
 
 in
 
 {
-  linux32-env = linux32.env;
-  linux32 = linux32.deps // {
-    ent         = ent         linux32;
-    urbit       = urbit       linux32;
-    urbit-debug = urbit-debug linux32;
-  };
-
-  linux64-env = linux64.env;
-  linux64 = linux64.deps // {
-    ent         = ent         linux64;
-    urbit       = urbit       linux64;
-    urbit-debug = urbit-debug linux64;
-  };
-
-  darwin-env = darwin.env;
-  darwin = darwin.deps // {
-    ent         = ent         darwin;
-    urbit       = urbit       darwin;
-    urbit-debug = urbit-debug darwin;
-  };
+  linux64 = builds-for-platform linux64;
+  darwin  = darwin_extra // builds-for-platform darwin;
 }

pkg/urbit/Makefile

@@ -20,7 +20,7 @@ all_exes = ./hash_tests ./hashtable_tests ./urbit ./urbit-worker
 
 # -Werror promotes all warnings that are enabled into errors (this is on)
 # -Wall issues all types of errors.  This is off (for now)
-CFLAGS := $(CFLAGS) -Werror
+CFLAGS := $(CFLAGS)
 
 
 ################################################################################

pkg/urbit/vere/http.c

@@ -461,7 +461,7 @@ _http_rec_accept(h2o_handler_t* han_u, h2o_req_t* rec_u)
     req_u->tim_u = c3_malloc(sizeof(*req_u->tim_u));
     req_u->tim_u->data = req_u;
     uv_timer_init(u3L, req_u->tim_u);
-    uv_timer_start(req_u->tim_u, _http_req_timer_cb, 300 * 1000, 0);
+    uv_timer_start(req_u->tim_u, _http_req_timer_cb, 900 * 1000, 0);
 
     _http_req_dispatch(req_u, req);
   }

sh/boot

@@ -3,9 +3,10 @@
 set -e
 
 case $# in
-  0) target=./bootbus;;
-  1) target=$1;;
-  *) echo "Usage: $0 [target]" >&2
+  0) ship=zod; target=./zod;;
+  1) ship=$1; target=./$1;;
+  2) ship=$1; target=$2;;
+  *) echo "Usage: $0 [ship] [target-dir]" >&2
      exit 1;;
 esac
 
@@ -14,7 +15,7 @@ then echo "Target directory '$target' already exists." >&2
      exit 1
 fi
 
-pkg=$(nix-build nix/ops -A bootbus --no-out-link)
+pkg=$(nix-build nix/ops/boot-ship.nix --arg ship "\"$ship\"" --no-out-link)
 
 mkdir -p "$(dirname "$target")"
 cp -r $pkg/ $target

sh/cachix

@@ -19,8 +19,15 @@ cleanup () {
 
 trap cleanup EXIT
 
+set -ex
+
 cachix authtoken "$CACHIX_AUTH_TOKEN" >/dev/null
 cachix use urbit2
-nix-build --no-out-link nix/dep-repos.nix > .cache.list
-nix-build --no-out-link default.nix      >> .cache.list
+
+nix-build --no-out-link          \
+          --max-jobs 2           \
+          nix/cachix/local.nix   \
+          nix/cachix/release.nix \
+  > .cache.list
+
 cachix push urbit2 < .cache.list

sh/ci-tests

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [ "$TRAVIS_PULL_REQUEST" = false ]
+then
+
+    sh/test --arg debug false
+
+else
+
+    nix-build nix/ops       \
+              -A test       \
+              -A brass      \
+              --max-jobs 2  \
+              --no-out-link
+
+    sh/test --arg debug true
+    sh/update-brass-pill
+
+fi

sh/cross

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+set -e
+
+err () { echo "$@" >&2; exit 1; }
+
+[ $# = 2 ] || {
+  sh/cross urbit linux64
+  sh/cross urbit darwin
+  exit 0
+}
+
+traced () {
+  echo '$' "$@" >&2; "$@"
+}
+
+release () {
+  traced nix-build nix/release.nix --no-out-link -A "$1"
+}
+
+pkg="$1"
+env="$2"
+
+res=$(release "$env.$pkg")
+
+mkdir -p ./cross/$env
+traced cp -f $res/bin/$pkg ./cross/$env/$pkg
+traced cp -f $res/bin/$pkg-worker ./cross/$env/$pkg-worker
+traced cp -r $res/bin/$pkg-terminfo ./cross/$env/$pkg-terminfo
+
+chmod -R u+wr ./cross/$env/$pkg-terminfo

sh/mk-bootzod

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-case $# in
-  0) target=./bootzod;;
-  1) target=$1;;
-  *) echo "Usage: $0 [target]" >&2
-     exit 1;;
-esac
-
-if [ -e "$target" ]
-then echo "Target directory '$target' already exists." >&2
-     exit 1
-fi
-
-pkg=$(nix-build nix/ops -A bootzod --no-out-link)
-
-mkdir -p "$(dirname "$target")"
-cp -r $pkg/ $target
-chmod -R u+rw $target

sh/mk-fakezod

@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-case $# in
-  0) target=./fakezod;;
-  1) target=$1;;
-  *) echo "Usage: $0 [target]" >&2
-     exit 1;;
-esac
-
-if [ -e "$target" ]
-then echo "Target directory '$target' already exists." >&2
-     exit 1
-fi
-
-pkg=$(nix-build nix/ops -A fakezod --no-out-link)
-
-mkdir -p "$(dirname "$target")"
-cp -r $pkg/ $target
-chmod -R u+rw $target

sh/release

@@ -2,26 +2,22 @@
 
 set -e
 
-err () { echo "$@" >&2; exit 1; }
+if [ -n "${TRAVIS_TAG-}" ]
+then
+  ver="$TRAVIS_TAG"
+elif [ -n "${TRAVIS_COMMIT-}" ]
+then
+  ver="$TRAVIS_COMMIT"
+else
+  ver="$(git rev-parse HEAD)"
+fi
 
-[ $# = 2 ] || err "usage: $0 pkg host"
+mkdir -p release
 
-traced () {
-  echo '$' "$@" >&2; "$@"
-}
+for plat in linux64 darwin
+do
+  sh/cross urbit "$plat"
 
-release () {
-  traced nix-build nix/release.nix --no-out-link -A "$1"
-}
-
-pkg="$1"
-env="$2"
-
-res=$(release "$env.$pkg")
-
-mkdir -p ./bin/$env
-traced cp -f $res/bin/$pkg ./bin/$env/$pkg
-traced cp -f $res/bin/$pkg-worker ./bin/$env/$pkg-worker
-traced cp -r $res/bin/$pkg-terminfo ./bin/$env/$pkg-terminfo
-
-chmod -R u+wr ./bin/$env/$pkg-terminfo
+  echo "packaging release/urbit-$plat-$ver.tgz"
+  (cd cross/$plat; tar cz .) > release/urbit-$plat-$ver.tgz
+done

sh/test

@@ -2,7 +2,7 @@
 
 set -e
 
-pkg=$(nix-build nix/ops -A test --no-out-link --arg debug true)
+pkg=$(nix-build nix/ops -A test --no-out-link "$@")
 
 hdr () {
   echo =====$(sed 's/./=/g' <<< "$1")=====

sh/update-brass-pill

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
-set -e
+set -ex
 
 pkg=$(nix-build nix/ops -A brass --no-out-link)
-
 cp $pkg bin/brass.pill

sh/update-solid-pill

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+
+pkg=$(nix-build nix/ops -A solid --no-out-link)
+cp $pkg bin/solid.pill