Merge pull request #317 from samueldr-wip/feature/cross-canary

cross-canary: Execute cross-built binaries using qemu-user

modules/cross-workarounds.nix

@@ -9,7 +9,10 @@ let
     config.nixpkgs.localSystem.system != null &&
     config.nixpkgs.crossSystem.system != config.nixpkgs.localSystem.system;
 
-  AArch32Overlay = final: super: {
+  AArch32Overlay = final: super: 
+    # Ensure pkgsBuildBuild ends up unmodified, otherwise the canary test will
+    # get super expensive to build.
+    if super.stdenv.buildPlatform == super.stdenv.hostPlatform then {} else {
     # Works around libselinux failure with python on armv7l.
     # LONG_BIT definition appears wrong for platform
     libselinux = (super.libselinux

overlay/mobile-nixos/cross-canary/test.nix

@@ -0,0 +1,86 @@
+{ stdenv, lib, runCommandNoCC, runtimeShell, busybox, hello, hello-mruby, pkgsBuildBuild, mruby, mrbgems, mobile-nixos }:
+
+let
+  static = stdenv.hostPlatform.isStatic;
+
+  inherit (pkgsBuildBuild) file;
+  inherit (lib) optionalString;
+  inherit (stdenv) system;
+  emulators = {
+    "aarch64-linux" = "qemu-aarch64";
+    "armv7l-linux" = "qemu-arm";
+    "x86_64-linux" = "qemu-x86_64";
+  };
+  emulator =
+    if stdenv.buildPlatform == stdenv.hostPlatform then ""
+    else "${pkgsBuildBuild.qemu}/bin/${emulators.${system}}"
+  ;
+  mkTest = what: script: runCommandNoCC "cross-canary-${what}-${stdenv.system}" {} ''
+    assert_static() {
+      if ! ${file}/bin/file "$1" | grep -q 'statically linked'; then
+        printf "Assertion failed: '%s' is not a static binary\n" "$1"
+        ${file}/bin/file "$1"
+        exit 2
+      fi
+    }
+
+    (
+    PS4=" $ "
+    set -x
+
+    ${script}
+
+    )
+
+    # Everything went okay, mark the build as a success!
+    touch $out
+  '';
+
+  # Enables a couple mrbgems that are known to be fickle.
+  mrubyWithGems = mruby.override({
+    gems = with mrbgems; [
+      mruby-file-stat
+    ];
+  });
+in
+
+# We're not creating a "useless" canary when there is no cross compilation.
+if stdenv.buildPlatform == stdenv.hostPlatform then {} else (
+# We're not creating known-failing static builds.
+(if static then {} else
+{
+  # On armv7l, known to fails with `error: C compiler cannot create executables`
+  runtimeShell = mkTest "runtimeShell" ''
+    ${emulator} ${runtimeShell} -c 'echo runtimeShell works...'
+  '';
+
+  # This is more of an integrated test. It ends up exercising the systemd build.
+  # But this is still a _canary_ for us as it is at the root of our dependencies.
+  mobile-nixos-script-loader = mkTest "mobile-nixos-script-loader" ''
+    echo 'puts ARGV.inspect' > test.rb
+    ${emulator} ${mobile-nixos.stage-1.script-loader}/bin/loader
+    ${emulator} ${mobile-nixos.stage-1.script-loader}/bin/loader ./test.rb okay
+  '';
+}) //
+# Builds expected to work in both normal and static package sets.
+{
+
+  busybox = mkTest "busybox" ''
+    ${emulator} ${busybox}/bin/busybox uname -a
+    ${emulator} ${busybox}/bin/busybox sh -c 'echo busybox works...'
+  '';
+
+  hello = mkTest "hello" ''
+    ${optionalString static "assert_static ${hello}/bin/hello"}
+    ${emulator} ${hello}/bin/hello
+  '';
+
+  hello-mruby = mkTest "hello-mruby" ''
+    ${emulator} ${hello-mruby}/bin/hello
+  '';
+
+  mruby-with-gems = mkTest "mruby-with-gems" ''
+    ${emulator} ${mrubyWithGems}/bin/mruby --version
+  '';
+}
+)

overlay/mruby-builder/mruby/default.nix

@@ -12,9 +12,9 @@ in
 { stdenv
 , lib
 , buildPackages
+, pkgsBuildBuild
 , ruby
 , bison
-, rake
 , fetchFromGitHub
 , file
 , mruby
@@ -189,7 +189,7 @@ let
         --replace '//#define MRB_INT64' '#define MRB_INT64'
     '';
 
-    nativeBuildInputs = [ pkgconfig-helper ruby bison rake ] ++ gemNativeBuildInputs;
+    nativeBuildInputs = [ pkgconfig-helper ruby bison pkgsBuildBuild.rake ] ++ gemNativeBuildInputs;
     buildInputs = gemBuildInputs;
 
     # Necessary so it uses `gcc` instead of `ld` for linking.

overlay/overlay.nix

@@ -136,6 +136,9 @@ in
       make-flashable-zip = callPackage ./mobile-nixos/android-flashable-zip/make-flashable-zip.nix {};
 
       map-dtbs = callPackage ./mobile-nixos/map-dtbs {};
+
+      cross-canary-test = callPackage ./mobile-nixos/cross-canary/test.nix {};
+      cross-canary-test-static = self.pkgsStatic.callPackage ./mobile-nixos/cross-canary/test.nix {};
     };
 
     imageBuilder = callPackage ../lib/image-builder {};

release.nix

@@ -51,6 +51,9 @@ let
     ];
   };
 
+  onlyDerivations = lib.filterAttrs (k: v: lib.isDerivation v);
+  onlyDerivationsAndAttrsets = lib.filterAttrs (k: v: lib.isDerivation v || (lib.isAttrs v && !lib.isFunction v));
+
   # Given an evaluated "device", filters `pkgs` down to only our packages
   # unique to the overaly.
   # Also removes some non-packages from the overlay.
@@ -70,11 +73,11 @@ let
       # lib-like attributes...
       # How should we handle these?
       imageBuilder = null;
-      mobile-nixos = overlay.mobile-nixos // {
-        kernel-builder = null;
-        kernel-builder-clang_9 = null;
-        kernel-builder-gcc49 = null;
-        kernel-builder-gcc6 = null;
+      mobile-nixos = (onlyDerivationsAndAttrsets overlay.mobile-nixos) // {
+        # The cross canaries attrsets will be used as constituents.
+        # Filter out `override` and `overrideAttrs` early.
+        cross-canary-test = onlyDerivations overlay.mobile-nixos.cross-canary-test;
+        cross-canary-test-static = onlyDerivations overlay.mobile-nixos.cross-canary-test-static;
       };
 
       # Also lib-like, but a "global" like attribute :/
@@ -139,11 +142,28 @@ rec {
     (evalForSystem system)
   );
 
+  cross-canaries = lib.genAttrs ["aarch64-linux" "armv7l-linux"] (system:
+    releaseTools.aggregate {
+      name = "cross-canaries-${system}";
+      constituents =
+        let
+          overlay' = overlay.x86_64-linux."${system}-cross";
+        in
+        (builtins.attrValues overlay'.mobile-nixos.cross-canary-test)
+        ++ (builtins.attrValues overlay'.mobile-nixos.cross-canary-test-static)
+      ;
+      meta = {
+        description = "Useful checks for cross-compilation.";
+      };
+    }
+  );
+
   tested = let
     hasSystem = name: lib.lists.any (el: el == name) systems;
 
     constituents =
-      lib.optionals (hasSystem "x86_64-linux") [
+      cross-canaries.aarch64-linux.constituents
+      ++ lib.optionals (hasSystem "x86_64-linux") [
         device.uefi-x86_64.x86_64-linux              # UEFI system
         # Cross builds
         device.asus-z00t.x86_64-linux                # Android
@@ -174,6 +194,7 @@ rec {
     hasSystem = name: lib.lists.any (el: el == name) systems;
 
     constituents = tested.constituents
+      ++ cross-canaries.armv7l-linux.constituents
       ++ lib.optionals (hasSystem "x86_64-linux") [
         device.asus-flo.x86_64-linux
         overlay.x86_64-linux.armv7l-linux-cross.mobile-nixos.android-flashable-zip-binaries
@@ -187,7 +208,7 @@ rec {
       ;
   in
   releaseTools.aggregate {
-    name = "mobile-nixos-tested";
+    name = "mobile-nixos-tested-plus";
     inherit constituents;
     meta = {
       description = ''