on iko/ghcjs-patch-master help sign in
Make DLLs in deps available to wine TH runner (#1405)

This is a better fix to the problem of making DLLs available to the process running in wine that is used to evaluate TH code when cross compiling for Windows. Also fixes macOS loadArchive in TH for ghc 9 Co-authored-by: Michael Peyton Jones <michael.peyton-jones@iohk.io>

commit4b6ee9767daaf5fc1d5419e07733ab006b95ec93 authorHamish Mackenzie <Hamish.K.Mackenzie@gmail.com> dateFri 25 Mar 2022, 21:44 (5 years ago) parent2c895a2
10 files changed+116−40
builder/comp-builder.nix+20−12 overlays/bootstrap.nix+2 overlays/mingw_w64.nix+19−21 overlays/patches/ghc/ghc-9.0-macOS-loadArchive-fix.patch+9 overlays/patches/ghc/ghc-9.2-macOS-loadArchive-fix.patch+9 overlays/windows.nix+1−7 test/default.nix+1 test/th-dlls/default.nix+23 test/th-dlls/src/Lib.hs+14 test/th-dlls/th-dlls.cabal+18
builder/comp-builder.nix+20−12
@@ -382,18 +382,26 @@ let
382382 runHook postConfigure
383383 '';
384384
385− buildPhase = if stdenv.hostPlatform.isGhcjs then ''
386− runHook preBuild
387− # https://gitlab.haskell.org/ghc/ghc/issues/9221
388− $SETUP_HS build ${haskellLib.componentTarget componentId} ${lib.concatStringsSep " " setupBuildFlags}
389− runHook postBuild
390− '' else ''
391− runHook preBuild
392− # https://gitlab.haskell.org/ghc/ghc/issues/9221
393− $SETUP_HS build ${haskellLib.componentTarget componentId} -j$(($NIX_BUILD_CORES > 4 ? 4 : $NIX_BUILD_CORES)) ${lib.concatStringsSep " " setupBuildFlags}
394− runHook postBuild
395− ''
396− ;
385+ buildPhase =
386+ # It seems that by the time the iserv wrapper specifiec by `--ghc-option=-pgmi` runs
387+ # all the array environment variables are removed from the environment. To get a list
388+ # of all the locations a DLLs might be present we need access to pkgsHostTarget.
389+ # Adding a string version of the list array of nix store paths allows us to get that
390+ # list when we need it.
391+ (lib.optionalString stdenv.hostPlatform.isWindows ''
392+ export pkgsHostTargetAsString="''${pkgsHostTarget[@]}"
393+ '') +
394+ (if stdenv.hostPlatform.isGhcjs then ''
395+ runHook preBuild
396+ # https://gitlab.haskell.org/ghc/ghc/issues/9221
397+ $SETUP_HS build ${haskellLib.componentTarget componentId} ${lib.concatStringsSep " " setupBuildFlags}
398+ runHook postBuild
399+ '' else ''
400+ runHook preBuild
401+ # https://gitlab.haskell.org/ghc/ghc/issues/9221
402+ $SETUP_HS build ${haskellLib.componentTarget componentId} -j$(($NIX_BUILD_CORES > 4 ? 4 : $NIX_BUILD_CORES)) ${lib.concatStringsSep " " setupBuildFlags}
403+ runHook postBuild
404+ '');
397405
398406 # Note: Cabal does *not* copy test executables during the `install` phase.
399407 #
overlays/bootstrap.nix+2
@@ -169,6 +169,8 @@ in {
169169 ++ fromUntil "8.8.2" "8.9" ./patches/ghc/ghc-8.8.2-reinstallable-lib-ghc.patch
170170 ++ final.lib.optional (version == "8.6.4") ./patches/ghc/ghc-8.6.4-better-plusSimplCountErrors.patch
171171 ++ final.lib.optional (versionAtLeast "8.6.4" && versionLessThan "9.0" && final.stdenv.isDarwin) ./patches/ghc/ghc-macOS-loadArchive-fix.patch
172+ ++ final.lib.optional (versionAtLeast "9.0.0" && versionLessThan "9.2" && final.stdenv.isDarwin) ./patches/ghc/ghc-9.0-macOS-loadArchive-fix.patch
173+ ++ final.lib.optional (versionAtLeast "9.2.0" && versionLessThan "9.3" && final.stdenv.isDarwin) ./patches/ghc/ghc-9.2-macOS-loadArchive-fix.patch
172174 ++ final.lib.optional (versionAtLeast "8.4.4" && versionLessThan "8.10" && final.stdenv.isDarwin) ./patches/ghc/ghc-darwin-gcc-version-fix.patch
173175 ++ final.lib.optional (versionAtLeast "8.10.1" && versionLessThan "9.0.2" && final.stdenv.isDarwin) ./patches/ghc/ghc-8.10-darwin-gcc-version-fix.patch
174176 # backport of https://gitlab.haskell.org/ghc/ghc/-/merge_requests/3227
overlays/mingw_w64.nix+19−21
@@ -18,7 +18,7 @@ let
1818
1919 configureFlags = lib.optional hostPlatform.isWindows "--disable-split-sections";
2020
21− wineIservWrapperVanilla = writeScriptBin "iserv-wrapper" ''
21+ wineIservWrapperScript = enableProfiling: writeScriptBin ("iserv-wrapper" + lib.optionalString enableProfiling "-prof") ''
2222 #!${stdenv.shell}
2323 set -euo pipefail
2424 # unset the configureFlags.
@@ -28,25 +28,23 @@ let
2828 unset configureFlags
2929 PORT=$((5000 + $RANDOM % 5000))
3030 (>&2 echo "---> Starting remote-iserv on port $PORT")
31− WINEDLLOVERRIDES="winemac.drv=d" WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag WINEPREFIX=$TMP ${wine}/bin/wine64 ${remote-iserv}/bin/remote-iserv.exe tmp $PORT &
32− (>&2 echo "---| remote-iserv should have started on $PORT")
33− RISERV_PID="$!"
34− ${iserv-proxy}/bin/iserv-proxy $@ 127.0.0.1 "$PORT"
35− (>&2 echo "---> killing remote-iserv...")
36− kill $RISERV_PID
37− '';
38−
39− wineIservWrapperProf = writeScriptBin "iserv-wrapper-prof" ''
40− #!${stdenv.shell}
41− set -euo pipefail
42− # unset the configureFlags.
43− # configure should have run already
44− # without restting it, wine might fail
45− # due to a too large environment.
46− unset configureFlags
47− PORT=$((5000 + $RANDOM % 5000))
48− (>&2 echo "---> Starting remote-iserv on port $PORT")
49− WINEDLLOVERRIDES="winemac.drv=d" WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag WINEPREFIX=$TMP ${wine}/bin/wine64 ${remote-iserv.override { enableProfiling = true; }}/bin/remote-iserv.exe tmp $PORT &
31+ REMOTE_ISERV=$(mktemp -d)
32+ ln -s ${if enableProfiling then remote-iserv.override { inherit enableProfiling; } else remote-iserv}/bin/* $REMOTE_ISERV
33+ # See coment in comp-builder.nix for where this comes from and why it's here
34+ for p in $pkgsHostTargetAsString; do
35+ find "$p" -iname '*.dll' -exec ln -s {} $REMOTE_ISERV \;
36+ find "$p" -iname '*.dll.a' -exec ln -s {} $REMOTE_ISERV \;
37+ done
38+ # Some DLLs have a `lib` prefix but we attempt to load them without the prefix.
39+ # This was a problem for `double-conversion` package when used in TH code.
40+ # Creating links from the `X.dll` to `libX.dll` works around this issue.
41+ (
42+ cd $REMOTE_ISERV
43+ for l in lib*.dll; do
44+ ln -s "$l" "''${l#lib}"
45+ done
46+ )
47+ WINEDLLOVERRIDES="winemac.drv=d" WINEDEBUG=warn-all,fixme-all,-menubuilder,-mscoree,-ole,-secur32,-winediag WINEPREFIX=$TMP ${wine}/bin/wine64 $REMOTE_ISERV/remote-iserv.exe tmp $PORT &
5048 (>&2 echo "---| remote-iserv should have started on $PORT")
5149 RISERV_PID="$!"
5250 ${iserv-proxy}/bin/iserv-proxy $@ 127.0.0.1 "$PORT"
@@ -54,7 +52,7 @@ let
5452 kill $RISERV_PID
5553 '';
5654
57− wineIservWrapper = symlinkJoin { name = "iserv-wrapper"; paths = [ wineIservWrapperVanilla wineIservWrapperProf ]; };
55+ wineIservWrapper = symlinkJoin { name = "iserv-wrapper"; paths = [ (wineIservWrapperScript false) (wineIservWrapperScript true) ]; };
5856
5957 ################################################################################
6058 # Build logic (TH support via remote iserv via wine)
overlays/patches/ghc/ghc-9.0-macOS-loadArchive-fix.patchnew file+9
@@ -0,0 +1,9 @@
1+--- a/compiler/GHC/Runtime/Linker.hs
2++++ b/compiler/GHC/Runtime/Linker.hs
3+@@ -1487,7 +1487,6 @@ locateLib hsc_env is_hs lib_dirs gcc_dirs lib
4+ dyn_obj_file = lib <.> "dyn_o"
5+ arch_files = [ "lib" ++ lib ++ lib_tag <.> "a"
6+ , lib <.> "a" -- native code has no lib_tag
7+- , "lib" ++ lib, lib
8+ ]
9+ lib_tag = if is_hs && loading_profiled_hs_libs then "_p" else ""
overlays/patches/ghc/ghc-9.2-macOS-loadArchive-fix.patchnew file+9
@@ -0,0 +1,9 @@
1+--- a/compiler/GHC/Linker/Loader.hs
2++++ b/compiler/GHC/Linker/Loader.hs
3+@@ -1487,7 +1487,6 @@ locateLib hsc_env is_hs lib_dirs gcc_dirs lib
4+ dyn_obj_file = lib <.> "dyn_o"
5+ arch_files = [ "lib" ++ lib ++ lib_tag <.> "a"
6+ , lib <.> "a" -- native code has no lib_tag
7+- , "lib" ++ lib, lib
8+ ]
9+ lib_tag = if is_hs && loading_profiled_hs_libs then "_p" else ""
overlays/windows.nix+1−7
@@ -73,17 +73,11 @@ final: prev:
7373 # dependencies) and then placing them somewhere where wine+remote-iserv
7474 # will find them.
7575 remote-iserv.postInstall = pkgs.lib.optionalString pkgs.stdenv.hostPlatform.isWindows (
76− let extra-libs = [ pkgs.openssl.bin pkgs.libffi pkgs.gmp (pkgs.libsodium-vrf or pkgs.libsodium) pkgs.windows.mcfgthreads pkgs.buildPackages.gcc.cc pkgs.secp256k1 ]; in ''
76+ let extra-libs = [ pkgs.libffi pkgs.gmp pkgs.windows.mcfgthreads pkgs.buildPackages.gcc.cc ]; in ''
7777 for p in ${lib.concatStringsSep " "extra-libs}; do
7878 find "$p" -iname '*.dll' -exec cp {} $out/bin/ \;
7979 find "$p" -iname '*.dll.a' -exec cp {} $out/bin/ \;
8080 done
81− (
82− cd $out/bin
83− for l in lib*.dll; do
84− ln -s "$l" "''${l#lib}"
85− done
86− )
8781 '');
8882
8983 # Apply https://github.com/haskell/cabal/pull/6055
test/default.nix+1
@@ -202,6 +202,7 @@ let
202202 sublib-docs = callTest ./sublib-docs { inherit util compiler-nix-name; };
203203 githash = haskell-nix.callPackage ./githash { inherit compiler-nix-name; testSrc = testSrcWithGitDir; };
204204 c-ffi = callTest ./c-ffi { inherit util compiler-nix-name; };
205+ th-dlls = callTest ./th-dlls { inherit util compiler-nix-name; };
205206
206207 unit = unitTests;
207208 };
test/th-dlls/default.nixnew file+23
@@ -0,0 +1,23 @@
1+# Test building TH code that needs DLLs when cross compiling for windows
2+{ stdenv, lib, util, project', haskellLib, recurseIntoAttrs, testSrc, compiler-nix-name }:
3+
4+with lib;
5+
6+let
7+ project = project' {
8+ inherit compiler-nix-name;
9+ src = testSrc "th-dlls";
10+ };
11+
12+ packages = project.hsPkgs;
13+
14+in recurseIntoAttrs {
15+ meta.disabled = stdenv.hostPlatform.isGhcjs;
16+
17+ ifdInputs = {
18+ inherit (project) plan-nix;
19+ };
20+
21+ build = packages.th-dlls.components.library;
22+ build-profiled = packages.th-dlls.components.library.profiled;
23+}
test/th-dlls/src/Lib.hsnew file+14
@@ -0,0 +1,14 @@
1+{-# LANGUAGE TemplateHaskell #-}
2+module Lib where
3+
4+import Control.Monad.IO.Class (liftIO)
5+import OpenSSL (withOpenSSL)
6+import OpenSSL.BN (withBN)
7+import Libsodium (sodium_init)
8+import Language.Haskell.TH.Syntax (Exp(..), Lit(..))
9+import Data.Text as T
10+import Data.Double.Conversion.Text (toShortest)
11+
12+x = $(liftIO (withOpenSSL (withBN 0 (\_ -> return (LitE (IntegerL 0))))))
13+y = $(liftIO (sodium_init >> return (LitE (IntegerL 0))))
14+z = $(liftIO (return (LitE (IntegerL (fromIntegral (T.length (toShortest 1.0)))))))
test/th-dlls/th-dlls.cabalnew file+18
@@ -0,0 +1,18 @@
1+cabal-version: >=1.10
2+name: th-dlls
3+version: 0.1.0.0
4+license: PublicDomain
5+author: Hamish Mackenzie
6+maintainer: Hamish.K.Mackenzie@gmail.com
7+build-type: Simple
8+
9+library
10+ build-depends: base
11+ , HsOpenSSL
12+ , libsodium
13+ , template-haskell
14+ , text
15+ , double-conversion
16+ exposed-modules: Lib
17+ hs-source-dirs: src
18+ default-language: Haskell2010