digital-asset/daml
1
1
Fork 0
mirror of https://github.com/digital-asset/daml.git synced 2024-09-20 09:17:43 +03:00
Code Issues Projects Releases Wiki Activity
d530b2cdeb
daml/bazel_tools/haskell-cc-wrapper.patch
Andreas Herrmann cf6814e93b Use cc_wrapper to shorten library and include paths and rpaths (#2791) 
* Fix bazel query deps(//...)

* Add rules_haskell cc_wrapper

Updates to latest rules_haskell master and adds the cc_wrapper PR as a
patch, see https://github.com/tweag/rules_haskell/pull/1039.

* Shorten include dirs in cc-wrapper

When using `haskell_cabal_library` GHC constructs unnecessarily long
include directories which can quickly overflow the maximum command-line
length. This patch avoids the issue by normalizing include paths.

* glob --> breadth_first_walk
2019-09-09 15:50:51 +00:00

3682 lines
128 KiB
Diff
Raw Blame History

From 15ec6f2bfe891b6437e760ba9cc72af4df30b278 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 4 Jul 2019 16:12:05 +0200
Subject: [PATCH 01/31] Windows use cc.tools.cc
---
haskell/toolchain.bzl | 40 +++++++++++++++++++---------------------
1 file changed, 19 insertions(+), 21 deletions(-)
diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl
index 02d2ca3a..5984cf08 100644
--- a/haskell/toolchain.bzl
+++ b/haskell/toolchain.bzl
@@ -38,29 +38,27 @@ def _run_ghc(hs, cc, inputs, outputs, mnemonic, arguments, params_file = None, e
args.add(hs.tools.ghc)
extra_inputs += [hs.tools.ghc]
- # Do not use Bazel's CC toolchain on Windows, as it leads to linker and librarty compatibility issues.
# XXX: We should also tether Bazel's CC toolchain to GHC's, so that we can properly mix Bazel-compiled
# C libraries with Haskell targets.
- if not hs.toolchain.is_windows:
- args.add_all([
- # GHC uses C compiler for assemly, linking and preprocessing as well.
- "-pgma",
- cc.tools.cc,
- "-pgmc",
- cc.tools.cc,
- "-pgml",
- cc.tools.cc,
- "-pgmP",
- cc.tools.cc,
- # Setting -pgm* flags explicitly has the unfortunate side effect
- # of resetting any program flags in the GHC settings file. So we
- # restore them here. See
- # https://ghc.haskell.org/trac/ghc/ticket/7929.
- "-optc-fno-stack-protector",
- "-optP-E",
- "-optP-undef",
- "-optP-traditional",
- ])
+ args.add_all([
+ # GHC uses C compiler for assemly, linking and preprocessing as well.
+ "-pgma",
+ cc.tools.cc,
+ "-pgmc",
+ cc.tools.cc,
+ "-pgml",
+ cc.tools.cc,
+ "-pgmP",
+ cc.tools.cc,
+ # Setting -pgm* flags explicitly has the unfortunate side effect
+ # of resetting any program flags in the GHC settings file. So we
+ # restore them here. See
+ # https://ghc.haskell.org/trac/ghc/ticket/7929.
+ "-optc-fno-stack-protector",
+ "-optP-E",
+ "-optP-undef",
+ "-optP-traditional",
+ ])
compile_flags_file = hs.actions.declare_file("compile_flags_%s_%s" % (hs.name, mnemonic))
extra_args_file = hs.actions.declare_file("extra_args_%s_%s" % (hs.name, mnemonic))
From ce2d7516d574997dade55505388cf600f1c0f5dc Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 4 Jul 2019 14:16:29 +0200
Subject: [PATCH 02/31] run_ghc: locale_archive as default input
Always pass locale_archive to run_ghc if provided.
---
haskell/private/actions/compile.bzl | 5 -----
haskell/toolchain.bzl | 3 +++
2 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/haskell/private/actions/compile.bzl b/haskell/private/actions/compile.bzl
index ac8725f5..3dea77ce 100644
--- a/haskell/private/actions/compile.bzl
+++ b/haskell/private/actions/compile.bzl
@@ -230,10 +230,6 @@ def _compilation_defaults(hs, cc, java, dep_info, plugin_dep_info, cc_info, srcs
compile_flags += cc.include_args
- locale_archive_depset = (
- depset([hs.toolchain.locale_archive]) if hs.toolchain.locale_archive != None else depset()
- )
-
# This is absolutely required otherwise GHC doesn't know what package it's
# creating `Name`s for to put them in Haddock interface files which then
# results in Haddock not being able to find names for linking in
@@ -342,7 +338,6 @@ def _compilation_defaults(hs, cc, java, dep_info, plugin_dep_info, cc_info, srcs
plugin_dep_info.dynamic_libraries,
ghci_extra_libs,
java.inputs,
- locale_archive_depset,
preprocessors.inputs,
plugin_tool_inputs,
]),
diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl
index 5984cf08..1e42b024 100644
--- a/haskell/toolchain.bzl
+++ b/haskell/toolchain.bzl
@@ -73,6 +73,9 @@ def _run_ghc(hs, cc, inputs, outputs, mnemonic, arguments, params_file = None, e
extra_args_file,
] + cc.files
+ if hs.toolchain.locale_archive != None:
+ extra_inputs.append(hs.toolchain.locale_archive)
+
flagsfile = extra_args_file
if params_file:
flagsfile = merge_parameter_files(hs, extra_args_file, params_file)
From b567d276201ca8de5bb94a7b36f1d38141b742bf Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 4 Jul 2019 10:27:47 +0200
Subject: [PATCH 03/31] Combined cc_wrapper for Windows and Unix
- Shortens library search paths to stay below maximum path length on Windows.
GHC generates library search paths that contain redundant up-level
references (..). This can exceed the maximum path length on Windows, which
will cause linking failures. This wrapper shortens library search paths to
avoid that issue.
- Shortens rpaths and load commands on macOS.
The rpaths and load commands generated by GHC and Bazel can quickly exceed
the MACH-O header size limit on macOS. This wrapper shortens and combines
rpaths and load commands to avoid exceeding that limit.
- Finds .so files if only .dylib are searched on macOS.
Bazel's cc_library will generate .so files for dynamic libraries even on
macOS. GHC strictly expects .dylib files on macOS. This wrapper hooks into
gcc's --print-file-name feature to work around this mismatch in file
extension.
---
haskell/BUILD.bazel | 10 +
haskell/cc.bzl | 22 +-
haskell/private/actions/compile.bzl | 2 +-
haskell/private/actions/link.bzl | 7 +-
haskell/private/cc_wrapper.bzl | 56 ++
haskell/private/cc_wrapper.py.tpl | 812 ++++++++++++++++++++++++++++
haskell/providers.bzl | 14 +-
haskell/toolchain.bzl | 17 +
8 files changed, 907 insertions(+), 33 deletions(-)
create mode 100644 haskell/private/cc_wrapper.bzl
create mode 100644 haskell/private/cc_wrapper.py.tpl
diff --git a/haskell/BUILD.bazel b/haskell/BUILD.bazel
index a7e39f75..41afb641 100644
--- a/haskell/BUILD.bazel
+++ b/haskell/BUILD.bazel
@@ -2,6 +2,10 @@ load(
"@rules_haskell//haskell:private/haskell_impl.bzl",
"haskell_toolchain_libraries",
)
+load(
+ "@rules_haskell//haskell:private/cc_wrapper.bzl",
+ "cc_wrapper",
+)
exports_files(
glob(["*.bzl"]) + [
@@ -10,6 +14,7 @@ exports_files(
"private/coverage_wrapper.sh.tpl",
"private/ghci_repl_wrapper.sh",
"private/haddock_wrapper.sh.tpl",
+ "private/cc_wrapper.py.tpl",
"private/osx_cc_wrapper.sh.tpl",
"private/pkgdb_to_bzl.py",
],
@@ -21,6 +26,11 @@ exports_files(
visibility = ["//tests/unit-tests:__pkg__"],
)
+cc_wrapper(
+ name = "cc_wrapper",
+ visibility = ["//visibility:public"],
+)
+
py_binary(
name = "pkgdb_to_bzl",
srcs = ["private/pkgdb_to_bzl.py"],
diff --git a/haskell/cc.bzl b/haskell/cc.bzl
index cd426135..dfed4de3 100644
--- a/haskell/cc.bzl
+++ b/haskell/cc.bzl
@@ -20,6 +20,7 @@ CcInteropInfo = provider(
# See the following for why this is needed:
# https://stackoverflow.com/questions/52769846/custom-c-rule-with-the-cc-common-api
"files": "Files for all tools (input to any action that uses tools)",
+ "manifests": "Input manifests for all tools (input to any action that uses tools)",
"hdrs": "CC headers",
"cpp_flags": "Preprocessor flags",
"compiler_flags": "Flags for compilation",
@@ -99,22 +100,10 @@ def cc_interop_info(ctx):
# Generate cc wrapper script on Darwin that adjusts load commands.
hs_toolchain = ctx.toolchains["@rules_haskell//haskell:toolchain"]
- if hs_toolchain.is_darwin:
- cc_wrapper = ctx.actions.declare_file("osx_cc_wrapper")
- cc = cc_wrapper.path
- ctx.actions.expand_template(
- template = hs_toolchain.osx_cc_wrapper_tpl,
- output = cc_wrapper,
- substitutions = {
- "%{cc}": cc_toolchain.compiler_executable(),
- },
- )
- cc_files = ctx.files._cc_toolchain + [
- cc_wrapper,
- ]
- else:
- cc = cc_toolchain.compiler_executable()
- cc_files = ctx.files._cc_toolchain
+ cc_wrapper = hs_toolchain.cc_wrapper
+ cc = cc_wrapper.executable.path
+ cc_files = ctx.files._cc_toolchain + cc_wrapper.inputs.to_list()
+ cc_manifests = cc_wrapper.manifests
# XXX Workaround https://github.com/bazelbuild/bazel/issues/6876.
linker_flags = [flag for flag in linker_flags if flag not in ["-shared"]]
@@ -139,6 +128,7 @@ def cc_interop_info(ctx):
return CcInteropInfo(
tools = struct(**tools),
files = cc_files,
+ manifests = cc_manifests,
hdrs = hdrs.to_list(),
cpp_flags = cpp_flags,
include_args = include_args,
diff --git a/haskell/private/actions/compile.bzl b/haskell/private/actions/compile.bzl
index 3dea77ce..ea0b5946 100644
--- a/haskell/private/actions/compile.bzl
+++ b/haskell/private/actions/compile.bzl
@@ -319,7 +319,7 @@ def _compilation_defaults(hs, cc, java, dep_info, plugin_dep_info, cc_info, srcs
)
# Transitive library dependencies for runtime.
- (ghci_extra_libs, ghc_env) = get_ghci_extra_libs(hs, cc_info, dynamic = False)
+ (ghci_extra_libs, ghc_env) = get_ghci_extra_libs(hs, cc_info)
return struct(
args = args,
diff --git a/haskell/private/actions/link.bzl b/haskell/private/actions/link.bzl
index 6f74bd23..996af781 100644
--- a/haskell/private/actions/link.bzl
+++ b/haskell/private/actions/link.bzl
@@ -352,15 +352,10 @@ def link_library_dynamic(hs, cc, dep_info, cc_info, extra_srcs, objects_dir, my_
)
args.add_all(pkg_info_args)
- # When linking a dynamic library we still collect static libraries for
- # dependencies where possible. This is so that a final binary that depends
- # on this dynamic library, is linked statically itself, will not fail at
- # link time due to missing transitive dynamic library dependencies. In this
- # case transitive dependencies will still be linked in statically.
(cache_file, static_libs, dynamic_libs) = create_link_config(
hs = hs,
cc_info = cc_info,
- dynamic = False,
+ dynamic = True,
pic = True,
binary = dynamic_library,
args = args,
diff --git a/haskell/private/cc_wrapper.bzl b/haskell/private/cc_wrapper.bzl
new file mode 100644
index 00000000..9913d806
--- /dev/null
+++ b/haskell/private/cc_wrapper.bzl
@@ -0,0 +1,56 @@
+load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
+load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES")
+load("@os_info//:os_info.bzl", "is_linux")
+
+def _cc_wrapper_impl(ctx):
+ cc_toolchain = find_cpp_toolchain(ctx)
+ feature_configuration = cc_common.configure_features(
+ ctx = ctx,
+ cc_toolchain = cc_toolchain,
+ requested_features = ctx.features,
+ unsupported_features = ctx.disabled_features,
+ )
+ cc = cc_common.get_tool_for_action(
+ feature_configuration = feature_configuration,
+ action_name = ACTION_NAMES.c_compile,
+ )
+ cc_wrapper = ctx.actions.declare_file(ctx.label.name + ".py")
+ ctx.actions.expand_template(
+ template = ctx.file.template,
+ output = cc_wrapper,
+ is_executable = True,
+ substitutions = {
+ "{:cc:}": cc,
+ },
+ )
+ return [DefaultInfo(files = depset([cc_wrapper]))]
+
+_cc_wrapper = rule(
+ implementation = _cc_wrapper_impl,
+ attrs = {
+ "template": attr.label(
+ allow_single_file = True,
+ default = Label("@rules_haskell//haskell:private/cc_wrapper.py.tpl"),
+ ),
+ "_cc_toolchain": attr.label(
+ default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
+ ),
+ },
+ fragments = ["cpp"],
+)
+
+def cc_wrapper(name, template = None, **kwargs):
+ _cc_wrapper(
+ name = name + "-source",
+ template = template,
+ )
+ native.py_binary(
+ name = name,
+ srcs = [name + "-source"],
+ main = name + "-source.py",
+ python_version = "PY3",
+ deps = [
+ "@bazel_tools//tools/python/runfiles",
+ ],
+ **kwargs
+ )
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
new file mode 100644
index 00000000..2f71a23c
--- /dev/null
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -0,0 +1,812 @@
+#!/usr/bin/env python3
+"""CC toolchain wrapper
+
+Usage: cc_wrapper [ARG]...
+
+Wraps the C compiler of the Bazel CC toolchain. Transforms arguments to work
+around limitations of Bazel and GHC and passes those via response file to the C
+compiler.
+
+- Shortens library search paths to stay below maximum path length on Windows.
+
+ GHC generates library search paths that contain redundant up-level
+ references (..). This can exceed the maximum path length on Windows, which
+ will cause linking failures. This wrapper shortens library search paths to
+ avoid that issue.
+
+- Shortens rpaths and load commands on macOS.
+
+ The rpaths and load commands generated by GHC and Bazel can quickly exceed
+ the MACH-O header size limit on macOS. This wrapper shortens and combines
+ rpaths and load commands to avoid exceeding that limit.
+
+- Finds .so files if only .dylib are searched on macOS.
+
+ Bazel's cc_library will generate .so files for dynamic libraries even on
+ macOS. GHC strictly expects .dylib files on macOS. This wrapper hooks into
+ gcc's --print-file-name feature to work around this mismatch in file
+ extension.
+
+"""
+
+from bazel_tools.tools.python.runfiles import runfiles as bazel_runfiles
+from contextlib import contextmanager
+import glob
+import itertools
+import os
+import platform
+import shlex
+import subprocess
+import sys
+import tempfile
+
+CC = "{:cc:}"
+INSTALL_NAME_TOOL = "/usr/bin/install_name_tool"
+OTOOL = "/usr/bin/otool"
+
+
+def main():
+ parsed = Args(load_response_files(sys.argv[1:]))
+
+ if parsed.linking:
+ link(parsed.output, parsed.libraries, parsed.rpaths, parsed.args)
+ elif parsed.printing_file_name:
+ print_file_name(parsed.print_file_name, parsed.args)
+ else:
+ run_cc(parsed.args, exit_on_error=True)
+
+
+# --------------------------------------------------------------------
+# Parse arguments
+
+
+class Args:
+ """Parsed command-line arguments.
+
+ Attrs:
+ args: The collected and transformed arguments.
+
+ linking: The action is linking.
+ printing_file_name: The action is print-file-name.
+
+ output: The output binary or library when linking.
+ library_paths: The library search paths when linking.
+ libraries: The required libraries when linking.
+ rpaths: The provided rpaths when linking.
+
+ print_file_name: The queried file name on print-file-name.
+
+ """
+ LINK = "link"
+ COMPILE = "compile"
+ PRINT_FILE_NAME = "print-file-name"
+
+ def __init__(self, args):
+ """Parse the given arguments into an Args object.
+
+ - Shortens library search paths.
+ - Detects the requested action.
+ - Keeps rpath arguments for further processing when linking.
+ - Keeps print-file-name arguments for further processing.
+
+ Args:
+ args: Iterable over command-line arguments.
+
+ """
+ self.action = Args.LINK
+ self.print_file_name = None
+ self.libraries = []
+ self.library_paths = []
+ self.rpaths = []
+ self.output = None
+ self._prev_ld_arg = None
+
+ self.args = list(self._handle_args(args))
+
+ if not self.linking:
+ # We don't expect rpath arguments if not linking, however, just in
+ # case, forward them if we don't mean to modify them.
+ self.args.extend(rpath_args(self.rpaths))
+
+ @property
+ def linking(self):
+ """Whether this is a link invocation."""
+ return self.action == Args.LINK and self.output is not None
+
+ @property
+ def compiling(self):
+ """Whether this is a compile invocation."""
+ return self.action == Args.COMPILE
+
+ @property
+ def printing_file_name(self):
+ """Whether this is a print-file-name invocation."""
+ return self.action == Args.PRINT_FILE_NAME and self.print_file_name is not None
+
+ def _handle_args(self, args):
+ """Argument handling pipeline.
+
+ Args:
+ args: Iterable, command-line arguments.
+
+ Yields:
+ Transformed command-line arguments.
+
+ """
+ args = iter(args)
+ for arg in args:
+ out = []
+ # Poor man's pattern matching: Each handler function takes the
+ # current argument, the stream of up-coming arguments, and a
+ # reference to the list of arguments to forward. The handler must
+ # return True if it consumes the argument, and return False if
+ # another handler should consume the argument.
+ if self._handle_output(arg, args, out):
+ pass
+ elif self._handle_library(arg, args, out):
+ pass
+ elif self._handle_library_path(arg, args, out):
+ pass
+ elif self._handle_linker_arg(arg, args, out):
+ pass
+ elif self._handle_print_file_name(arg, args, out):
+ pass
+ elif self._handle_compile(arg, args, out):
+ pass
+ else:
+ yield arg
+
+ for out_arg in out:
+ yield out_arg
+
+ def _handle_output(self, arg, args, out):
+ if arg == "-o":
+ # Remember the output filename.
+ self.output = next(args)
+ out.extend(["-o", self.output])
+ return True
+ else:
+ return False
+
+ def _handle_library(self, arg, args, out):
+ if arg == "-l" or arg == "--library":
+ library = next(args)
+ elif arg.startswith("-l"):
+ library = arg[2:]
+ elif arg.startswith("--library="):
+ library = arg[len("--library="):]
+ else:
+ return False
+
+ # Remember the required libraries.
+ self.libraries.append(library)
+ out.append("-l{}".format(library))
+
+ return True
+
+ def _handle_library_path(self, arg, args, out):
+ if arg == "-L" or arg == "--library-path":
+ library_path = next(args)
+ elif arg.startswith("-L"):
+ library_path = arg[2:]
+ elif arg.startswith("--library-path="):
+ library_path = arg[len("--library-path="):]
+ else:
+ return False
+
+ # Shorten the library search paths. On Windows library search paths may
+ # exceed the maximum path length.
+ shortened = shorten_path(library_path)
+ # Remember the library search paths.
+ self.library_paths.append(shortened)
+ out.append("-L{}".format(shortened))
+
+ return True
+
+ def _handle_linker_arg(self, arg, args, out):
+ if arg == "-Xlinker":
+ ld_arg = next(args)
+ if self._prev_ld_arg is None:
+ if ld_arg == "-rpath":
+ self._prev_ld_arg = ld_arg
+ else:
+ out.extend(["-Xlinker", ld_arg])
+ elif self._prev_ld_arg == "-rpath":
+ self._prev_ld_arg = None
+ self._handle_rpath(ld_arg, out)
+ else:
+ # This indicates a programmer error and should not happen.
+ raise RuntimeError("Unhandled _prev_ld_arg '{}'.".format(self._prev_ld_arg))
+ return True
+ elif arg.startswith("-Wl,"):
+ ld_args = arg.split(",")[1:]
+ if len(ld_args) == 2 and ld_args[0] == "-rpath":
+ self._handle_rpath(ld_args[1], out)
+ return True
+ else:
+ out.append(arg)
+ return True
+ else:
+ return False
+
+ def _handle_rpath(self, rpath, out):
+ # Filter out all RPATH flags for now and manually add the needed ones
+ # later on.
+ self.rpaths.append(rpath)
+
+ def _handle_print_file_name(self, arg, args, out):
+ if arg == "--print-file-name":
+ print_file_name = next(args)
+ elif arg.startswith("--print-file-name="):
+ print_file_name = arg[len("--print-file-name="):]
+ else:
+ return False
+
+ # Remember print-file-name action. Don't forward to allow for later
+ # manipulation.
+ self.print_file_name = print_file_name
+ self.action = Args.PRINT_FILE_NAME
+
+ return True
+
+ def _handle_compile(self, arg, args, out):
+ if arg == "-c":
+ self.action = Args.COMPILE
+ out.append(arg)
+ else:
+ return False
+
+ return True
+
+
+def load_response_files(args):
+ """Generator that loads arguments from response files.
+
+ Passes through any regular arguments.
+
+ Args:
+ args: Iterable of arguments.
+
+ Yields:
+ All arguments, with response files replaced by their contained arguments.
+
+ """
+ args = iter(args)
+ for arg in args:
+ if arg == "-install_name":
+ # macOS only: The install_name may start with an '@' character.
+ yield arg
+ yield next(args)
+ elif arg.startswith("@"):
+ with open(arg[1:], "r") as rsp:
+ for line in rsp:
+ for rsp_arg in parse_response_line(line):
+ yield rsp_arg
+ else:
+ yield arg
+
+
+def parse_response_line(s):
+ # GHC writes response files with quoted lines.
+ return shlex.split(s)
+
+
+def shorten_path(input_path):
+ """Shorten the given path if possible.
+
+ Applies the following transformations if they shorten the path length:
+ - Make path relative to CWD.
+ - Remove redundant up-level references.
+ - Resolve symbolic links.
+
+ Args:
+ input_path: The path to shorten.
+
+ Returns:
+ The shortened path.
+
+ """
+ exists = os.path.exists(input_path)
+ shortened = input_path
+
+ # Try relativizing to current working directory.
+ rel = os.path.relpath(shortened)
+ if len(rel) < len(shortened):
+ shortened = rel
+
+ # Try normalizing the path if possible.
+ norm = os.path.normpath(shortened)
+ if len(norm) < len(shortened):
+ # Ensure that the path is still correct. Reducing up-level references
+ # may change the meaning of the path in the presence of symbolic links.
+ try:
+ if not exists or os.path.samefile(norm, shortened):
+ shortened = norm
+ except IOError:
+ # stat may fail if the path became invalid or does not exist.
+ pass
+
+ # Try resolving symlinks.
+ try:
+ real = os.path.relpath(os.path.realpath(shortened))
+ if len(real) < len(shortened):
+ shortened = real
+ except IOError:
+ # realpath may fail if the path does not exist.
+ pass
+
+ return shortened
+
+
+def rpath_args(rpaths):
+ """Generate arguments for RUNPATHs."""
+ for rpath in rpaths:
+ yield "-Xlinker"
+ yield "-rpath"
+ yield "-Xlinker"
+ yield rpath
+
+
+# --------------------------------------------------------------------
+# Link binary or library
+
+
+def link(output, libraries, rpaths, args):
+ """Execute the link action.
+
+ Args:
+ output: The output binary or library.
+ libraries: Library dependencies.
+ rpaths: The provided rpaths.
+ args: The command-line arguments.
+
+ """
+ if is_darwin():
+ # Reserve space in load commands for later replacement.
+ args.append("-headerpad_max_install_names")
+ rpaths, darwin_rewrites = darwin_shorten_rpaths(
+ rpaths, libraries, output)
+ else:
+ rpaths = shorten_rpaths(rpaths, libraries, output)
+
+ args.extend(rpath_args(rpaths))
+ run_cc(args, exit_on_error=True)
+
+ if is_darwin():
+ darwin_rewrite_load_commands(darwin_rewrites, output)
+
+
+def shorten_rpaths(rpaths, libraries, output):
+ """Avoid redundant rpaths.
+
+ Filters out rpaths that are not required to load any library dependency.
+
+ Args:
+ rpaths: List of given rpaths.
+ libraries: List of library dependencies.
+ output: The output binary, used to resolve rpaths.
+
+ Returns:
+ List of required rpaths.
+
+ """
+ input_rpaths = sort_rpaths(rpaths)
+ missing = set(libraries)
+
+ rpaths = []
+
+ for rpath in input_rpaths:
+ if not missing:
+ break
+ rpath, rpath_dir = resolve_rpath(rpath, output)
+ found, missing = find_library(missing, rpath_dir)
+ if found:
+ rpaths.append(rpath)
+
+ return rpaths
+
+
+def darwin_shorten_rpaths(rpaths, libraries, output):
+ """Avoid redundant rpaths and adapt library load commands.
+
+ Avoids redundant rpaths by detecting the solib directory and making load
+ commands relative to the solib directory where applicable.
+
+ Args:
+ rpaths: List of given rpaths.
+ libraries: List of library dependencies.
+ output: The output binary, used to resolve rpaths.
+
+ Returns:
+ (rpaths, rewrites):
+ rpaths: List of required rpaths.
+ rewrites: List of load command rewrites.
+
+ """
+ input_rpaths = sort_rpaths(rpaths)
+ missing = set(libraries)
+
+ rpaths = []
+ rewrites = []
+
+ # References to core libs take up much space. Consider detecting the GHC
+ # libdir and adding an rpath for that and making load commands relative to
+ # that. Alternatively, https://github.com/bazelbuild/bazel/pull/8888 would
+ # also avoid this issue.
+
+ # Determine solib dir and rewrite load commands relative to solib dir.
+ # This allows to replace potentially many rpaths by one.
+ solib_rpath = find_solib_rpath(input_rpaths, output)
+ if missing and solib_rpath is not None:
+ solib_rpath, solib_dir = resolve_rpath(solib_rpath, output)
+
+ found, missing = find_library_recursive(missing, solib_dir)
+ if found:
+ rpaths.append(solib_rpath)
+ for f in found.values():
+ soname = darwin_get_install_name(os.path.join(solib_dir, f))
+ rewrites.append((soname, f))
+
+ # For the remaining missing libraries, determine which rpaths are required.
+ for rpath in input_rpaths:
+ if not missing:
+ break
+ rpath, rpath_dir = resolve_rpath(rpath, output)
+ found, missing = find_library(missing, rpath_dir)
+ # Libraries with an absolute install_name don't require an rpath entry.
+ found = dict(itertools.filterfalse(
+ lambda item: os.path.isabs(darwin_get_install_name(os.path.join(rpath_dir, item[1]))),
+ found.items()))
+ if len(found) == 1:
+ # Avoid unnecessary rpath if it is only relevant for one load command.
+ [filename] = found.values()
+ soname = darwin_get_install_name(os.path.join(rpath_dir, filename))
+ rewrites.append((soname, os.path.join(rpath, filename)))
+ elif found:
+ rpaths.append(rpath)
+
+ return rpaths, rewrites
+
+
+def sort_rpaths(rpaths):
+ """Sort RUNPATHs by preference.
+
+ Preference in decsending order:
+ - Relative to target
+ - Absolute path
+ - Relative to CWD
+
+ """
+ def rpath_priority(rpath):
+ system = platform.system()
+ if system == "Darwin":
+ if rpath.startswith("@loader_path"):
+ return 0
+ elif system == "Linux":
+ if rpath.startswith("$ORIGIN"):
+ return 0
+ if os.path.isabs(rpath):
+ return 1
+ return 2
+
+ return sorted(rpaths, key=rpath_priority)
+
+
+def find_solib_rpath(rpaths, output):
+ """Find the solib directory rpath entry.
+
+ The solib directory is the directory under which Bazel places dynamic
+ library symbolic links on Unix. It has the form `_solib_<cpu>`.
+
+ """
+ for rpath in rpaths:
+ components = rpath.replace("\\", "/").split("/")
+ solib_rpath = []
+ for comp in components:
+ solib_rpath.append(comp)
+ if comp.startswith("_solib_"):
+ return "/".join(solib_rpath)
+
+ if is_temporary_output(output):
+ # GHC generates temporary libraries outside the execroot. In that case
+ # the Bazel generated RPATHs are not forwarded, and the solib directory
+ # is not visible on the command-line.
+ candidates = glob.glob("**/bin/_solib_*", recursive=True)
+ if candidates:
+ return min(candidates)
+
+ return None
+
+
+def find_library_recursive(libraries, directory):
+ """Find libraries in given directory tree.
+
+ Args:
+ libraries: List of missing libraries.
+ directory: Root of directory tree.
+
+ Returns:
+ (found, missing):
+ found: Dict of found libraries {libname: path} relative to directory.
+ missing: Set of remaining missing libraries.
+
+ """
+ missing = set(libraries)
+ found = {}
+ for root, _, files in os.walk(directory, followlinks=True):
+ prefix = os.path.relpath(root, directory)
+ if not missing:
+ break
+ for f in files:
+ libname = get_lib_name(f)
+ if libname and libname in missing:
+ found[libname] = os.path.join(prefix, f) if prefix != "." else f
+ missing.discard(libname)
+ if not missing:
+ break
+
+ return found, missing
+
+
+def find_library(libraries, directory):
+ """Find libraries in the given directory.
+
+ Args:
+ libraries: List of missing libraries.
+ directory: The directory in which to search for libraries.
+
+ Returns:
+ (found, missing):
+ found: Dict of found libraries {libname: path} relative to directory.
+ missing: Set of remaining missing libraries.
+
+ """
+ missing = set(libraries)
+ found = {}
+ for _, _, files in itertools.islice(os.walk(directory), 1):
+ if not missing:
+ break
+ for f in files:
+ libname = get_lib_name(f)
+ if libname and libname in missing:
+ found[libname] = f
+ missing.discard(libname)
+
+ return found, missing
+
+
+def get_lib_name(filename):
+ """Determine the library name of the given library file.
+
+ The library name is the name by which the library is referred to in a -l
+ argument to the linker.
+
+ """
+ if not filename.startswith("lib"):
+ return None
+
+ libname = filename[3:]
+ dotsodot = libname.find(".so.")
+ if dotsodot != -1:
+ return libname[:dotsodot]
+
+ libname, ext = os.path.splitext(libname)
+ if ext in [".dll", ".dylib", ".so"]:
+ return libname
+
+ return None
+
+
+def resolve_rpath(rpath, output):
+ """Resolve the given rpath, replacing references to the binary."""
+ def has_origin(rpath):
+ return rpath.startswith("$ORIGIN") or rpath.startswith("@loader_path")
+
+ def replace_origin(rpath, origin):
+ rpath = rpath.replace("$ORIGIN/", origin)
+ rpath = rpath.replace("$ORIGIN", origin)
+ rpath = rpath.replace("@loader_path/", origin)
+ rpath = rpath.replace("@loader_path", origin)
+ return rpath
+
+ if is_temporary_output(output):
+ # GHC generates temporary libraries outside the execroot. The regular
+ # relative rpaths don't work in that case and have to be converted to
+ # absolute paths.
+ if has_origin(rpath):
+ # We don't know what $ORIGIN/@loader_path was meant to refer to.
+ # Try to find an existing, matching rpath by globbing.
+ stripped = replace_origin(rpath, "")
+ candidates = glob.glob(os.path.join("**", stripped), recursive=True)
+ if not candidates:
+ # Path does not exist. It will be sorted out later, since no
+ # library will be found underneath it.
+ rpath = stripped
+ else:
+ rpath = os.path.abspath(shorten_path(min(candidates)))
+ else:
+ rpath = os.path.abspath(shorten_path(rpath))
+
+ return rpath, rpath
+ else:
+ # Consider making relative rpaths relative to output.
+ # E.g. bazel-out/.../some/dir to @loader_path/.../some/dir
+ outdir = os.path.dirname(output) + "/"
+ resolved = replace_origin(rpath, outdir)
+ return rpath, resolved
+
+
+def darwin_get_install_name(lib):
+ """Read the install_name of the given library."""
+ lines = subprocess.check_output([OTOOL, "-D", lib]).splitlines()
+ if len(lines) >= 2:
+ return lines[1]
+ else:
+ return os.path.basename(lib)
+
+
+def darwin_rewrite_load_commands(rewrites, output):
+ """Rewrite the load commands in the given binary."""
+ args = []
+ for old, new in rewrites:
+ args.extend(["-change", old, os.path.join("@rpath", new)])
+ if args:
+ subprocess.check_call([INSTALL_NAME_TOOL] + args + [output])
+
+
+# --------------------------------------------------------------------
+# print-file-name
+
+
+def print_file_name(filename, args):
+ """Execute the print-file-name action.
+
+ Args:
+ filename: The queried filename.
+ args: The remaining arguments.
+
+ """
+ (basename, ext) = os.path.splitext(filename)
+ if is_darwin() and ext == ".dylib":
+ # Bazel generates dynamic libraries with .so extension on Darwin.
+ # However, GHC only looks for files with .dylib extension.
+
+ # Try with the .dylib extension first.
+ found, res = run_cc_print_file_name(filename, args)
+ if not found:
+ # Retry with .so extension.
+ found, so_res = run_cc_print_file_name("%s.so" % basename, args)
+ if found:
+ res = so_res
+ else:
+ _, res = run_cc_print_file_name(filename, args)
+
+ sys.stdout.write(res.stdout.decode())
+ sys.stderr.write(res.stderr.decode())
+ sys.exit(res.returncode)
+
+
+def run_cc_print_file_name(filename, args):
+ """Run cc --print-file-name on the given file name.
+
+ Args:
+ filename: The filename to query for.
+ args: Remaining command-line arguments. Relevant for -B flags.
+
+ Returns:
+ filename, res:
+ filename: The returned filename, if it exists, otherwise None.
+ res: CompletedProcess
+
+ """
+ args = args + ["--print-file-name", filename]
+ res = run_cc(args, capture_output=True, exit_on_error=True)
+ filename = res.stdout.decode().strip()
+ # Note, gcc --print-file-name does not fail if the file was not found, but
+ # instead just returns the input filename.
+ if os.path.isfile(filename):
+ return filename, res
+ else:
+ return None, res
+
+
+# --------------------------------------------------------------------
+
+
+def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
+ """Execute cc with a response file holding the given arguments.
+
+ Args:
+ args: Iterable of arguments to pass to cc.
+ capture_output: Whether to capture stdout and stderr.
+ exit_on_error: Whether to exit on error. Will print captured output first.
+
+ Returns:
+ CompletedProcess
+
+ """
+ if capture_output:
+ # The capture_output argument to subprocess.run was only added in 3.7.
+ new_kwargs = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ new_kwargs.update(kwargs)
+ kwargs = new_kwargs
+
+ with response_file(args) as rsp:
+ res = subprocess.run([CC, "@" + rsp], **kwargs)
+
+ if exit_on_error and res.returncode != 0:
+ if capture_output:
+ sys.stdout.write(res.stdout.decode())
+ sys.stderr.write(res.stderr.decode())
+ sys.exit(res.returncode)
+
+ return res
+
+
+@contextmanager
+def response_file(args):
+ """Create a response file for the given arguments.
+
+ Context manager, use in a with statement. The file will be deleted at the
+ end of scope.
+
+ Args:
+ args: Iterable, the arguments to write in to the response file.
+
+ Yields:
+ The file name of the response file.
+
+ """
+ try:
+ with tempfile.NamedTemporaryFile(mode="w", prefix="rsp", delete=False) as f:
+ for arg in args:
+ line = generate_response_line(arg)
+ f.write(line)
+ f.close()
+ yield f.name
+ finally:
+ try:
+ os.remove(f.name)
+ except OSError:
+ pass
+
+
+def generate_response_line(arg):
+ # Gcc expects one argument per line, surrounded by double quotes, with
+ # inner double quotes escaped with backslash, and backslashes themselves
+ # escaped. shlex.quote conflicts with this format.
+ return '"{}"\n'.format(arg.replace("\\", "\\\\").replace('"', '\\"'))
+
+
+def is_darwin():
+ """Whether the execution platform is Darwin."""
+ return platform.system() == "Darwin"
+
+
+def is_temporary_output(output):
+ """Whether the target is temporary.
+
+ GHC generates temporary libraries in certain cases related to Template
+ Haskell outside the execroot. This means that rpaths relative to $ORIGIN or
+ @loader_path are going to be invalid.
+
+ """
+ # Assumes that the temporary directory is set to an absolute path, while
+ # the outputs under the execroot are referred to by relative path. This
+ # should be a valid assumption as the temporary directory needs to be
+ # available irrespective of the current working directory, while Bazel uses
+ # paths relative to the execroot to avoid things like user names creeping
+ # into cache keys. If this turns out to be wrong we could instead look for
+ # path components matching Bazel's output directory hierarchy.
+ # See https://docs.bazel.build/versions/master/output_directories.html
+ return os.path.isabs(output)
+
+
+# --------------------------------------------------------------------
+
+
+if __name__ == "__main__":
+ main()
+
+
+# vim: ft=python
diff --git a/haskell/providers.bzl b/haskell/providers.bzl
index 3e423848..f5bfce0e 100644
--- a/haskell/providers.bzl
+++ b/haskell/providers.bzl
@@ -186,7 +186,7 @@ def _get_unique_lib_files(cc_info):
for filename in filenames
]
-def get_ghci_extra_libs(hs, cc_info, dynamic = True, path_prefix = None):
+def get_ghci_extra_libs(hs, cc_info, path_prefix = None):
"""Get libraries appropriate for GHCi's linker.
GHC expects dynamic and static versions of the same library to have the
@@ -197,16 +197,12 @@ def get_ghci_extra_libs(hs, cc_info, dynamic = True, path_prefix = None):
directory to allow for less RPATH entries and to fix file extensions that
GHCi does not support.
- GHCi can load PIC static libraries (-fPIC -fexternal-dynamic-refs) and
- dynamic libraries. Preferring static libraries can be useful to reduce the
- risk of exceeding the MACH-O header size limit on macOS, and to reduce
- build times by avoiding to generate dynamic libraries. However, this
- requires GHCi to run with the statically linked rts library.
+ GHCi can load PIC static libraries (-fPIC -fexternal-dynamic-refs) with a
+ dynamic RTS and dynamic libraries with a dynamic RTS.
Args:
hs: Haskell context.
cc_info: Combined CcInfo provider of dependencies.
- dynamic: (optional) Whether to prefer dynamic libraries.
path_prefix: (optional) Prefix for the entries in the generated library path.
Returns:
@@ -218,7 +214,7 @@ def get_ghci_extra_libs(hs, cc_info, dynamic = True, path_prefix = None):
(static_libs, dynamic_libs) = get_extra_libs(
hs,
cc_info,
- dynamic = dynamic,
+ dynamic = not hs.toolchain.is_static,
pic = True,
fixup_dir = "_ghci_libs",
)
@@ -278,8 +274,6 @@ def get_extra_libs(hs, cc_info, dynamic = False, pic = None, fixup_dir = "_libs"
elif lib_to_link.static_library and not pic_required:
static_lib = lib_to_link.static_library
- if dynamic_lib:
- dynamic_lib = symlink_dynamic_library(hs, dynamic_lib, fixed_lib_dir)
static_lib = mangle_static_library(hs, dynamic_lib, static_lib, fixed_lib_dir)
if static_lib and not (dynamic and dynamic_lib):
diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl
index 1e42b024..dc14a2f7 100644
--- a/haskell/toolchain.bzl
+++ b/haskell/toolchain.bzl
@@ -86,6 +86,11 @@ def _run_ghc(hs, cc, inputs, outputs, mnemonic, arguments, params_file = None, e
else:
inputs += extra_inputs
+ if input_manifests != None:
+ input_manifests = input_manifests + cc.manifests
+ else:
+ input_manifests = cc.manifests
+
hs.actions.run(
inputs = inputs,
tools = tools,
@@ -145,6 +150,8 @@ def _haskell_toolchain_impl(ctx):
for lib in ctx.attr.libraries
}
+ (cc_wrapper_inputs, cc_wrapper_manifest) = ctx.resolve_tools(tools = [ctx.attr._cc_wrapper])
+
return [
platform_common.ToolchainInfo(
name = ctx.label.name,
@@ -154,6 +161,11 @@ def _haskell_toolchain_impl(ctx):
haddock_flags = ctx.attr.haddock_flags,
locale = ctx.attr.locale,
locale_archive = locale_archive,
+ cc_wrapper = struct(
+ executable = ctx.executable._cc_wrapper,
+ inputs = cc_wrapper_inputs,
+ manifests = cc_wrapper_manifest,
+ ),
osx_cc_wrapper_tpl = ctx.file._osx_cc_wrapper_tpl,
mode = ctx.var["COMPILATION_MODE"],
actions = struct(
@@ -219,6 +231,11 @@ _haskell_toolchain = rule(
Label pointing to the locale archive file to use. Mostly useful on NixOS.
""",
),
+ "_cc_wrapper": attr.label(
+ cfg = "host",
+ default = Label("@rules_haskell//haskell:cc_wrapper"),
+ executable = True,
+ ),
"_osx_cc_wrapper_tpl": attr.label(
allow_single_file = True,
default = Label("@rules_haskell//haskell:private/osx_cc_wrapper.sh.tpl"),
From 329373a34311bb281b9bf39e66cb235a3df60802 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Tue, 16 Jul 2019 14:32:17 +0200
Subject: [PATCH 04/31] Remove unused osx_cc_wrapper
---
haskell/BUILD.bazel | 1 -
haskell/private/osx_cc_wrapper.sh.tpl | 313 --------------------------
haskell/toolchain.bzl | 5 -
3 files changed, 319 deletions(-)
delete mode 100644 haskell/private/osx_cc_wrapper.sh.tpl
diff --git a/haskell/BUILD.bazel b/haskell/BUILD.bazel
index 41afb641..e67c98f3 100644
--- a/haskell/BUILD.bazel
+++ b/haskell/BUILD.bazel
@@ -15,7 +15,6 @@ exports_files(
"private/ghci_repl_wrapper.sh",
"private/haddock_wrapper.sh.tpl",
"private/cc_wrapper.py.tpl",
- "private/osx_cc_wrapper.sh.tpl",
"private/pkgdb_to_bzl.py",
],
)
diff --git a/haskell/private/osx_cc_wrapper.sh.tpl b/haskell/private/osx_cc_wrapper.sh.tpl
deleted file mode 100644
index 9abf9ce9..00000000
--- a/haskell/private/osx_cc_wrapper.sh.tpl
+++ /dev/null
@@ -1,313 +0,0 @@
-#!/bin/bash
-#
-# Copyright 2015 The Bazel Authors. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# This is a wrapper script around gcc/clang that adjusts linker flags for
-# Haskell library and binary targets.
-#
-# Load commands that attempt to load dynamic libraries relative to the working
-# directory in their package output path (bazel-out/...) are converted to load
-# commands relative to @rpath. rules_haskell passes the corresponding
-# -Wl,-rpath,... flags itself.
-#
-# rpath commands that attempt to add rpaths relative to the working directory
-# to look for libraries in their package output path (bazel-out/...) are
-# omitted, since rules_haskell adds more appropriate rpaths itself.
-#
-# GHC generates intermediate dynamic libraries outside the build tree.
-# Additional RPATH entries are provided for those to make dynamic library
-# dependencies in the Bazel build tree available at runtime.
-#
-# See https://blogs.oracle.com/dipol/entry/dynamic_libraries_rpath_and_mac
-# on how to set those paths for Mach-O binaries.
-#
-set -euo pipefail
-
-INSTALL_NAME_TOOL="/usr/bin/install_name_tool"
-OTOOL="/usr/bin/otool"
-
-# Collect arguments to forward in a fresh response file.
-RESPONSE_FILE="$(mktemp osx_cc_args_XXXX.rsp)"
-rm_response_file() {
- rm -f "$RESPONSE_FILE"
-}
-trap rm_response_file EXIT
-
-add_args() {
- # Add the given arguments to the fresh response file. We follow GHC's
- # example in storing one argument per line, wrapped in double quotes. Double
- # quotes in the argument itself are escaped.
- for arg in "$@"; do
- printf '"%s"\n' "${arg//\"/\\\"}" >> "$RESPONSE_FILE"
- done
-}
-
-# Collect library, library dir, and rpath arguments.
-LIBS=()
-LIB_DIRS=()
-RPATHS=()
-
-# Parser state.
-# Parsing response file - unquote arguments.
-QUOTES=
-# Upcoming linker argument.
-LINKER=
-# Upcoming rpath argument.
-RPATH=
-# Upcoming install-name argument.
-INSTALL=
-# Upcoming output argument.
-OUTPUT=
-
-parse_arg() {
- # Parse the given argument. Decide whether to pass it on to the compiler,
- # and how it affects the parser state.
- local arg="$1"
- # Unquote response file arguments.
- if [[ "$QUOTES" = "1" && "$arg" =~ ^\"(.*)\"$ ]]; then
- # Take GHC's argument quoting into account when parsing a response
- # file. Note, no indication was found that GHC would pass multiline
- # arguments, or insert escape codes into the quoted arguments. If you
- # observe ill-formed arguments being passed to the compiler, then this
- # logic may need to be extended.
- arg="${BASH_REMATCH[1]}"
- fi
- # Parse given argument.
- if [[ "$OUTPUT" = "1" ]]; then
- # The previous argument was -o. Read output file.
- OUTPUT="$arg"
- add_args "$arg"
- elif [[ "$LINKER" = "1" ]]; then
- # The previous argument was -Xlinker. Read linker argument.
- if [[ "$RPATH" = "1" ]]; then
- # The previous argument was -rpath. Read RPATH.
- parse_rpath "$arg"
- RPATH=0
- elif [[ "$arg" = "-rpath" ]]; then
- # rpath is coming
- RPATH=1
- else
- # Unrecognized linker argument. Pass it on.
- add_args "-Xlinker" "$arg"
- fi
- LINKER=
- elif [[ "$INSTALL" = "1" ]]; then
- INSTALL=
- add_args "$arg"
- elif [[ "$arg" =~ ^@(.*)$ ]]; then
- # Handle response file argument. Parse the arguments contained in the
- # response file one by one. Take GHC's argument quoting into account.
- # Note, assumes that response file arguments are not nested in other
- # response files.
- QUOTES=1
- while read line; do
- parse_arg "$line"
- done < "${BASH_REMATCH[1]}"
- QUOTES=
- elif [[ "$arg" = "-install_name" ]]; then
- # Install name is coming. We don't use it, but it can start with an @
- # and be mistaken for a response file.
- INSTALL=1
- add_args "$arg"
- elif [[ "$arg" = "-o" ]]; then
- # output is coming
- OUTPUT=1
- add_args "$arg"
- elif [[ "$arg" = "-Xlinker" ]]; then
- # linker flag is coming
- LINKER=1
- elif [[ "$arg" =~ ^-l(.*)$ ]]; then
- LIBS+=("${BASH_REMATCH[1]}")
- add_args "$arg"
- elif [[ "$arg" =~ ^-L(.*)$ ]]; then
- LIB_DIRS+=("${BASH_REMATCH[1]}")
- add_args "$arg"
- elif [[ "$arg" =~ ^-Wl,-rpath,(.*)$ ]]; then
- parse_rpath "${BASH_REMATCH[1]}"
- else
- # Unrecognized argument. Pass it on.
- add_args "$arg"
- fi
-}
-
-parse_rpath() {
- # Parse the given -rpath argument and decide whether it should be
- # forwarded to the compiler/linker.
- local rpath="$1"
- if [[ "$rpath" =~ ^/ || "$rpath" =~ ^@ ]]; then
- # Absolute rpaths or rpaths relative to @loader_path or similar, are
- # passed on to the linker. Other relative rpaths are dropped, these
- # are auto-generated by GHC, but are useless because rules_haskell
- # constructs dedicated rpaths to the _solib or _hssolib directory.
- # See https://github.com/tweag/rules_haskell/issues/689
- add_args "-Wl,-rpath,$rpath"
- RPATHS+=("$rpath")
- fi
-}
-
-# Parse all given arguments.
-for arg in "$@"; do
- parse_arg "$arg"
-done
-
-get_library_in() {
- # Find the given library in the given directory.
- # Returns empty string if the library is not found.
- local lib="$1"
- local dir="$2"
- local solib="${dir}${dir:+/}lib${lib}.so"
- local dylib="${dir}${dir:+/}lib${lib}.dylib"
- if [[ -f "$solib" ]]; then
- echo "$solib"
- elif [[ -f "$dylib" ]]; then
- echo "$dylib"
- fi
-}
-
-get_library_path() {
- # Find the given library in the specified library search paths.
- # Returns empty string if the library is not found.
- if [[ ${#LIB_DIRS[@]} -gt 0 ]]; then
- local libpath
- for libdir in "${LIB_DIRS[@]}"; do
- libpath="$(get_library_in "$1" "$libdir")"
- if [[ -n "$libpath" ]]; then
- echo "$libpath"
- return
- fi
- done
- fi
-}
-
-resolve_rpath() {
- # Resolve the given rpath. I.e. if it is an absolute path, just return it.
- # If it is relative to the output, then prepend the output path.
- local rpath="$1"
- if [[ "$rpath" =~ ^/ ]]; then
- echo "$rpath"
- elif [[ "$rpath" =~ ^@loader_path/(.*)$ || "$rpath" =~ ^@executable_path/(.*)$ ]]; then
- echo "$(dirname "$OUTPUT")/${BASH_REMATCH[1]}"
- else
- echo "$rpath"
- fi
-}
-
-get_library_rpath() {
- # Find the given library in the specified rpaths.
- # Returns empty string if the library is not found.
- if [[ ${#RPATHS[@]} -gt 0 ]]; then
- local libdir libpath
- for rpath in "${RPATHS[@]}"; do
- libdir="$(resolve_rpath "$rpath")"
- libpath="$(get_library_in "$1" "$libdir")"
- if [[ -n "$libpath" ]]; then
- echo "$libpath"
- return
- fi
- done
- fi
-}
-
-get_library_name() {
- # Get the "library name" of the given library.
- "$OTOOL" -D "$1" | tail -1
-}
-
-relpath() {
- # Find relative path from the first to the second path. Assuming the first
- # is a directory. If either is an absolute path, then we return the
- # absolute path to the second.
- local from="$1"
- local to="$2"
- if [[ "$to" =~ ^/ ]]; then
- echo "$to"
- elif [[ "$from" =~ ^/ ]]; then
- echo "$PWD/$to"
- else
- # Split path and store components in bash array.
- IFS=/ read -a fromarr <<<"$from"
- IFS=/ read -a toarr <<<"$to"
- # Drop common prefix.
- for ((i=0; i < ${#fromarr[@]}; ++i)); do
- if [[ "${fromarr[$i]}" != "${toarr[$i]}" ]]; then
- break
- fi
- done
- # Construct relative path.
- local common=$i
- local out=
- for ((i=$common; i < ${#fromarr[@]}; ++i)); do
- out="$out${out:+/}.."
- done
- for ((i=$common; i < ${#toarr[@]}; ++i)); do
- out="$out${out:+/}${toarr[$i]}"
- done
- echo $out
- fi
-}
-
-generate_rpath() {
- # Generate an rpath entry for the given library path.
- local rpath="$(relpath "$(dirname "$OUTPUT")" "$(dirname "$1")")"
- if [[ "$rpath" =~ ^/ ]]; then
- echo "$rpath"
- else
- # Relative rpaths are relative to the binary.
- echo "@loader_path${rpath:+/}$rpath"
- fi
-}
-
-if [[ ! "$OUTPUT" =~ ^bazel-out/ && ${#LIBS[@]} -gt 0 ]]; then
- # GHC generates temporary dynamic libraries during compilation outside of
- # the build directory. References to dynamic C libraries are broken in this
- # case. Here we add additional RPATHs to fix these references. The Hazel
- # package for swagger2 is an example that triggers this issue.
- for lib in "${LIBS[@]}"; do
- librpath="$(get_library_rpath "$lib")"
- if [[ -z "$librpath" ]]; then
- # The given library was not found in any of the rpaths.
- # Find it in the library search paths.
- libpath="$(get_library_path "$lib")"
- if [[ "$libpath" =~ ^bazel-out/ ]]; then
- # The library is Bazel generated and loaded relative to PWD.
- # Add an RPATH entry, so it is found at runtime.
- rpath="$(generate_rpath "$libpath")"
- parse_rpath "$rpath"
- fi
- fi
- done
-fi
-
-# Call the C++ compiler with the fresh response file.
-%{cc} "@$RESPONSE_FILE"
-
-if [[ ${#LIBS[@]} -gt 0 ]]; then
- # Replace load commands relative to the working directory, by load commands
- # relative to the rpath, if the library can be found relative to an rpath.
- for lib in "${LIBS[@]}"; do
- librpath="$(get_library_rpath "$lib")"
- if [[ -n "$librpath" ]]; then
- libname="$(get_library_name "$librpath")"
- if [[ "$libname" =~ ^bazel-out/ ]]; then
- "${INSTALL_NAME_TOOL}" -change \
- "$libname" \
- "@rpath/$(basename "$librpath")" \
- "$OUTPUT"
- fi
- fi
- done
-fi
-
-# vim: ft=sh
diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl
index dc14a2f7..9b8372ed 100644
--- a/haskell/toolchain.bzl
+++ b/haskell/toolchain.bzl
@@ -166,7 +166,6 @@ def _haskell_toolchain_impl(ctx):
inputs = cc_wrapper_inputs,
manifests = cc_wrapper_manifest,
),
- osx_cc_wrapper_tpl = ctx.file._osx_cc_wrapper_tpl,
mode = ctx.var["COMPILATION_MODE"],
actions = struct(
compile_binary = compile_binary,
@@ -236,10 +235,6 @@ Label pointing to the locale archive file to use. Mostly useful on NixOS.
default = Label("@rules_haskell//haskell:cc_wrapper"),
executable = True,
),
- "_osx_cc_wrapper_tpl": attr.label(
- allow_single_file = True,
- default = Label("@rules_haskell//haskell:private/osx_cc_wrapper.sh.tpl"),
- ),
},
)
From e4167125c996f94bdcd35a74975df866d2284252 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Tue, 23 Jul 2019 09:42:38 +0200
Subject: [PATCH 05/31] Use cc_wrapper in ghci
Otherwise we would still require symbolic links for dynamic library
dependencies on macOS for REPL targets.
---
haskell/private/actions/repl.bzl | 2 ++
haskell/private/actions/runghc.bzl | 2 ++
haskell/private/cc_wrapper.bzl | 10 ++++++++--
haskell/private/cc_wrapper.py.tpl | 19 ++++++++++++++++++-
haskell/private/ghci_repl_wrapper.sh | 7 ++++++-
haskell/repl.bzl | 3 +++
haskell/toolchain.bzl | 5 +++++
7 files changed, 44 insertions(+), 4 deletions(-)
diff --git a/haskell/private/actions/repl.bzl b/haskell/private/actions/repl.bzl
index 483a0172..c452a33f 100644
--- a/haskell/private/actions/repl.bzl
+++ b/haskell/private/actions/repl.bzl
@@ -118,6 +118,7 @@ def build_haskell_repl(
substitutions = {
"{ENV}": render_env(ghc_env),
"{TOOL}": hs.tools.ghci.path,
+ "{CC}": hs.toolchain.cc_wrapper.executable.path,
"{ARGS}": " ".join(
[
"-ghci-script",
@@ -174,5 +175,6 @@ def build_haskell_repl(
pkg_info_inputs,
ghci_extra_libs,
hs_info.source_files,
+ hs.toolchain.cc_wrapper.runfiles.files,
])
ln(hs, repl_file, output, extra_inputs)
diff --git a/haskell/private/actions/runghc.bzl b/haskell/private/actions/runghc.bzl
index ca2c41e2..71b31d76 100644
--- a/haskell/private/actions/runghc.bzl
+++ b/haskell/private/actions/runghc.bzl
@@ -87,6 +87,7 @@ def build_haskell_runghc(
substitutions = {
"{ENV}": render_env(ghc_env),
"{TOOL}": hs.tools.runghc.path,
+ "{CC}": hs.toolchain.cc_wrapper.executable.path,
"{ARGS}": " ".join([shell.quote(a) for a in runcompile_flags]),
},
is_executable = True,
@@ -105,5 +106,6 @@ def build_haskell_runghc(
pkg_info_inputs,
ghci_extra_libs,
hs_info.source_files,
+ hs.toolchain.cc_wrapper.runfiles.files,
])
ln(hs, runghc_file, output, extra_inputs)
diff --git a/haskell/private/cc_wrapper.bzl b/haskell/private/cc_wrapper.bzl
index 9913d806..4e7d3957 100644
--- a/haskell/private/cc_wrapper.bzl
+++ b/haskell/private/cc_wrapper.bzl
@@ -1,6 +1,5 @@
load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain")
load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES")
-load("@os_info//:os_info.bzl", "is_linux")
def _cc_wrapper_impl(ctx):
cc_toolchain = find_cpp_toolchain(ctx)
@@ -21,9 +20,16 @@ def _cc_wrapper_impl(ctx):
is_executable = True,
substitutions = {
"{:cc:}": cc,
+ "{:workspace:}": ctx.workspace_name,
},
)
- return [DefaultInfo(files = depset([cc_wrapper]))]
+ return [DefaultInfo(
+ files = depset([cc_wrapper]),
+ runfiles = ctx.runfiles(
+ transitive_files = cc_toolchain.all_files,
+ collect_data = True,
+ ),
+ )]
_cc_wrapper = rule(
implementation = _cc_wrapper_impl,
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 2f71a23c..ffdb9f6f 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -40,6 +40,7 @@ import subprocess
import sys
import tempfile
+WORKSPACE = "{:workspace:}"
CC = "{:cc:}"
INSTALL_NAME_TOOL = "/usr/bin/install_name_tool"
OTOOL = "/usr/bin/otool"
@@ -731,8 +732,24 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
new_kwargs.update(kwargs)
kwargs = new_kwargs
+ if os.path.isfile(CC):
+ cc = CC
+ else:
+ # On macOS CC is a relative path to a wrapper script. If we're
+ # being called from a GHCi REPL then we need to find this wrapper
+ # script using Bazel runfiles.
+ r = bazel_runfiles.Create()
+ cc = r.Rlocation("/".join([WORKSPACE, CC]))
+ if cc is None and platform.system() == "Windows":
+ # We must use "/" instead of os.path.join on Windows, because the
+ # Bazel runfiles_manifest file uses "/" separators.
+ cc = r.Rlocation("/".join([WORKSPACE, CC + ".exe"]))
+ if cc is None:
+ print("CC not found '{}'.".format(CC), file=sys.stderr)
+ sys.exit(1)
+
with response_file(args) as rsp:
- res = subprocess.run([CC, "@" + rsp], **kwargs)
+ res = subprocess.run([cc, "@" + rsp], **kwargs)
if exit_on_error and res.returncode != 0:
if capture_output:
diff --git a/haskell/private/ghci_repl_wrapper.sh b/haskell/private/ghci_repl_wrapper.sh
index cd6acefc..f672671a 100644
--- a/haskell/private/ghci_repl_wrapper.sh
+++ b/haskell/private/ghci_repl_wrapper.sh
@@ -54,6 +54,11 @@ cd "$BUILD_WORKSPACE_DIRECTORY"
RULES_HASKELL_EXEC_ROOT=$(dirname $(readlink ${BUILD_WORKSPACE_DIRECTORY}/bazel-out))
TOOL_LOCATION="$RULES_HASKELL_EXEC_ROOT/{TOOL}"
+# Setting -pgm* flags explicitly has the unfortunate side effect
+# of resetting any program flags in the GHC settings file. So we
+# restore them here. See
+# https://ghc.haskell.org/trac/ghc/ticket/7929.
+PGM_ARGS="-pgma {CC} -pgmc {CC} -pgml {CC} -pgmP {CC} -optc-fno-stack-protector -optP-E -optP-undef -optP-traditional"
{ENV}
-"$TOOL_LOCATION" {ARGS} "$@"
+"$TOOL_LOCATION" $PGM_ARGS {ARGS} "$@"
diff --git a/haskell/repl.bzl b/haskell/repl.bzl
index 06e2c4f3..4dd87163 100644
--- a/haskell/repl.bzl
+++ b/haskell/repl.bzl
@@ -293,6 +293,7 @@ def _create_repl(hs, ctx, repl_info, output):
substitutions = {
"{ENV}": render_env(dicts.add(hs.env, ghc_env)),
"{TOOL}": hs.tools.ghci.path,
+ "{CC}": hs.toolchain.cc_wrapper.executable.path,
"{ARGS}": " ".join(
args + [
shell.quote(a)
@@ -316,6 +317,8 @@ def _create_repl(hs, ctx, repl_info, output):
depset([hs.toolchain.locale_archive] if hs.toolchain.locale_archive else []),
]),
collect_data = ctx.attr.collect_data,
+ ).merge(
+ hs.toolchain.cc_wrapper.runfiles,
),
)]
diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl
index 9b8372ed..68570284 100644
--- a/haskell/toolchain.bzl
+++ b/haskell/toolchain.bzl
@@ -151,6 +151,10 @@ def _haskell_toolchain_impl(ctx):
}
(cc_wrapper_inputs, cc_wrapper_manifest) = ctx.resolve_tools(tools = [ctx.attr._cc_wrapper])
+ cc_wrapper_info = ctx.attr._cc_wrapper[DefaultInfo]
+ cc_wrapper_runfiles = cc_wrapper_info.default_runfiles.merge(
+ cc_wrapper_info.data_runfiles,
+ )
return [
platform_common.ToolchainInfo(
@@ -165,6 +169,7 @@ def _haskell_toolchain_impl(ctx):
executable = ctx.executable._cc_wrapper,
inputs = cc_wrapper_inputs,
manifests = cc_wrapper_manifest,
+ runfiles = cc_wrapper_runfiles,
),
mode = ctx.var["COMPILATION_MODE"],
actions = struct(
From 7836e68439d4a4e96928b167049683ecbd306422 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Tue, 23 Jul 2019 17:36:37 +0200
Subject: [PATCH 06/31] doctest: use cc_wrapper as well
---
haskell/doctest.bzl | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/haskell/doctest.bzl b/haskell/doctest.bzl
index 0a75eea4..88144865 100644
--- a/haskell/doctest.bzl
+++ b/haskell/doctest.bzl
@@ -1,6 +1,7 @@
"""Doctest support"""
load("@bazel_skylib//lib:dicts.bzl", "dicts")
+load(":cc.bzl", "cc_interop_info")
load(":private/context.bzl", "haskell_context", "render_env")
load(":private/path_utils.bzl", "link_libraries")
load(":private/set.bzl", "set")
@@ -86,6 +87,27 @@ def _haskell_doctest_single(target, ctx):
args = ctx.actions.args()
args.add("--no-magic")
+ cc = cc_interop_info(ctx)
+ args.add_all([
+ # GHC uses C compiler for assemly, linking and preprocessing as well.
+ "-pgma",
+ cc.tools.cc,
+ "-pgmc",
+ cc.tools.cc,
+ "-pgml",
+ cc.tools.cc,
+ "-pgmP",
+ cc.tools.cc,
+ # Setting -pgm* flags explicitly has the unfortunate side effect
+ # of resetting any program flags in the GHC settings file. So we
+ # restore them here. See
+ # https://ghc.haskell.org/trac/ghc/ticket/7929.
+ "-optc-fno-stack-protector",
+ "-optP-E",
+ "-optP-undef",
+ "-optP-traditional",
+ ])
+
doctest_log = ctx.actions.declare_file(
"doctest-log-" + ctx.label.name + "-" + target.label.name,
)
@@ -118,6 +140,7 @@ def _haskell_doctest_single(target, ctx):
ghci_extra_libs,
depset(
toolchain.doctest +
+ cc.files +
[hs.tools.ghc],
),
]),
@@ -183,7 +206,11 @@ haskell_doctest = rule(
omitted, all exposed modules provided by `deps` will be tested.
""",
),
+ "_cc_toolchain": attr.label(
+ default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
+ ),
},
+ fragments = ["cpp"],
toolchains = [
"@rules_haskell//haskell:toolchain",
"@rules_haskell//haskell:doctest-toolchain",
From 8294c7dc9106947b98c6a3ad7421f3dffb814642 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Tue, 23 Jul 2019 18:26:40 +0200
Subject: [PATCH 07/31] print(..., file=...) not supported on bindist
---
haskell/private/cc_wrapper.py.tpl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index ffdb9f6f..ba722cea 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -745,7 +745,7 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
# Bazel runfiles_manifest file uses "/" separators.
cc = r.Rlocation("/".join([WORKSPACE, CC + ".exe"]))
if cc is None:
- print("CC not found '{}'.".format(CC), file=sys.stderr)
+ sys.stderr.write("CC not found '{}'.\n".format(CC))
sys.exit(1)
with response_file(args) as rsp:
From 8d14b4022eddf1eb8998c23f23e4f994cef58325 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Wed, 24 Jul 2019 10:24:13 +0200
Subject: [PATCH 08/31] subprocess.run not supported on bindist
---
haskell/private/cc_wrapper.py.tpl | 56 ++++++++++++++++++-------------
1 file changed, 33 insertions(+), 23 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index ba722cea..561aace7 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -668,23 +668,22 @@ def print_file_name(filename, args):
"""
(basename, ext) = os.path.splitext(filename)
- if is_darwin() and ext == ".dylib":
+ found = run_cc_print_file_name(filename, args)
+ if not found and is_darwin() and ext == ".dylib":
# Bazel generates dynamic libraries with .so extension on Darwin.
# However, GHC only looks for files with .dylib extension.
- # Try with the .dylib extension first.
- found, res = run_cc_print_file_name(filename, args)
- if not found:
- # Retry with .so extension.
- found, so_res = run_cc_print_file_name("%s.so" % basename, args)
- if found:
- res = so_res
+ # Retry with .so extension.
+ found = run_cc_print_file_name("%s.so" % basename, args)
+
+ # Note, gcc --print-file-name does not fail if the file was not found, but
+ # instead just returns the input filename.
+ if found:
+ print(found)
else:
- _, res = run_cc_print_file_name(filename, args)
+ print(filename)
- sys.stdout.write(res.stdout.decode())
- sys.stderr.write(res.stderr.decode())
- sys.exit(res.returncode)
+ sys.exit()
def run_cc_print_file_name(filename, args):
@@ -701,14 +700,14 @@ def run_cc_print_file_name(filename, args):
"""
args = args + ["--print-file-name", filename]
- res = run_cc(args, capture_output=True, exit_on_error=True)
- filename = res.stdout.decode().strip()
+ _, stdoutbuf, _ = run_cc(args, capture_output=True, exit_on_error=True)
+ filename = stdoutbuf.decode().strip()
# Note, gcc --print-file-name does not fail if the file was not found, but
# instead just returns the input filename.
if os.path.isfile(filename):
- return filename, res
+ return filename
else:
- return None, res
+ return None
# --------------------------------------------------------------------
@@ -723,7 +722,10 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
exit_on_error: Whether to exit on error. Will print captured output first.
Returns:
- CompletedProcess
+ (returncode, stdoutbuf, stderrbuf):
+ returncode: The exit code of the the process.
+ stdoutbuf: The captured standard output, None if not capture_output.
+ stderrbuf: The captured standard error, None if not capture_output.
"""
if capture_output:
@@ -748,16 +750,24 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
sys.stderr.write("CC not found '{}'.\n".format(CC))
sys.exit(1)
+ stdoutbuf = None
+ stderrbuf = None
+
with response_file(args) as rsp:
- res = subprocess.run([cc, "@" + rsp], **kwargs)
+ # subprocess.run is not supported in the bindist CI setup.
+ with subprocess.Popen([cc, "@" + rsp], **kwargs) as proc:
+ if capture_output:
+ (stdoutbuf, stderrbuf) = proc.communicate()
+
+ returncode = proc.wait()
- if exit_on_error and res.returncode != 0:
+ if exit_on_error and returncode != 0:
if capture_output:
- sys.stdout.write(res.stdout.decode())
- sys.stderr.write(res.stderr.decode())
- sys.exit(res.returncode)
+ sys.stdout.write(stdout.decode())
+ sys.stderr.write(stderr.decode())
+ sys.exit(returncode)
- return res
+ return (returncode, stdoutbuf, stderrbuf)
@contextmanager
From ad7c9a796086eace35a6b72eb1a9443070fe1f65 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Wed, 24 Jul 2019 10:47:05 +0200
Subject: [PATCH 09/31] Popen context manager not supported on bindist
---
haskell/private/cc_wrapper.py.tpl | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 561aace7..01876afe 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -755,11 +755,13 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
with response_file(args) as rsp:
# subprocess.run is not supported in the bindist CI setup.
- with subprocess.Popen([cc, "@" + rsp], **kwargs) as proc:
- if capture_output:
- (stdoutbuf, stderrbuf) = proc.communicate()
+ # subprocess.Popen does not support context manager on CI setup.
+ proc = subprocess.Popen([cc, "@" + rsp], **kwargs)
- returncode = proc.wait()
+ if capture_output:
+ (stdoutbuf, stderrbuf) = proc.communicate()
+
+ returncode = proc.wait()
if exit_on_error and returncode != 0:
if capture_output:
From f07bd2233835270e5e81f3b388b772c69a7b9da2 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Wed, 24 Jul 2019 10:55:57 +0200
Subject: [PATCH 10/31] NamedTempraryFile(..., delete=False) unsupported on
bindist
---
haskell/private/cc_wrapper.py.tpl | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 01876afe..0f854a91 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -787,15 +787,16 @@ def response_file(args):
"""
try:
- with tempfile.NamedTemporaryFile(mode="w", prefix="rsp", delete=False) as f:
- for arg in args:
- line = generate_response_line(arg)
- f.write(line)
- f.close()
- yield f.name
+ (fd, filename) = tempfile.mkstemp(prefix="rsp", text=True)
+ handle = os.fdopen(fd, "w")
+ for arg in args:
+ line = generate_response_line(arg)
+ handle.write(line)
+ handle.close()
+ yield filename
finally:
try:
- os.remove(f.name)
+ os.remove(filename)
except OSError:
pass
From f4301ca15ae02259d9cf7b81b898730fc78d671f Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Fri, 26 Jul 2019 18:36:01 +0200
Subject: [PATCH 11/31] Windows: cc_wrapper in bash
The python cc_wrapper is too slow on Windows. See
https://github.com/tweag/rules_haskell/pull/1002
Rewriting it in C++ would be a lot of work as we cannot rely on C++17's
std::filesystem, yet. Including a C++ filesystem library would impose a
large dependency. Fortunately, the Windows cc_wrapper only needs to
shorten library paths, not handle dynamic libraries. It is easier to use
a dedicated bash cc_wrapper on Windows instead.
---
haskell/BUILD.bazel | 1 +
haskell/private/cc_wrapper.bzl | 40 ++++++--
haskell/private/cc_wrapper.sh.tpl | 162 ++++++++++++++++++++++++++++++
3 files changed, 195 insertions(+), 8 deletions(-)
create mode 100644 haskell/private/cc_wrapper.sh.tpl
diff --git a/haskell/BUILD.bazel b/haskell/BUILD.bazel
index e67c98f3..dd3cfc44 100644
--- a/haskell/BUILD.bazel
+++ b/haskell/BUILD.bazel
@@ -15,6 +15,7 @@ exports_files(
"private/ghci_repl_wrapper.sh",
"private/haddock_wrapper.sh.tpl",
"private/cc_wrapper.py.tpl",
+ "private/cc_wrapper.sh.tpl",
"private/pkgdb_to_bzl.py",
],
)
diff --git a/haskell/private/cc_wrapper.bzl b/haskell/private/cc_wrapper.bzl
index 4e7d3957..ee185aab 100644
--- a/haskell/private/cc_wrapper.bzl
+++ b/haskell/private/cc_wrapper.bzl
@@ -13,7 +13,7 @@ def _cc_wrapper_impl(ctx):
feature_configuration = feature_configuration,
action_name = ACTION_NAMES.c_compile,
)
- cc_wrapper = ctx.actions.declare_file(ctx.label.name + ".py")
+ cc_wrapper = ctx.actions.declare_file(ctx.label.name)
ctx.actions.expand_template(
template = ctx.file.template,
output = cc_wrapper,
@@ -36,7 +36,6 @@ _cc_wrapper = rule(
attrs = {
"template": attr.label(
allow_single_file = True,
- default = Label("@rules_haskell//haskell:private/cc_wrapper.py.tpl"),
),
"_cc_toolchain": attr.label(
default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
@@ -45,18 +44,43 @@ _cc_wrapper = rule(
fragments = ["cpp"],
)
-def cc_wrapper(name, template = None, **kwargs):
+def cc_wrapper(name, **kwargs):
_cc_wrapper(
- name = name + "-source",
- template = template,
+ name = name + ".py",
+ template = "@rules_haskell//haskell:private/cc_wrapper.py.tpl",
+ )
+ _cc_wrapper(
+ name = name + ".sh",
+ template = "@rules_haskell//haskell:private/cc_wrapper.sh.tpl",
)
native.py_binary(
- name = name,
- srcs = [name + "-source"],
- main = name + "-source.py",
+ name = name + "-python",
+ srcs = [name + ".py"],
python_version = "PY3",
+ main = name + ".py",
deps = [
"@bazel_tools//tools/python/runfiles",
],
**kwargs
)
+
+ # This is a workaround for py_binary being too slow on Windows.
+ # See https://github.com/bazelbuild/bazel/issues/8981
+ # In principle the python cc_wrapper would be sufficient for all platforms,
+ # however, execution is too slow on Windows to be practical.
+ native.sh_binary(
+ name = name + "-bash",
+ srcs = [name + ".sh"],
+ deps = [
+ "@bazel_tools//tools/bash/runfiles",
+ ],
+ **kwargs
+ )
+ native.alias(
+ name = name,
+ actual = select({
+ "@rules_haskell//haskell/platforms:mingw32": name + "-bash",
+ "//conditions:default": name + "-python",
+ }),
+ **kwargs
+ )
diff --git a/haskell/private/cc_wrapper.sh.tpl b/haskell/private/cc_wrapper.sh.tpl
new file mode 100644
index 00000000..96fa0d54
--- /dev/null
+++ b/haskell/private/cc_wrapper.sh.tpl
@@ -0,0 +1,162 @@
+# CC toolchain wrapper
+#
+# Usage: cc_wrapper [ARG]...
+#
+# Wraps the C compiler of the Bazel CC toolchain. Transforms arguments to work
+# around limitations of Bazel and GHC and passes those via response file to the C
+# compiler.
+#
+# - Shortens library search paths to stay below maximum path length on Windows.
+#
+# GHC generates library search paths that contain redundant up-level
+# references (..). This can exceed the maximum path length on Windows, which
+# will cause linking failures. This wrapper shortens library search paths to
+# avoid that issue.
+
+set -euo pipefail
+
+# ----------------------------------------------------------
+# Find compiler
+
+find_exe() {
+ local exe="$1"
+ local location
+
+ location="$exe"
+ if [[ -f "$location" ]]; then
+ echo "$location"
+ return
+ fi
+
+ location="${exe}.exe"
+ if [[ -f "$location" ]]; then
+ echo "$location"
+ return
+ fi
+
+ # --- begin runfiles.bash initialization v2 ---
+ # Copy-pasted from the Bazel Bash runfiles library v2.
+ set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+ source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+ source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+ source "$0.runfiles/$f" 2>/dev/null || \
+ source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+ source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+ { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+ # --- end runfiles.bash initialization v2 ---
+
+ location="$(rlocation "{:workspace:}/$exe")"
+ if [[ -f "$location" ]]; then
+ echo "$location"
+ return
+ fi
+
+ location="$(rlocation "{:workspace:}/${exe}.exe")"
+ if [[ -f "$location" ]]; then
+ echo "$location"
+ return
+ fi
+}
+
+CC="$(find_exe "{:cc:}")"
+
+# ----------------------------------------------------------
+# Handle response file
+
+RESPONSE_FILE="$(mktemp rspXXXX)"
+rm_response_file() {
+ rm -f "$RESPONSE_FILE"
+}
+trap rm_response_file EXIT
+
+quote_arg() {
+ # Gcc expects one argument per line, surrounded by double quotes, with
+ # inner double quotes escaped with backslash, and backslashes themselves
+ # escaped.
+ local arg="$1"
+ arg="${arg//\\/\\\\}"
+ arg="${arg//\"/\\\"}"
+ printf '"%s"\n' "$arg"
+}
+
+unquote_arg() {
+ local arg="$1"
+ if [[ "$arg" =~ ^\"(.*)\"[[:space:]]*$ ]]; then
+ arg="${BASH_REMATCH[1]}"
+ arg="${arg//\\\"/\"}"
+ arg="${arg//\\\\/\\}"
+ fi
+ echo "$arg"
+}
+
+add_arg() {
+ quote_arg "$1" >> "$RESPONSE_FILE"
+}
+
+# ----------------------------------------------------------
+# Parse arguments
+
+IN_RESPONSE_FILE=
+LIB_DIR_COMING=
+
+shorten_path() {
+ local input="$1"
+ local shortest="$input"
+
+ if [[ ! -e "$shortest" ]]; then
+ # realpath fails if the file does not exist.
+ echo "$shortest"
+ return
+ fi
+
+ local normalized="$(realpath "$shortest")"
+ if [[ ${#normalized} -lt ${#shortest} ]]; then
+ shortest="$normalized"
+ fi
+
+ local relative="$(realpath --relative-to="$PWD" "$shortest")"
+ if [[ ${#relative} -lt ${#shortest} ]]; then
+ shortest="$relative"
+ fi
+
+ echo "$shortest"
+}
+
+handle_lib_dir() {
+ local lib_dir="$1"
+ add_arg "-L$(shorten_path "$lib_dir")"
+}
+
+handle_arg() {
+ local arg="$1"
+ if [[ $IN_RESPONSE_FILE = 1 ]]; then
+ arg="$(unquote_arg "$arg")"
+ fi
+ if [[ $LIB_DIR_COMING = 1 ]]; then
+ LIB_DIR_COMING=
+ handle_lib_dir "$arg"
+ elif [[ "$arg" =~ ^@(.*)$ ]]; then
+ IN_RESPONSE_FILE=1
+ while read line; do
+ handle_arg "$line"
+ done < "${BASH_REMATCH[1]}"
+ IN_RESPONSE_FILE=
+ elif [[ "$arg" =~ ^-L(.*)$ || "$arg" =~ ^--library-path=(.*)$ ]]; then
+ handle_lib_dir "${BASH_REMATCH[1]}"
+ elif [[ "$arg" = -L || "$arg" = --library-path ]]; then
+ LIB_DIR_COMING=1
+ else
+ add_arg "$arg"
+ fi
+}
+
+for arg in "$@"; do
+ handle_arg "$arg"
+done
+
+# ----------------------------------------------------------
+# Call compiler
+
+"$CC" "@$RESPONSE_FILE"
+
+# vim: ft=sh
From c0d0f11aa7fc8f85f2adccdde5378694d791c3ae Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Tue, 13 Aug 2019 16:15:53 +0200
Subject: [PATCH 12/31] Remove .dll.a on Windows
These libraries cause linking errors on Windows when linking pthreads.
---
haskell/ghc_bindist.bzl | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/haskell/ghc_bindist.bzl b/haskell/ghc_bindist.bzl
index 535f0e37..884b2c31 100644
--- a/haskell/ghc_bindist.bzl
+++ b/haskell/ghc_bindist.bzl
@@ -266,6 +266,14 @@ haskell_toolchain(
haddock_flags = ctx.attr.haddock_flags,
repl_ghci_args = ctx.attr.repl_ghci_args,
)
+
+ if os == "windows":
+ # These libraries cause linking errors on Windows when linking
+ # pthreads, due to libwinpthread-1.dll not being loaded.
+ _execute_fail_loudly(ctx, ["rm", "mingw/lib/gcc/x86_64-w64-mingw32/7.2.0/libstdc++.dll.a"])
+ _execute_fail_loudly(ctx, ["rm", "mingw/x86_64-w64-mingw32/lib/libpthread.dll.a"])
+ _execute_fail_loudly(ctx, ["rm", "mingw/x86_64-w64-mingw32/lib/libwinpthread.dll.a"])
+
ctx.template(
"BUILD",
ghc_build,
From fe5464ac9b4c334f1c20f95710215257fefc04ca Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Wed, 14 Aug 2019 11:41:27 +0200
Subject: [PATCH 13/31] Bash: Use nameref variables
Use nameref variables to return values from functions instead of echoing
strings. This allows to avoid command substitution, which would create a
subprocess for each function call, which is costly on Windows.
---
haskell/private/cc_wrapper.sh.tpl | 38 +++++++++++++++----------------
1 file changed, 18 insertions(+), 20 deletions(-)
diff --git a/haskell/private/cc_wrapper.sh.tpl b/haskell/private/cc_wrapper.sh.tpl
index 96fa0d54..86245085 100644
--- a/haskell/private/cc_wrapper.sh.tpl
+++ b/haskell/private/cc_wrapper.sh.tpl
@@ -19,18 +19,16 @@ set -euo pipefail
# Find compiler
find_exe() {
- local exe="$1"
- local location
+ local -n location="$1"
+ local exe="$2"
location="$exe"
if [[ -f "$location" ]]; then
- echo "$location"
return
fi
location="${exe}.exe"
if [[ -f "$location" ]]; then
- echo "$location"
return
fi
@@ -47,18 +45,17 @@ find_exe() {
location="$(rlocation "{:workspace:}/$exe")"
if [[ -f "$location" ]]; then
- echo "$location"
return
fi
location="$(rlocation "{:workspace:}/${exe}.exe")"
if [[ -f "$location" ]]; then
- echo "$location"
return
fi
}
-CC="$(find_exe "{:cc:}")"
+declare CC
+find_exe CC "{:cc:}"
# ----------------------------------------------------------
# Handle response file
@@ -80,13 +77,14 @@ quote_arg() {
}
unquote_arg() {
- local arg="$1"
- if [[ "$arg" =~ ^\"(.*)\"[[:space:]]*$ ]]; then
- arg="${BASH_REMATCH[1]}"
- arg="${arg//\\\"/\"}"
- arg="${arg//\\\\/\\}"
+ local -n output="$1"
+ local input="$2"
+ if [[ "$input" =~ ^\"(.*)\"[[:space:]]*$ ]]; then
+ input="${BASH_REMATCH[1]}"
+ input="${input//\\\"/\"}"
+ input="${input//\\\\/\\}"
fi
- echo "$arg"
+ output="$input"
}
add_arg() {
@@ -100,12 +98,12 @@ IN_RESPONSE_FILE=
LIB_DIR_COMING=
shorten_path() {
- local input="$1"
- local shortest="$input"
+ local -n shortest="$1"
+ local input="$2"
+ shortest="$input"
if [[ ! -e "$shortest" ]]; then
# realpath fails if the file does not exist.
- echo "$shortest"
return
fi
@@ -118,19 +116,19 @@ shorten_path() {
if [[ ${#relative} -lt ${#shortest} ]]; then
shortest="$relative"
fi
-
- echo "$shortest"
}
handle_lib_dir() {
local lib_dir="$1"
- add_arg "-L$(shorten_path "$lib_dir")"
+ local shortened
+ shorten_path shortened "$lib_dir"
+ add_arg "-L$shortened"
}
handle_arg() {
local arg="$1"
if [[ $IN_RESPONSE_FILE = 1 ]]; then
- arg="$(unquote_arg "$arg")"
+ unquote_arg arg "$arg"
fi
if [[ $LIB_DIR_COMING = 1 ]]; then
LIB_DIR_COMING=
From c6f4da909848366d9cecae8a6c9680c85df6324e Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Wed, 28 Aug 2019 13:34:59 +0200
Subject: [PATCH 14/31] Define response file
https://github.com/tweag/rules_haskell/pull/1039/files/71c2a6d5286a0e0802c78be3a1a8cf6e6ae8061d..a10b82374bc7b6f86844835b1656e06b449a0b0a#r318497584
---
haskell/private/cc_wrapper.py.tpl | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 0f854a91..4dc2300d 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -5,7 +5,8 @@ Usage: cc_wrapper [ARG]...
Wraps the C compiler of the Bazel CC toolchain. Transforms arguments to work
around limitations of Bazel and GHC and passes those via response file to the C
-compiler.
+compiler. A response file is a text file listing command-line arguments. It is
+used to avoid command line length limitations.
- Shortens library search paths to stay below maximum path length on Windows.
From 90778d1f20de499d5d947c2a14f432746ffd8008 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Wed, 28 Aug 2019 13:53:43 +0200
Subject: [PATCH 15/31] Factor out command line argument matching
https://github.com/tweag/rules_haskell/pull/1039/files/71c2a6d5286a0e0802c78be3a1a8cf6e6ae8061d..a10b82374bc7b6f86844835b1656e06b449a0b0a#r318490807
---
haskell/private/cc_wrapper.py.tpl | 107 ++++++++++++++++++------------
1 file changed, 64 insertions(+), 43 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 4dc2300d..9fedad97 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -162,48 +162,37 @@ class Args:
yield out_arg
def _handle_output(self, arg, args, out):
- if arg == "-o":
+ consumed, output = argument(arg, args, short = "-o")
+
+ if consumed:
# Remember the output filename.
- self.output = next(args)
+ self.output = output
out.extend(["-o", self.output])
- return True
- else:
- return False
+
+ return consumed
def _handle_library(self, arg, args, out):
- if arg == "-l" or arg == "--library":
- library = next(args)
- elif arg.startswith("-l"):
- library = arg[2:]
- elif arg.startswith("--library="):
- library = arg[len("--library="):]
- else:
- return False
+ consumed, library = argument(arg, args, short = "-l", long = "--library")
- # Remember the required libraries.
- self.libraries.append(library)
- out.append("-l{}".format(library))
+ if consumed:
+ # Remember the required libraries.
+ self.libraries.append(library)
+ out.append("-l{}".format(library))
- return True
+ return consumed
def _handle_library_path(self, arg, args, out):
- if arg == "-L" or arg == "--library-path":
- library_path = next(args)
- elif arg.startswith("-L"):
- library_path = arg[2:]
- elif arg.startswith("--library-path="):
- library_path = arg[len("--library-path="):]
- else:
- return False
+ consumed, library_path = argument(arg, args, short = "-L", long = "--library-path")
- # Shorten the library search paths. On Windows library search paths may
- # exceed the maximum path length.
- shortened = shorten_path(library_path)
- # Remember the library search paths.
- self.library_paths.append(shortened)
- out.append("-L{}".format(shortened))
+ if consumed:
+ # Shorten the library search paths. On Windows library search paths may
+ # exceed the maximum path length.
+ shortened = shorten_path(library_path)
+ # Remember the library search paths.
+ self.library_paths.append(shortened)
+ out.append("-L{}".format(shortened))
- return True
+ return consumed
def _handle_linker_arg(self, arg, args, out):
if arg == "-Xlinker":
@@ -237,19 +226,15 @@ class Args:
self.rpaths.append(rpath)
def _handle_print_file_name(self, arg, args, out):
- if arg == "--print-file-name":
- print_file_name = next(args)
- elif arg.startswith("--print-file-name="):
- print_file_name = arg[len("--print-file-name="):]
- else:
- return False
+ consumed, print_file_name = argument(arg, args, long = "--print-file-name")
- # Remember print-file-name action. Don't forward to allow for later
- # manipulation.
- self.print_file_name = print_file_name
- self.action = Args.PRINT_FILE_NAME
+ if consumed:
+ # Remember print-file-name action. Don't forward to allow for later
+ # manipulation.
+ self.print_file_name = print_file_name
+ self.action = Args.PRINT_FILE_NAME
- return True
+ return consumed
def _handle_compile(self, arg, args, out):
if arg == "-c":
@@ -261,6 +246,42 @@ class Args:
return True
+def argument(arg, args, short = None, long = None):
+ """Parse an argument that takes a parameter.
+
+ I.e. arguments such as
+ -l <library>
+ -l<library>
+ --library <library>
+ --library=<library>
+
+ Args:
+ arg: The current command-line argument.
+ args: Iterator over the remaining arguments.
+ short: The short argument name, e.g. "-l".
+ long: The long argument name, e.g. "--library".
+
+ Returns:
+ consumed, value
+ consumed: bool, Whether the argument matched.
+ value: string, The value parameter or None.
+
+ """
+ if short:
+ if arg == short:
+ return True, next(args)
+ elif arg.startswith(short):
+ return True, arg[len(short):]
+
+ if long:
+ if arg == long:
+ return True, next(args)
+ elif arg.startswith(long + "="):
+ return True, arg[len(long + "="):]
+
+ return False, None
+
+
def load_response_files(args):
"""Generator that loads arguments from response files.
From 570f47be9c30e3580f45581adf3c230a252141d1 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Wed, 28 Aug 2019 14:00:56 +0200
Subject: [PATCH 16/31] Handle -rpath= syntax
https://github.com/tweag/rules_haskell/pull/1039/files/71c2a6d5286a0e0802c78be3a1a8cf6e6ae8061d..a10b82374bc7b6f86844835b1656e06b449a0b0a#r318496719
---
haskell/private/cc_wrapper.py.tpl | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 9fedad97..d37d1d53 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -200,6 +200,8 @@ class Args:
if self._prev_ld_arg is None:
if ld_arg == "-rpath":
self._prev_ld_arg = ld_arg
+ elif ld_arg.startswith("-rpath="):
+ self._handle_rpath(ld_arg[len("-rpath="):], out)
else:
out.extend(["-Xlinker", ld_arg])
elif self._prev_ld_arg == "-rpath":
@@ -214,6 +216,8 @@ class Args:
if len(ld_args) == 2 and ld_args[0] == "-rpath":
self._handle_rpath(ld_args[1], out)
return True
+ elif len(ld_args) == 1 and ld_args[0].startswith("-rpath="):
+ self._handle_rpath(ld_args[0][len("-rpath="):])
else:
out.append(arg)
return True
From f0337047331bfb318734264f70a9171ed99762aa Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Wed, 28 Aug 2019 14:06:51 +0200
Subject: [PATCH 17/31] Explain -Xlinker and -Wl,...
https://github.com/tweag/rules_haskell/pull/1039/files/71c2a6d5286a0e0802c78be3a1a8cf6e6ae8061d..a10b82374bc7b6f86844835b1656e06b449a0b0a#r318495192
---
haskell/private/cc_wrapper.py.tpl | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index d37d1d53..595743d2 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -195,6 +195,10 @@ class Args:
return consumed
def _handle_linker_arg(self, arg, args, out):
+ # gcc allows to forward flags to the linker using either
+ # -Xlinker <flag>
+ # or
+ # -Wl,<flag1>,<flag2>...
if arg == "-Xlinker":
ld_arg = next(args)
if self._prev_ld_arg is None:
From 10672f840e302c17ba2e9bfbf4215ecf098705f5 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Wed, 28 Aug 2019 14:38:09 +0200
Subject: [PATCH 18/31] shorten_path: Path must exist
https://github.com/tweag/rules_haskell/pull/1039/files/71c2a6d5286a0e0802c78be3a1a8cf6e6ae8061d..a10b82374bc7b6f86844835b1656e06b449a0b0a#r318499680
---
haskell/private/cc_wrapper.py.tpl | 27 ++++++++++++++++-----------
1 file changed, 16 insertions(+), 11 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 595743d2..d92225eb 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -185,12 +185,16 @@ class Args:
consumed, library_path = argument(arg, args, short = "-L", long = "--library-path")
if consumed:
- # Shorten the library search paths. On Windows library search paths may
- # exceed the maximum path length.
- shortened = shorten_path(library_path)
- # Remember the library search paths.
- self.library_paths.append(shortened)
- out.append("-L{}".format(shortened))
+ # Skip non-existent library search paths. These can occur in static
+ # linking mode where dynamic libraries are not present in the
+ # sandbox, or with Cabal packages with bogus library-path entries.
+ if os.path.exists(library_path):
+ # Shorten the library search paths. On Windows library search
+ # paths may exceed the maximum path length.
+ shortened = shorten_path(library_path)
+ # Remember the library search paths.
+ self.library_paths.append(shortened)
+ out.append("-L{}".format(shortened))
return consumed
@@ -331,13 +335,12 @@ def shorten_path(input_path):
- Resolve symbolic links.
Args:
- input_path: The path to shorten.
+ input_path: The path to shorten, must exist.
Returns:
The shortened path.
"""
- exists = os.path.exists(input_path)
shortened = input_path
# Try relativizing to current working directory.
@@ -351,7 +354,7 @@ def shorten_path(input_path):
# Ensure that the path is still correct. Reducing up-level references
# may change the meaning of the path in the presence of symbolic links.
try:
- if not exists or os.path.samefile(norm, shortened):
+ if os.path.samefile(norm, shortened):
shortened = norm
except IOError:
# stat may fail if the path became invalid or does not exist.
@@ -363,7 +366,7 @@ def shorten_path(input_path):
if len(real) < len(shortened):
shortened = real
except IOError:
- # realpath may fail if the path does not exist.
+ # may fail if the path does not exist or on dangling symlinks.
pass
return shortened
@@ -656,7 +659,9 @@ def resolve_rpath(rpath, output):
else:
rpath = os.path.abspath(shorten_path(min(candidates)))
else:
- rpath = os.path.abspath(shorten_path(rpath))
+ if os.path.exists(rpath):
+ rpath = shorten_path(rpath)
+ rpath = os.path.abspath(rpath)
return rpath, rpath
else:
From eeac689c205a6d39227f35b19e34b7e8cb46eb75 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 09:51:17 +0200
Subject: [PATCH 19/31] cc_wrapper: Args.action --> Args._action
It's an internal attribute and should not be accessed directly, instead
clients should use `Args.linking`, `Args.compiling`, or
`Args.printing_file_name`.
https://github.com/tweag/rules_haskell/pull/1039/files#r318549754
---
haskell/private/cc_wrapper.py.tpl | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index d92225eb..fdfd0fa6 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -68,8 +68,9 @@ class Args:
Attrs:
args: The collected and transformed arguments.
- linking: The action is linking.
- printing_file_name: The action is print-file-name.
+ linking: Gcc is called for linking (default).
+ compiling: Gcc is called for compiling (-c).
+ printing_file_name: Gcc is called with --print-file-name.
output: The output binary or library when linking.
library_paths: The library search paths when linking.
@@ -95,12 +96,16 @@ class Args:
args: Iterable over command-line arguments.
"""
- self.action = Args.LINK
self.print_file_name = None
self.libraries = []
self.library_paths = []
self.rpaths = []
self.output = None
+ # gcc action, print-file-name (--print-file-name), compile (-c), or
+ # link (default)
+ self._action = Args.LINK
+ # The currently active linker option that expects an argument. E.g. if
+ # `-Xlinker -rpath` was encountered, then `-rpath`.
self._prev_ld_arg = None
self.args = list(self._handle_args(args))
@@ -113,17 +118,17 @@ class Args:
@property
def linking(self):
"""Whether this is a link invocation."""
- return self.action == Args.LINK and self.output is not None
+ return self._action == Args.LINK and self.output is not None
@property
def compiling(self):
"""Whether this is a compile invocation."""
- return self.action == Args.COMPILE
+ return self._action == Args.COMPILE
@property
def printing_file_name(self):
"""Whether this is a print-file-name invocation."""
- return self.action == Args.PRINT_FILE_NAME and self.print_file_name is not None
+ return self._action == Args.PRINT_FILE_NAME and self.print_file_name is not None
def _handle_args(self, args):
"""Argument handling pipeline.
@@ -244,13 +249,13 @@ class Args:
# Remember print-file-name action. Don't forward to allow for later
# manipulation.
self.print_file_name = print_file_name
- self.action = Args.PRINT_FILE_NAME
+ self._action = Args.PRINT_FILE_NAME
return consumed
def _handle_compile(self, arg, args, out):
if arg == "-c":
- self.action = Args.COMPILE
+ self._action = Args.COMPILE
out.append(arg)
else:
return False
From b7744580d558ae9ce24a9e6ae3543cc0276fec76 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 10:02:45 +0200
Subject: [PATCH 20/31] fix typo
---
haskell/private/cc_wrapper.py.tpl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index fdfd0fa6..d82ec9e1 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -510,7 +510,7 @@ def darwin_shorten_rpaths(rpaths, libraries, output):
def sort_rpaths(rpaths):
"""Sort RUNPATHs by preference.
- Preference in decsending order:
+ Preference in descending order:
- Relative to target
- Absolute path
- Relative to CWD
From 4e35278afb0bc8f970efa9a0c9771ddf6ce2b435 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 11:01:53 +0200
Subject: [PATCH 21/31] Improve library search code
Use clearer variable names and add comments to improve readability.
https://github.com/tweag/rules_haskell/pull/1039/files#r318540827
---
haskell/private/cc_wrapper.py.tpl | 89 +++++++++++++++++++------------
1 file changed, 55 insertions(+), 34 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index d82ec9e1..0bf4029b 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -430,19 +430,22 @@ def shorten_rpaths(rpaths, libraries, output):
"""
input_rpaths = sort_rpaths(rpaths)
- missing = set(libraries)
- rpaths = []
+ # Keeps track of libraries that were not yet found in an rpath.
+ libs_still_missing = set(libraries)
+ # Keeps track of rpaths in which we found libraries.
+ required_rpaths = []
+ # Iterate over the given rpaths until all libraries are found.
for rpath in input_rpaths:
- if not missing:
+ if not libs_still_missing:
break
rpath, rpath_dir = resolve_rpath(rpath, output)
- found, missing = find_library(missing, rpath_dir)
+ found, libs_still_missing = find_library(libs_still_missing, rpath_dir)
if found:
- rpaths.append(rpath)
+ required_rpaths.append(rpath)
- return rpaths
+ return required_rpaths
def darwin_shorten_rpaths(rpaths, libraries, output):
@@ -463,9 +466,12 @@ def darwin_shorten_rpaths(rpaths, libraries, output):
"""
input_rpaths = sort_rpaths(rpaths)
- missing = set(libraries)
- rpaths = []
+ # Keeps track of libraries that were not yet found in an rpath.
+ libs_still_missing = set(libraries)
+ # Keeps track of rpaths in which we found libraries.
+ required_rpaths = []
+ # Keeps track of required rewrites of load commands.
rewrites = []
# References to core libs take up much space. Consider detecting the GHC
@@ -473,38 +479,45 @@ def darwin_shorten_rpaths(rpaths, libraries, output):
# that. Alternatively, https://github.com/bazelbuild/bazel/pull/8888 would
# also avoid this issue.
- # Determine solib dir and rewrite load commands relative to solib dir.
- # This allows to replace potentially many rpaths by one.
+ # Determine solib dir and rewrite load commands relative to solib dir. This
+ # allows to replace potentially many rpaths by a single one. On macOS load
+ # commands can use paths relative to rpath entries, e.g.
+ # `@rpath/some_dir/libsome_lib.dylib`.
solib_rpath = find_solib_rpath(input_rpaths, output)
- if missing and solib_rpath is not None:
+ if libs_still_missing and solib_rpath is not None:
solib_rpath, solib_dir = resolve_rpath(solib_rpath, output)
- found, missing = find_library_recursive(missing, solib_dir)
+ found, libs_still_missing = find_library_recursive(libs_still_missing, solib_dir)
if found:
- rpaths.append(solib_rpath)
+ required_rpaths.append(solib_rpath)
for f in found.values():
+ # Determine rewrites of load commands to load libraries
+ # relative to the solib dir rpath entry.
soname = darwin_get_install_name(os.path.join(solib_dir, f))
rewrites.append((soname, f))
# For the remaining missing libraries, determine which rpaths are required.
+ # Iterate over the given rpaths until all libraries are found.
for rpath in input_rpaths:
- if not missing:
+ if not libs_still_missing:
break
rpath, rpath_dir = resolve_rpath(rpath, output)
- found, missing = find_library(missing, rpath_dir)
- # Libraries with an absolute install_name don't require an rpath entry.
+ found, libs_still_missing = find_library(libs_still_missing, rpath_dir)
+ # Libraries with an absolute install_name don't require an rpath entry
+ # and can be filtered out.
found = dict(itertools.filterfalse(
lambda item: os.path.isabs(darwin_get_install_name(os.path.join(rpath_dir, item[1]))),
found.items()))
if len(found) == 1:
- # Avoid unnecessary rpath if it is only relevant for one load command.
+ # If the rpath is only needed for one load command, then we can
+ # avoid the rpath entry by fusing the rpath into the load command.
[filename] = found.values()
soname = darwin_get_install_name(os.path.join(rpath_dir, filename))
rewrites.append((soname, os.path.join(rpath, filename)))
elif found:
- rpaths.append(rpath)
+ required_rpaths.append(rpath)
- return rpaths, rewrites
+ return required_rpaths, rewrites
def sort_rpaths(rpaths):
@@ -565,26 +578,30 @@ def find_library_recursive(libraries, directory):
directory: Root of directory tree.
Returns:
- (found, missing):
+ (found, libs_still_missing):
found: Dict of found libraries {libname: path} relative to directory.
- missing: Set of remaining missing libraries.
+ libs_still_missing: Set of remaining missing libraries.
"""
- missing = set(libraries)
+ # Keeps track of libraries that were not yet found underneath directory.
+ libs_still_missing = set(libraries)
+ # Keeps track of libraries that were already found.
found = {}
+ # Iterate over the directory tree until all libraries are found.
for root, _, files in os.walk(directory, followlinks=True):
prefix = os.path.relpath(root, directory)
- if not missing:
+ if not libs_still_missing:
break
for f in files:
libname = get_lib_name(f)
- if libname and libname in missing:
+ if libname and libname in libs_still_missing:
found[libname] = os.path.join(prefix, f) if prefix != "." else f
- missing.discard(libname)
- if not missing:
+ libs_still_missing.discard(libname)
+ if not libs_still_missing:
+ # Short-cut files iteration if no more libs are missing.
break
- return found, missing
+ return found, libs_still_missing
def find_library(libraries, directory):
@@ -595,23 +612,27 @@ def find_library(libraries, directory):
directory: The directory in which to search for libraries.
Returns:
- (found, missing):
+ (found, libs_still_missing):
found: Dict of found libraries {libname: path} relative to directory.
- missing: Set of remaining missing libraries.
+ libs_still_missing: Set of remaining missing libraries.
"""
- missing = set(libraries)
+ # Keeps track of libraries that were not yet found within directory.
+ libs_still_missing = set(libraries)
+ # Keeps track of libraries that were already found.
found = {}
+ # Iterate over the files within directory until all libraries are found.
+ # This corresponds to a one level deep os.walk.
for _, _, files in itertools.islice(os.walk(directory), 1):
- if not missing:
+ if not libs_still_missing:
break
for f in files:
libname = get_lib_name(f)
- if libname and libname in missing:
+ if libname and libname in libs_still_missing:
found[libname] = f
- missing.discard(libname)
+ libs_still_missing.discard(libname)
- return found, missing
+ return found, libs_still_missing
def get_lib_name(filename):
From 178db6da0a04bef66e35d554da69e5bae44ef3bd Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 11:45:00 +0200
Subject: [PATCH 22/31] Add test-case to verify _solib_<cpu> assumption
https://github.com/tweag/rules_haskell/pull/1039/files#r318542321
---
tests/solib_dir/BUILD.bazel | 10 +++++
tests/solib_dir/solib_test.bzl | 71 ++++++++++++++++++++++++++++++++++
2 files changed, 81 insertions(+)
create mode 100644 tests/solib_dir/BUILD.bazel
create mode 100644 tests/solib_dir/solib_test.bzl
diff --git a/tests/solib_dir/BUILD.bazel b/tests/solib_dir/BUILD.bazel
new file mode 100644
index 00000000..e708e687
--- /dev/null
+++ b/tests/solib_dir/BUILD.bazel
@@ -0,0 +1,10 @@
+load(":solib_test.bzl", "solib_test")
+
+# See rule docstring in solib_test.bzl for details.
+solib_test(
+ name = "solib_dir",
+ is_windows = select({
+ "@rules_haskell//haskell/platforms:mingw32": True,
+ "//conditions:default": False,
+ }),
+)
diff --git a/tests/solib_dir/solib_test.bzl b/tests/solib_dir/solib_test.bzl
new file mode 100644
index 00000000..a671f4c3
--- /dev/null
+++ b/tests/solib_dir/solib_test.bzl
@@ -0,0 +1,71 @@
+_test_script_template = """#!/usr/bin/env bash
+library_path={library_path}
+is_windows={is_windows}
+expected="bin/_solib_{cpu}"
+if [[ "$is_windows" != 1 && ! $library_path =~ ^.*"/$expected/".*$ ]]; then
+ echo "Expected library path containing directory '$expected'," >&2
+ echo "but got: '$library_path'." >&2
+ exit 1
+fi
+"""
+
+def _solib_test_impl(ctx):
+ # Write a dummy dynamic library. It will never be loaded, we're only
+ # interested in the paths that Bazel generates.
+ dynamic_library = ctx.actions.declare_file("lib{}.so".format(ctx.label.name))
+ ctx.actions.write(
+ content = "",
+ is_executable = False,
+ output = dynamic_library,
+ )
+
+ # XXX Workaround https://github.com/bazelbuild/bazel/issues/6874.
+ # Should be find_cpp_toolchain() instead.
+ cc_toolchain = ctx.attr._cc_toolchain[cc_common.CcToolchainInfo]
+ feature_configuration = cc_common.configure_features(
+ ctx = ctx,
+ cc_toolchain = cc_toolchain,
+ requested_features = ctx.features,
+ unsupported_features = ctx.disabled_features,
+ )
+ lib_to_link = cc_common.create_library_to_link(
+ actions = ctx.actions,
+ feature_configuration = feature_configuration,
+ dynamic_library = dynamic_library,
+ cc_toolchain = cc_toolchain,
+ )
+
+ # Write the test script.
+ test_script = ctx.actions.declare_file(ctx.label.name)
+ ctx.actions.write(
+ content = _test_script_template.format(
+ cpu = cc_toolchain.cpu,
+ library_path = lib_to_link.dynamic_library.path,
+ is_windows = "1" if ctx.attr.is_windows else "",
+ ),
+ is_executable = True,
+ output = test_script,
+ )
+ return [DefaultInfo(
+ executable = test_script,
+ )]
+
+solib_test = rule(
+ _solib_test_impl,
+ attrs = {
+ "is_windows": attr.bool(),
+ "_cc_toolchain": attr.label(
+ default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
+ ),
+ },
+ executable = True,
+ fragments = ["cpp"],
+ test = True,
+)
+"""Test that Bazel's solib directory matches our expectations.
+
+The cc_wrapper used by rules_haskell (haskell/private/cc_wrapper.py.tpl)
+assumes that Bazel generates symbolic links for dynamic libraries under a
+directory called `bin/_solib_<cpu>` on Darwin and Linux. This rule generates a
+test-case that fails if this assumption is not met.
+"""
From 84ef3de172c59c1d1f40d4f32261503e951006c7 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 12:04:59 +0200
Subject: [PATCH 23/31] Explain print-file-name
https://github.com/tweag/rules_haskell/pull/1039/files#r318547022
---
haskell/private/cc_wrapper.py.tpl | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 0bf4029b..84766d5f 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -723,6 +723,14 @@ def darwin_rewrite_load_commands(rewrites, output):
def print_file_name(filename, args):
"""Execute the print-file-name action.
+ From gcc(1)
+
+ -print-file-name=library
+ Print the full absolute name of the library file library that would
+ be used when linking---and don't do anything else. With this
+ option, GCC does not compile or link anything; it just prints the
+ file name.
+
Args:
filename: The queried filename.
args: The remaining arguments.
From 05300f170fd7f28868237d377b9f45cc32ff94c9 Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 12:10:09 +0200
Subject: [PATCH 24/31] Pass platform from Bazel
https://github.com/tweag/rules_haskell/pull/1039/files#r318662838
---
haskell/private/cc_wrapper.bzl | 12 ++++++++++++
haskell/private/cc_wrapper.py.tpl | 21 +++++++++++++++------
2 files changed, 27 insertions(+), 6 deletions(-)
diff --git a/haskell/private/cc_wrapper.bzl b/haskell/private/cc_wrapper.bzl
index ee185aab..3aefd1f1 100644
--- a/haskell/private/cc_wrapper.bzl
+++ b/haskell/private/cc_wrapper.bzl
@@ -21,6 +21,7 @@ def _cc_wrapper_impl(ctx):
substitutions = {
"{:cc:}": cc,
"{:workspace:}": ctx.workspace_name,
+ "{:platform:}": ctx.attr.platform,
},
)
return [DefaultInfo(
@@ -37,6 +38,7 @@ _cc_wrapper = rule(
"template": attr.label(
allow_single_file = True,
),
+ "platform": attr.string(),
"_cc_toolchain": attr.label(
default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"),
),
@@ -48,10 +50,20 @@ def cc_wrapper(name, **kwargs):
_cc_wrapper(
name = name + ".py",
template = "@rules_haskell//haskell:private/cc_wrapper.py.tpl",
+ platform = select({
+ "@rules_haskell//haskell/platforms:darwin": "darwin",
+ "@rules_haskell//haskell/platforms:mingw32": "windows",
+ "//conditions:default": "linux",
+ }),
)
_cc_wrapper(
name = name + ".sh",
template = "@rules_haskell//haskell:private/cc_wrapper.sh.tpl",
+ platform = select({
+ "@rules_haskell//haskell/platforms:darwin": "darwin",
+ "@rules_haskell//haskell/platforms:mingw32": "windows",
+ "//conditions:default": "linux",
+ }),
)
native.py_binary(
name = name + "-python",
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 84766d5f..beb983c5 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -35,7 +35,6 @@ from contextlib import contextmanager
import glob
import itertools
import os
-import platform
import shlex
import subprocess
import sys
@@ -43,6 +42,7 @@ import tempfile
WORKSPACE = "{:workspace:}"
CC = "{:cc:}"
+PLATFORM = "{:platform:}"
INSTALL_NAME_TOOL = "/usr/bin/install_name_tool"
OTOOL = "/usr/bin/otool"
@@ -530,11 +530,10 @@ def sort_rpaths(rpaths):
"""
def rpath_priority(rpath):
- system = platform.system()
- if system == "Darwin":
+ if is_darwin():
if rpath.startswith("@loader_path"):
return 0
- elif system == "Linux":
+ elif is_linux():
if rpath.startswith("$ORIGIN"):
return 0
if os.path.isabs(rpath):
@@ -811,7 +810,7 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
# script using Bazel runfiles.
r = bazel_runfiles.Create()
cc = r.Rlocation("/".join([WORKSPACE, CC]))
- if cc is None and platform.system() == "Windows":
+ if cc is None and is_windows():
# We must use "/" instead of os.path.join on Windows, because the
# Bazel runfiles_manifest file uses "/" separators.
cc = r.Rlocation("/".join([WORKSPACE, CC + ".exe"]))
@@ -879,7 +878,17 @@ def generate_response_line(arg):
def is_darwin():
"""Whether the execution platform is Darwin."""
- return platform.system() == "Darwin"
+ return PLATFORM == "darwin"
+
+
+def is_linux():
+ """Whether the execution platform is Linux."""
+ return PLATFORM == "linux"
+
+
+def is_windows():
+ """Whether the execution platform is Windows."""
+ return PLATFORM == "windows"
def is_temporary_output(output):
From 232fb6e5b255844fcdb0b1f98af800f83a80359b Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 12:21:33 +0200
Subject: [PATCH 25/31] Write to buffer to avoid decode
https://github.com/tweag/rules_haskell/pull/1039#discussion_r318545150
---
haskell/private/cc_wrapper.py.tpl | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index beb983c5..33d4bd95 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -833,8 +833,8 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
if exit_on_error and returncode != 0:
if capture_output:
- sys.stdout.write(stdout.decode())
- sys.stderr.write(stderr.decode())
+ sys.stdout.buffer.write(stdout)
+ sys.stderr.buffer.write(stderr)
sys.exit(returncode)
return (returncode, stdoutbuf, stderrbuf)
From 91bd8c2fcd2ff52b008c5912f57b1c7a85047c0f Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 13:22:23 +0200
Subject: [PATCH 26/31] cc_wrapper.sh --> cc_wrapper_windows.sh
https://github.com/tweag/rules_haskell/pull/1039/files#r318669187
---
haskell/BUILD.bazel | 2 +-
haskell/private/cc_wrapper.bzl | 2 +-
.../private/{cc_wrapper.sh.tpl => cc_wrapper_windows.sh.tpl} | 0
3 files changed, 2 insertions(+), 2 deletions(-)
rename haskell/private/{cc_wrapper.sh.tpl => cc_wrapper_windows.sh.tpl} (100%)
diff --git a/haskell/BUILD.bazel b/haskell/BUILD.bazel
index dd3cfc44..be61737a 100644
--- a/haskell/BUILD.bazel
+++ b/haskell/BUILD.bazel
@@ -15,7 +15,7 @@ exports_files(
"private/ghci_repl_wrapper.sh",
"private/haddock_wrapper.sh.tpl",
"private/cc_wrapper.py.tpl",
- "private/cc_wrapper.sh.tpl",
+ "private/cc_wrapper_windows.sh.tpl",
"private/pkgdb_to_bzl.py",
],
)
diff --git a/haskell/private/cc_wrapper.bzl b/haskell/private/cc_wrapper.bzl
index 3aefd1f1..10102961 100644
--- a/haskell/private/cc_wrapper.bzl
+++ b/haskell/private/cc_wrapper.bzl
@@ -58,7 +58,7 @@ def cc_wrapper(name, **kwargs):
)
_cc_wrapper(
name = name + ".sh",
- template = "@rules_haskell//haskell:private/cc_wrapper.sh.tpl",
+ template = "@rules_haskell//haskell:private/cc_wrapper_windows.sh.tpl",
platform = select({
"@rules_haskell//haskell/platforms:darwin": "darwin",
"@rules_haskell//haskell/platforms:mingw32": "windows",
diff --git a/haskell/private/cc_wrapper.sh.tpl b/haskell/private/cc_wrapper_windows.sh.tpl
similarity index 100%
rename from haskell/private/cc_wrapper.sh.tpl
rename to haskell/private/cc_wrapper_windows.sh.tpl
From a97c521226c22a5b73c9792718b7daee18f8899b Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 13:32:01 +0200
Subject: [PATCH 27/31] Expand sort_rpaths docstring
https://github.com/tweag/rules_haskell/pull/1039/files#r319002515
---
haskell/private/cc_wrapper.py.tpl | 19 +++++++++++++++----
1 file changed, 15 insertions(+), 4 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 33d4bd95..a2f1e1ab 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -523,10 +523,21 @@ def darwin_shorten_rpaths(rpaths, libraries, output):
def sort_rpaths(rpaths):
"""Sort RUNPATHs by preference.
- Preference in descending order:
- - Relative to target
- - Absolute path
- - Relative to CWD
+ We classify three types of rpaths (in descending order of preference):
+ - relative to output, i.e. $ORIGIN/... or @loader_path/...
+ - absolute, e.g. /nix/store/...
+ - relative, e.g. bazel-out/....
+
+ We prefer rpaths relative to the output. They tend to be shorter, and they
+ typically involve Bazel's _solib_* directory which bundles lots of
+ libraries (meaning less rpaths required). They're also less likely to leak
+ information about the local installation into the Bazel cache.
+
+ Next, we prefer absolute paths. They function regardless of execution
+ directory, and they are still likely to play well with the cache, e.g.
+ /nix/store/... or /usr/lib/....
+
+ Finally, we fall back to relative rpaths.
"""
def rpath_priority(rpath):
From a1ef96b6aaada165fbb8303ff3808e5b1924f62c Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 13:36:11 +0200
Subject: [PATCH 28/31] Explain @rpath load commands
https://github.com/tweag/rules_haskell/pull/1039/files#r319002635
---
haskell/private/cc_wrapper.py.tpl | 25 +++++++++++++++++++++----
1 file changed, 21 insertions(+), 4 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index a2f1e1ab..3a0c2c5b 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -479,10 +479,27 @@ def darwin_shorten_rpaths(rpaths, libraries, output):
# that. Alternatively, https://github.com/bazelbuild/bazel/pull/8888 would
# also avoid this issue.
- # Determine solib dir and rewrite load commands relative to solib dir. This
- # allows to replace potentially many rpaths by a single one. On macOS load
- # commands can use paths relative to rpath entries, e.g.
- # `@rpath/some_dir/libsome_lib.dylib`.
+ # Determine solib dir and rewrite load commands relative to solib dir.
+ #
+ # This allows to replace potentially many rpaths by a single one on Darwin.
+ # Namely, Darwin allows to explicitly refer to the rpath in load commands.
+ # E.g.
+ #
+ # LOAD @rpath/somedir/libsomelib.dylib
+ #
+ # With that we can avoid multiple rpath entries of the form
+ #
+ # RPATH @loader_path/.../_solib_*/mangled_a
+ # RPATH @loader_path/.../_solib_*/mangled_b
+ # RPATH @loader_path/.../_solib_*/mangled_c
+ #
+ # And instead have a single rpath and load commands as follows
+ #
+ # RPATH @loader_path/.../_solib_*
+ # LOAD @rpath/mangled_a/lib_a
+ # LOAD @rpath/mangled_b/lib_b
+ # LOAD @rpath/mangled_c/lib_c
+ #
solib_rpath = find_solib_rpath(input_rpaths, output)
if libs_still_missing and solib_rpath is not None:
solib_rpath, solib_dir = resolve_rpath(solib_rpath, output)
From 88aa0d45daae5cea0870ffb25e0281dbe430f97e Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 13:42:34 +0200
Subject: [PATCH 29/31] Factor out find_cc
https://github.com/tweag/rules_haskell/pull/1039/files#r318665931
---
haskell/private/cc_wrapper.py.tpl | 37 ++++++++++++++++++-------------
1 file changed, 22 insertions(+), 15 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 3a0c2c5b..8e76c53e 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -809,6 +809,27 @@ def run_cc_print_file_name(filename, args):
# --------------------------------------------------------------------
+def find_cc():
+ """Find the path to the actual compiler executable."""
+ if os.path.isfile(CC):
+ cc = CC
+ else:
+ # On macOS CC is a relative path to a wrapper script. If we're
+ # being called from a GHCi REPL then we need to find this wrapper
+ # script using Bazel runfiles.
+ r = bazel_runfiles.Create()
+ cc = r.Rlocation("/".join([WORKSPACE, CC]))
+ if cc is None and is_windows():
+ # We must use "/" instead of os.path.join on Windows, because the
+ # Bazel runfiles_manifest file uses "/" separators.
+ cc = r.Rlocation("/".join([WORKSPACE, CC + ".exe"]))
+ if cc is None:
+ sys.stderr.write("CC not found '{}'.\n".format(CC))
+ sys.exit(1)
+
+ return cc
+
+
def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
"""Execute cc with a response file holding the given arguments.
@@ -830,21 +851,7 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
new_kwargs.update(kwargs)
kwargs = new_kwargs
- if os.path.isfile(CC):
- cc = CC
- else:
- # On macOS CC is a relative path to a wrapper script. If we're
- # being called from a GHCi REPL then we need to find this wrapper
- # script using Bazel runfiles.
- r = bazel_runfiles.Create()
- cc = r.Rlocation("/".join([WORKSPACE, CC]))
- if cc is None and is_windows():
- # We must use "/" instead of os.path.join on Windows, because the
- # Bazel runfiles_manifest file uses "/" separators.
- cc = r.Rlocation("/".join([WORKSPACE, CC + ".exe"]))
- if cc is None:
- sys.stderr.write("CC not found '{}'.\n".format(CC))
- sys.exit(1)
+ cc = find_cc()
stdoutbuf = None
stderrbuf = None
From 99d601f92541fbf6ad9d86096c426b72c6acdc2d Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Thu, 29 Aug 2019 13:47:26 +0200
Subject: [PATCH 30/31] read --> read -r
https://github.com/tweag/rules_haskell/pull/1039/files#r318989507
---
haskell/private/cc_wrapper_windows.sh.tpl | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/haskell/private/cc_wrapper_windows.sh.tpl b/haskell/private/cc_wrapper_windows.sh.tpl
index 86245085..eb7c363e 100644
--- a/haskell/private/cc_wrapper_windows.sh.tpl
+++ b/haskell/private/cc_wrapper_windows.sh.tpl
@@ -135,7 +135,7 @@ handle_arg() {
handle_lib_dir "$arg"
elif [[ "$arg" =~ ^@(.*)$ ]]; then
IN_RESPONSE_FILE=1
- while read line; do
+ while read -r line; do
handle_arg "$line"
done < "${BASH_REMATCH[1]}"
IN_RESPONSE_FILE=
From a4f40a81d7bcc3fdfbcb33f810189f4f23580e2e Mon Sep 17 00:00:00 2001
From: Andreas Herrmann <andreas.herrmann@tweag.io>
Date: Mon, 2 Sep 2019 15:23:14 +0200
Subject: [PATCH 31/31] Avoid response files for performance
Reduces the runtime of the cc_wrapper in nix-shell by 60% in case of
short overall cc_wrapper execution time.
---
haskell/private/cc_wrapper.py.tpl | 36 ++++++++++++++++++++-----------
1 file changed, 24 insertions(+), 12 deletions(-)
diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl
index 8e76c53e..bbfbea33 100644
--- a/haskell/private/cc_wrapper.py.tpl
+++ b/haskell/private/cc_wrapper.py.tpl
@@ -853,26 +853,38 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs):
cc = find_cc()
- stdoutbuf = None
- stderrbuf = None
-
- with response_file(args) as rsp:
+ def _run_cc(args):
# subprocess.run is not supported in the bindist CI setup.
# subprocess.Popen does not support context manager on CI setup.
- proc = subprocess.Popen([cc, "@" + rsp], **kwargs)
+ proc = subprocess.Popen([cc] + args, **kwargs)
if capture_output:
(stdoutbuf, stderrbuf) = proc.communicate()
+ else:
+ stdoutbuf = None
+ stderrbuf = None
returncode = proc.wait()
- if exit_on_error and returncode != 0:
- if capture_output:
- sys.stdout.buffer.write(stdout)
- sys.stderr.buffer.write(stderr)
- sys.exit(returncode)
-
- return (returncode, stdoutbuf, stderrbuf)
+ if exit_on_error and returncode != 0:
+ if capture_output:
+ sys.stdout.buffer.write(stdoutbuf)
+ sys.stderr.buffer.write(stderrbuf)
+ sys.exit(returncode)
+
+ return (returncode, stdoutbuf, stderrbuf)
+
+ # Too avoid exceeding the OS command-line length limit we use response
+ # files. However, creating and removing temporary files causes overhead.
+ # For performance reasons we only create response files if there is a risk
+ # of exceeding the OS command-line length limit. For short cc_wrapper calls
+ # avoiding the response file reduces the runtime by about 60%.
+ if sum(map(len, args)) < 8000:
+ # Windows has the shortest command-line length limit at 8191 characters.
+ return _run_cc(args)
+ else:
+ with response_file(args) as rsp:
+ return _run_cc(["@" + rsp])
@contextmanager
Reference in New Issue View Git Blame Copy Permalink