From 15ec6f2bfe891b6437e760ba9cc72af4df30b278 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 4 Jul 2019 16:12:05 +0200 Subject: [PATCH 01/31] Windows use cc.tools.cc --- haskell/toolchain.bzl | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl index 02d2ca3a..5984cf08 100644 --- a/haskell/toolchain.bzl +++ b/haskell/toolchain.bzl @@ -38,29 +38,27 @@ def _run_ghc(hs, cc, inputs, outputs, mnemonic, arguments, params_file = None, e args.add(hs.tools.ghc) extra_inputs += [hs.tools.ghc] - # Do not use Bazel's CC toolchain on Windows, as it leads to linker and librarty compatibility issues. # XXX: We should also tether Bazel's CC toolchain to GHC's, so that we can properly mix Bazel-compiled # C libraries with Haskell targets. - if not hs.toolchain.is_windows: - args.add_all([ - # GHC uses C compiler for assemly, linking and preprocessing as well. - "-pgma", - cc.tools.cc, - "-pgmc", - cc.tools.cc, - "-pgml", - cc.tools.cc, - "-pgmP", - cc.tools.cc, - # Setting -pgm* flags explicitly has the unfortunate side effect - # of resetting any program flags in the GHC settings file. So we - # restore them here. See - # https://ghc.haskell.org/trac/ghc/ticket/7929. - "-optc-fno-stack-protector", - "-optP-E", - "-optP-undef", - "-optP-traditional", - ]) + args.add_all([ + # GHC uses C compiler for assemly, linking and preprocessing as well. + "-pgma", + cc.tools.cc, + "-pgmc", + cc.tools.cc, + "-pgml", + cc.tools.cc, + "-pgmP", + cc.tools.cc, + # Setting -pgm* flags explicitly has the unfortunate side effect + # of resetting any program flags in the GHC settings file. So we + # restore them here. See + # https://ghc.haskell.org/trac/ghc/ticket/7929. + "-optc-fno-stack-protector", + "-optP-E", + "-optP-undef", + "-optP-traditional", + ]) compile_flags_file = hs.actions.declare_file("compile_flags_%s_%s" % (hs.name, mnemonic)) extra_args_file = hs.actions.declare_file("extra_args_%s_%s" % (hs.name, mnemonic)) From ce2d7516d574997dade55505388cf600f1c0f5dc Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 4 Jul 2019 14:16:29 +0200 Subject: [PATCH 02/31] run_ghc: locale_archive as default input Always pass locale_archive to run_ghc if provided. --- haskell/private/actions/compile.bzl | 5 ----- haskell/toolchain.bzl | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/haskell/private/actions/compile.bzl b/haskell/private/actions/compile.bzl index ac8725f5..3dea77ce 100644 --- a/haskell/private/actions/compile.bzl +++ b/haskell/private/actions/compile.bzl @@ -230,10 +230,6 @@ def _compilation_defaults(hs, cc, java, dep_info, plugin_dep_info, cc_info, srcs compile_flags += cc.include_args - locale_archive_depset = ( - depset([hs.toolchain.locale_archive]) if hs.toolchain.locale_archive != None else depset() - ) - # This is absolutely required otherwise GHC doesn't know what package it's # creating `Name`s for to put them in Haddock interface files which then # results in Haddock not being able to find names for linking in @@ -342,7 +338,6 @@ def _compilation_defaults(hs, cc, java, dep_info, plugin_dep_info, cc_info, srcs plugin_dep_info.dynamic_libraries, ghci_extra_libs, java.inputs, - locale_archive_depset, preprocessors.inputs, plugin_tool_inputs, ]), diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl index 5984cf08..1e42b024 100644 --- a/haskell/toolchain.bzl +++ b/haskell/toolchain.bzl @@ -73,6 +73,9 @@ def _run_ghc(hs, cc, inputs, outputs, mnemonic, arguments, params_file = None, e extra_args_file, ] + cc.files + if hs.toolchain.locale_archive != None: + extra_inputs.append(hs.toolchain.locale_archive) + flagsfile = extra_args_file if params_file: flagsfile = merge_parameter_files(hs, extra_args_file, params_file) From b567d276201ca8de5bb94a7b36f1d38141b742bf Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 4 Jul 2019 10:27:47 +0200 Subject: [PATCH 03/31] Combined cc_wrapper for Windows and Unix - Shortens library search paths to stay below maximum path length on Windows. GHC generates library search paths that contain redundant up-level references (..). This can exceed the maximum path length on Windows, which will cause linking failures. This wrapper shortens library search paths to avoid that issue. - Shortens rpaths and load commands on macOS. The rpaths and load commands generated by GHC and Bazel can quickly exceed the MACH-O header size limit on macOS. This wrapper shortens and combines rpaths and load commands to avoid exceeding that limit. - Finds .so files if only .dylib are searched on macOS. Bazel's cc_library will generate .so files for dynamic libraries even on macOS. GHC strictly expects .dylib files on macOS. This wrapper hooks into gcc's --print-file-name feature to work around this mismatch in file extension. --- haskell/BUILD.bazel | 10 + haskell/cc.bzl | 22 +- haskell/private/actions/compile.bzl | 2 +- haskell/private/actions/link.bzl | 7 +- haskell/private/cc_wrapper.bzl | 56 ++ haskell/private/cc_wrapper.py.tpl | 812 ++++++++++++++++++++++++++++ haskell/providers.bzl | 14 +- haskell/toolchain.bzl | 17 + 8 files changed, 907 insertions(+), 33 deletions(-) create mode 100644 haskell/private/cc_wrapper.bzl create mode 100644 haskell/private/cc_wrapper.py.tpl diff --git a/haskell/BUILD.bazel b/haskell/BUILD.bazel index a7e39f75..41afb641 100644 --- a/haskell/BUILD.bazel +++ b/haskell/BUILD.bazel @@ -2,6 +2,10 @@ load( "@rules_haskell//haskell:private/haskell_impl.bzl", "haskell_toolchain_libraries", ) +load( + "@rules_haskell//haskell:private/cc_wrapper.bzl", + "cc_wrapper", +) exports_files( glob(["*.bzl"]) + [ @@ -10,6 +14,7 @@ exports_files( "private/coverage_wrapper.sh.tpl", "private/ghci_repl_wrapper.sh", "private/haddock_wrapper.sh.tpl", + "private/cc_wrapper.py.tpl", "private/osx_cc_wrapper.sh.tpl", "private/pkgdb_to_bzl.py", ], @@ -21,6 +26,11 @@ exports_files( visibility = ["//tests/unit-tests:__pkg__"], ) +cc_wrapper( + name = "cc_wrapper", + visibility = ["//visibility:public"], +) + py_binary( name = "pkgdb_to_bzl", srcs = ["private/pkgdb_to_bzl.py"], diff --git a/haskell/cc.bzl b/haskell/cc.bzl index cd426135..dfed4de3 100644 --- a/haskell/cc.bzl +++ b/haskell/cc.bzl @@ -20,6 +20,7 @@ CcInteropInfo = provider( # See the following for why this is needed: # https://stackoverflow.com/questions/52769846/custom-c-rule-with-the-cc-common-api "files": "Files for all tools (input to any action that uses tools)", + "manifests": "Input manifests for all tools (input to any action that uses tools)", "hdrs": "CC headers", "cpp_flags": "Preprocessor flags", "compiler_flags": "Flags for compilation", @@ -99,22 +100,10 @@ def cc_interop_info(ctx): # Generate cc wrapper script on Darwin that adjusts load commands. hs_toolchain = ctx.toolchains["@rules_haskell//haskell:toolchain"] - if hs_toolchain.is_darwin: - cc_wrapper = ctx.actions.declare_file("osx_cc_wrapper") - cc = cc_wrapper.path - ctx.actions.expand_template( - template = hs_toolchain.osx_cc_wrapper_tpl, - output = cc_wrapper, - substitutions = { - "%{cc}": cc_toolchain.compiler_executable(), - }, - ) - cc_files = ctx.files._cc_toolchain + [ - cc_wrapper, - ] - else: - cc = cc_toolchain.compiler_executable() - cc_files = ctx.files._cc_toolchain + cc_wrapper = hs_toolchain.cc_wrapper + cc = cc_wrapper.executable.path + cc_files = ctx.files._cc_toolchain + cc_wrapper.inputs.to_list() + cc_manifests = cc_wrapper.manifests # XXX Workaround https://github.com/bazelbuild/bazel/issues/6876. linker_flags = [flag for flag in linker_flags if flag not in ["-shared"]] @@ -139,6 +128,7 @@ def cc_interop_info(ctx): return CcInteropInfo( tools = struct(**tools), files = cc_files, + manifests = cc_manifests, hdrs = hdrs.to_list(), cpp_flags = cpp_flags, include_args = include_args, diff --git a/haskell/private/actions/compile.bzl b/haskell/private/actions/compile.bzl index 3dea77ce..ea0b5946 100644 --- a/haskell/private/actions/compile.bzl +++ b/haskell/private/actions/compile.bzl @@ -319,7 +319,7 @@ def _compilation_defaults(hs, cc, java, dep_info, plugin_dep_info, cc_info, srcs ) # Transitive library dependencies for runtime. - (ghci_extra_libs, ghc_env) = get_ghci_extra_libs(hs, cc_info, dynamic = False) + (ghci_extra_libs, ghc_env) = get_ghci_extra_libs(hs, cc_info) return struct( args = args, diff --git a/haskell/private/actions/link.bzl b/haskell/private/actions/link.bzl index 6f74bd23..996af781 100644 --- a/haskell/private/actions/link.bzl +++ b/haskell/private/actions/link.bzl @@ -352,15 +352,10 @@ def link_library_dynamic(hs, cc, dep_info, cc_info, extra_srcs, objects_dir, my_ ) args.add_all(pkg_info_args) - # When linking a dynamic library we still collect static libraries for - # dependencies where possible. This is so that a final binary that depends - # on this dynamic library, is linked statically itself, will not fail at - # link time due to missing transitive dynamic library dependencies. In this - # case transitive dependencies will still be linked in statically. (cache_file, static_libs, dynamic_libs) = create_link_config( hs = hs, cc_info = cc_info, - dynamic = False, + dynamic = True, pic = True, binary = dynamic_library, args = args, diff --git a/haskell/private/cc_wrapper.bzl b/haskell/private/cc_wrapper.bzl new file mode 100644 index 00000000..9913d806 --- /dev/null +++ b/haskell/private/cc_wrapper.bzl @@ -0,0 +1,56 @@ +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") +load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") +load("@os_info//:os_info.bzl", "is_linux") + +def _cc_wrapper_impl(ctx): + cc_toolchain = find_cpp_toolchain(ctx) + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + cc = cc_common.get_tool_for_action( + feature_configuration = feature_configuration, + action_name = ACTION_NAMES.c_compile, + ) + cc_wrapper = ctx.actions.declare_file(ctx.label.name + ".py") + ctx.actions.expand_template( + template = ctx.file.template, + output = cc_wrapper, + is_executable = True, + substitutions = { + "{:cc:}": cc, + }, + ) + return [DefaultInfo(files = depset([cc_wrapper]))] + +_cc_wrapper = rule( + implementation = _cc_wrapper_impl, + attrs = { + "template": attr.label( + allow_single_file = True, + default = Label("@rules_haskell//haskell:private/cc_wrapper.py.tpl"), + ), + "_cc_toolchain": attr.label( + default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), + ), + }, + fragments = ["cpp"], +) + +def cc_wrapper(name, template = None, **kwargs): + _cc_wrapper( + name = name + "-source", + template = template, + ) + native.py_binary( + name = name, + srcs = [name + "-source"], + main = name + "-source.py", + python_version = "PY3", + deps = [ + "@bazel_tools//tools/python/runfiles", + ], + **kwargs + ) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl new file mode 100644 index 00000000..2f71a23c --- /dev/null +++ b/haskell/private/cc_wrapper.py.tpl @@ -0,0 +1,812 @@ +#!/usr/bin/env python3 +"""CC toolchain wrapper + +Usage: cc_wrapper [ARG]... + +Wraps the C compiler of the Bazel CC toolchain. Transforms arguments to work +around limitations of Bazel and GHC and passes those via response file to the C +compiler. + +- Shortens library search paths to stay below maximum path length on Windows. + + GHC generates library search paths that contain redundant up-level + references (..). This can exceed the maximum path length on Windows, which + will cause linking failures. This wrapper shortens library search paths to + avoid that issue. + +- Shortens rpaths and load commands on macOS. + + The rpaths and load commands generated by GHC and Bazel can quickly exceed + the MACH-O header size limit on macOS. This wrapper shortens and combines + rpaths and load commands to avoid exceeding that limit. + +- Finds .so files if only .dylib are searched on macOS. + + Bazel's cc_library will generate .so files for dynamic libraries even on + macOS. GHC strictly expects .dylib files on macOS. This wrapper hooks into + gcc's --print-file-name feature to work around this mismatch in file + extension. + +""" + +from bazel_tools.tools.python.runfiles import runfiles as bazel_runfiles +from contextlib import contextmanager +import glob +import itertools +import os +import platform +import shlex +import subprocess +import sys +import tempfile + +CC = "{:cc:}" +INSTALL_NAME_TOOL = "/usr/bin/install_name_tool" +OTOOL = "/usr/bin/otool" + + +def main(): + parsed = Args(load_response_files(sys.argv[1:])) + + if parsed.linking: + link(parsed.output, parsed.libraries, parsed.rpaths, parsed.args) + elif parsed.printing_file_name: + print_file_name(parsed.print_file_name, parsed.args) + else: + run_cc(parsed.args, exit_on_error=True) + + +# -------------------------------------------------------------------- +# Parse arguments + + +class Args: + """Parsed command-line arguments. + + Attrs: + args: The collected and transformed arguments. + + linking: The action is linking. + printing_file_name: The action is print-file-name. + + output: The output binary or library when linking. + library_paths: The library search paths when linking. + libraries: The required libraries when linking. + rpaths: The provided rpaths when linking. + + print_file_name: The queried file name on print-file-name. + + """ + LINK = "link" + COMPILE = "compile" + PRINT_FILE_NAME = "print-file-name" + + def __init__(self, args): + """Parse the given arguments into an Args object. + + - Shortens library search paths. + - Detects the requested action. + - Keeps rpath arguments for further processing when linking. + - Keeps print-file-name arguments for further processing. + + Args: + args: Iterable over command-line arguments. + + """ + self.action = Args.LINK + self.print_file_name = None + self.libraries = [] + self.library_paths = [] + self.rpaths = [] + self.output = None + self._prev_ld_arg = None + + self.args = list(self._handle_args(args)) + + if not self.linking: + # We don't expect rpath arguments if not linking, however, just in + # case, forward them if we don't mean to modify them. + self.args.extend(rpath_args(self.rpaths)) + + @property + def linking(self): + """Whether this is a link invocation.""" + return self.action == Args.LINK and self.output is not None + + @property + def compiling(self): + """Whether this is a compile invocation.""" + return self.action == Args.COMPILE + + @property + def printing_file_name(self): + """Whether this is a print-file-name invocation.""" + return self.action == Args.PRINT_FILE_NAME and self.print_file_name is not None + + def _handle_args(self, args): + """Argument handling pipeline. + + Args: + args: Iterable, command-line arguments. + + Yields: + Transformed command-line arguments. + + """ + args = iter(args) + for arg in args: + out = [] + # Poor man's pattern matching: Each handler function takes the + # current argument, the stream of up-coming arguments, and a + # reference to the list of arguments to forward. The handler must + # return True if it consumes the argument, and return False if + # another handler should consume the argument. + if self._handle_output(arg, args, out): + pass + elif self._handle_library(arg, args, out): + pass + elif self._handle_library_path(arg, args, out): + pass + elif self._handle_linker_arg(arg, args, out): + pass + elif self._handle_print_file_name(arg, args, out): + pass + elif self._handle_compile(arg, args, out): + pass + else: + yield arg + + for out_arg in out: + yield out_arg + + def _handle_output(self, arg, args, out): + if arg == "-o": + # Remember the output filename. + self.output = next(args) + out.extend(["-o", self.output]) + return True + else: + return False + + def _handle_library(self, arg, args, out): + if arg == "-l" or arg == "--library": + library = next(args) + elif arg.startswith("-l"): + library = arg[2:] + elif arg.startswith("--library="): + library = arg[len("--library="):] + else: + return False + + # Remember the required libraries. + self.libraries.append(library) + out.append("-l{}".format(library)) + + return True + + def _handle_library_path(self, arg, args, out): + if arg == "-L" or arg == "--library-path": + library_path = next(args) + elif arg.startswith("-L"): + library_path = arg[2:] + elif arg.startswith("--library-path="): + library_path = arg[len("--library-path="):] + else: + return False + + # Shorten the library search paths. On Windows library search paths may + # exceed the maximum path length. + shortened = shorten_path(library_path) + # Remember the library search paths. + self.library_paths.append(shortened) + out.append("-L{}".format(shortened)) + + return True + + def _handle_linker_arg(self, arg, args, out): + if arg == "-Xlinker": + ld_arg = next(args) + if self._prev_ld_arg is None: + if ld_arg == "-rpath": + self._prev_ld_arg = ld_arg + else: + out.extend(["-Xlinker", ld_arg]) + elif self._prev_ld_arg == "-rpath": + self._prev_ld_arg = None + self._handle_rpath(ld_arg, out) + else: + # This indicates a programmer error and should not happen. + raise RuntimeError("Unhandled _prev_ld_arg '{}'.".format(self._prev_ld_arg)) + return True + elif arg.startswith("-Wl,"): + ld_args = arg.split(",")[1:] + if len(ld_args) == 2 and ld_args[0] == "-rpath": + self._handle_rpath(ld_args[1], out) + return True + else: + out.append(arg) + return True + else: + return False + + def _handle_rpath(self, rpath, out): + # Filter out all RPATH flags for now and manually add the needed ones + # later on. + self.rpaths.append(rpath) + + def _handle_print_file_name(self, arg, args, out): + if arg == "--print-file-name": + print_file_name = next(args) + elif arg.startswith("--print-file-name="): + print_file_name = arg[len("--print-file-name="):] + else: + return False + + # Remember print-file-name action. Don't forward to allow for later + # manipulation. + self.print_file_name = print_file_name + self.action = Args.PRINT_FILE_NAME + + return True + + def _handle_compile(self, arg, args, out): + if arg == "-c": + self.action = Args.COMPILE + out.append(arg) + else: + return False + + return True + + +def load_response_files(args): + """Generator that loads arguments from response files. + + Passes through any regular arguments. + + Args: + args: Iterable of arguments. + + Yields: + All arguments, with response files replaced by their contained arguments. + + """ + args = iter(args) + for arg in args: + if arg == "-install_name": + # macOS only: The install_name may start with an '@' character. + yield arg + yield next(args) + elif arg.startswith("@"): + with open(arg[1:], "r") as rsp: + for line in rsp: + for rsp_arg in parse_response_line(line): + yield rsp_arg + else: + yield arg + + +def parse_response_line(s): + # GHC writes response files with quoted lines. + return shlex.split(s) + + +def shorten_path(input_path): + """Shorten the given path if possible. + + Applies the following transformations if they shorten the path length: + - Make path relative to CWD. + - Remove redundant up-level references. + - Resolve symbolic links. + + Args: + input_path: The path to shorten. + + Returns: + The shortened path. + + """ + exists = os.path.exists(input_path) + shortened = input_path + + # Try relativizing to current working directory. + rel = os.path.relpath(shortened) + if len(rel) < len(shortened): + shortened = rel + + # Try normalizing the path if possible. + norm = os.path.normpath(shortened) + if len(norm) < len(shortened): + # Ensure that the path is still correct. Reducing up-level references + # may change the meaning of the path in the presence of symbolic links. + try: + if not exists or os.path.samefile(norm, shortened): + shortened = norm + except IOError: + # stat may fail if the path became invalid or does not exist. + pass + + # Try resolving symlinks. + try: + real = os.path.relpath(os.path.realpath(shortened)) + if len(real) < len(shortened): + shortened = real + except IOError: + # realpath may fail if the path does not exist. + pass + + return shortened + + +def rpath_args(rpaths): + """Generate arguments for RUNPATHs.""" + for rpath in rpaths: + yield "-Xlinker" + yield "-rpath" + yield "-Xlinker" + yield rpath + + +# -------------------------------------------------------------------- +# Link binary or library + + +def link(output, libraries, rpaths, args): + """Execute the link action. + + Args: + output: The output binary or library. + libraries: Library dependencies. + rpaths: The provided rpaths. + args: The command-line arguments. + + """ + if is_darwin(): + # Reserve space in load commands for later replacement. + args.append("-headerpad_max_install_names") + rpaths, darwin_rewrites = darwin_shorten_rpaths( + rpaths, libraries, output) + else: + rpaths = shorten_rpaths(rpaths, libraries, output) + + args.extend(rpath_args(rpaths)) + run_cc(args, exit_on_error=True) + + if is_darwin(): + darwin_rewrite_load_commands(darwin_rewrites, output) + + +def shorten_rpaths(rpaths, libraries, output): + """Avoid redundant rpaths. + + Filters out rpaths that are not required to load any library dependency. + + Args: + rpaths: List of given rpaths. + libraries: List of library dependencies. + output: The output binary, used to resolve rpaths. + + Returns: + List of required rpaths. + + """ + input_rpaths = sort_rpaths(rpaths) + missing = set(libraries) + + rpaths = [] + + for rpath in input_rpaths: + if not missing: + break + rpath, rpath_dir = resolve_rpath(rpath, output) + found, missing = find_library(missing, rpath_dir) + if found: + rpaths.append(rpath) + + return rpaths + + +def darwin_shorten_rpaths(rpaths, libraries, output): + """Avoid redundant rpaths and adapt library load commands. + + Avoids redundant rpaths by detecting the solib directory and making load + commands relative to the solib directory where applicable. + + Args: + rpaths: List of given rpaths. + libraries: List of library dependencies. + output: The output binary, used to resolve rpaths. + + Returns: + (rpaths, rewrites): + rpaths: List of required rpaths. + rewrites: List of load command rewrites. + + """ + input_rpaths = sort_rpaths(rpaths) + missing = set(libraries) + + rpaths = [] + rewrites = [] + + # References to core libs take up much space. Consider detecting the GHC + # libdir and adding an rpath for that and making load commands relative to + # that. Alternatively, https://github.com/bazelbuild/bazel/pull/8888 would + # also avoid this issue. + + # Determine solib dir and rewrite load commands relative to solib dir. + # This allows to replace potentially many rpaths by one. + solib_rpath = find_solib_rpath(input_rpaths, output) + if missing and solib_rpath is not None: + solib_rpath, solib_dir = resolve_rpath(solib_rpath, output) + + found, missing = find_library_recursive(missing, solib_dir) + if found: + rpaths.append(solib_rpath) + for f in found.values(): + soname = darwin_get_install_name(os.path.join(solib_dir, f)) + rewrites.append((soname, f)) + + # For the remaining missing libraries, determine which rpaths are required. + for rpath in input_rpaths: + if not missing: + break + rpath, rpath_dir = resolve_rpath(rpath, output) + found, missing = find_library(missing, rpath_dir) + # Libraries with an absolute install_name don't require an rpath entry. + found = dict(itertools.filterfalse( + lambda item: os.path.isabs(darwin_get_install_name(os.path.join(rpath_dir, item[1]))), + found.items())) + if len(found) == 1: + # Avoid unnecessary rpath if it is only relevant for one load command. + [filename] = found.values() + soname = darwin_get_install_name(os.path.join(rpath_dir, filename)) + rewrites.append((soname, os.path.join(rpath, filename))) + elif found: + rpaths.append(rpath) + + return rpaths, rewrites + + +def sort_rpaths(rpaths): + """Sort RUNPATHs by preference. + + Preference in decsending order: + - Relative to target + - Absolute path + - Relative to CWD + + """ + def rpath_priority(rpath): + system = platform.system() + if system == "Darwin": + if rpath.startswith("@loader_path"): + return 0 + elif system == "Linux": + if rpath.startswith("$ORIGIN"): + return 0 + if os.path.isabs(rpath): + return 1 + return 2 + + return sorted(rpaths, key=rpath_priority) + + +def find_solib_rpath(rpaths, output): + """Find the solib directory rpath entry. + + The solib directory is the directory under which Bazel places dynamic + library symbolic links on Unix. It has the form `_solib_`. + + """ + for rpath in rpaths: + components = rpath.replace("\\", "/").split("/") + solib_rpath = [] + for comp in components: + solib_rpath.append(comp) + if comp.startswith("_solib_"): + return "/".join(solib_rpath) + + if is_temporary_output(output): + # GHC generates temporary libraries outside the execroot. In that case + # the Bazel generated RPATHs are not forwarded, and the solib directory + # is not visible on the command-line. + candidates = glob.glob("**/bin/_solib_*", recursive=True) + if candidates: + return min(candidates) + + return None + + +def find_library_recursive(libraries, directory): + """Find libraries in given directory tree. + + Args: + libraries: List of missing libraries. + directory: Root of directory tree. + + Returns: + (found, missing): + found: Dict of found libraries {libname: path} relative to directory. + missing: Set of remaining missing libraries. + + """ + missing = set(libraries) + found = {} + for root, _, files in os.walk(directory, followlinks=True): + prefix = os.path.relpath(root, directory) + if not missing: + break + for f in files: + libname = get_lib_name(f) + if libname and libname in missing: + found[libname] = os.path.join(prefix, f) if prefix != "." else f + missing.discard(libname) + if not missing: + break + + return found, missing + + +def find_library(libraries, directory): + """Find libraries in the given directory. + + Args: + libraries: List of missing libraries. + directory: The directory in which to search for libraries. + + Returns: + (found, missing): + found: Dict of found libraries {libname: path} relative to directory. + missing: Set of remaining missing libraries. + + """ + missing = set(libraries) + found = {} + for _, _, files in itertools.islice(os.walk(directory), 1): + if not missing: + break + for f in files: + libname = get_lib_name(f) + if libname and libname in missing: + found[libname] = f + missing.discard(libname) + + return found, missing + + +def get_lib_name(filename): + """Determine the library name of the given library file. + + The library name is the name by which the library is referred to in a -l + argument to the linker. + + """ + if not filename.startswith("lib"): + return None + + libname = filename[3:] + dotsodot = libname.find(".so.") + if dotsodot != -1: + return libname[:dotsodot] + + libname, ext = os.path.splitext(libname) + if ext in [".dll", ".dylib", ".so"]: + return libname + + return None + + +def resolve_rpath(rpath, output): + """Resolve the given rpath, replacing references to the binary.""" + def has_origin(rpath): + return rpath.startswith("$ORIGIN") or rpath.startswith("@loader_path") + + def replace_origin(rpath, origin): + rpath = rpath.replace("$ORIGIN/", origin) + rpath = rpath.replace("$ORIGIN", origin) + rpath = rpath.replace("@loader_path/", origin) + rpath = rpath.replace("@loader_path", origin) + return rpath + + if is_temporary_output(output): + # GHC generates temporary libraries outside the execroot. The regular + # relative rpaths don't work in that case and have to be converted to + # absolute paths. + if has_origin(rpath): + # We don't know what $ORIGIN/@loader_path was meant to refer to. + # Try to find an existing, matching rpath by globbing. + stripped = replace_origin(rpath, "") + candidates = glob.glob(os.path.join("**", stripped), recursive=True) + if not candidates: + # Path does not exist. It will be sorted out later, since no + # library will be found underneath it. + rpath = stripped + else: + rpath = os.path.abspath(shorten_path(min(candidates))) + else: + rpath = os.path.abspath(shorten_path(rpath)) + + return rpath, rpath + else: + # Consider making relative rpaths relative to output. + # E.g. bazel-out/.../some/dir to @loader_path/.../some/dir + outdir = os.path.dirname(output) + "/" + resolved = replace_origin(rpath, outdir) + return rpath, resolved + + +def darwin_get_install_name(lib): + """Read the install_name of the given library.""" + lines = subprocess.check_output([OTOOL, "-D", lib]).splitlines() + if len(lines) >= 2: + return lines[1] + else: + return os.path.basename(lib) + + +def darwin_rewrite_load_commands(rewrites, output): + """Rewrite the load commands in the given binary.""" + args = [] + for old, new in rewrites: + args.extend(["-change", old, os.path.join("@rpath", new)]) + if args: + subprocess.check_call([INSTALL_NAME_TOOL] + args + [output]) + + +# -------------------------------------------------------------------- +# print-file-name + + +def print_file_name(filename, args): + """Execute the print-file-name action. + + Args: + filename: The queried filename. + args: The remaining arguments. + + """ + (basename, ext) = os.path.splitext(filename) + if is_darwin() and ext == ".dylib": + # Bazel generates dynamic libraries with .so extension on Darwin. + # However, GHC only looks for files with .dylib extension. + + # Try with the .dylib extension first. + found, res = run_cc_print_file_name(filename, args) + if not found: + # Retry with .so extension. + found, so_res = run_cc_print_file_name("%s.so" % basename, args) + if found: + res = so_res + else: + _, res = run_cc_print_file_name(filename, args) + + sys.stdout.write(res.stdout.decode()) + sys.stderr.write(res.stderr.decode()) + sys.exit(res.returncode) + + +def run_cc_print_file_name(filename, args): + """Run cc --print-file-name on the given file name. + + Args: + filename: The filename to query for. + args: Remaining command-line arguments. Relevant for -B flags. + + Returns: + filename, res: + filename: The returned filename, if it exists, otherwise None. + res: CompletedProcess + + """ + args = args + ["--print-file-name", filename] + res = run_cc(args, capture_output=True, exit_on_error=True) + filename = res.stdout.decode().strip() + # Note, gcc --print-file-name does not fail if the file was not found, but + # instead just returns the input filename. + if os.path.isfile(filename): + return filename, res + else: + return None, res + + +# -------------------------------------------------------------------- + + +def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): + """Execute cc with a response file holding the given arguments. + + Args: + args: Iterable of arguments to pass to cc. + capture_output: Whether to capture stdout and stderr. + exit_on_error: Whether to exit on error. Will print captured output first. + + Returns: + CompletedProcess + + """ + if capture_output: + # The capture_output argument to subprocess.run was only added in 3.7. + new_kwargs = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) + new_kwargs.update(kwargs) + kwargs = new_kwargs + + with response_file(args) as rsp: + res = subprocess.run([CC, "@" + rsp], **kwargs) + + if exit_on_error and res.returncode != 0: + if capture_output: + sys.stdout.write(res.stdout.decode()) + sys.stderr.write(res.stderr.decode()) + sys.exit(res.returncode) + + return res + + +@contextmanager +def response_file(args): + """Create a response file for the given arguments. + + Context manager, use in a with statement. The file will be deleted at the + end of scope. + + Args: + args: Iterable, the arguments to write in to the response file. + + Yields: + The file name of the response file. + + """ + try: + with tempfile.NamedTemporaryFile(mode="w", prefix="rsp", delete=False) as f: + for arg in args: + line = generate_response_line(arg) + f.write(line) + f.close() + yield f.name + finally: + try: + os.remove(f.name) + except OSError: + pass + + +def generate_response_line(arg): + # Gcc expects one argument per line, surrounded by double quotes, with + # inner double quotes escaped with backslash, and backslashes themselves + # escaped. shlex.quote conflicts with this format. + return '"{}"\n'.format(arg.replace("\\", "\\\\").replace('"', '\\"')) + + +def is_darwin(): + """Whether the execution platform is Darwin.""" + return platform.system() == "Darwin" + + +def is_temporary_output(output): + """Whether the target is temporary. + + GHC generates temporary libraries in certain cases related to Template + Haskell outside the execroot. This means that rpaths relative to $ORIGIN or + @loader_path are going to be invalid. + + """ + # Assumes that the temporary directory is set to an absolute path, while + # the outputs under the execroot are referred to by relative path. This + # should be a valid assumption as the temporary directory needs to be + # available irrespective of the current working directory, while Bazel uses + # paths relative to the execroot to avoid things like user names creeping + # into cache keys. If this turns out to be wrong we could instead look for + # path components matching Bazel's output directory hierarchy. + # See https://docs.bazel.build/versions/master/output_directories.html + return os.path.isabs(output) + + +# -------------------------------------------------------------------- + + +if __name__ == "__main__": + main() + + +# vim: ft=python diff --git a/haskell/providers.bzl b/haskell/providers.bzl index 3e423848..f5bfce0e 100644 --- a/haskell/providers.bzl +++ b/haskell/providers.bzl @@ -186,7 +186,7 @@ def _get_unique_lib_files(cc_info): for filename in filenames ] -def get_ghci_extra_libs(hs, cc_info, dynamic = True, path_prefix = None): +def get_ghci_extra_libs(hs, cc_info, path_prefix = None): """Get libraries appropriate for GHCi's linker. GHC expects dynamic and static versions of the same library to have the @@ -197,16 +197,12 @@ def get_ghci_extra_libs(hs, cc_info, dynamic = True, path_prefix = None): directory to allow for less RPATH entries and to fix file extensions that GHCi does not support. - GHCi can load PIC static libraries (-fPIC -fexternal-dynamic-refs) and - dynamic libraries. Preferring static libraries can be useful to reduce the - risk of exceeding the MACH-O header size limit on macOS, and to reduce - build times by avoiding to generate dynamic libraries. However, this - requires GHCi to run with the statically linked rts library. + GHCi can load PIC static libraries (-fPIC -fexternal-dynamic-refs) with a + dynamic RTS and dynamic libraries with a dynamic RTS. Args: hs: Haskell context. cc_info: Combined CcInfo provider of dependencies. - dynamic: (optional) Whether to prefer dynamic libraries. path_prefix: (optional) Prefix for the entries in the generated library path. Returns: @@ -218,7 +214,7 @@ def get_ghci_extra_libs(hs, cc_info, dynamic = True, path_prefix = None): (static_libs, dynamic_libs) = get_extra_libs( hs, cc_info, - dynamic = dynamic, + dynamic = not hs.toolchain.is_static, pic = True, fixup_dir = "_ghci_libs", ) @@ -278,8 +274,6 @@ def get_extra_libs(hs, cc_info, dynamic = False, pic = None, fixup_dir = "_libs" elif lib_to_link.static_library and not pic_required: static_lib = lib_to_link.static_library - if dynamic_lib: - dynamic_lib = symlink_dynamic_library(hs, dynamic_lib, fixed_lib_dir) static_lib = mangle_static_library(hs, dynamic_lib, static_lib, fixed_lib_dir) if static_lib and not (dynamic and dynamic_lib): diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl index 1e42b024..dc14a2f7 100644 --- a/haskell/toolchain.bzl +++ b/haskell/toolchain.bzl @@ -86,6 +86,11 @@ def _run_ghc(hs, cc, inputs, outputs, mnemonic, arguments, params_file = None, e else: inputs += extra_inputs + if input_manifests != None: + input_manifests = input_manifests + cc.manifests + else: + input_manifests = cc.manifests + hs.actions.run( inputs = inputs, tools = tools, @@ -145,6 +150,8 @@ def _haskell_toolchain_impl(ctx): for lib in ctx.attr.libraries } + (cc_wrapper_inputs, cc_wrapper_manifest) = ctx.resolve_tools(tools = [ctx.attr._cc_wrapper]) + return [ platform_common.ToolchainInfo( name = ctx.label.name, @@ -154,6 +161,11 @@ def _haskell_toolchain_impl(ctx): haddock_flags = ctx.attr.haddock_flags, locale = ctx.attr.locale, locale_archive = locale_archive, + cc_wrapper = struct( + executable = ctx.executable._cc_wrapper, + inputs = cc_wrapper_inputs, + manifests = cc_wrapper_manifest, + ), osx_cc_wrapper_tpl = ctx.file._osx_cc_wrapper_tpl, mode = ctx.var["COMPILATION_MODE"], actions = struct( @@ -219,6 +231,11 @@ _haskell_toolchain = rule( Label pointing to the locale archive file to use. Mostly useful on NixOS. """, ), + "_cc_wrapper": attr.label( + cfg = "host", + default = Label("@rules_haskell//haskell:cc_wrapper"), + executable = True, + ), "_osx_cc_wrapper_tpl": attr.label( allow_single_file = True, default = Label("@rules_haskell//haskell:private/osx_cc_wrapper.sh.tpl"), From 329373a34311bb281b9bf39e66cb235a3df60802 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 16 Jul 2019 14:32:17 +0200 Subject: [PATCH 04/31] Remove unused osx_cc_wrapper --- haskell/BUILD.bazel | 1 - haskell/private/osx_cc_wrapper.sh.tpl | 313 -------------------------- haskell/toolchain.bzl | 5 - 3 files changed, 319 deletions(-) delete mode 100644 haskell/private/osx_cc_wrapper.sh.tpl diff --git a/haskell/BUILD.bazel b/haskell/BUILD.bazel index 41afb641..e67c98f3 100644 --- a/haskell/BUILD.bazel +++ b/haskell/BUILD.bazel @@ -15,7 +15,6 @@ exports_files( "private/ghci_repl_wrapper.sh", "private/haddock_wrapper.sh.tpl", "private/cc_wrapper.py.tpl", - "private/osx_cc_wrapper.sh.tpl", "private/pkgdb_to_bzl.py", ], ) diff --git a/haskell/private/osx_cc_wrapper.sh.tpl b/haskell/private/osx_cc_wrapper.sh.tpl deleted file mode 100644 index 9abf9ce9..00000000 --- a/haskell/private/osx_cc_wrapper.sh.tpl +++ /dev/null @@ -1,313 +0,0 @@ -#!/bin/bash -# -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# This is a wrapper script around gcc/clang that adjusts linker flags for -# Haskell library and binary targets. -# -# Load commands that attempt to load dynamic libraries relative to the working -# directory in their package output path (bazel-out/...) are converted to load -# commands relative to @rpath. rules_haskell passes the corresponding -# -Wl,-rpath,... flags itself. -# -# rpath commands that attempt to add rpaths relative to the working directory -# to look for libraries in their package output path (bazel-out/...) are -# omitted, since rules_haskell adds more appropriate rpaths itself. -# -# GHC generates intermediate dynamic libraries outside the build tree. -# Additional RPATH entries are provided for those to make dynamic library -# dependencies in the Bazel build tree available at runtime. -# -# See https://blogs.oracle.com/dipol/entry/dynamic_libraries_rpath_and_mac -# on how to set those paths for Mach-O binaries. -# -set -euo pipefail - -INSTALL_NAME_TOOL="/usr/bin/install_name_tool" -OTOOL="/usr/bin/otool" - -# Collect arguments to forward in a fresh response file. -RESPONSE_FILE="$(mktemp osx_cc_args_XXXX.rsp)" -rm_response_file() { - rm -f "$RESPONSE_FILE" -} -trap rm_response_file EXIT - -add_args() { - # Add the given arguments to the fresh response file. We follow GHC's - # example in storing one argument per line, wrapped in double quotes. Double - # quotes in the argument itself are escaped. - for arg in "$@"; do - printf '"%s"\n' "${arg//\"/\\\"}" >> "$RESPONSE_FILE" - done -} - -# Collect library, library dir, and rpath arguments. -LIBS=() -LIB_DIRS=() -RPATHS=() - -# Parser state. -# Parsing response file - unquote arguments. -QUOTES= -# Upcoming linker argument. -LINKER= -# Upcoming rpath argument. -RPATH= -# Upcoming install-name argument. -INSTALL= -# Upcoming output argument. -OUTPUT= - -parse_arg() { - # Parse the given argument. Decide whether to pass it on to the compiler, - # and how it affects the parser state. - local arg="$1" - # Unquote response file arguments. - if [[ "$QUOTES" = "1" && "$arg" =~ ^\"(.*)\"$ ]]; then - # Take GHC's argument quoting into account when parsing a response - # file. Note, no indication was found that GHC would pass multiline - # arguments, or insert escape codes into the quoted arguments. If you - # observe ill-formed arguments being passed to the compiler, then this - # logic may need to be extended. - arg="${BASH_REMATCH[1]}" - fi - # Parse given argument. - if [[ "$OUTPUT" = "1" ]]; then - # The previous argument was -o. Read output file. - OUTPUT="$arg" - add_args "$arg" - elif [[ "$LINKER" = "1" ]]; then - # The previous argument was -Xlinker. Read linker argument. - if [[ "$RPATH" = "1" ]]; then - # The previous argument was -rpath. Read RPATH. - parse_rpath "$arg" - RPATH=0 - elif [[ "$arg" = "-rpath" ]]; then - # rpath is coming - RPATH=1 - else - # Unrecognized linker argument. Pass it on. - add_args "-Xlinker" "$arg" - fi - LINKER= - elif [[ "$INSTALL" = "1" ]]; then - INSTALL= - add_args "$arg" - elif [[ "$arg" =~ ^@(.*)$ ]]; then - # Handle response file argument. Parse the arguments contained in the - # response file one by one. Take GHC's argument quoting into account. - # Note, assumes that response file arguments are not nested in other - # response files. - QUOTES=1 - while read line; do - parse_arg "$line" - done < "${BASH_REMATCH[1]}" - QUOTES= - elif [[ "$arg" = "-install_name" ]]; then - # Install name is coming. We don't use it, but it can start with an @ - # and be mistaken for a response file. - INSTALL=1 - add_args "$arg" - elif [[ "$arg" = "-o" ]]; then - # output is coming - OUTPUT=1 - add_args "$arg" - elif [[ "$arg" = "-Xlinker" ]]; then - # linker flag is coming - LINKER=1 - elif [[ "$arg" =~ ^-l(.*)$ ]]; then - LIBS+=("${BASH_REMATCH[1]}") - add_args "$arg" - elif [[ "$arg" =~ ^-L(.*)$ ]]; then - LIB_DIRS+=("${BASH_REMATCH[1]}") - add_args "$arg" - elif [[ "$arg" =~ ^-Wl,-rpath,(.*)$ ]]; then - parse_rpath "${BASH_REMATCH[1]}" - else - # Unrecognized argument. Pass it on. - add_args "$arg" - fi -} - -parse_rpath() { - # Parse the given -rpath argument and decide whether it should be - # forwarded to the compiler/linker. - local rpath="$1" - if [[ "$rpath" =~ ^/ || "$rpath" =~ ^@ ]]; then - # Absolute rpaths or rpaths relative to @loader_path or similar, are - # passed on to the linker. Other relative rpaths are dropped, these - # are auto-generated by GHC, but are useless because rules_haskell - # constructs dedicated rpaths to the _solib or _hssolib directory. - # See https://github.com/tweag/rules_haskell/issues/689 - add_args "-Wl,-rpath,$rpath" - RPATHS+=("$rpath") - fi -} - -# Parse all given arguments. -for arg in "$@"; do - parse_arg "$arg" -done - -get_library_in() { - # Find the given library in the given directory. - # Returns empty string if the library is not found. - local lib="$1" - local dir="$2" - local solib="${dir}${dir:+/}lib${lib}.so" - local dylib="${dir}${dir:+/}lib${lib}.dylib" - if [[ -f "$solib" ]]; then - echo "$solib" - elif [[ -f "$dylib" ]]; then - echo "$dylib" - fi -} - -get_library_path() { - # Find the given library in the specified library search paths. - # Returns empty string if the library is not found. - if [[ ${#LIB_DIRS[@]} -gt 0 ]]; then - local libpath - for libdir in "${LIB_DIRS[@]}"; do - libpath="$(get_library_in "$1" "$libdir")" - if [[ -n "$libpath" ]]; then - echo "$libpath" - return - fi - done - fi -} - -resolve_rpath() { - # Resolve the given rpath. I.e. if it is an absolute path, just return it. - # If it is relative to the output, then prepend the output path. - local rpath="$1" - if [[ "$rpath" =~ ^/ ]]; then - echo "$rpath" - elif [[ "$rpath" =~ ^@loader_path/(.*)$ || "$rpath" =~ ^@executable_path/(.*)$ ]]; then - echo "$(dirname "$OUTPUT")/${BASH_REMATCH[1]}" - else - echo "$rpath" - fi -} - -get_library_rpath() { - # Find the given library in the specified rpaths. - # Returns empty string if the library is not found. - if [[ ${#RPATHS[@]} -gt 0 ]]; then - local libdir libpath - for rpath in "${RPATHS[@]}"; do - libdir="$(resolve_rpath "$rpath")" - libpath="$(get_library_in "$1" "$libdir")" - if [[ -n "$libpath" ]]; then - echo "$libpath" - return - fi - done - fi -} - -get_library_name() { - # Get the "library name" of the given library. - "$OTOOL" -D "$1" | tail -1 -} - -relpath() { - # Find relative path from the first to the second path. Assuming the first - # is a directory. If either is an absolute path, then we return the - # absolute path to the second. - local from="$1" - local to="$2" - if [[ "$to" =~ ^/ ]]; then - echo "$to" - elif [[ "$from" =~ ^/ ]]; then - echo "$PWD/$to" - else - # Split path and store components in bash array. - IFS=/ read -a fromarr <<<"$from" - IFS=/ read -a toarr <<<"$to" - # Drop common prefix. - for ((i=0; i < ${#fromarr[@]}; ++i)); do - if [[ "${fromarr[$i]}" != "${toarr[$i]}" ]]; then - break - fi - done - # Construct relative path. - local common=$i - local out= - for ((i=$common; i < ${#fromarr[@]}; ++i)); do - out="$out${out:+/}.." - done - for ((i=$common; i < ${#toarr[@]}; ++i)); do - out="$out${out:+/}${toarr[$i]}" - done - echo $out - fi -} - -generate_rpath() { - # Generate an rpath entry for the given library path. - local rpath="$(relpath "$(dirname "$OUTPUT")" "$(dirname "$1")")" - if [[ "$rpath" =~ ^/ ]]; then - echo "$rpath" - else - # Relative rpaths are relative to the binary. - echo "@loader_path${rpath:+/}$rpath" - fi -} - -if [[ ! "$OUTPUT" =~ ^bazel-out/ && ${#LIBS[@]} -gt 0 ]]; then - # GHC generates temporary dynamic libraries during compilation outside of - # the build directory. References to dynamic C libraries are broken in this - # case. Here we add additional RPATHs to fix these references. The Hazel - # package for swagger2 is an example that triggers this issue. - for lib in "${LIBS[@]}"; do - librpath="$(get_library_rpath "$lib")" - if [[ -z "$librpath" ]]; then - # The given library was not found in any of the rpaths. - # Find it in the library search paths. - libpath="$(get_library_path "$lib")" - if [[ "$libpath" =~ ^bazel-out/ ]]; then - # The library is Bazel generated and loaded relative to PWD. - # Add an RPATH entry, so it is found at runtime. - rpath="$(generate_rpath "$libpath")" - parse_rpath "$rpath" - fi - fi - done -fi - -# Call the C++ compiler with the fresh response file. -%{cc} "@$RESPONSE_FILE" - -if [[ ${#LIBS[@]} -gt 0 ]]; then - # Replace load commands relative to the working directory, by load commands - # relative to the rpath, if the library can be found relative to an rpath. - for lib in "${LIBS[@]}"; do - librpath="$(get_library_rpath "$lib")" - if [[ -n "$librpath" ]]; then - libname="$(get_library_name "$librpath")" - if [[ "$libname" =~ ^bazel-out/ ]]; then - "${INSTALL_NAME_TOOL}" -change \ - "$libname" \ - "@rpath/$(basename "$librpath")" \ - "$OUTPUT" - fi - fi - done -fi - -# vim: ft=sh diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl index dc14a2f7..9b8372ed 100644 --- a/haskell/toolchain.bzl +++ b/haskell/toolchain.bzl @@ -166,7 +166,6 @@ def _haskell_toolchain_impl(ctx): inputs = cc_wrapper_inputs, manifests = cc_wrapper_manifest, ), - osx_cc_wrapper_tpl = ctx.file._osx_cc_wrapper_tpl, mode = ctx.var["COMPILATION_MODE"], actions = struct( compile_binary = compile_binary, @@ -236,10 +235,6 @@ Label pointing to the locale archive file to use. Mostly useful on NixOS. default = Label("@rules_haskell//haskell:cc_wrapper"), executable = True, ), - "_osx_cc_wrapper_tpl": attr.label( - allow_single_file = True, - default = Label("@rules_haskell//haskell:private/osx_cc_wrapper.sh.tpl"), - ), }, ) From e4167125c996f94bdcd35a74975df866d2284252 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 23 Jul 2019 09:42:38 +0200 Subject: [PATCH 05/31] Use cc_wrapper in ghci Otherwise we would still require symbolic links for dynamic library dependencies on macOS for REPL targets. --- haskell/private/actions/repl.bzl | 2 ++ haskell/private/actions/runghc.bzl | 2 ++ haskell/private/cc_wrapper.bzl | 10 ++++++++-- haskell/private/cc_wrapper.py.tpl | 19 ++++++++++++++++++- haskell/private/ghci_repl_wrapper.sh | 7 ++++++- haskell/repl.bzl | 3 +++ haskell/toolchain.bzl | 5 +++++ 7 files changed, 44 insertions(+), 4 deletions(-) diff --git a/haskell/private/actions/repl.bzl b/haskell/private/actions/repl.bzl index 483a0172..c452a33f 100644 --- a/haskell/private/actions/repl.bzl +++ b/haskell/private/actions/repl.bzl @@ -118,6 +118,7 @@ def build_haskell_repl( substitutions = { "{ENV}": render_env(ghc_env), "{TOOL}": hs.tools.ghci.path, + "{CC}": hs.toolchain.cc_wrapper.executable.path, "{ARGS}": " ".join( [ "-ghci-script", @@ -174,5 +175,6 @@ def build_haskell_repl( pkg_info_inputs, ghci_extra_libs, hs_info.source_files, + hs.toolchain.cc_wrapper.runfiles.files, ]) ln(hs, repl_file, output, extra_inputs) diff --git a/haskell/private/actions/runghc.bzl b/haskell/private/actions/runghc.bzl index ca2c41e2..71b31d76 100644 --- a/haskell/private/actions/runghc.bzl +++ b/haskell/private/actions/runghc.bzl @@ -87,6 +87,7 @@ def build_haskell_runghc( substitutions = { "{ENV}": render_env(ghc_env), "{TOOL}": hs.tools.runghc.path, + "{CC}": hs.toolchain.cc_wrapper.executable.path, "{ARGS}": " ".join([shell.quote(a) for a in runcompile_flags]), }, is_executable = True, @@ -105,5 +106,6 @@ def build_haskell_runghc( pkg_info_inputs, ghci_extra_libs, hs_info.source_files, + hs.toolchain.cc_wrapper.runfiles.files, ]) ln(hs, runghc_file, output, extra_inputs) diff --git a/haskell/private/cc_wrapper.bzl b/haskell/private/cc_wrapper.bzl index 9913d806..4e7d3957 100644 --- a/haskell/private/cc_wrapper.bzl +++ b/haskell/private/cc_wrapper.bzl @@ -1,6 +1,5 @@ load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") -load("@os_info//:os_info.bzl", "is_linux") def _cc_wrapper_impl(ctx): cc_toolchain = find_cpp_toolchain(ctx) @@ -21,9 +20,16 @@ def _cc_wrapper_impl(ctx): is_executable = True, substitutions = { "{:cc:}": cc, + "{:workspace:}": ctx.workspace_name, }, ) - return [DefaultInfo(files = depset([cc_wrapper]))] + return [DefaultInfo( + files = depset([cc_wrapper]), + runfiles = ctx.runfiles( + transitive_files = cc_toolchain.all_files, + collect_data = True, + ), + )] _cc_wrapper = rule( implementation = _cc_wrapper_impl, diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 2f71a23c..ffdb9f6f 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -40,6 +40,7 @@ import subprocess import sys import tempfile +WORKSPACE = "{:workspace:}" CC = "{:cc:}" INSTALL_NAME_TOOL = "/usr/bin/install_name_tool" OTOOL = "/usr/bin/otool" @@ -731,8 +732,24 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): new_kwargs.update(kwargs) kwargs = new_kwargs + if os.path.isfile(CC): + cc = CC + else: + # On macOS CC is a relative path to a wrapper script. If we're + # being called from a GHCi REPL then we need to find this wrapper + # script using Bazel runfiles. + r = bazel_runfiles.Create() + cc = r.Rlocation("/".join([WORKSPACE, CC])) + if cc is None and platform.system() == "Windows": + # We must use "/" instead of os.path.join on Windows, because the + # Bazel runfiles_manifest file uses "/" separators. + cc = r.Rlocation("/".join([WORKSPACE, CC + ".exe"])) + if cc is None: + print("CC not found '{}'.".format(CC), file=sys.stderr) + sys.exit(1) + with response_file(args) as rsp: - res = subprocess.run([CC, "@" + rsp], **kwargs) + res = subprocess.run([cc, "@" + rsp], **kwargs) if exit_on_error and res.returncode != 0: if capture_output: diff --git a/haskell/private/ghci_repl_wrapper.sh b/haskell/private/ghci_repl_wrapper.sh index cd6acefc..f672671a 100644 --- a/haskell/private/ghci_repl_wrapper.sh +++ b/haskell/private/ghci_repl_wrapper.sh @@ -54,6 +54,11 @@ cd "$BUILD_WORKSPACE_DIRECTORY" RULES_HASKELL_EXEC_ROOT=$(dirname $(readlink ${BUILD_WORKSPACE_DIRECTORY}/bazel-out)) TOOL_LOCATION="$RULES_HASKELL_EXEC_ROOT/{TOOL}" +# Setting -pgm* flags explicitly has the unfortunate side effect +# of resetting any program flags in the GHC settings file. So we +# restore them here. See +# https://ghc.haskell.org/trac/ghc/ticket/7929. +PGM_ARGS="-pgma {CC} -pgmc {CC} -pgml {CC} -pgmP {CC} -optc-fno-stack-protector -optP-E -optP-undef -optP-traditional" {ENV} -"$TOOL_LOCATION" {ARGS} "$@" +"$TOOL_LOCATION" $PGM_ARGS {ARGS} "$@" diff --git a/haskell/repl.bzl b/haskell/repl.bzl index 06e2c4f3..4dd87163 100644 --- a/haskell/repl.bzl +++ b/haskell/repl.bzl @@ -293,6 +293,7 @@ def _create_repl(hs, ctx, repl_info, output): substitutions = { "{ENV}": render_env(dicts.add(hs.env, ghc_env)), "{TOOL}": hs.tools.ghci.path, + "{CC}": hs.toolchain.cc_wrapper.executable.path, "{ARGS}": " ".join( args + [ shell.quote(a) @@ -316,6 +317,8 @@ def _create_repl(hs, ctx, repl_info, output): depset([hs.toolchain.locale_archive] if hs.toolchain.locale_archive else []), ]), collect_data = ctx.attr.collect_data, + ).merge( + hs.toolchain.cc_wrapper.runfiles, ), )] diff --git a/haskell/toolchain.bzl b/haskell/toolchain.bzl index 9b8372ed..68570284 100644 --- a/haskell/toolchain.bzl +++ b/haskell/toolchain.bzl @@ -151,6 +151,10 @@ def _haskell_toolchain_impl(ctx): } (cc_wrapper_inputs, cc_wrapper_manifest) = ctx.resolve_tools(tools = [ctx.attr._cc_wrapper]) + cc_wrapper_info = ctx.attr._cc_wrapper[DefaultInfo] + cc_wrapper_runfiles = cc_wrapper_info.default_runfiles.merge( + cc_wrapper_info.data_runfiles, + ) return [ platform_common.ToolchainInfo( @@ -165,6 +169,7 @@ def _haskell_toolchain_impl(ctx): executable = ctx.executable._cc_wrapper, inputs = cc_wrapper_inputs, manifests = cc_wrapper_manifest, + runfiles = cc_wrapper_runfiles, ), mode = ctx.var["COMPILATION_MODE"], actions = struct( From 7836e68439d4a4e96928b167049683ecbd306422 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 23 Jul 2019 17:36:37 +0200 Subject: [PATCH 06/31] doctest: use cc_wrapper as well --- haskell/doctest.bzl | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/haskell/doctest.bzl b/haskell/doctest.bzl index 0a75eea4..88144865 100644 --- a/haskell/doctest.bzl +++ b/haskell/doctest.bzl @@ -1,6 +1,7 @@ """Doctest support""" load("@bazel_skylib//lib:dicts.bzl", "dicts") +load(":cc.bzl", "cc_interop_info") load(":private/context.bzl", "haskell_context", "render_env") load(":private/path_utils.bzl", "link_libraries") load(":private/set.bzl", "set") @@ -86,6 +87,27 @@ def _haskell_doctest_single(target, ctx): args = ctx.actions.args() args.add("--no-magic") + cc = cc_interop_info(ctx) + args.add_all([ + # GHC uses C compiler for assemly, linking and preprocessing as well. + "-pgma", + cc.tools.cc, + "-pgmc", + cc.tools.cc, + "-pgml", + cc.tools.cc, + "-pgmP", + cc.tools.cc, + # Setting -pgm* flags explicitly has the unfortunate side effect + # of resetting any program flags in the GHC settings file. So we + # restore them here. See + # https://ghc.haskell.org/trac/ghc/ticket/7929. + "-optc-fno-stack-protector", + "-optP-E", + "-optP-undef", + "-optP-traditional", + ]) + doctest_log = ctx.actions.declare_file( "doctest-log-" + ctx.label.name + "-" + target.label.name, ) @@ -118,6 +140,7 @@ def _haskell_doctest_single(target, ctx): ghci_extra_libs, depset( toolchain.doctest + + cc.files + [hs.tools.ghc], ), ]), @@ -183,7 +206,11 @@ haskell_doctest = rule( omitted, all exposed modules provided by `deps` will be tested. """, ), + "_cc_toolchain": attr.label( + default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), + ), }, + fragments = ["cpp"], toolchains = [ "@rules_haskell//haskell:toolchain", "@rules_haskell//haskell:doctest-toolchain", From 8294c7dc9106947b98c6a3ad7421f3dffb814642 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 23 Jul 2019 18:26:40 +0200 Subject: [PATCH 07/31] print(..., file=...) not supported on bindist --- haskell/private/cc_wrapper.py.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index ffdb9f6f..ba722cea 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -745,7 +745,7 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): # Bazel runfiles_manifest file uses "/" separators. cc = r.Rlocation("/".join([WORKSPACE, CC + ".exe"])) if cc is None: - print("CC not found '{}'.".format(CC), file=sys.stderr) + sys.stderr.write("CC not found '{}'.\n".format(CC)) sys.exit(1) with response_file(args) as rsp: From 8d14b4022eddf1eb8998c23f23e4f994cef58325 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 24 Jul 2019 10:24:13 +0200 Subject: [PATCH 08/31] subprocess.run not supported on bindist --- haskell/private/cc_wrapper.py.tpl | 56 ++++++++++++++++++------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index ba722cea..561aace7 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -668,23 +668,22 @@ def print_file_name(filename, args): """ (basename, ext) = os.path.splitext(filename) - if is_darwin() and ext == ".dylib": + found = run_cc_print_file_name(filename, args) + if not found and is_darwin() and ext == ".dylib": # Bazel generates dynamic libraries with .so extension on Darwin. # However, GHC only looks for files with .dylib extension. - # Try with the .dylib extension first. - found, res = run_cc_print_file_name(filename, args) - if not found: - # Retry with .so extension. - found, so_res = run_cc_print_file_name("%s.so" % basename, args) - if found: - res = so_res + # Retry with .so extension. + found = run_cc_print_file_name("%s.so" % basename, args) + + # Note, gcc --print-file-name does not fail if the file was not found, but + # instead just returns the input filename. + if found: + print(found) else: - _, res = run_cc_print_file_name(filename, args) + print(filename) - sys.stdout.write(res.stdout.decode()) - sys.stderr.write(res.stderr.decode()) - sys.exit(res.returncode) + sys.exit() def run_cc_print_file_name(filename, args): @@ -701,14 +700,14 @@ def run_cc_print_file_name(filename, args): """ args = args + ["--print-file-name", filename] - res = run_cc(args, capture_output=True, exit_on_error=True) - filename = res.stdout.decode().strip() + _, stdoutbuf, _ = run_cc(args, capture_output=True, exit_on_error=True) + filename = stdoutbuf.decode().strip() # Note, gcc --print-file-name does not fail if the file was not found, but # instead just returns the input filename. if os.path.isfile(filename): - return filename, res + return filename else: - return None, res + return None # -------------------------------------------------------------------- @@ -723,7 +722,10 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): exit_on_error: Whether to exit on error. Will print captured output first. Returns: - CompletedProcess + (returncode, stdoutbuf, stderrbuf): + returncode: The exit code of the the process. + stdoutbuf: The captured standard output, None if not capture_output. + stderrbuf: The captured standard error, None if not capture_output. """ if capture_output: @@ -748,16 +750,24 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): sys.stderr.write("CC not found '{}'.\n".format(CC)) sys.exit(1) + stdoutbuf = None + stderrbuf = None + with response_file(args) as rsp: - res = subprocess.run([cc, "@" + rsp], **kwargs) + # subprocess.run is not supported in the bindist CI setup. + with subprocess.Popen([cc, "@" + rsp], **kwargs) as proc: + if capture_output: + (stdoutbuf, stderrbuf) = proc.communicate() + + returncode = proc.wait() - if exit_on_error and res.returncode != 0: + if exit_on_error and returncode != 0: if capture_output: - sys.stdout.write(res.stdout.decode()) - sys.stderr.write(res.stderr.decode()) - sys.exit(res.returncode) + sys.stdout.write(stdout.decode()) + sys.stderr.write(stderr.decode()) + sys.exit(returncode) - return res + return (returncode, stdoutbuf, stderrbuf) @contextmanager From ad7c9a796086eace35a6b72eb1a9443070fe1f65 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 24 Jul 2019 10:47:05 +0200 Subject: [PATCH 09/31] Popen context manager not supported on bindist --- haskell/private/cc_wrapper.py.tpl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 561aace7..01876afe 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -755,11 +755,13 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): with response_file(args) as rsp: # subprocess.run is not supported in the bindist CI setup. - with subprocess.Popen([cc, "@" + rsp], **kwargs) as proc: - if capture_output: - (stdoutbuf, stderrbuf) = proc.communicate() + # subprocess.Popen does not support context manager on CI setup. + proc = subprocess.Popen([cc, "@" + rsp], **kwargs) - returncode = proc.wait() + if capture_output: + (stdoutbuf, stderrbuf) = proc.communicate() + + returncode = proc.wait() if exit_on_error and returncode != 0: if capture_output: From f07bd2233835270e5e81f3b388b772c69a7b9da2 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 24 Jul 2019 10:55:57 +0200 Subject: [PATCH 10/31] NamedTempraryFile(..., delete=False) unsupported on bindist --- haskell/private/cc_wrapper.py.tpl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 01876afe..0f854a91 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -787,15 +787,16 @@ def response_file(args): """ try: - with tempfile.NamedTemporaryFile(mode="w", prefix="rsp", delete=False) as f: - for arg in args: - line = generate_response_line(arg) - f.write(line) - f.close() - yield f.name + (fd, filename) = tempfile.mkstemp(prefix="rsp", text=True) + handle = os.fdopen(fd, "w") + for arg in args: + line = generate_response_line(arg) + handle.write(line) + handle.close() + yield filename finally: try: - os.remove(f.name) + os.remove(filename) except OSError: pass From f4301ca15ae02259d9cf7b81b898730fc78d671f Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Fri, 26 Jul 2019 18:36:01 +0200 Subject: [PATCH 11/31] Windows: cc_wrapper in bash The python cc_wrapper is too slow on Windows. See https://github.com/tweag/rules_haskell/pull/1002 Rewriting it in C++ would be a lot of work as we cannot rely on C++17's std::filesystem, yet. Including a C++ filesystem library would impose a large dependency. Fortunately, the Windows cc_wrapper only needs to shorten library paths, not handle dynamic libraries. It is easier to use a dedicated bash cc_wrapper on Windows instead. --- haskell/BUILD.bazel | 1 + haskell/private/cc_wrapper.bzl | 40 ++++++-- haskell/private/cc_wrapper.sh.tpl | 162 ++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 8 deletions(-) create mode 100644 haskell/private/cc_wrapper.sh.tpl diff --git a/haskell/BUILD.bazel b/haskell/BUILD.bazel index e67c98f3..dd3cfc44 100644 --- a/haskell/BUILD.bazel +++ b/haskell/BUILD.bazel @@ -15,6 +15,7 @@ exports_files( "private/ghci_repl_wrapper.sh", "private/haddock_wrapper.sh.tpl", "private/cc_wrapper.py.tpl", + "private/cc_wrapper.sh.tpl", "private/pkgdb_to_bzl.py", ], ) diff --git a/haskell/private/cc_wrapper.bzl b/haskell/private/cc_wrapper.bzl index 4e7d3957..ee185aab 100644 --- a/haskell/private/cc_wrapper.bzl +++ b/haskell/private/cc_wrapper.bzl @@ -13,7 +13,7 @@ def _cc_wrapper_impl(ctx): feature_configuration = feature_configuration, action_name = ACTION_NAMES.c_compile, ) - cc_wrapper = ctx.actions.declare_file(ctx.label.name + ".py") + cc_wrapper = ctx.actions.declare_file(ctx.label.name) ctx.actions.expand_template( template = ctx.file.template, output = cc_wrapper, @@ -36,7 +36,6 @@ _cc_wrapper = rule( attrs = { "template": attr.label( allow_single_file = True, - default = Label("@rules_haskell//haskell:private/cc_wrapper.py.tpl"), ), "_cc_toolchain": attr.label( default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), @@ -45,18 +44,43 @@ _cc_wrapper = rule( fragments = ["cpp"], ) -def cc_wrapper(name, template = None, **kwargs): +def cc_wrapper(name, **kwargs): _cc_wrapper( - name = name + "-source", - template = template, + name = name + ".py", + template = "@rules_haskell//haskell:private/cc_wrapper.py.tpl", + ) + _cc_wrapper( + name = name + ".sh", + template = "@rules_haskell//haskell:private/cc_wrapper.sh.tpl", ) native.py_binary( - name = name, - srcs = [name + "-source"], - main = name + "-source.py", + name = name + "-python", + srcs = [name + ".py"], python_version = "PY3", + main = name + ".py", deps = [ "@bazel_tools//tools/python/runfiles", ], **kwargs ) + + # This is a workaround for py_binary being too slow on Windows. + # See https://github.com/bazelbuild/bazel/issues/8981 + # In principle the python cc_wrapper would be sufficient for all platforms, + # however, execution is too slow on Windows to be practical. + native.sh_binary( + name = name + "-bash", + srcs = [name + ".sh"], + deps = [ + "@bazel_tools//tools/bash/runfiles", + ], + **kwargs + ) + native.alias( + name = name, + actual = select({ + "@rules_haskell//haskell/platforms:mingw32": name + "-bash", + "//conditions:default": name + "-python", + }), + **kwargs + ) diff --git a/haskell/private/cc_wrapper.sh.tpl b/haskell/private/cc_wrapper.sh.tpl new file mode 100644 index 00000000..96fa0d54 --- /dev/null +++ b/haskell/private/cc_wrapper.sh.tpl @@ -0,0 +1,162 @@ +# CC toolchain wrapper +# +# Usage: cc_wrapper [ARG]... +# +# Wraps the C compiler of the Bazel CC toolchain. Transforms arguments to work +# around limitations of Bazel and GHC and passes those via response file to the C +# compiler. +# +# - Shortens library search paths to stay below maximum path length on Windows. +# +# GHC generates library search paths that contain redundant up-level +# references (..). This can exceed the maximum path length on Windows, which +# will cause linking failures. This wrapper shortens library search paths to +# avoid that issue. + +set -euo pipefail + +# ---------------------------------------------------------- +# Find compiler + +find_exe() { + local exe="$1" + local location + + location="$exe" + if [[ -f "$location" ]]; then + echo "$location" + return + fi + + location="${exe}.exe" + if [[ -f "$location" ]]; then + echo "$location" + return + fi + + # --- begin runfiles.bash initialization v2 --- + # Copy-pasted from the Bazel Bash runfiles library v2. + set -uo pipefail; f=bazel_tools/tools/bash/runfiles/runfiles.bash + source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e + # --- end runfiles.bash initialization v2 --- + + location="$(rlocation "{:workspace:}/$exe")" + if [[ -f "$location" ]]; then + echo "$location" + return + fi + + location="$(rlocation "{:workspace:}/${exe}.exe")" + if [[ -f "$location" ]]; then + echo "$location" + return + fi +} + +CC="$(find_exe "{:cc:}")" + +# ---------------------------------------------------------- +# Handle response file + +RESPONSE_FILE="$(mktemp rspXXXX)" +rm_response_file() { + rm -f "$RESPONSE_FILE" +} +trap rm_response_file EXIT + +quote_arg() { + # Gcc expects one argument per line, surrounded by double quotes, with + # inner double quotes escaped with backslash, and backslashes themselves + # escaped. + local arg="$1" + arg="${arg//\\/\\\\}" + arg="${arg//\"/\\\"}" + printf '"%s"\n' "$arg" +} + +unquote_arg() { + local arg="$1" + if [[ "$arg" =~ ^\"(.*)\"[[:space:]]*$ ]]; then + arg="${BASH_REMATCH[1]}" + arg="${arg//\\\"/\"}" + arg="${arg//\\\\/\\}" + fi + echo "$arg" +} + +add_arg() { + quote_arg "$1" >> "$RESPONSE_FILE" +} + +# ---------------------------------------------------------- +# Parse arguments + +IN_RESPONSE_FILE= +LIB_DIR_COMING= + +shorten_path() { + local input="$1" + local shortest="$input" + + if [[ ! -e "$shortest" ]]; then + # realpath fails if the file does not exist. + echo "$shortest" + return + fi + + local normalized="$(realpath "$shortest")" + if [[ ${#normalized} -lt ${#shortest} ]]; then + shortest="$normalized" + fi + + local relative="$(realpath --relative-to="$PWD" "$shortest")" + if [[ ${#relative} -lt ${#shortest} ]]; then + shortest="$relative" + fi + + echo "$shortest" +} + +handle_lib_dir() { + local lib_dir="$1" + add_arg "-L$(shorten_path "$lib_dir")" +} + +handle_arg() { + local arg="$1" + if [[ $IN_RESPONSE_FILE = 1 ]]; then + arg="$(unquote_arg "$arg")" + fi + if [[ $LIB_DIR_COMING = 1 ]]; then + LIB_DIR_COMING= + handle_lib_dir "$arg" + elif [[ "$arg" =~ ^@(.*)$ ]]; then + IN_RESPONSE_FILE=1 + while read line; do + handle_arg "$line" + done < "${BASH_REMATCH[1]}" + IN_RESPONSE_FILE= + elif [[ "$arg" =~ ^-L(.*)$ || "$arg" =~ ^--library-path=(.*)$ ]]; then + handle_lib_dir "${BASH_REMATCH[1]}" + elif [[ "$arg" = -L || "$arg" = --library-path ]]; then + LIB_DIR_COMING=1 + else + add_arg "$arg" + fi +} + +for arg in "$@"; do + handle_arg "$arg" +done + +# ---------------------------------------------------------- +# Call compiler + +"$CC" "@$RESPONSE_FILE" + +# vim: ft=sh From c0d0f11aa7fc8f85f2adccdde5378694d791c3ae Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Tue, 13 Aug 2019 16:15:53 +0200 Subject: [PATCH 12/31] Remove .dll.a on Windows These libraries cause linking errors on Windows when linking pthreads. --- haskell/ghc_bindist.bzl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/haskell/ghc_bindist.bzl b/haskell/ghc_bindist.bzl index 535f0e37..884b2c31 100644 --- a/haskell/ghc_bindist.bzl +++ b/haskell/ghc_bindist.bzl @@ -266,6 +266,14 @@ haskell_toolchain( haddock_flags = ctx.attr.haddock_flags, repl_ghci_args = ctx.attr.repl_ghci_args, ) + + if os == "windows": + # These libraries cause linking errors on Windows when linking + # pthreads, due to libwinpthread-1.dll not being loaded. + _execute_fail_loudly(ctx, ["rm", "mingw/lib/gcc/x86_64-w64-mingw32/7.2.0/libstdc++.dll.a"]) + _execute_fail_loudly(ctx, ["rm", "mingw/x86_64-w64-mingw32/lib/libpthread.dll.a"]) + _execute_fail_loudly(ctx, ["rm", "mingw/x86_64-w64-mingw32/lib/libwinpthread.dll.a"]) + ctx.template( "BUILD", ghc_build, From fe5464ac9b4c334f1c20f95710215257fefc04ca Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 14 Aug 2019 11:41:27 +0200 Subject: [PATCH 13/31] Bash: Use nameref variables Use nameref variables to return values from functions instead of echoing strings. This allows to avoid command substitution, which would create a subprocess for each function call, which is costly on Windows. --- haskell/private/cc_wrapper.sh.tpl | 38 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/haskell/private/cc_wrapper.sh.tpl b/haskell/private/cc_wrapper.sh.tpl index 96fa0d54..86245085 100644 --- a/haskell/private/cc_wrapper.sh.tpl +++ b/haskell/private/cc_wrapper.sh.tpl @@ -19,18 +19,16 @@ set -euo pipefail # Find compiler find_exe() { - local exe="$1" - local location + local -n location="$1" + local exe="$2" location="$exe" if [[ -f "$location" ]]; then - echo "$location" return fi location="${exe}.exe" if [[ -f "$location" ]]; then - echo "$location" return fi @@ -47,18 +45,17 @@ find_exe() { location="$(rlocation "{:workspace:}/$exe")" if [[ -f "$location" ]]; then - echo "$location" return fi location="$(rlocation "{:workspace:}/${exe}.exe")" if [[ -f "$location" ]]; then - echo "$location" return fi } -CC="$(find_exe "{:cc:}")" +declare CC +find_exe CC "{:cc:}" # ---------------------------------------------------------- # Handle response file @@ -80,13 +77,14 @@ quote_arg() { } unquote_arg() { - local arg="$1" - if [[ "$arg" =~ ^\"(.*)\"[[:space:]]*$ ]]; then - arg="${BASH_REMATCH[1]}" - arg="${arg//\\\"/\"}" - arg="${arg//\\\\/\\}" + local -n output="$1" + local input="$2" + if [[ "$input" =~ ^\"(.*)\"[[:space:]]*$ ]]; then + input="${BASH_REMATCH[1]}" + input="${input//\\\"/\"}" + input="${input//\\\\/\\}" fi - echo "$arg" + output="$input" } add_arg() { @@ -100,12 +98,12 @@ IN_RESPONSE_FILE= LIB_DIR_COMING= shorten_path() { - local input="$1" - local shortest="$input" + local -n shortest="$1" + local input="$2" + shortest="$input" if [[ ! -e "$shortest" ]]; then # realpath fails if the file does not exist. - echo "$shortest" return fi @@ -118,19 +116,19 @@ shorten_path() { if [[ ${#relative} -lt ${#shortest} ]]; then shortest="$relative" fi - - echo "$shortest" } handle_lib_dir() { local lib_dir="$1" - add_arg "-L$(shorten_path "$lib_dir")" + local shortened + shorten_path shortened "$lib_dir" + add_arg "-L$shortened" } handle_arg() { local arg="$1" if [[ $IN_RESPONSE_FILE = 1 ]]; then - arg="$(unquote_arg "$arg")" + unquote_arg arg "$arg" fi if [[ $LIB_DIR_COMING = 1 ]]; then LIB_DIR_COMING= From c6f4da909848366d9cecae8a6c9680c85df6324e Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 28 Aug 2019 13:34:59 +0200 Subject: [PATCH 14/31] Define response file https://github.com/tweag/rules_haskell/pull/1039/files/71c2a6d5286a0e0802c78be3a1a8cf6e6ae8061d..a10b82374bc7b6f86844835b1656e06b449a0b0a#r318497584 --- haskell/private/cc_wrapper.py.tpl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 0f854a91..4dc2300d 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -5,7 +5,8 @@ Usage: cc_wrapper [ARG]... Wraps the C compiler of the Bazel CC toolchain. Transforms arguments to work around limitations of Bazel and GHC and passes those via response file to the C -compiler. +compiler. A response file is a text file listing command-line arguments. It is +used to avoid command line length limitations. - Shortens library search paths to stay below maximum path length on Windows. From 90778d1f20de499d5d947c2a14f432746ffd8008 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 28 Aug 2019 13:53:43 +0200 Subject: [PATCH 15/31] Factor out command line argument matching https://github.com/tweag/rules_haskell/pull/1039/files/71c2a6d5286a0e0802c78be3a1a8cf6e6ae8061d..a10b82374bc7b6f86844835b1656e06b449a0b0a#r318490807 --- haskell/private/cc_wrapper.py.tpl | 107 ++++++++++++++++++------------ 1 file changed, 64 insertions(+), 43 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 4dc2300d..9fedad97 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -162,48 +162,37 @@ class Args: yield out_arg def _handle_output(self, arg, args, out): - if arg == "-o": + consumed, output = argument(arg, args, short = "-o") + + if consumed: # Remember the output filename. - self.output = next(args) + self.output = output out.extend(["-o", self.output]) - return True - else: - return False + + return consumed def _handle_library(self, arg, args, out): - if arg == "-l" or arg == "--library": - library = next(args) - elif arg.startswith("-l"): - library = arg[2:] - elif arg.startswith("--library="): - library = arg[len("--library="):] - else: - return False + consumed, library = argument(arg, args, short = "-l", long = "--library") - # Remember the required libraries. - self.libraries.append(library) - out.append("-l{}".format(library)) + if consumed: + # Remember the required libraries. + self.libraries.append(library) + out.append("-l{}".format(library)) - return True + return consumed def _handle_library_path(self, arg, args, out): - if arg == "-L" or arg == "--library-path": - library_path = next(args) - elif arg.startswith("-L"): - library_path = arg[2:] - elif arg.startswith("--library-path="): - library_path = arg[len("--library-path="):] - else: - return False + consumed, library_path = argument(arg, args, short = "-L", long = "--library-path") - # Shorten the library search paths. On Windows library search paths may - # exceed the maximum path length. - shortened = shorten_path(library_path) - # Remember the library search paths. - self.library_paths.append(shortened) - out.append("-L{}".format(shortened)) + if consumed: + # Shorten the library search paths. On Windows library search paths may + # exceed the maximum path length. + shortened = shorten_path(library_path) + # Remember the library search paths. + self.library_paths.append(shortened) + out.append("-L{}".format(shortened)) - return True + return consumed def _handle_linker_arg(self, arg, args, out): if arg == "-Xlinker": @@ -237,19 +226,15 @@ class Args: self.rpaths.append(rpath) def _handle_print_file_name(self, arg, args, out): - if arg == "--print-file-name": - print_file_name = next(args) - elif arg.startswith("--print-file-name="): - print_file_name = arg[len("--print-file-name="):] - else: - return False + consumed, print_file_name = argument(arg, args, long = "--print-file-name") - # Remember print-file-name action. Don't forward to allow for later - # manipulation. - self.print_file_name = print_file_name - self.action = Args.PRINT_FILE_NAME + if consumed: + # Remember print-file-name action. Don't forward to allow for later + # manipulation. + self.print_file_name = print_file_name + self.action = Args.PRINT_FILE_NAME - return True + return consumed def _handle_compile(self, arg, args, out): if arg == "-c": @@ -261,6 +246,42 @@ class Args: return True +def argument(arg, args, short = None, long = None): + """Parse an argument that takes a parameter. + + I.e. arguments such as + -l + -l + --library + --library= + + Args: + arg: The current command-line argument. + args: Iterator over the remaining arguments. + short: The short argument name, e.g. "-l". + long: The long argument name, e.g. "--library". + + Returns: + consumed, value + consumed: bool, Whether the argument matched. + value: string, The value parameter or None. + + """ + if short: + if arg == short: + return True, next(args) + elif arg.startswith(short): + return True, arg[len(short):] + + if long: + if arg == long: + return True, next(args) + elif arg.startswith(long + "="): + return True, arg[len(long + "="):] + + return False, None + + def load_response_files(args): """Generator that loads arguments from response files. From 570f47be9c30e3580f45581adf3c230a252141d1 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 28 Aug 2019 14:00:56 +0200 Subject: [PATCH 16/31] Handle -rpath= syntax https://github.com/tweag/rules_haskell/pull/1039/files/71c2a6d5286a0e0802c78be3a1a8cf6e6ae8061d..a10b82374bc7b6f86844835b1656e06b449a0b0a#r318496719 --- haskell/private/cc_wrapper.py.tpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 9fedad97..d37d1d53 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -200,6 +200,8 @@ class Args: if self._prev_ld_arg is None: if ld_arg == "-rpath": self._prev_ld_arg = ld_arg + elif ld_arg.startswith("-rpath="): + self._handle_rpath(ld_arg[len("-rpath="):], out) else: out.extend(["-Xlinker", ld_arg]) elif self._prev_ld_arg == "-rpath": @@ -214,6 +216,8 @@ class Args: if len(ld_args) == 2 and ld_args[0] == "-rpath": self._handle_rpath(ld_args[1], out) return True + elif len(ld_args) == 1 and ld_args[0].startswith("-rpath="): + self._handle_rpath(ld_args[0][len("-rpath="):]) else: out.append(arg) return True From f0337047331bfb318734264f70a9171ed99762aa Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 28 Aug 2019 14:06:51 +0200 Subject: [PATCH 17/31] Explain -Xlinker and -Wl,... https://github.com/tweag/rules_haskell/pull/1039/files/71c2a6d5286a0e0802c78be3a1a8cf6e6ae8061d..a10b82374bc7b6f86844835b1656e06b449a0b0a#r318495192 --- haskell/private/cc_wrapper.py.tpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index d37d1d53..595743d2 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -195,6 +195,10 @@ class Args: return consumed def _handle_linker_arg(self, arg, args, out): + # gcc allows to forward flags to the linker using either + # -Xlinker + # or + # -Wl,,... if arg == "-Xlinker": ld_arg = next(args) if self._prev_ld_arg is None: From 10672f840e302c17ba2e9bfbf4215ecf098705f5 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Wed, 28 Aug 2019 14:38:09 +0200 Subject: [PATCH 18/31] shorten_path: Path must exist https://github.com/tweag/rules_haskell/pull/1039/files/71c2a6d5286a0e0802c78be3a1a8cf6e6ae8061d..a10b82374bc7b6f86844835b1656e06b449a0b0a#r318499680 --- haskell/private/cc_wrapper.py.tpl | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 595743d2..d92225eb 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -185,12 +185,16 @@ class Args: consumed, library_path = argument(arg, args, short = "-L", long = "--library-path") if consumed: - # Shorten the library search paths. On Windows library search paths may - # exceed the maximum path length. - shortened = shorten_path(library_path) - # Remember the library search paths. - self.library_paths.append(shortened) - out.append("-L{}".format(shortened)) + # Skip non-existent library search paths. These can occur in static + # linking mode where dynamic libraries are not present in the + # sandbox, or with Cabal packages with bogus library-path entries. + if os.path.exists(library_path): + # Shorten the library search paths. On Windows library search + # paths may exceed the maximum path length. + shortened = shorten_path(library_path) + # Remember the library search paths. + self.library_paths.append(shortened) + out.append("-L{}".format(shortened)) return consumed @@ -331,13 +335,12 @@ def shorten_path(input_path): - Resolve symbolic links. Args: - input_path: The path to shorten. + input_path: The path to shorten, must exist. Returns: The shortened path. """ - exists = os.path.exists(input_path) shortened = input_path # Try relativizing to current working directory. @@ -351,7 +354,7 @@ def shorten_path(input_path): # Ensure that the path is still correct. Reducing up-level references # may change the meaning of the path in the presence of symbolic links. try: - if not exists or os.path.samefile(norm, shortened): + if os.path.samefile(norm, shortened): shortened = norm except IOError: # stat may fail if the path became invalid or does not exist. @@ -363,7 +366,7 @@ def shorten_path(input_path): if len(real) < len(shortened): shortened = real except IOError: - # realpath may fail if the path does not exist. + # may fail if the path does not exist or on dangling symlinks. pass return shortened @@ -656,7 +659,9 @@ def resolve_rpath(rpath, output): else: rpath = os.path.abspath(shorten_path(min(candidates))) else: - rpath = os.path.abspath(shorten_path(rpath)) + if os.path.exists(rpath): + rpath = shorten_path(rpath) + rpath = os.path.abspath(rpath) return rpath, rpath else: From eeac689c205a6d39227f35b19e34b7e8cb46eb75 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 09:51:17 +0200 Subject: [PATCH 19/31] cc_wrapper: Args.action --> Args._action It's an internal attribute and should not be accessed directly, instead clients should use `Args.linking`, `Args.compiling`, or `Args.printing_file_name`. https://github.com/tweag/rules_haskell/pull/1039/files#r318549754 --- haskell/private/cc_wrapper.py.tpl | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index d92225eb..fdfd0fa6 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -68,8 +68,9 @@ class Args: Attrs: args: The collected and transformed arguments. - linking: The action is linking. - printing_file_name: The action is print-file-name. + linking: Gcc is called for linking (default). + compiling: Gcc is called for compiling (-c). + printing_file_name: Gcc is called with --print-file-name. output: The output binary or library when linking. library_paths: The library search paths when linking. @@ -95,12 +96,16 @@ class Args: args: Iterable over command-line arguments. """ - self.action = Args.LINK self.print_file_name = None self.libraries = [] self.library_paths = [] self.rpaths = [] self.output = None + # gcc action, print-file-name (--print-file-name), compile (-c), or + # link (default) + self._action = Args.LINK + # The currently active linker option that expects an argument. E.g. if + # `-Xlinker -rpath` was encountered, then `-rpath`. self._prev_ld_arg = None self.args = list(self._handle_args(args)) @@ -113,17 +118,17 @@ class Args: @property def linking(self): """Whether this is a link invocation.""" - return self.action == Args.LINK and self.output is not None + return self._action == Args.LINK and self.output is not None @property def compiling(self): """Whether this is a compile invocation.""" - return self.action == Args.COMPILE + return self._action == Args.COMPILE @property def printing_file_name(self): """Whether this is a print-file-name invocation.""" - return self.action == Args.PRINT_FILE_NAME and self.print_file_name is not None + return self._action == Args.PRINT_FILE_NAME and self.print_file_name is not None def _handle_args(self, args): """Argument handling pipeline. @@ -244,13 +249,13 @@ class Args: # Remember print-file-name action. Don't forward to allow for later # manipulation. self.print_file_name = print_file_name - self.action = Args.PRINT_FILE_NAME + self._action = Args.PRINT_FILE_NAME return consumed def _handle_compile(self, arg, args, out): if arg == "-c": - self.action = Args.COMPILE + self._action = Args.COMPILE out.append(arg) else: return False From b7744580d558ae9ce24a9e6ae3543cc0276fec76 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 10:02:45 +0200 Subject: [PATCH 20/31] fix typo --- haskell/private/cc_wrapper.py.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index fdfd0fa6..d82ec9e1 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -510,7 +510,7 @@ def darwin_shorten_rpaths(rpaths, libraries, output): def sort_rpaths(rpaths): """Sort RUNPATHs by preference. - Preference in decsending order: + Preference in descending order: - Relative to target - Absolute path - Relative to CWD From 4e35278afb0bc8f970efa9a0c9771ddf6ce2b435 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 11:01:53 +0200 Subject: [PATCH 21/31] Improve library search code Use clearer variable names and add comments to improve readability. https://github.com/tweag/rules_haskell/pull/1039/files#r318540827 --- haskell/private/cc_wrapper.py.tpl | 89 +++++++++++++++++++------------ 1 file changed, 55 insertions(+), 34 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index d82ec9e1..0bf4029b 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -430,19 +430,22 @@ def shorten_rpaths(rpaths, libraries, output): """ input_rpaths = sort_rpaths(rpaths) - missing = set(libraries) - rpaths = [] + # Keeps track of libraries that were not yet found in an rpath. + libs_still_missing = set(libraries) + # Keeps track of rpaths in which we found libraries. + required_rpaths = [] + # Iterate over the given rpaths until all libraries are found. for rpath in input_rpaths: - if not missing: + if not libs_still_missing: break rpath, rpath_dir = resolve_rpath(rpath, output) - found, missing = find_library(missing, rpath_dir) + found, libs_still_missing = find_library(libs_still_missing, rpath_dir) if found: - rpaths.append(rpath) + required_rpaths.append(rpath) - return rpaths + return required_rpaths def darwin_shorten_rpaths(rpaths, libraries, output): @@ -463,9 +466,12 @@ def darwin_shorten_rpaths(rpaths, libraries, output): """ input_rpaths = sort_rpaths(rpaths) - missing = set(libraries) - rpaths = [] + # Keeps track of libraries that were not yet found in an rpath. + libs_still_missing = set(libraries) + # Keeps track of rpaths in which we found libraries. + required_rpaths = [] + # Keeps track of required rewrites of load commands. rewrites = [] # References to core libs take up much space. Consider detecting the GHC @@ -473,38 +479,45 @@ def darwin_shorten_rpaths(rpaths, libraries, output): # that. Alternatively, https://github.com/bazelbuild/bazel/pull/8888 would # also avoid this issue. - # Determine solib dir and rewrite load commands relative to solib dir. - # This allows to replace potentially many rpaths by one. + # Determine solib dir and rewrite load commands relative to solib dir. This + # allows to replace potentially many rpaths by a single one. On macOS load + # commands can use paths relative to rpath entries, e.g. + # `@rpath/some_dir/libsome_lib.dylib`. solib_rpath = find_solib_rpath(input_rpaths, output) - if missing and solib_rpath is not None: + if libs_still_missing and solib_rpath is not None: solib_rpath, solib_dir = resolve_rpath(solib_rpath, output) - found, missing = find_library_recursive(missing, solib_dir) + found, libs_still_missing = find_library_recursive(libs_still_missing, solib_dir) if found: - rpaths.append(solib_rpath) + required_rpaths.append(solib_rpath) for f in found.values(): + # Determine rewrites of load commands to load libraries + # relative to the solib dir rpath entry. soname = darwin_get_install_name(os.path.join(solib_dir, f)) rewrites.append((soname, f)) # For the remaining missing libraries, determine which rpaths are required. + # Iterate over the given rpaths until all libraries are found. for rpath in input_rpaths: - if not missing: + if not libs_still_missing: break rpath, rpath_dir = resolve_rpath(rpath, output) - found, missing = find_library(missing, rpath_dir) - # Libraries with an absolute install_name don't require an rpath entry. + found, libs_still_missing = find_library(libs_still_missing, rpath_dir) + # Libraries with an absolute install_name don't require an rpath entry + # and can be filtered out. found = dict(itertools.filterfalse( lambda item: os.path.isabs(darwin_get_install_name(os.path.join(rpath_dir, item[1]))), found.items())) if len(found) == 1: - # Avoid unnecessary rpath if it is only relevant for one load command. + # If the rpath is only needed for one load command, then we can + # avoid the rpath entry by fusing the rpath into the load command. [filename] = found.values() soname = darwin_get_install_name(os.path.join(rpath_dir, filename)) rewrites.append((soname, os.path.join(rpath, filename))) elif found: - rpaths.append(rpath) + required_rpaths.append(rpath) - return rpaths, rewrites + return required_rpaths, rewrites def sort_rpaths(rpaths): @@ -565,26 +578,30 @@ def find_library_recursive(libraries, directory): directory: Root of directory tree. Returns: - (found, missing): + (found, libs_still_missing): found: Dict of found libraries {libname: path} relative to directory. - missing: Set of remaining missing libraries. + libs_still_missing: Set of remaining missing libraries. """ - missing = set(libraries) + # Keeps track of libraries that were not yet found underneath directory. + libs_still_missing = set(libraries) + # Keeps track of libraries that were already found. found = {} + # Iterate over the directory tree until all libraries are found. for root, _, files in os.walk(directory, followlinks=True): prefix = os.path.relpath(root, directory) - if not missing: + if not libs_still_missing: break for f in files: libname = get_lib_name(f) - if libname and libname in missing: + if libname and libname in libs_still_missing: found[libname] = os.path.join(prefix, f) if prefix != "." else f - missing.discard(libname) - if not missing: + libs_still_missing.discard(libname) + if not libs_still_missing: + # Short-cut files iteration if no more libs are missing. break - return found, missing + return found, libs_still_missing def find_library(libraries, directory): @@ -595,23 +612,27 @@ def find_library(libraries, directory): directory: The directory in which to search for libraries. Returns: - (found, missing): + (found, libs_still_missing): found: Dict of found libraries {libname: path} relative to directory. - missing: Set of remaining missing libraries. + libs_still_missing: Set of remaining missing libraries. """ - missing = set(libraries) + # Keeps track of libraries that were not yet found within directory. + libs_still_missing = set(libraries) + # Keeps track of libraries that were already found. found = {} + # Iterate over the files within directory until all libraries are found. + # This corresponds to a one level deep os.walk. for _, _, files in itertools.islice(os.walk(directory), 1): - if not missing: + if not libs_still_missing: break for f in files: libname = get_lib_name(f) - if libname and libname in missing: + if libname and libname in libs_still_missing: found[libname] = f - missing.discard(libname) + libs_still_missing.discard(libname) - return found, missing + return found, libs_still_missing def get_lib_name(filename): From 178db6da0a04bef66e35d554da69e5bae44ef3bd Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 11:45:00 +0200 Subject: [PATCH 22/31] Add test-case to verify _solib_ assumption https://github.com/tweag/rules_haskell/pull/1039/files#r318542321 --- tests/solib_dir/BUILD.bazel | 10 +++++ tests/solib_dir/solib_test.bzl | 71 ++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 tests/solib_dir/BUILD.bazel create mode 100644 tests/solib_dir/solib_test.bzl diff --git a/tests/solib_dir/BUILD.bazel b/tests/solib_dir/BUILD.bazel new file mode 100644 index 00000000..e708e687 --- /dev/null +++ b/tests/solib_dir/BUILD.bazel @@ -0,0 +1,10 @@ +load(":solib_test.bzl", "solib_test") + +# See rule docstring in solib_test.bzl for details. +solib_test( + name = "solib_dir", + is_windows = select({ + "@rules_haskell//haskell/platforms:mingw32": True, + "//conditions:default": False, + }), +) diff --git a/tests/solib_dir/solib_test.bzl b/tests/solib_dir/solib_test.bzl new file mode 100644 index 00000000..a671f4c3 --- /dev/null +++ b/tests/solib_dir/solib_test.bzl @@ -0,0 +1,71 @@ +_test_script_template = """#!/usr/bin/env bash +library_path={library_path} +is_windows={is_windows} +expected="bin/_solib_{cpu}" +if [[ "$is_windows" != 1 && ! $library_path =~ ^.*"/$expected/".*$ ]]; then + echo "Expected library path containing directory '$expected'," >&2 + echo "but got: '$library_path'." >&2 + exit 1 +fi +""" + +def _solib_test_impl(ctx): + # Write a dummy dynamic library. It will never be loaded, we're only + # interested in the paths that Bazel generates. + dynamic_library = ctx.actions.declare_file("lib{}.so".format(ctx.label.name)) + ctx.actions.write( + content = "", + is_executable = False, + output = dynamic_library, + ) + + # XXX Workaround https://github.com/bazelbuild/bazel/issues/6874. + # Should be find_cpp_toolchain() instead. + cc_toolchain = ctx.attr._cc_toolchain[cc_common.CcToolchainInfo] + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain, + requested_features = ctx.features, + unsupported_features = ctx.disabled_features, + ) + lib_to_link = cc_common.create_library_to_link( + actions = ctx.actions, + feature_configuration = feature_configuration, + dynamic_library = dynamic_library, + cc_toolchain = cc_toolchain, + ) + + # Write the test script. + test_script = ctx.actions.declare_file(ctx.label.name) + ctx.actions.write( + content = _test_script_template.format( + cpu = cc_toolchain.cpu, + library_path = lib_to_link.dynamic_library.path, + is_windows = "1" if ctx.attr.is_windows else "", + ), + is_executable = True, + output = test_script, + ) + return [DefaultInfo( + executable = test_script, + )] + +solib_test = rule( + _solib_test_impl, + attrs = { + "is_windows": attr.bool(), + "_cc_toolchain": attr.label( + default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), + ), + }, + executable = True, + fragments = ["cpp"], + test = True, +) +"""Test that Bazel's solib directory matches our expectations. + +The cc_wrapper used by rules_haskell (haskell/private/cc_wrapper.py.tpl) +assumes that Bazel generates symbolic links for dynamic libraries under a +directory called `bin/_solib_` on Darwin and Linux. This rule generates a +test-case that fails if this assumption is not met. +""" From 84ef3de172c59c1d1f40d4f32261503e951006c7 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 12:04:59 +0200 Subject: [PATCH 23/31] Explain print-file-name https://github.com/tweag/rules_haskell/pull/1039/files#r318547022 --- haskell/private/cc_wrapper.py.tpl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 0bf4029b..84766d5f 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -723,6 +723,14 @@ def darwin_rewrite_load_commands(rewrites, output): def print_file_name(filename, args): """Execute the print-file-name action. + From gcc(1) + + -print-file-name=library + Print the full absolute name of the library file library that would + be used when linking---and don't do anything else. With this + option, GCC does not compile or link anything; it just prints the + file name. + Args: filename: The queried filename. args: The remaining arguments. From 05300f170fd7f28868237d377b9f45cc32ff94c9 Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 12:10:09 +0200 Subject: [PATCH 24/31] Pass platform from Bazel https://github.com/tweag/rules_haskell/pull/1039/files#r318662838 --- haskell/private/cc_wrapper.bzl | 12 ++++++++++++ haskell/private/cc_wrapper.py.tpl | 21 +++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/haskell/private/cc_wrapper.bzl b/haskell/private/cc_wrapper.bzl index ee185aab..3aefd1f1 100644 --- a/haskell/private/cc_wrapper.bzl +++ b/haskell/private/cc_wrapper.bzl @@ -21,6 +21,7 @@ def _cc_wrapper_impl(ctx): substitutions = { "{:cc:}": cc, "{:workspace:}": ctx.workspace_name, + "{:platform:}": ctx.attr.platform, }, ) return [DefaultInfo( @@ -37,6 +38,7 @@ _cc_wrapper = rule( "template": attr.label( allow_single_file = True, ), + "platform": attr.string(), "_cc_toolchain": attr.label( default = Label("@bazel_tools//tools/cpp:current_cc_toolchain"), ), @@ -48,10 +50,20 @@ def cc_wrapper(name, **kwargs): _cc_wrapper( name = name + ".py", template = "@rules_haskell//haskell:private/cc_wrapper.py.tpl", + platform = select({ + "@rules_haskell//haskell/platforms:darwin": "darwin", + "@rules_haskell//haskell/platforms:mingw32": "windows", + "//conditions:default": "linux", + }), ) _cc_wrapper( name = name + ".sh", template = "@rules_haskell//haskell:private/cc_wrapper.sh.tpl", + platform = select({ + "@rules_haskell//haskell/platforms:darwin": "darwin", + "@rules_haskell//haskell/platforms:mingw32": "windows", + "//conditions:default": "linux", + }), ) native.py_binary( name = name + "-python", diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 84766d5f..beb983c5 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -35,7 +35,6 @@ from contextlib import contextmanager import glob import itertools import os -import platform import shlex import subprocess import sys @@ -43,6 +42,7 @@ import tempfile WORKSPACE = "{:workspace:}" CC = "{:cc:}" +PLATFORM = "{:platform:}" INSTALL_NAME_TOOL = "/usr/bin/install_name_tool" OTOOL = "/usr/bin/otool" @@ -530,11 +530,10 @@ def sort_rpaths(rpaths): """ def rpath_priority(rpath): - system = platform.system() - if system == "Darwin": + if is_darwin(): if rpath.startswith("@loader_path"): return 0 - elif system == "Linux": + elif is_linux(): if rpath.startswith("$ORIGIN"): return 0 if os.path.isabs(rpath): @@ -811,7 +810,7 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): # script using Bazel runfiles. r = bazel_runfiles.Create() cc = r.Rlocation("/".join([WORKSPACE, CC])) - if cc is None and platform.system() == "Windows": + if cc is None and is_windows(): # We must use "/" instead of os.path.join on Windows, because the # Bazel runfiles_manifest file uses "/" separators. cc = r.Rlocation("/".join([WORKSPACE, CC + ".exe"])) @@ -879,7 +878,17 @@ def generate_response_line(arg): def is_darwin(): """Whether the execution platform is Darwin.""" - return platform.system() == "Darwin" + return PLATFORM == "darwin" + + +def is_linux(): + """Whether the execution platform is Linux.""" + return PLATFORM == "linux" + + +def is_windows(): + """Whether the execution platform is Windows.""" + return PLATFORM == "windows" def is_temporary_output(output): From 232fb6e5b255844fcdb0b1f98af800f83a80359b Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 12:21:33 +0200 Subject: [PATCH 25/31] Write to buffer to avoid decode https://github.com/tweag/rules_haskell/pull/1039#discussion_r318545150 --- haskell/private/cc_wrapper.py.tpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index beb983c5..33d4bd95 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -833,8 +833,8 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): if exit_on_error and returncode != 0: if capture_output: - sys.stdout.write(stdout.decode()) - sys.stderr.write(stderr.decode()) + sys.stdout.buffer.write(stdout) + sys.stderr.buffer.write(stderr) sys.exit(returncode) return (returncode, stdoutbuf, stderrbuf) From 91bd8c2fcd2ff52b008c5912f57b1c7a85047c0f Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 13:22:23 +0200 Subject: [PATCH 26/31] cc_wrapper.sh --> cc_wrapper_windows.sh https://github.com/tweag/rules_haskell/pull/1039/files#r318669187 --- haskell/BUILD.bazel | 2 +- haskell/private/cc_wrapper.bzl | 2 +- .../private/{cc_wrapper.sh.tpl => cc_wrapper_windows.sh.tpl} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename haskell/private/{cc_wrapper.sh.tpl => cc_wrapper_windows.sh.tpl} (100%) diff --git a/haskell/BUILD.bazel b/haskell/BUILD.bazel index dd3cfc44..be61737a 100644 --- a/haskell/BUILD.bazel +++ b/haskell/BUILD.bazel @@ -15,7 +15,7 @@ exports_files( "private/ghci_repl_wrapper.sh", "private/haddock_wrapper.sh.tpl", "private/cc_wrapper.py.tpl", - "private/cc_wrapper.sh.tpl", + "private/cc_wrapper_windows.sh.tpl", "private/pkgdb_to_bzl.py", ], ) diff --git a/haskell/private/cc_wrapper.bzl b/haskell/private/cc_wrapper.bzl index 3aefd1f1..10102961 100644 --- a/haskell/private/cc_wrapper.bzl +++ b/haskell/private/cc_wrapper.bzl @@ -58,7 +58,7 @@ def cc_wrapper(name, **kwargs): ) _cc_wrapper( name = name + ".sh", - template = "@rules_haskell//haskell:private/cc_wrapper.sh.tpl", + template = "@rules_haskell//haskell:private/cc_wrapper_windows.sh.tpl", platform = select({ "@rules_haskell//haskell/platforms:darwin": "darwin", "@rules_haskell//haskell/platforms:mingw32": "windows", diff --git a/haskell/private/cc_wrapper.sh.tpl b/haskell/private/cc_wrapper_windows.sh.tpl similarity index 100% rename from haskell/private/cc_wrapper.sh.tpl rename to haskell/private/cc_wrapper_windows.sh.tpl From a97c521226c22a5b73c9792718b7daee18f8899b Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 13:32:01 +0200 Subject: [PATCH 27/31] Expand sort_rpaths docstring https://github.com/tweag/rules_haskell/pull/1039/files#r319002515 --- haskell/private/cc_wrapper.py.tpl | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 33d4bd95..a2f1e1ab 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -523,10 +523,21 @@ def darwin_shorten_rpaths(rpaths, libraries, output): def sort_rpaths(rpaths): """Sort RUNPATHs by preference. - Preference in descending order: - - Relative to target - - Absolute path - - Relative to CWD + We classify three types of rpaths (in descending order of preference): + - relative to output, i.e. $ORIGIN/... or @loader_path/... + - absolute, e.g. /nix/store/... + - relative, e.g. bazel-out/.... + + We prefer rpaths relative to the output. They tend to be shorter, and they + typically involve Bazel's _solib_* directory which bundles lots of + libraries (meaning less rpaths required). They're also less likely to leak + information about the local installation into the Bazel cache. + + Next, we prefer absolute paths. They function regardless of execution + directory, and they are still likely to play well with the cache, e.g. + /nix/store/... or /usr/lib/.... + + Finally, we fall back to relative rpaths. """ def rpath_priority(rpath): From a1ef96b6aaada165fbb8303ff3808e5b1924f62c Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 13:36:11 +0200 Subject: [PATCH 28/31] Explain @rpath load commands https://github.com/tweag/rules_haskell/pull/1039/files#r319002635 --- haskell/private/cc_wrapper.py.tpl | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index a2f1e1ab..3a0c2c5b 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -479,10 +479,27 @@ def darwin_shorten_rpaths(rpaths, libraries, output): # that. Alternatively, https://github.com/bazelbuild/bazel/pull/8888 would # also avoid this issue. - # Determine solib dir and rewrite load commands relative to solib dir. This - # allows to replace potentially many rpaths by a single one. On macOS load - # commands can use paths relative to rpath entries, e.g. - # `@rpath/some_dir/libsome_lib.dylib`. + # Determine solib dir and rewrite load commands relative to solib dir. + # + # This allows to replace potentially many rpaths by a single one on Darwin. + # Namely, Darwin allows to explicitly refer to the rpath in load commands. + # E.g. + # + # LOAD @rpath/somedir/libsomelib.dylib + # + # With that we can avoid multiple rpath entries of the form + # + # RPATH @loader_path/.../_solib_*/mangled_a + # RPATH @loader_path/.../_solib_*/mangled_b + # RPATH @loader_path/.../_solib_*/mangled_c + # + # And instead have a single rpath and load commands as follows + # + # RPATH @loader_path/.../_solib_* + # LOAD @rpath/mangled_a/lib_a + # LOAD @rpath/mangled_b/lib_b + # LOAD @rpath/mangled_c/lib_c + # solib_rpath = find_solib_rpath(input_rpaths, output) if libs_still_missing and solib_rpath is not None: solib_rpath, solib_dir = resolve_rpath(solib_rpath, output) From 88aa0d45daae5cea0870ffb25e0281dbe430f97e Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 13:42:34 +0200 Subject: [PATCH 29/31] Factor out find_cc https://github.com/tweag/rules_haskell/pull/1039/files#r318665931 --- haskell/private/cc_wrapper.py.tpl | 37 ++++++++++++++++++------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 3a0c2c5b..8e76c53e 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -809,6 +809,27 @@ def run_cc_print_file_name(filename, args): # -------------------------------------------------------------------- +def find_cc(): + """Find the path to the actual compiler executable.""" + if os.path.isfile(CC): + cc = CC + else: + # On macOS CC is a relative path to a wrapper script. If we're + # being called from a GHCi REPL then we need to find this wrapper + # script using Bazel runfiles. + r = bazel_runfiles.Create() + cc = r.Rlocation("/".join([WORKSPACE, CC])) + if cc is None and is_windows(): + # We must use "/" instead of os.path.join on Windows, because the + # Bazel runfiles_manifest file uses "/" separators. + cc = r.Rlocation("/".join([WORKSPACE, CC + ".exe"])) + if cc is None: + sys.stderr.write("CC not found '{}'.\n".format(CC)) + sys.exit(1) + + return cc + + def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): """Execute cc with a response file holding the given arguments. @@ -830,21 +851,7 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): new_kwargs.update(kwargs) kwargs = new_kwargs - if os.path.isfile(CC): - cc = CC - else: - # On macOS CC is a relative path to a wrapper script. If we're - # being called from a GHCi REPL then we need to find this wrapper - # script using Bazel runfiles. - r = bazel_runfiles.Create() - cc = r.Rlocation("/".join([WORKSPACE, CC])) - if cc is None and is_windows(): - # We must use "/" instead of os.path.join on Windows, because the - # Bazel runfiles_manifest file uses "/" separators. - cc = r.Rlocation("/".join([WORKSPACE, CC + ".exe"])) - if cc is None: - sys.stderr.write("CC not found '{}'.\n".format(CC)) - sys.exit(1) + cc = find_cc() stdoutbuf = None stderrbuf = None From 99d601f92541fbf6ad9d86096c426b72c6acdc2d Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Thu, 29 Aug 2019 13:47:26 +0200 Subject: [PATCH 30/31] read --> read -r https://github.com/tweag/rules_haskell/pull/1039/files#r318989507 --- haskell/private/cc_wrapper_windows.sh.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/haskell/private/cc_wrapper_windows.sh.tpl b/haskell/private/cc_wrapper_windows.sh.tpl index 86245085..eb7c363e 100644 --- a/haskell/private/cc_wrapper_windows.sh.tpl +++ b/haskell/private/cc_wrapper_windows.sh.tpl @@ -135,7 +135,7 @@ handle_arg() { handle_lib_dir "$arg" elif [[ "$arg" =~ ^@(.*)$ ]]; then IN_RESPONSE_FILE=1 - while read line; do + while read -r line; do handle_arg "$line" done < "${BASH_REMATCH[1]}" IN_RESPONSE_FILE= From a4f40a81d7bcc3fdfbcb33f810189f4f23580e2e Mon Sep 17 00:00:00 2001 From: Andreas Herrmann Date: Mon, 2 Sep 2019 15:23:14 +0200 Subject: [PATCH 31/31] Avoid response files for performance Reduces the runtime of the cc_wrapper in nix-shell by 60% in case of short overall cc_wrapper execution time. --- haskell/private/cc_wrapper.py.tpl | 36 ++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/haskell/private/cc_wrapper.py.tpl b/haskell/private/cc_wrapper.py.tpl index 8e76c53e..bbfbea33 100644 --- a/haskell/private/cc_wrapper.py.tpl +++ b/haskell/private/cc_wrapper.py.tpl @@ -853,26 +853,38 @@ def run_cc(args, capture_output=False, exit_on_error=False, **kwargs): cc = find_cc() - stdoutbuf = None - stderrbuf = None - - with response_file(args) as rsp: + def _run_cc(args): # subprocess.run is not supported in the bindist CI setup. # subprocess.Popen does not support context manager on CI setup. - proc = subprocess.Popen([cc, "@" + rsp], **kwargs) + proc = subprocess.Popen([cc] + args, **kwargs) if capture_output: (stdoutbuf, stderrbuf) = proc.communicate() + else: + stdoutbuf = None + stderrbuf = None returncode = proc.wait() - if exit_on_error and returncode != 0: - if capture_output: - sys.stdout.buffer.write(stdout) - sys.stderr.buffer.write(stderr) - sys.exit(returncode) - - return (returncode, stdoutbuf, stderrbuf) + if exit_on_error and returncode != 0: + if capture_output: + sys.stdout.buffer.write(stdoutbuf) + sys.stderr.buffer.write(stderrbuf) + sys.exit(returncode) + + return (returncode, stdoutbuf, stderrbuf) + + # Too avoid exceeding the OS command-line length limit we use response + # files. However, creating and removing temporary files causes overhead. + # For performance reasons we only create response files if there is a risk + # of exceeding the OS command-line length limit. For short cc_wrapper calls + # avoiding the response file reduces the runtime by about 60%. + if sum(map(len, args)) < 8000: + # Windows has the shortest command-line length limit at 8191 characters. + return _run_cc(args) + else: + with response_file(args) as rsp: + return _run_cc(["@" + rsp]) @contextmanager