pruneLibtoolFiles: init setup hook (#41819)

A .la file specifies linker flags to link with the library it describes. Its
"dependency_libs" field lists the libraries that this library depends upon.
This list often contains "-l" flags without corresponding "-L" flags. Many
packages in Nixpkgs deal with this in one of these ways:
- delete .la file [1]
- clear dependency_libs [2]
- add -L flags to dependency_libs [3]
- propagate dependencies [4]

Sometimes "dependency_libs" contain wrong "-L" flags pointing to the "dev"
output with headers rather than to the main output with libraries. They have to
be edited or deleted to reduce closure size [5].

Deleting .la files is often but not always safe [6].  Atomatically deleting as
many of them as possible is complex [7].  Deleting .la files that describe
shared rather than static libraries is probably safe; but clearing their
"dependency_libs" field achieves the same effect with less potential for
unintended consequences.  This is the approach that may be enabled for all
Nixpkgs.

[1] 2a79d296d3
[2] c83a530985
[3] 9e0dcf3bd9
[4] 01134e698f
[5] f6c73f1e37
[6] https://wiki.gentoo.org/wiki/Project:Quality_Assurance/Handling_Libtool_Archives
[7] https://github.com/gentoo/gentoo/blob/fb1f2435/eclass/ltprune.eclass

pkgs/build-support/setup-hooks/prune-libtool-files.sh

@@ -0,0 +1,22 @@
+# Clear dependency_libs in libtool files for shared libraries.
+
+# Shared libraries already encode their dependencies with locations.  .la
+# files do not always encode those locations, and sometimes encode the
+# locations in the wrong Nix output. .la files are not needed for shared
+# libraries, but without dependency_libs they do not hurt either.
+
+fixupOutputHooks+=(_pruneLibtoolFiles)
+
+_pruneLibtoolFiles() {
+    if [ "$dontPruneLibtoolFiles" ]; then
+       return
+    fi
+
+    # Libtool uses "dlname" and "library_names" fields for shared libraries and
+    # the "old_library" field for static libraries.  We are processing only
+    # those .la files that do not describe static libraries.
+    find "$prefix" -type f -name '*.la' \
+         -exec grep -q '^# Generated by libtool' {} \; \
+         -exec grep -q "^old_library=''" {} \; \
+         -exec sed -i {} -e "/^dependency_libs='[^']/ c dependency_libs='' #pruned" \;
+}

pkgs/top-level/all-packages.nix

@@ -344,6 +344,9 @@ with pkgs;
 
   pathsFromGraph = ../build-support/kernel/paths-from-graph.pl;
 
+  pruneLibtoolFiles = makeSetupHook { name = "prune-libtool-files"; }
+    ../build-support/setup-hooks/prune-libtool-files.sh;
+
   closureInfo = callPackage ../build-support/closure-info.nix { };
 
   setupSystemdUnits = callPackage ../build-support/setup-systemd-units.nix { };