Merge pull request #301354 from reckenrode/libiconv-switch-mk2

darwin.libiconv: 50 -> 99 and move to pkgs/by-name
2024-10-05 20:19:16 +03:00 · 2024-04-28 23:21:37 +02:00 · 2024-04-28 23:21:37 +02:00 · 84df02abdb
commit 84df02abdb
parent 8e098acf1d e7b5b44f9d
19 changed files with 824 additions and 73 deletions
--- a/pkgs/applications/editors/vim/macvim.nix
+++ b/pkgs/applications/editors/vim/macvim.nix
@ -11,6 +11,7 @@
 , perl
 , luajit
 , darwin
+, libiconv
 , python3
 }:

@ -127,7 +128,7 @@ stdenv.mkDerivation {
      --replace " -L${stdenv.cc.libc}/lib" "" \
      --replace " -L${darwin.libobjc}/lib" "" \
      --replace " -L${darwin.libunwind}/lib" "" \
-      --replace " -L${darwin.libiconv}/lib" ""
+      --replace " -L${libiconv}/lib" ""

    # All the libraries we stripped have -osx- in their name as of this time.
    # Assert now that this pattern no longer appears in config.mk.
--- a/pkgs/by-name/at/atf/package.nix
+++ b/pkgs/by-name/at/atf/package.nix
@ -0,0 +1,89 @@
+{
+  lib,
+  stdenv,
+  fetchFromGitHub,
+  fetchpatch,
+  autoreconfHook,
+  kyua,
+  pkg-config,
+  gitUpdater,
+}:
+
+stdenv.mkDerivation (finalAttrs: {
+  pname = "atf";
+  version = "0.21-unstable-2021-09-01"; # Match the commit used in FreeBSD’s port.
+
+  src = fetchFromGitHub {
+    owner = "freebsd";
+    repo = "atf";
+    rev = "55c21b2c5fb189bbdfccb2b297bfa89236502542";
+    hash = "sha256-u0YBPcoIBvqBVaytaO9feBaRnQygtzEPGJV0ItI1Vco=";
+  };
+
+  patches = [
+    # Fixes use after free that causes failures in Kyua’s test suite.
+    # https://github.com/freebsd/atf/pull/57
+    # https://github.com/freebsd/kyua/issues/223
+    (fetchpatch {
+      name = "fix-use-after-free.patch";
+      url = "https://github.com/freebsd/atf/commit/fb22f3837bcfdce5ce8b3c0e18af131bb6902a02.patch";
+      hash = "sha256-p4L3sxSYfMSzwKrUDlEZpoJydbaK3Hcbvn90KlPHkic=";
+    })
+  ];
+
+  postPatch =
+    lib.optionalString finalAttrs.doInstallCheck ''
+      # https://github.com/freebsd/atf/issues/61
+      substituteInPlace atf-c/check_test.c \
+        --replace-fail 'ATF_TP_ADD_TC(tp, build_cpp)' ""
+      substituteInPlace atf-c++/check_test.cpp \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, build_cpp);' ""
+      # Can’t find `c_helpers` in the work folder.
+      substituteInPlace test-programs/Kyuafile \
+        --replace-fail 'atf_test_program{name="srcdir_test"}' ""
+    ''
+    # These tests fail on Darwin.
+    + lib.optionalString (finalAttrs.doInstallCheck && stdenv.isDarwin) ''
+      substituteInPlace atf-c/detail/process_test.c \
+        --replace-fail 'ATF_TP_ADD_TC(tp, status_coredump);' ""
+    ''
+    # This test fails on Linux.
+    + lib.optionalString (finalAttrs.doInstallCheck && stdenv.isLinux) ''
+      substituteInPlace atf-c/detail/fs_test.c \
+        --replace-fail 'ATF_TP_ADD_TC(tp, eaccess);' ""
+    '';
+
+  strictDeps = true;
+
+  nativeBuildInputs = [ autoreconfHook ];
+
+  enableParallelBuilding = true;
+
+  makeFlags = [
+    # ATF isn’t compatible with C++17, which is the default on current clang and GCC.
+    "CXXFLAGS=-std=c++11"
+  ];
+
+  doInstallCheck = true;
+
+  nativeInstallCheckInputs = [ kyua ];
+
+  installCheckPhase = ''
+    runHook preInstallCheck
+    HOME=$TMPDIR PATH=$out/bin:$PATH kyua test
+    runHook postInstallCheck
+  '';
+
+  passthru.updateScript = gitUpdater { rev-prefix = "atf-"; };
+
+  __structuredAttrs = true;
+
+  meta = {
+    description = "Libraries to write tests in C, C++, and shell";
+    homepage = "https://github.com/freebsd/atf/";
+    license = lib.licenses.bsd3;
+    mainProgram = "atf-sh";
+    maintainers = with lib.maintainers; [ reckenrode ];
+    platforms = lib.platforms.unix;
+  };
+})
--- a/pkgs/by-name/fr/frankenphp/package.nix
+++ b/pkgs/by-name/fr/frankenphp/package.nix
@ -7,6 +7,7 @@
 , testers
 , frankenphp
 , darwin
+, libiconv
 , pkg-config
 , makeBinaryWrapper
 , runCommand
@ -64,7 +65,7 @@ in buildGoModule rec {
  '' + lib.optionalString stdenv.isDarwin ''
    # replace hard-code homebrew path
    substituteInPlace ../frankenphp.go \
-      --replace "-L/opt/homebrew/opt/libiconv/lib" "-L${darwin.libiconv}/lib"
+      --replace "-L/opt/homebrew/opt/libiconv/lib" "-L${libiconv}/lib"
  '';

  preFixup = ''
--- a/pkgs/by-name/ky/kyua/kyua-check-hook.sh
+++ b/pkgs/by-name/ky/kyua/kyua-check-hook.sh
@ -0,0 +1,10 @@
+kyuaCheckPhase() {
+    runHook preCheck
+    # Kyua expects to save test results in ~/.kyua/store
+    HOME=$TMPDIR kyua test
+    runHook postCheck
+}
+
+if [ -z "${dontUseKyuaCheck-}" ] && [ -z "${checkPhase-}" ]; then
+    checkPhase=kyuaCheckPhase
+fi
--- a/pkgs/by-name/ky/kyua/package.nix
+++ b/pkgs/by-name/ky/kyua/package.nix
@ -0,0 +1,119 @@
+{
+  lib,
+  stdenv,
+  fetchFromGitHub,
+  atf,
+  autoreconfHook,
+  lutok,
+  pkg-config,
+  sqlite,
+  gitUpdater,
+}:
+
+let
+  # Avoid an infinite recursion (because ATF uses Kyua for testing).
+  atf' = atf.overrideAttrs (_: {
+    doInstallCheck = false;
+  });
+in
+stdenv.mkDerivation (finalAttrs: {
+  pname = "kyua";
+  version = "0.13-unstable-2024-01-22"; # Match the commit used in FreeBSD’s port.
+
+  src = fetchFromGitHub {
+    owner = "freebsd";
+    repo = "kyua";
+    rev = "c85354e09ad93a902c9e8a701c042c045ec2a5b7";
+    hash = "sha256-fZ0WFgOTj8Gw8IT5O8DnuaNyZscKpg6B94m+l5UoZGc";
+  };
+
+  setupHooks = ./kyua-check-hook.sh;
+
+  postPatch =
+    ''
+      # Fix a linking error on Darwin. Embedding an archive in an archive isn’t portable.
+      substituteInPlace cli/Makefile.am.inc \
+        --replace-fail 'libcli_a_LIBADD = libutils.a' "" \
+        --replace-fail 'CLI_LIBS = ' 'CLI_LIBS = libutils.a '
+    ''
+    # These tests fail on Darwin or are unreliable.
+    + lib.optionalString (finalAttrs.doInstallCheck && stdenv.isDarwin) ''
+      sed -i utils/process/Makefile.am.inc -e '/executor_pid_test/d'
+      substituteInPlace utils/process/Kyuafile \
+        --replace-fail 'atf_test_program{name="executor_pid_test"}' ""
+      substituteInPlace engine/atf_test.cpp \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, test__body_only__crashes);' ""
+      substituteInPlace engine/scheduler_test.cpp \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, integration__stacktrace);' ""
+      substituteInPlace utils/stacktrace_test.cpp \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, unlimit_core_size);' ""
+      substituteInPlace utils/process/isolation_test.cpp \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, isolate_child__enable_core_dumps);' ""
+      substituteInPlace utils/process/operations_test.cpp \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, terminate_self_with__termsig_and_core);' ""
+      substituteInPlace utils/process/status_test.cpp \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, integration__coredump);' ""
+      substituteInPlace integration/cmd_test_test.sh \
+        --replace-fail 'atf_add_test_case premature_exit' ""
+    ''
+    # fchflags and UF_NOUNLINK are not supported on Linux. Other tests also fail.
+    + lib.optionalString (finalAttrs.doInstallCheck && stdenv.isLinux) ''
+      sed -i utils/process/Makefile.am.inc -e '/executor_pid_test/d'
+      substituteInPlace utils/process/Kyuafile \
+        --replace-fail 'atf_test_program{name="executor_pid_test"}' ""
+      substituteInPlace engine/atf_test.cpp \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, test__body_only__crashes);' ""
+      substituteInPlace utils/stacktrace_test.cpp \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, dump_stacktrace__ok);' "" \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, dump_stacktrace_if_available__append);' "" \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, find_core__found__long);' "" \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, find_core__found__short);' "" \
+        --replace-fail 'ATF_ADD_TEST_CASE(tcs, unlimit_core_size__hard_is_zero);' ""
+      substituteInPlace integration/cmd_test_test.sh \
+        --replace-fail 'atf_add_test_case premature_exit' ""
+    '';
+
+  strictDeps = true;
+
+  buildInputs = [
+    lutok
+    sqlite
+  ];
+
+  nativeBuildInputs = [
+    atf'
+    autoreconfHook
+    pkg-config
+  ];
+
+  enableParallelBuilding = true;
+
+  makeFlags = [
+    # Kyua isn’t compatible with C++17, which is the default on current clang and GCC.
+    "CXXFLAGS=-std=c++11"
+  ];
+
+  doInstallCheck = true;
+
+  installCheckInputs = [ atf' ];
+  nativeInstallCheckInputs = [ sqlite ];
+
+  installCheckPhase = ''
+    runHook preInstallCheck
+    HOME=$TMPDIR PATH=$out/bin:$PATH kyua test
+    runHook postInstallCheck
+  '';
+
+  passthru.updateScript = gitUpdater { rev-prefix = "kyua-"; };
+
+  __structuredAttrs = true;
+
+  meta = {
+    description = "Testing framework for infrastructure software";
+    homepage = "https://github.com/freebsd/kyua/";
+    license = lib.licenses.bsd3;
+    mainProgram = "kyua";
+    maintainers = with lib.maintainers; [ reckenrode ];
+    platforms = lib.platforms.unix;
+  };
+})
--- a/pkgs/by-name/li/libiconv-darwin/meson.build
+++ b/pkgs/by-name/li/libiconv-darwin/meson.build
@ -0,0 +1,303 @@
+# Build settings based on the upstream Xcode project.
+# See: https://github.com/apple-oss-distributions/libiconv/blob/main/libiconv.xcodeproj/project.pbxproj
+
+# Project settings
+project('libiconv', 'c', version : '@version@')
+
+fs = import('fs')
+
+
+# Dependencies
+cc = meson.get_compiler('c')
+
+
+# Definitions
+prefix_libdir = get_option('prefix') / get_option('libdir')
+prefix_datadir = get_option('prefix') / get_option('datadir')
+
+i18nmoduledir = prefix_libdir / 'i18n'
+esdbdir = prefix_datadir / 'i18n/esdb'
+csmapperdir = prefix_datadir / 'i18n/csmapper'
+
+
+# Libraries
+libcharset = library(
+    'charset',
+    darwin_versions : '1',
+    install : true,
+    include_directories : ['libcharset'],
+    sources : [
+        'libcharset/libcharset.c'
+    ],
+    soversion : '1'
+)
+install_headers(
+    'libcharset/libcharset.h',
+    'libcharset/localcharset.h'
+)
+
+libiconv = library(
+    'iconv',
+    build_rpath : fs.parent(libcharset.full_path()),
+    c_args : [
+        f'-D_PATH_I18NMODULE="@i18nmoduledir@"',
+        f'-D_PATH_ESDB="@esdbdir@"',
+        f'-D_PATH_CSMAPPER="@csmapperdir@"'
+    ],
+    darwin_versions : '7',
+    install : true,
+    include_directories : ['citrus', 'libcharset'],
+    link_args : ['-Wl,-reexport_library', fs.name(libcharset.full_path())],
+    link_depends : [libcharset],
+    override_options : {'b_asneeded' : false}, # Make sure the libcharset reexport is not stripped
+    sources : [
+        'citrus/__iconv_get_list.c',
+        'citrus/__iconv_free_list.c',
+        'citrus/__iconv.c',
+        'citrus/bsd_iconv.c',
+        'citrus/citrus_bcs_strtol.c',
+        'citrus/citrus_bcs_strtoul.c',
+        'citrus/citrus_bcs.c',
+        'citrus/citrus_csmapper.c',
+        'citrus/citrus_db.c',
+        'citrus/citrus_db_factory.c',
+        'citrus/citrus_db_hash.c',
+        'citrus/citrus_esdb.c',
+        'citrus/citrus_hash.c',
+        'citrus/citrus_iconv.c',
+        'citrus/citrus_lookup_factory.c',
+        'citrus/citrus_lookup.c',
+        'citrus/citrus_mapper.c',
+        'citrus/citrus_memstream.c',
+        'citrus/citrus_mmap.c',
+        'citrus/citrus_module.c',
+        'citrus/citrus_none.c',
+        'citrus/citrus_pivot_factory.c',
+        'citrus/citrus_prop.c',
+        'citrus/citrus_stdenc.c',
+        'citrus/iconv_canonicalize.c',
+        'citrus/iconv_close.c',
+        'citrus/iconv_compat.c',
+        'citrus/iconv_open_into.c',
+        'citrus/iconv_open.c',
+        'citrus/iconv_set_relocation_prefix.c',
+        'citrus/iconvctl.c',
+        'citrus/iconvlist.c',
+        'citrus/iconv.c',
+    ],
+    soversion : '2'
+)
+install_headers(
+    'citrus/iconv.h'
+)
+install_man(
+    'citrus/__iconv_get_list.3',
+    'citrus/iconv_canonicalize.3',
+    'citrus/iconv.3',
+    'citrus/iconvctl.3',
+    'citrus/iconvlist.3',
+)
+
+
+# Binaries
+executable(
+    'iconv',
+    install : true,
+    include_directories : ['citrus', 'libcharset'],
+    link_with : [libiconv],
+    sources : [
+        'iconv/iconv.c'
+    ]
+)
+install_man('iconv/iconv.1')
+
+
+# Data
+## csmapper
+csmapper_modules = [
+    'APPLE',
+    'AST',
+    'BIG5',
+    'CNS',
+    'CP',
+    'EBCDIC',
+    'GB',
+    'GEORGIAN',
+    'ISO-8859',
+    'ISO646',
+    'JIS',
+    'KAZAKH',
+    'KOI',
+    'KS',
+    'MISC',
+    'TCVN'
+]
+
+foreach module : csmapper_modules
+    mps_files = run_command(
+        'find', 'i18n/csmapper' / module, '-regex', '.*\\.\\(mps\\|646\\)',
+        check : true
+    ).stdout().strip().split('\n')
+    install_data(mps_files, install_dir : csmapperdir / module)
+endforeach
+
+install_data(
+    'i18n/csmapper/charset.pivot',
+    'i18n/csmapper/charset.pivot.pvdb',
+    'i18n/csmapper/mapper.dir',
+    'i18n/csmapper/mapper.dir.db',
+    install_dir : csmapperdir
+)
+
+## esdb
+esdb_modules = [
+    'APPLE',
+    'AST',
+    'BIG5',
+    'CP',
+    'DEC',
+    'EBCDIC',
+    'EUC',
+    'GB',
+    'GEORGIAN',
+    'ISO-2022',
+    'ISO-8859',
+    'ISO646',
+    'KAZAKH',
+    'KOI',
+    'MISC',
+    'TCVN',
+    'UTF'
+]
+
+foreach module : esdb_modules
+    esdb_files = run_command(
+        'find', 'i18n/esdb' / module, '-name', '*.esdb',
+        check : true
+    ).stdout().strip().split('\n')
+    install_data(esdb_files, install_dir : esdbdir / module)
+endforeach
+
+install_data(
+    'i18n/esdb/esdb.alias',
+    'i18n/esdb/esdb.alias.db',
+    'i18n/esdb/esdb.dir',
+    'i18n/esdb/esdb.dir.db',
+    install_dir : esdbdir
+)
+
+
+# Modules
+libiconv_modules = [
+    'BIG5',
+    'DECHanyu',
+    'DECKanji',
+    'EUC',
+    'EUCTW',
+    'GBK2K',
+    'HZ',
+    'ISO2022',
+    'JOHAB',
+    'MSKanji',
+    'UES',
+    'UTF1632',
+    'UTF7',
+    'UTF8',
+    'UTF8MAC',
+    'VIQR',
+    'ZW',
+    'iconv_none',
+    'iconv_std',
+    'mapper_646',
+    'mapper_none',
+    'mapper_serial',
+    'mapper_parallel',
+    'mapper_std',
+    'mapper_zone'
+]
+
+foreach module : libiconv_modules
+    module_source = module.to_lower()
+    module_path = 'libiconv_modules' / module
+
+    if module == 'UTF8MAC'
+        extra_headers = 'libiconv_modules/UTF8'
+    else
+        extra_headers = [ ]
+    endif
+
+    # Upstream builds this module under both names.
+    # See: https://github.com/apple-oss-distributions/libiconv/blob/81be60a93521c931a01aab9c747dd2b078bc0679/libiconv.xcodeproj/project.pbxproj#L2549-L2556
+    # See also: https://cgit.freebsd.org/src/tree/lib/libiconv_modules/mapper_parallel/Makefile?id=9241ebc796c11cf133c550f188f324bd2c12d89a
+    if module == 'mapper_parallel'
+        module_source = 'mapper_serial'
+        module_path = 'libiconv_modules/mapper_serial'
+    endif
+
+    library(
+        module,
+        darwin_versions : '1',
+        install : true,
+        install_dir : i18nmoduledir,
+        include_directories : [module_path, 'citrus', 'libcharset'] + extra_headers,
+        link_with : [libiconv],
+        override_options : {'b_asneeded' : false}, # Upstream always links libiconv
+        sources : [
+            module_path / f'citrus_@module_source@.c'
+        ]
+    )
+endforeach
+
+
+# Tests
+if get_option('tests') == true
+## Only required for running the tests
+    atf = dependency('atf-c')
+    foreach suite : ['libiconv_test', 'mbopt_test', 'nixpkgs_test']
+        test_src = f'tests/libiconv/@suite@.c'
+        test_exe = executable(
+            suite,
+            dependencies : [atf],
+            include_directories : ['citrus', 'libcharset'],
+            link_with : [libiconv],
+            sources : [test_src]
+        )
+
+        # Extract the tests to run from the test source code.
+        tests = run_command(
+            'sed', '-n', '-E', 's|.*ATF_TP_ADD_TC\\([^,]*, ([^)]*).*$|\\1|p', test_src,
+            check : true
+        ).stdout().strip().split('\n')
+
+        foreach test : tests
+            test(test, test_exe, args : [test], suite : suite, timeout : 300)
+        endforeach
+    endforeach
+
+# These tests depend on `os_variant_has_internal_content`, which is stubbed out.
+#     atf_sh = find_program('atf-sh')
+#     print_charset = executable(
+#         'print_charset',
+#         include_directories : ['citrus', 'libcharset'],
+#         link_with : [libiconv],
+#         sources : 'tests/libcharset/print_charset.c'
+#     )
+#
+#     test_charset = custom_target(
+#         'test_charset.sh',
+#         command : ['cp', '@INPUT@', '@OUTPUT@'],
+#         depends : print_charset,
+#         input : 'tests/libcharset/test_charset.sh',
+#         output : 'test_charset.sh'
+#     )
+#
+#     # Extract the tests to run from the test source code.
+#     tests = run_command(
+#         'sed', '-n', '-E', 's|.*atf_add_test_case (.*$)|\\1|p', 'tests/libcharset/test_charset.sh',
+#         check : true
+#     ).stdout().strip().split('\n')
+#
+#     foreach test : tests
+#         test(test, atf_sh, args : [test_charset, test], suite : 'libcharset', timeout : 300)
+#     endforeach
+endif
--- a/pkgs/by-name/li/libiconv-darwin/meson.options
+++ b/pkgs/by-name/li/libiconv-darwin/meson.options
@ -0,0 +1 @@
+option('tests', type : 'boolean')
--- a/pkgs/by-name/li/libiconv-darwin/nixpkgs_test.c
+++ b/pkgs/by-name/li/libiconv-darwin/nixpkgs_test.c
@ -0,0 +1,82 @@
+#include <atf-c.h>
+#include <iconv.h>
+#include <stdint.h>
+
+// The following tests were failing in libarchive due to libiconv issues.
+//   218: test_read_format_cab_filename (4 failures)
+//   415: test_read_format_zip_filename_CP932_eucJP (4 failures)
+//   426: test_read_format_zip_filename_CP932_CP932 (2 failures)
+
+ATF_TC(test_cp932_eucjp);
+ATF_TC_HEAD(test_cp932_eucjp, tc)
+{
+    atf_tc_set_md_var(tc, "descr", "regression test for CP932 to EUC-JP conversion");
+}
+ATF_TC_BODY(test_cp932_eucjp, tc)
+{
+    char expected[] = "\xc9\xbd\xa4\xc0\xa4\xe8\x5c\xb4\xc1\xbb\xfa\x2e\x74\x78\x74";
+    size_t expected_length = sizeof(expected) - 1;
+
+    char input[] = "\x95\x5c\x82\xbe\x82\xe6\x5c\x8a\xbf\x8e\x9a\x2e\x74\x78\x74";
+    size_t input_length = sizeof(input) - 1;
+
+    size_t output_available = sizeof(expected) - 1 ;
+    char output[sizeof(expected)] = { 0 };
+
+    iconv_t cd = iconv_open("eucJP", "CP932");
+    ATF_REQUIRE((size_t)cd != -1);
+
+    char* input_buf = input;
+    char* output_buf = output;
+
+    size_t res = iconv(cd, &input_buf, &input_length, &output_buf, &output_available);
+    iconv_close(cd);
+
+    ATF_CHECK(res != -1);
+
+    size_t output_length = sizeof(output) - output_available - 1;
+
+    ATF_CHECK_INTEQ(expected_length, output_length);
+    ATF_CHECK_STREQ(expected, output);
+}
+
+ATF_TC(test_cp932_cp932);
+ATF_TC_HEAD(test_cp932_cp932, tc)
+{
+    atf_tc_set_md_var(tc, "descr", "regression test for CP932 to CP932 conversion");
+}
+ATF_TC_BODY(test_cp932_cp932, tc)
+{
+    char expected[] = "\x95\x5c\x82\xbe\x82\xe6\x5c\x8a\xbf\x8e\x9a\x2e\x74\x78\x74";
+    size_t expected_length = sizeof(expected) - 1;
+
+    char input[] = "\x95\x5c\x82\xbe\x82\xe6\x5c\x8a\xbf\x8e\x9a\x2e\x74\x78\x74";
+    size_t input_length = sizeof(input) - 1;
+
+    size_t output_available = sizeof(expected) - 1 ;
+    char output[sizeof(expected)] = { 0 };
+
+    iconv_t cd = iconv_open("CP932", "CP932");
+    ATF_REQUIRE((size_t)cd != -1);
+
+    char* input_buf = input;
+    char* output_buf = output;
+
+    size_t res = iconv(cd, &input_buf, &input_length, &output_buf, &output_available);
+    iconv_close(cd);
+
+    ATF_CHECK(res != -1);
+
+    size_t output_length = sizeof(output) - output_available - 1;
+
+    ATF_CHECK_INTEQ(expected_length, output_length);
+    ATF_CHECK_STREQ(expected, output);
+}
+
+ATF_TP_ADD_TCS(tp)
+{
+    ATF_TP_ADD_TC(tp, test_cp932_eucjp);
+    ATF_TP_ADD_TC(tp, test_cp932_cp932);
+
+    return atf_no_error();
+}
--- a/pkgs/by-name/li/libiconv-darwin/package.nix
+++ b/pkgs/by-name/li/libiconv-darwin/package.nix
@ -0,0 +1,95 @@
+{
+  lib,
+  stdenv,
+  fetchFromGitHub,
+  fetchpatch,
+  atf,
+  libiconvReal,
+  meson,
+  ninja,
+  pkg-config,
+  gitUpdater,
+}:
+
+stdenv.mkDerivation (finalAttrs: {
+  pname = "libiconv";
+  version = "99";
+
+  outputs = [
+    "out"
+    "dev"
+  ];
+
+  src = fetchFromGitHub {
+    owner = "apple-oss-distributions";
+    repo = "libiconv";
+    rev = "libiconv-${finalAttrs.version}";
+    hash = "sha256-TGt6rsU52ztfW2rCqwnhMAExLbexI/59IoDOGY+XGu0=";
+  };
+
+  inherit (libiconvReal) setupHooks;
+
+  postPatch =
+    ''
+      substitute ${./meson.build} meson.build --subst-var version
+      cp ${./meson.options} meson.options
+
+      # Work around unnecessary private API usage in libcharset
+      mkdir -p libcharset/os && cat <<-header > libcharset/os/variant_private.h
+        #pragma once
+        #include <stdbool.h>
+        static inline bool os_variant_has_internal_content(const char*) { return false; }
+      header
+
+      cp ${./nixpkgs_test.c} tests/libiconv/nixpkgs_test.c
+    '';
+
+  strictDeps = true;
+
+  nativeBuildInputs = [
+    meson
+    ninja
+  ];
+
+  mesonBuildType = "release";
+
+  mesonFlags = [ (lib.mesonBool "tests" finalAttrs.doInstallCheck) ];
+
+  postInstall = lib.optionalString stdenv.isDarwin ''
+    ${stdenv.cc.targetPrefix}install_name_tool "$out/lib/libiconv.2.dylib" \
+      -change '@rpath/libcharset.1.dylib' "$out/lib/libcharset.1.dylib"
+  '';
+
+  # Tests have to be run in `installCheckPhase` because libiconv expects to `dlopen`
+  # modules from `$out/lib/i18n`.
+  nativeInstallCheckInputs = [ pkg-config ];
+  installCheckInputs = [ atf ];
+
+  doInstallCheck = stdenv.buildPlatform.canExecute stdenv.hostPlatform;
+
+  # Can’t use `mesonCheckPhase` because it runs the wrong hooks for `installCheckPhase`.
+  installCheckPhase = ''
+    runHook preInstallCheck
+    meson test --no-rebuild
+    runHook postInstallCheck
+  '';
+
+  passthru.updateScript = gitUpdater { rev-prefix = "libiconv-"; };
+
+  __structuredAttrs = true;
+
+  meta = {
+    description = "An iconv(3) implementation";
+    homepage = "https://opensource.apple.com/releases/";
+    license =
+      with lib.licenses;
+      [
+        bsd2
+        bsd3
+      ]
+      ++ lib.optional finalAttrs.doInstallCheck apsl10;
+    mainProgram = "iconv";
+    maintainers = with lib.maintainers; [ reckenrode ];
+    platforms = lib.platforms.darwin;
+  };
+})
--- a/pkgs/by-name/lu/lutok/package.nix
+++ b/pkgs/by-name/lu/lutok/package.nix
@ -0,0 +1,87 @@
+{
+  lib,
+  stdenv,
+  fetchFromGitHub,
+  atf,
+  autoreconfHook,
+  kyua,
+  lua,
+  pkg-config,
+  gitUpdater,
+}:
+
+lib.fix (
+  drv:
+  let
+    # Avoid infinite recursions:
+    # - Lutok depends on ATF and Kyua for testing; but
+    # - ATF depends on Kyua for testing, and Kyua depends on Lutok as a build input.
+    # To break the cycle (ATF -> Kyua -> Lutok -> ATF and Kyua):
+    # - Build ATF without testing (avoiding the Kyua dependency); and
+    # - Build Kyua against a version of Lutok without testing (also avoiding the ATF and Kyua dependencies).
+    atf' = atf.overrideAttrs (_: {
+      doInstallCheck = false;
+    });
+    kyua' =
+      (kyua.override {
+        lutok = drv.overrideAttrs (_: {
+          doCheck = false;
+        });
+      }).overrideAttrs
+        (_: {
+          # Assume Kyua’s install check phase will run when Kyua is built. Don’t run it again
+          # while building Lutok because it can take four to five minutes to run.
+          doInstallCheck = false;
+        });
+  in
+  stdenv.mkDerivation (finalAttrs: {
+    pname = "lutok";
+    version = "0.4";
+
+    outputs = [
+      "out"
+      "dev"
+    ];
+
+    src = fetchFromGitHub {
+      owner = "freebsd";
+      repo = "lutok";
+      rev = "lutok-${finalAttrs.version}";
+      hash = "sha256-awAFxx9q8dZ6JO1/mShjhJnOPTLn1wCT4VrB4rlgWyg=";
+    };
+
+    strictDeps = true;
+
+    propagatedBuildInputs = [ lua ];
+
+    nativeBuildInputs = [
+      atf'
+      autoreconfHook
+      pkg-config
+    ];
+
+    enableParallelBuilding = true;
+
+    makeFlags = [
+      # Lutok isn’t compatible with C++17, which is the default on current clang and GCC.
+      "CXXFLAGS=-std=c++11"
+    ];
+
+    doCheck = true;
+
+    checkInputs = [ atf' ];
+    nativeCheckInputs = [ kyua' ];
+
+    passthru.updateScript = gitUpdater { rev-prefix = "lutok-"; };
+
+    __structuredAttrs = true;
+
+    meta = {
+      description = "Libraries to write tests in C, C++, and shell";
+      homepage = "https://github.com/freebsd/lutok/";
+      license = lib.licenses.bsd3;
+      maintainers = with lib.maintainers; [ reckenrode ];
+      platforms = lib.platforms.unix;
+    };
+  })
+)
--- a/pkgs/development/libraries/libiconv/default.nix
+++ b/pkgs/development/libraries/libiconv/default.nix
@ -63,6 +63,8 @@ stdenv.mkDerivation rec {
    (lib.enableFeature enableShared "shared")
  ] ++ lib.optional stdenv.isFreeBSD "--with-pic";

+  passthru = { inherit setupHooks; };
+
  meta = {
    description = "An iconv(3) implementation";

--- a/pkgs/development/python-modules/json-stream-rs-tokenizer/default.nix
+++ b/pkgs/development/python-modules/json-stream-rs-tokenizer/default.nix
@ -2,7 +2,7 @@
 , stdenv
 , buildPythonPackage
 , cargo
-, darwin
+, libiconv
 , fetchFromGitHub
 , json-stream
 , json-stream-rs-tokenizer
@ -45,7 +45,7 @@ buildPythonPackage rec {
  ];

  buildInputs = lib.optionals stdenv.isDarwin [
-    darwin.libiconv
+    libiconv
  ];

  # Tests depend on json-stream, which depends on this package.
--- a/pkgs/os-specific/darwin/apple-source-releases/default.nix
+++ b/pkgs/os-specific/darwin/apple-source-releases/default.nix
@ -11,7 +11,6 @@ let
    };
    "osx-10.12.6" = {
      xnu           = "3789.70.16";
-      libiconv      = "50";
      Libnotify     = "165.20.1";
      objc4         = "709.1";
      dyld          = "433.5";
@ -38,7 +37,6 @@ let
      dtrace        = "168";
      xnu           = "3248.60.10";
      libpthread    = "138.10.4";
-      libiconv      = "44";
      Libnotify     = "150.40.1";
      objc4         = "680";
      eap8021x      = "222.40.1";
@ -272,7 +270,6 @@ developerToolsPackages_11_3_1 // macosPackages_11_0_1 // {
    };
    libclosure      = applePackage "libclosure"        "osx-10.11.6"     "sha256-L5rQ+UBpf3B+W1U+gZKk7fXulslHsc8lxnCsplV+nr0=" {};
    libdispatch     = applePackage "libdispatch"       "osx-10.10.5"     "sha256-jfAEk0OLrJa9AIZVikIoHomd+l+4rCfc320Xh50qK5M=" {};
-    libiconv        = applePackage "libiconv"          "osx-10.12.6"     "sha256-ZzPFkchK3EU95UQUVVrR0t8iilhi/VnIkjjtP6KT2oI=" {};
    Libinfo         = applePackage "Libinfo"           "osx-10.11.6"     "sha256-6F7wiwerv4nz/xXHtp1qCHSaFzZgzcRN+jbmXA5oWOQ=" {};
    Libm            = applePackage "Libm"              "osx-10.7.4"      "sha256-KjMETfT4qJm0m0Ux/F6Rq8bI4Q4UVnFx6IKbKxXd+Es=" {};
    Libnotify       = applePackage "Libnotify"         "osx-10.12.6"     "sha256-6wvMBxAUfiYcQtmlfYCj1d3kFmFM/jdboTd7hRvi3e4=" {};
--- a/pkgs/os-specific/darwin/apple-source-releases/libiconv/default.nix
+++ b/pkgs/os-specific/darwin/apple-source-releases/libiconv/default.nix
@ -1,38 +0,0 @@
-{ stdenv, appleDerivation, lib
-, enableStatic ? stdenv.hostPlatform.isStatic
-, enableShared ? !stdenv.hostPlatform.isStatic
-}:
-
-appleDerivation {
-  postUnpack = "sourceRoot=$sourceRoot/libiconv";
-
-  preConfigure = lib.optionalString stdenv.hostPlatform.isiOS ''
-    sed -i 's/darwin\*/ios\*/g' configure libcharset/configure
-  '';
-
-  configureFlags = [
-    (lib.enableFeature enableStatic "static")
-    (lib.enableFeature enableShared "shared")
-  ];
-
-  postInstall = lib.optionalString enableShared ''
-    mv $out/lib/libiconv.dylib $out/lib/libiconv-nocharset.dylib
-    ${stdenv.cc.bintools.targetPrefix}install_name_tool -id $out/lib/libiconv-nocharset.dylib $out/lib/libiconv-nocharset.dylib
-
-    # re-export one useless symbol; ld will reject a dylib that only reexports other dylibs
-    echo 'void dont_use_this(){}' | ${stdenv.cc.bintools.targetPrefix}clang -dynamiclib -x c - -current_version 2.4.0 \
-      -compatibility_version 7.0.0 -current_version 7.0.0 -o $out/lib/libiconv.dylib \
-      -Wl,-reexport_library -Wl,$out/lib/libiconv-nocharset.dylib \
-      -Wl,-reexport_library -Wl,$out/lib/libcharset.dylib
-  '';
-
-  setupHooks = [
-    ../../../../build-support/setup-hooks/role.bash
-    ../../../../development/libraries/libiconv/setup-hook.sh
-  ];
-
-  meta = {
-    mainProgram = "iconv";
-    platforms = lib.platforms.darwin;
-  };
-}
--- a/pkgs/os-specific/darwin/apple-source-releases/macos-11.0.1.nix
+++ b/pkgs/os-specific/darwin/apple-source-releases/macos-11.0.1.nix
@ -27,7 +27,6 @@ file_cmds = applePackage' "file_cmds" "321.40.3" "macos-11.0.1" "0p077lnbcy8266m
 hfs = applePackage' "hfs" "556.41.1" "macos-11.0.1" "0a0s6b12b0q07wslfifna0bj51dml9v098i4crr2m1vivnx4xj75" {};
 libclosure = applePackage' "libclosure" "78" "macos-11.0.1" "0vf9n0k3m8dbprv1bf45zqg0g43bidy2i5z1v9a826bsf8lv7am7" {};
 libdispatch = applePackage' "libdispatch" "1271.40.12" "macos-11.0.1" "1ck5srcjapg18vqb8wl08gacs7ndc6xr067qjn3ngx39q1jdcywz" {};
-libiconv = applePackage' "libiconv" "59" "macos-11.0.1" "0lwa4brdwm4lvrdnxylzsn1yph4m7csgri2zkc4xb4xiisz32pwp" {};
 libmalloc = applePackage' "libmalloc" "317.40.8" "macos-11.0.1" "sha256-Tdhb0mq3w4Hwvp3xHB79Vr22hCOQK6h28HCsd7jvITI=" {};
 libplatform = applePackage' "libplatform" "254.40.4" "macos-11.0.1" "1qf3ri0yd8b1xjln1j1gyx7ks6k3a2jhd63blyvfby75y9s7flky" {};
 libpthread = applePackage' "libpthread" "454.40.3" "macos-11.0.1" "0zljbw8mpb80n1if65hhi9lkgwbgjr8vc9wvf7q1nl3mzyl35f8p" {};
--- a/pkgs/stdenv/darwin/default.nix
+++ b/pkgs/stdenv/darwin/default.nix
@ -428,6 +428,9 @@ in
      # Disable tests because they use dejagnu, which fails to run.
      libffi = super.libffi.override { doCheck = false; };

+      # Use libconvReal to break an infinite recursion. It will be dropped in the next stage.
+      libiconv = super.libiconvReal;
+
      # Avoid pulling in a full python and its extra dependencies for the llvm/clang builds.
      libxml2 = super.libxml2.override { pythonSupport = false; };

@ -539,7 +542,7 @@ in
      inherit (prevStage) ccWrapperStdenv
        autoconf automake bash binutils binutils-unwrapped bison brotli cmake cmakeMinimal
        coreutils cpio cyrus_sasl db ed expat flex gettext gmp gnugrep groff icu
-        libedit libffi libiconv libidn2 libkrb5 libssh2 libtool libunistring libxml2 m4
+        libedit libffi libidn2 libkrb5 libssh2 libtool libunistring libxml2 m4
        ncurses nghttp2 ninja openldap openssh openssl patchutils pbzx perl pkg-config
        python3Minimal scons sed serf sharutils sqlite subversion texinfo unzip which xz
        zlib zstd;
@ -614,8 +617,8 @@ in
    assert lib.all isFromBootstrapFiles (with prevStage; [ coreutils gnugrep ]);

    assert lib.all isBuiltByBootstrapFilesCompiler (with prevStage; [
-      autoconf automake bash binutils-unwrapped bison brotli cmake cpio cyrus_sasl db
-      ed expat flex gettext gmp groff icu libedit libffi libiconv libidn2 libkrb5 libssh2
+      atf autoconf automake bash binutils-unwrapped bison brotli cmake cpio cyrus_sasl db
+      ed expat flex gettext gmp groff icu kyua libedit libffi libiconv libidn2 libkrb5 libssh2
      libtool libunistring libxml2 m4 ncurses nghttp2 ninja openldap openssh openssl
      patchutils pbzx perl pkg-config.pkg-config python3 python3Minimal scons serf sqlite
      subversion sysctl.provider texinfo unzip which xz zlib zstd
@ -641,8 +644,8 @@ in

    overrides = self: super: {
      inherit (prevStage) ccWrapperStdenv
-        autoconf automake bash binutils binutils-unwrapped bison brotli cmake cmakeMinimal
-        cpio cyrus_sasl db ed expat flex gettext gmp groff icu libedit libffi libiconv
+        atf autoconf automake bash binutils binutils-unwrapped bison brotli cmake cmakeMinimal
+        cpio cyrus_sasl db ed expat flex gettext gmp groff icu kyua libedit libffi libiconv
        libidn2 libkrb5 libssh2 libtool libunistring libxml2 m4 ncurses nghttp2 ninja
        openldap openssh openssl patchutils pbzx perl pkg-config python3 python3Minimal
        scons sed serf sharutils sqlite subversion sysctl texinfo unzip which xz zlib zstd;
@ -703,8 +706,8 @@ in
  (prevStage:
    # previous stage-xclang stdenv:
    assert lib.all isBuiltByBootstrapFilesCompiler (with prevStage; [
-      autoconf automake bash binutils-unwrapped bison cmake cmakeMinimal coreutils cpio
-      cyrus_sasl db ed expat flex gettext gmp gnugrep groff icu libedit libtool m4 ninja
+      atf autoconf automake bash binutils-unwrapped bison cmake cmakeMinimal coreutils cpio
+      cyrus_sasl db ed expat flex gettext gmp gnugrep groff icu kyua libedit libtool m4 ninja
      openbsm openldap openpam openssh patchutils pbzx perl pkg-config.pkg-config python3
      python3Minimal scons serf sqlite subversion sysctl.provider texinfo unzip which xz
    ]);
@ -737,8 +740,8 @@ in

    overrides = self: super: {
      inherit (prevStage) ccWrapperStdenv
-        autoconf automake binutils-unwrapped bison brotli cmake cmakeMinimal coreutils
-        cpio cyrus_sasl db ed expat flex gettext gmp gnugrep groff icu libedit libffi
+        atf autoconf automake binutils-unwrapped bison brotli cmake cmakeMinimal coreutils
+        cpio cyrus_sasl db ed expat flex gettext gmp gnugrep groff icu kyua libedit libffi
        libiconv libidn2 libkrb5 libssh2 libtool libunistring libxml2 m4 ncurses nghttp2
        ninja openbsm openldap openpam openssh openssl patchutils pbzx perl pkg-config
        python3 python3Minimal scons serf sqlite subversion sysctl texinfo unzip which xz
@ -804,8 +807,8 @@ in
  (prevStage:
    # previous stage2-Libsystem stdenv:
    assert lib.all isBuiltByBootstrapFilesCompiler (with prevStage; [
-      autoconf automake binutils-unwrapped bison brotli cmake cmakeMinimal coreutils
-      cpio cyrus_sasl db ed expat flex gettext gmp gnugrep groff icu libedit libidn2
+      atf autoconf automake binutils-unwrapped bison brotli cmake cmakeMinimal coreutils
+      cpio cyrus_sasl db ed expat flex gettext gmp gnugrep groff icu kyua libedit libidn2
      libkrb5 libssh2 libtool libunistring m4 nghttp2 ninja openbsm openldap openpam openssh
      openssl patchutils pbzx perl pkg-config.pkg-config python3 python3Minimal scons serf
      sqlite subversion sysctl.provider texinfo unzip which xz zstd
@ -841,8 +844,8 @@ in

    overrides = self: super: {
      inherit (prevStage) ccWrapperStdenv
-        autoconf automake bash bison brotli cmake cmakeMinimal coreutils cpio
-        cyrus_sasl db ed expat flex gettext gmp gnugrep groff libedit libidn2 libkrb5
+        atf autoconf automake bash bison brotli cmake cmakeMinimal coreutils cpio
+        cyrus_sasl db ed expat flex gettext gmp gnugrep groff kyua libedit libidn2 libkrb5
        libssh2 libtool libunistring m4 ncurses nghttp2 ninja openbsm openldap openpam
        openssh openssl patchutils pbzx perl pkg-config python3 python3Minimal scons serf
        sqlite subversion sysctl texinfo unzip which xz zstd;
@ -870,7 +873,7 @@ in
      llvmPackages = super.llvmPackages // (
        let
          tools = super.llvmPackages.tools.extend (_: _: {
-            inherit (prevStage.llvmPackages) clang-unwrapped clangNoCompilerRtWithLibc libclang libllvm llvm;
+            inherit (prevStage.llvmPackages) clang-unwrapped clangNoCompilerRtWithLibc libclang lld libllvm llvm;
            clang = prevStage.stdenv.cc;
          });

@ -969,7 +972,7 @@ in
        sed serf sharutils sqlite subversion sysctl texinfo unzip which xz zstd

        # CF dependencies - don’t rebuild them.
-        icu libiconv libxml2 zlib;
+        icu libiconv libiconv-darwin libxml2 zlib;

      # Disable tests because they use dejagnu, which fails to run.
      libffi = super.libffi.override { doCheck = false; };
@ -1054,7 +1057,7 @@ in
        icu

        # LLVM dependencies - don’t rebuild them.
-        libffi libiconv libxml2 ncurses zlib;
+        libffi libiconv libiconv-darwin libxml2 ncurses zlib;

      darwin = super.darwin.overrideScope (selfDarwin: superDarwin: {
        inherit (prevStage.darwin) dyld CF Libsystem darwin-stubs
@ -1325,9 +1328,9 @@ in
      overrides = self: super: {
        inherit (prevStage)
          bash binutils brotli bzip2 coreutils cpio diffutils ed file findutils gawk
-          gettext gmp gnugrep gnumake gnused gnutar gzip icu libffi libiconv libidn2 libssh2
-          libunistring libxml2 libyaml ncurses nghttp2 openbsm openpam openssl patch pbzx
-          pcre python3Minimal xar xz zlib zstd;
+          gettext gmp gnugrep gnumake gnused gnutar gzip icu libffi libiconv libiconv-darwin
+          libidn2 libssh2 libunistring libxml2 libyaml ncurses nghttp2 openbsm openpam
+          openssl patch pbzx pcre python3Minimal xar xz zlib zstd;

        darwin = super.darwin.overrideScope (_: superDarwin: {
          inherit (prevStage.darwin)
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@ -22570,7 +22570,7 @@ with pkgs;

  # GNU libc provides libiconv so systems with glibc don't need to
  # build libiconv separately. Additionally, Apple forked/repackaged
-  # libiconv so we use that instead of the vanilla version on that OS,
+  # libiconv, so build and use the upstream one with a compatible ABI,
  # and BSDs include libiconv in libc.
  #
  # We also provide `libiconvReal`, which will always be a standalone libiconv,
@ -22581,7 +22581,7 @@ with pkgs;
        then libcCross
        else stdenv.cc.libc)
    else if stdenv.hostPlatform.isDarwin
-      then darwin.libiconv
+      then libiconv-darwin
    else libiconvReal;

  libcIconv = libc: let
@ -22598,7 +22598,7 @@ with pkgs;
    if lib.elem stdenv.hostPlatform.libc [ "glibc" "musl" ] then
      lib.getBin stdenv.cc.libc
    else if stdenv.hostPlatform.isDarwin then
-      lib.getBin darwin.libiconv
+      lib.getBin libiconv
    else
      lib.getBin libiconvReal;

--- a/pkgs/top-level/darwin-aliases.nix
+++ b/pkgs/top-level/darwin-aliases.nix
@ -51,6 +51,10 @@ mapAliases ({

  builder = throw "'darwin.builder' has been changed and renamed to 'darwin.linux-builder'. The default ssh port is now 31022. Please update your configuration or override the port back to 22. See https://nixos.org/manual/nixpkgs/unstable/#sec-darwin-builder"; # added 2023-07-06

+  ### L ###
+
+  libiconv = pkgs.libiconv; # 2024-03-27
+
  ### I ###

  insert_dylib = throw "'darwin.insert_dylib' has been renamed to 'insert-dylib'"; # added 2024-04-04
--- a/pkgs/top-level/python-packages.nix
+++ b/pkgs/top-level/python-packages.nix
@ -2566,7 +2566,6 @@ self: super: with self; {
  cryptodatahub = callPackage ../development/python-modules/cryptodatahub { };

  cryptography = callPackage ../development/python-modules/cryptography {
-    inherit (pkgs.darwin) libiconv;
    inherit (pkgs.darwin.apple_sdk.frameworks) Security;
  };

@ -2591,7 +2590,6 @@ self: super: with self; {
  css-html-js-minify = callPackage ../development/python-modules/css-html-js-minify { };

  css-inline = callPackage ../development/python-modules/css-inline {
-    inherit (pkgs.darwin) libiconv;
    inherit (pkgs.darwin.apple_sdk.frameworks) Security SystemConfiguration;
  };

@ -11631,9 +11629,7 @@ self: super: with self; {

  pyreaderwriterlock = callPackage ../development/python-modules/pyreaderwriterlock { };

-  pyreadstat = callPackage ../development/python-modules/pyreadstat {
-    inherit (pkgs.darwin) libiconv;
-  };
+  pyreadstat = callPackage ../development/python-modules/pyreadstat { };

  pyrealsense2 = toPythonModule (pkgs.librealsense.override {
    enablePython = true;