Adapt build deps to build universal libs

VimR/VimR.xcodeproj/project.pbxproj

@@ -1171,7 +1171,6 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				ALWAYS_SEARCH_USER_PATHS = NO;
-				ARCHS = x86_64;
 				CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES;
 				CLANG_ANALYZER_NONNULL = YES;
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";
@@ -1232,7 +1231,6 @@
 			isa = XCBuildConfiguration;
 			buildSettings = {
 				ALWAYS_SEARCH_USER_PATHS = NO;
-				ARCHS = x86_64;
 				CLANG_ANALYZER_LOCALIZABILITY_NONLOCALIZED = YES;
 				CLANG_ANALYZER_NONNULL = YES;
 				CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x";

bin/build_vimr_dev.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 set -Eeuo pipefail
 
-readonly target=${target:?"arm64 or x86_64"}
 readonly clean=${clean:?"true or false: when true, xcodebuild clean will be performed"}
 
 main() {
@@ -18,9 +17,8 @@ main() {
       -configuration Release \
       -scheme VimR -xcconfig \
       ./VimR/Dev.xcconfig \
-      ARCHS="${target}" \
       ${cmd}
   popd >/dev/null
 }
 
-main
+main

third-party/.gitignore

@@ -1,3 +1 @@
-/libag
-/libxz
-/libpcre
+/build

third-party/build.py

@@ -3,9 +3,10 @@ import pathlib
 import shutil
 
 from builder import Builder
-from config import Config, Target
+from config import Config
 from deps import ag, pcre, xz
 from deps.ag import AgBuilder
+from deps.pcre import PcreBuilder
 
 DEPS_FILE_NAME = ".deps"
 PACKAGE_NAME = "vimr-deps"
@@ -38,14 +39,6 @@ def parse_args() -> argparse.Namespace:
         required=True,
     )
 
-    parser.add_argument(
-        "--target",
-        action="store",
-        dest="target",
-        type=Target,
-        required=True,
-    )
-
     parser.add_argument(
         "--arm64-deployment-target",
         action="store",
@@ -69,14 +62,17 @@ if __name__ == "__main__":
     arm64_deployment_target = args.arm64_deployment_target
     x86_64_deployment_target = args.x86_64_deployment_target
 
-    cwd = pathlib.Path(__file__).parent.resolve()
-    install_path = cwd.joinpath(PACKAGE_NAME)
+    cwd = pathlib.Path(__file__).parent.resolve().joinpath("build")
+    shutil.rmtree(cwd, ignore_errors=True)
+    cwd.mkdir(parents=True, exist_ok=True)
+
+    install_path = cwd.parent.joinpath(PACKAGE_NAME)
+    shutil.rmtree(install_path, ignore_errors=True)
     install_path_lib = install_path.joinpath("lib")
     install_path_include = install_path.joinpath("include")
 
     xz_config = Config(
         version=args.xz_version,
-        target=args.target,
         arm64_deployment_target=arm64_deployment_target,
         x86_64_deployment_target=x86_64_deployment_target,
         default_cflags="-g -O2",
@@ -87,7 +83,6 @@ if __name__ == "__main__":
     )
     pcre_config = Config(
         version=args.pcre_version,
-        target=args.target,
         arm64_deployment_target=arm64_deployment_target,
         x86_64_deployment_target=x86_64_deployment_target,
         default_cflags="-D_THREAD_SAFE -pthread -g -O2",
@@ -96,42 +91,41 @@ if __name__ == "__main__":
         install_path_include=install_path_include,
         working_directory=cwd.joinpath(DEPS_FILE_NAME),
     )
+    ag_config = Config(
+        version=args.ag_version,
+        arm64_deployment_target=arm64_deployment_target,
+        x86_64_deployment_target=x86_64_deployment_target,
+        default_cflags="-g -O2 -D_THREAD_SAFE -pthread",
+        target_install_path_parent=cwd.joinpath("libag"),
+        install_path_lib=install_path_lib,
+        install_path_include=install_path_include,
+        working_directory=cwd.joinpath(DEPS_FILE_NAME),
+    )
     builders = {
         "xz": Builder(
             xz_config,
             download_command=xz.download_command,
+            extract_command=xz.extract_command,
             make_command=xz.make_command,
-            copy_command=xz.copy_command,
+            build_universal_and_install_command=xz.build_universal_and_install_command,
         ),
-        "pcre": Builder(
+        "pcre": PcreBuilder(
             pcre_config,
             download_command=pcre.download_command,
             make_command=pcre.make_command,
-            copy_command=pcre.copy_command,
+            extract_command=pcre.extract_command,
+            build_universal_and_install_command=pcre.build_universal_and_install_command,
         ),
         "ag": AgBuilder(
-            Config(
-                version=args.ag_version,
-                target=args.target,
-                arm64_deployment_target=arm64_deployment_target,
-                x86_64_deployment_target=x86_64_deployment_target,
-                default_cflags="-g -O2 -D_THREAD_SAFE -pthread",
-                target_install_path_parent=cwd.joinpath("libag"),
-                install_path_lib=install_path_lib,
-                install_path_include=install_path_include,
-                working_directory=cwd.joinpath(DEPS_FILE_NAME),
-            ),
+            ag_config,
             download_command=ag.download_command,
             make_command=ag.make_command,
-            copy_command=ag.copy_command,
             deps=[xz_config, pcre_config],
+            extract_command=ag.extract_command,
+            build_universal_and_install_command=ag.build_universal_and_install_command,
         ),
     }
 
-    shutil.rmtree(install_path, ignore_errors=True)
-    shutil.rmtree(install_path_lib, ignore_errors=True)
-    shutil.rmtree(install_path_include, ignore_errors=True)
-
     builders["xz"].build()
     builders["pcre"].build()
     builders["ag"].build()

third-party/builder.py

@@ -1,39 +1,60 @@
 from dataclasses import dataclass
 from string import Template
 
-from config import Config
+from config import Config, Target
 from utils.shell import shell
 
 
 @dataclass(frozen=True)
 class Builder:
     config: Config
+
     download_command: Template
+    extract_command: Template
     make_command: Template
-    copy_command: Template
+    build_universal_and_install_command: Template
 
     def download(self):
         cmd = self.download_command.substitute(dict(version=self.config.version))
         print(cmd)
         shell(cmd, cwd=self.config.working_directory)
 
-    def make(self):
-        cmd = self.make_command.substitute(
+    def extract(self, target: Target):
+        cmd = self.extract_command.substitute(
             dict(
-                cflags=self.config.target_specific_full_cflags,
-                deployment_target=self.config.target_specific_deployment_target,
-                install_path=self.config.target_specific_install_path,
+                target=target.value,
+                version=self.config.version,
             )
         )
         print(cmd)
         shell(cmd, cwd=self.config.working_directory)
 
-    def copy_to_install_path(self):
-        cmd = self.copy_command.substitute(
+    def make(self, target: Target):
+        cmd = self.make_command.substitute(
+            dict(
+                target=target.value,
+                cflags=self.config.target_specific_full_cflags(target),
+                deployment_target=self.config.target_specific_deployment_target(target),
+                install_path=self.config.target_specific_install_path(target),
+            )
+        )
+        print(cmd)
+        shell(cmd, cwd=self.config.working_directory)
+
+    def build_universal_and_install(self):
+        cmd = self.build_universal_and_install_command.substitute(
             dict(
-                target_specific_install_path=self.config.target_specific_install_path,
-                install_include_path=self.config.install_path_include,
                 install_lib_path=self.config.install_path_lib,
+                install_include_path=self.config.install_path_include,
+                arm64_lib_path=self.config.target_specific_install_path(Target.arm64).joinpath(
+                    "lib"
+                ),
+                arm64_include_path=self.config.target_specific_install_path(Target.arm64).joinpath(
+                    "include"
+                ),
+                x86_64_lib_path=self.config.target_specific_install_path(Target.x86_64).joinpath(
+                    "lib"
+                ),
             )
         )
         print(cmd)
@@ -44,5 +65,11 @@ class Builder:
         self.config.ensure_paths_exist()
 
         self.download()
-        self.make()
-        self.copy_to_install_path()
+
+        self.extract(Target.arm64)
+        self.make(Target.arm64)
+
+        self.extract(Target.x86_64)
+        self.make(Target.x86_64)
+
+        self.build_universal_and_install()

third-party/config.py

@@ -14,7 +14,6 @@ class Config:
     """The working_directory should be set for the particular library, e.g. ./.deps/xz"""
 
     version: str
-    target: Target
 
     arm64_deployment_target: str
     x86_64_deployment_target: str
@@ -26,29 +25,34 @@ class Config:
     install_path_lib: Path
     working_directory: Path
 
-    @property
-    def target_specific_deployment_target(self) -> str:
-        if self.target is Target.arm64:
+    def target_specific_host(self, target: Target) -> str:
+        if target is Target.arm64:
+            return "arm-apple-macos"
+        elif target is Target.x86_64:
+            return "x86_64-apple-macos"
+        else:
+            raise ValueError
+
+    def target_specific_deployment_target(self, target: Target) -> str:
+        if target is Target.arm64:
             return self.arm64_deployment_target
-        elif self.target is Target.x86_64:
+        elif target is Target.x86_64:
             return self.x86_64_deployment_target
         else:
             raise ValueError
 
-    @property
-    def target_specific_full_cflags(self) -> str:
-        if self.target is Target.arm64:
+    def target_specific_full_cflags(self, target: Target) -> str:
+        if target is Target.arm64:
             return self.arm64_full_cflags
-        elif self.target is Target.x86_64:
+        elif target is Target.x86_64:
             return self.x86_64_full_cflags
         else:
             raise ValueError
 
-    @property
-    def target_specific_install_path(self) -> Path:
-        if self.target is Target.arm64:
+    def target_specific_install_path(self, target: Target) -> Path:
+        if target is Target.arm64:
             return self.arm64_install_path
-        elif self.target is Target.x86_64:
+        elif target is Target.x86_64:
             return self.x86_64_install_path
         else:
             raise ValueError

third-party/deps/ag.py

@@ -2,35 +2,41 @@ from dataclasses import dataclass
 from string import Template
 
 from builder import Builder
-from config import Config
+from config import Config, Target
 from utils.shell import shell
 
 # language=bash
 download_command = Template(
     """
 curl -L -s -o ag.tar.gz https://github.com/ggreer/the_silver_searcher/archive/${version}.tar.gz
-rm -rf ag
+"""
+)
+
+# language=bash
+extract_command = Template(
+    """
+rm -rf "ag-${target}"
 tar xf ag.tar.gz
-mv the_silver_searcher-${version} ag
+mv "the_silver_searcher-${version}" "ag-${target}"
 """
 )
 
 # language=bash
 make_command = Template(
     """
-pushd ag >/dev/null
+pushd ag-${target} >/dev/null
   ./autogen.sh
   ./configure
       CFLAGS="${cflags} ${include_flags}" \
       LDFLAGS="${ldflags}" \
       MACOSX_DEPLOYMENT_TARGET="${deployment_target}"
-      
+
   pushd src > /dev/null
     cc ${cflags} ${include_flags} -c ignore.c log.c options.c print.c scandir.c search.c lang.c util.c decompress.c zfile.c
     ar -crs libag.a ignore.o log.o options.o print.o scandir.o search.o lang.o util.o decompress.o zfile.o
     mkdir -p "${install_path}/lib"
     mv libag.a "${install_path}/lib"
-  
+
     mkdir -p "${install_path}/include"
     cp *.h "${install_path}/include"
   popd >/dev/null
@@ -38,12 +44,11 @@ popd >/dev/null
 """
 )
 
-
 # language=bash
-copy_command = Template(
+build_universal_and_install_command = Template(
     """
-cp -r "${target_specific_install_path}/include"/* "${install_include_path}"
-cp -r "${target_specific_install_path}/lib"/* "${install_lib_path}"
+lipo -create -output "${install_lib_path}/libag.a" "${arm64_lib_path}/libag.a" "${x86_64_lib_path}/libag.a"
+cp -r "${arm64_include_path}"/* "${install_include_path}"
 """
 )
 
@@ -52,18 +57,19 @@ cp -r "${target_specific_install_path}/lib"/* "${install_lib_path}"
 class AgBuilder(Builder):
     deps: [Config]
 
-    def make(self):
+    def make(self, target: Target):
         include_flags = " ".join(
-            [f'-I{c.x86_64_install_path.joinpath("include")}' for c in self.deps]
+            [f'-I{c.install_path_include}' for c in self.deps]
         )
         ldflags = " ".join([f'-L{c.install_path_lib}' for c in self.deps])
         cmd = self.make_command.substitute(
             dict(
-                cflags=self.config.target_specific_full_cflags,
+                target=target.value,
+                cflags=self.config.target_specific_full_cflags(target),
                 ldflags=ldflags,
                 include_flags=include_flags,
-                deployment_target=self.config.target_specific_deployment_target,
-                install_path=self.config.target_specific_install_path,
+                deployment_target=self.config.target_specific_deployment_target(target),
+                install_path=self.config.target_specific_install_path(target),
             )
         )
         print(cmd)

third-party/deps/pcre.py

@@ -1,22 +1,32 @@
 from string import Template
+from config import Target
+from builder import Builder
+from utils.shell import shell
 
 
 # language=bash
 download_command = Template(
     """
 curl -L -s -o pcre.tar.bz2 "https://ftp.pcre.org/pub/pcre/pcre-${version}.tar.bz2"
-rm -rf pcre
+"""
+)
+
+# language=bash
+extract_command = Template(
+    """
+rm -rf "pcre-${target}"
 tar xf pcre.tar.bz2
-mv "pcre-${version}" pcre
+mv "pcre-${version}" "pcre-${target}"
 """
 )
 
 # language=bash
 make_command = Template(
     """
-pushd pcre >/dev/null
+pushd pcre-${target} >/dev/null
 ./configure \
     CFLAGS="${cflags}" \
+    CXXFLAGS="${cflags}" \
     MACOSX_DEPLOYMENT_TARGET="${deployment_target}" \
     --disable-dependency-tracking \
     --enable-utf8 \
@@ -28,6 +38,7 @@ pushd pcre >/dev/null
     --enable-pcregrep-libbz2 \
     --enable-jit \
     --disable-shared \
+    --host="${host}" \
     --prefix="${install_path}"
 make MACOSX_DEPLOYMENT_TARGET="${deployment_target}" install
 popd >/dev/null
@@ -35,9 +46,24 @@ popd >/dev/null
 )
 
 # language=bash
-copy_command = Template(
+build_universal_and_install_command = Template(
     """
-cp -r "${target_specific_install_path}/include"/* "${install_include_path}"
-cp -r "${target_specific_install_path}/lib"/* "${install_lib_path}"
+lipo -create -output "${install_lib_path}/libpcre.a" "${arm64_lib_path}/libpcre.a" "${x86_64_lib_path}/libpcre.a"
+cp -r "${arm64_include_path}"/* "${install_include_path}"
 """
 )
+
+
+class PcreBuilder(Builder):
+    def make(self, target: Target):
+        cmd = self.make_command.substitute(
+            dict(
+                target=target.value,
+                cflags=self.config.target_specific_full_cflags(target),
+                deployment_target=self.config.target_specific_deployment_target(target),
+                install_path=self.config.target_specific_install_path(target),
+                host=self.config.target_specific_host(target),
+            )
+        )
+        print(cmd)
+        shell(cmd, cwd=self.config.working_directory)

third-party/deps/xz.py

@@ -4,16 +4,22 @@ from string import Template
 download_command = Template(
     """
 curl -L -s -o xz.tar.gz "https://tukaani.org/xz/xz-${version}.tar.gz"
-rm -rf xz
+"""
+)
+
+# language=bash
+extract_command = Template(
+    """
+rm -rf "xz-${target}"
 tar xf xz.tar.gz
-mv "xz-${version}" xz
+mv "xz-${version}" "xz-${target}"
 """
 )
 
 # language=bash
 make_command = Template(
     """
-pushd ./xz >/dev/null
+pushd ./xz-${target} >/dev/null
 ./configure \
     CFLAGS="${cflags}" \
     MACOSX_DEPLOYMENT_TARGET="${deployment_target}" \
@@ -35,9 +41,9 @@ popd >/dev/null
 )
 
 # language=bash
-copy_command = Template(
+build_universal_and_install_command = Template(
     """
-cp -r "${target_specific_install_path}/include"/* "${install_include_path}"
-cp -r "${target_specific_install_path}/lib"/* "${install_lib_path}"
+lipo -create -output "${install_lib_path}/liblzma.a" "${arm64_lib_path}/liblzma.a" "${x86_64_lib_path}/liblzma.a"
+cp -r "${arm64_include_path}"/* "${install_include_path}"
 """
-)
+)

third-party/setup.py

@@ -6,5 +6,5 @@ setuptools.setup(
     author="Tae Won Ha",
     description="Scripts for building dependencies of VimR",
     packages=setuptools.find_packages(),
-    python_requires='>=3.8',
-)
+    python_requires=">=3.8",
+)