ladybird/Toolchain/BuildIt.sh

#!/usr/bin/env bash
set -eo pipefail
# This file will need to be run in bash, for now.


# === CONFIGURATION AND SETUP ===

DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"

echo "$DIR"

ARCH=${ARCH:-"x86_64"}
TARGET="$ARCH-pc-serenity"
PREFIX="$DIR/Local/$ARCH"
BUILD="$DIR/../Build/$ARCH"
SYSROOT="$BUILD/Root"

MAKE="make"
MD5SUM="md5sum"
NPROC="nproc"
REALPATH="realpath"

if command -v ginstall &>/dev/null; then
    INSTALL=ginstall
else
    INSTALL=install
fi

SYSTEM_NAME="$(uname -s)"

# We *most definitely* don't need debug symbols in the linker/compiler.
# This cuts the uncompressed size from 1.2 GiB per Toolchain down to about 120 MiB.
# Hence, this might actually cause marginal speedups, although the point is to not waste space as blatantly.
export CFLAGS="-g0 -O2 -mtune=native"
export CXXFLAGS="-g0 -O2 -mtune=native"

if [ "$SYSTEM_NAME" = "OpenBSD" ]; then
    MAKE=gmake
    MD5SUM="md5 -q"
    NPROC="sysctl -n hw.ncpuonline"
    REALPATH="readlink -f"
    export CC=egcc
    export CXX=eg++
    export with_gmp=/usr/local
    export LDFLAGS=-Wl,-z,notext
elif [ "$SYSTEM_NAME" = "FreeBSD" ]; then
    MAKE=gmake
    MD5SUM="md5 -q"
    NPROC="sysctl -n hw.ncpu"
    export with_gmp=/usr/local
    export with_mpfr=/usr/local
elif [ "$SYSTEM_NAME" = "Darwin" ]; then
    MD5SUM="md5 -q"
    NPROC="sysctl -n hw.ncpu"
fi

# On at least OpenBSD, the path must exist to call realpath(3) on it
if [ ! -d "$BUILD" ]; then
    mkdir -p "$BUILD"
fi
BUILD=$($REALPATH "$BUILD")

git_patch=
while [ "$1" != "" ]; do
    case $1 in
        --dev )           git_patch=1
                          ;;
    esac
    shift
done

echo PREFIX is "$PREFIX"
echo SYSROOT is "$SYSROOT"

mkdir -p "$DIR/Tarballs"

BINUTILS_VERSION="2.40"
BINUTILS_MD5SUM="b200db2cdd2f49019ced4016e1f9bfe7"
BINUTILS_NAME="binutils-$BINUTILS_VERSION"
BINUTILS_PKG="${BINUTILS_NAME}.tar.gz"
BINUTILS_BASE_URL="https://ftp.gnu.org/gnu/binutils"

GDB_VERSION="13.1"
GDB_MD5SUM="92b70971e81a450f6b3e1cf568671cfa"
GDB_NAME="gdb-$GDB_VERSION"
GDB_PKG="${GDB_NAME}.tar.gz"
GDB_BASE_URL="https://ftp.gnu.org/gnu/gdb"

# Note: If you bump the gcc version, you also have to update the matching
#       GCC_VERSION variable in the project's root CMakeLists.txt
GCC_VERSION="12.2.0"
GCC_MD5SUM="d7644b494246450468464ffc2c2b19c3"
GCC_NAME="gcc-$GCC_VERSION"
GCC_PKG="${GCC_NAME}.tar.gz"
GCC_BASE_URL="https://ftp.gnu.org/gnu/gcc"

buildstep() {
    NAME=$1
    shift
    "$@" 2>&1 | sed $'s|^|\x1b[34m['"${NAME}"$']\x1b[39m |'
}

has_gdb() {
    ARCH=$1
    ARCH_DASH="${ARCH//_/-}"
    if command -v gdb >/dev/null && gdb -ex 'set architecture' -ex 'quit' 2>&1 | grep "$ARCH_DASH"; then
        return 0
    else
        command -v "$ARCH"-elf-gdb >/dev/null
    fi
}

NEEDS_GDB=1
if has_gdb "$ARCH"; then
    NEEDS_GDB=0
fi

# === DEPENDENCIES ===
buildstep dependencies echo "Checking whether 'make' is available..."
if ! command -v ${MAKE:-make} >/dev/null; then
    buildstep dependencies echo "Please make sure to install GNU Make (for the '${MAKE:-make}' tool)."
    exit 1
fi

buildstep dependencies echo "Checking whether 'patch' is available..."
if ! command -v patch >/dev/null; then
    buildstep dependencies echo "Please make sure to install GNU patch (for the 'patch' tool)."
    exit 1
fi

buildstep dependencies echo "Checking whether your C compiler works..."
if ! ${CC:-cc} -o /dev/null -xc - >/dev/null <<'PROGRAM'
int main() {}
PROGRAM
then
    buildstep dependencies echo "Please make sure to install a working C compiler."
    exit 1
fi

if [ "$SYSTEM_NAME" != "Darwin" ]; then
    for lib in gmp mpc mpfr; do
        buildstep dependencies echo "Checking whether the $lib library and headers are available..."
        if ! ${CC:-cc} -I /usr/local/include -L /usr/local/lib -l$lib -o /dev/null -xc - >/dev/null <<PROGRAM
#include <$lib.h>
int main() {}
PROGRAM
        then
            echo "Please make sure to install the $lib library and headers."
            exit 1
        fi
    done
fi

# === CHECK CACHE AND REUSE ===

pushd "$DIR"
    if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then
        # The actual logic had to be moved to .github/workflows/cmake.yml.
        # Github Actions guarantees that Toolchain/Cache/ is empty on a cache
        # miss, and non-empty on a cache hit.
        # The following logic is correct *only* because of that.

        mkdir -p Cache
        echo "Cache (before):"
        ls -l Cache
        CACHED_TOOLCHAIN_ARCHIVE="Cache/ToolchainBinariesGithubActions.tar.gz"
        if [ -r "${CACHED_TOOLCHAIN_ARCHIVE}" ] ; then
            echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} exists!"
            echo "Extracting toolchain from cache:"
            if tar xzf "${CACHED_TOOLCHAIN_ARCHIVE}" ; then
                echo "Done 'building' the toolchain."
                echo "Cache unchanged."
                exit 0
            else
                echo
                echo
                echo
                echo "Could not extract cached toolchain archive."
                echo "This means the cache is broken and *should be removed*!"
                echo "As Github Actions cannot update a cache, this will unnecessarily"
                echo "slow down all future builds for this hash, until someone"
                echo "resets the cache."
                echo
                echo
                echo
                rm -f "${CACHED_TOOLCHAIN_ARCHIVE}"
            fi
        else
            echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} does not exist."
            echo "Will rebuild toolchain from scratch, and save the result."
        fi
        echo "::group::Actually building Toolchain"
    fi
popd


# === DOWNLOAD AND PATCH ===

pushd "$DIR/Tarballs"
    # Build gdb for cross-debugging support
    if [ $NEEDS_GDB -eq 1 ]; then
        echo "GDB not found for $ARCH. Will build it from source."
        md5=""
        if [ -e "$GDB_PKG" ]; then
            md5="$($MD5SUM $GDB_PKG | cut -f1 -d' ')"
            echo "gdb md5='$md5'"
        fi
        if [ "$md5" != ${GDB_MD5SUM} ] ; then
            rm -f $GDB_PKG
            curl -LO "$GDB_BASE_URL/$GDB_PKG"
        else
            echo "Skipped downloading gdb"
        fi
    fi

    md5=""
    if [ -e "$BINUTILS_PKG" ]; then
        md5="$($MD5SUM $BINUTILS_PKG | cut -f1 -d' ')"
        echo "binutils md5='$md5'"
    fi
    if [ "$md5" != ${BINUTILS_MD5SUM} ] ; then
        rm -f $BINUTILS_PKG
        curl -LO "$BINUTILS_BASE_URL/$BINUTILS_PKG"
    else
        echo "Skipped downloading binutils"
    fi

    md5=""
    if [ -e "$GCC_PKG" ]; then
        md5="$($MD5SUM ${GCC_PKG} | cut -f1 -d' ')"
        echo "gcc md5='$md5'"
    fi
    if [ "$md5" != ${GCC_MD5SUM} ] ; then
        rm -f $GCC_PKG
        curl -LO "$GCC_BASE_URL/$GCC_NAME/$GCC_PKG"
    else
        echo "Skipped downloading gcc"
    fi

    if [ $NEEDS_GDB -eq 1 ]; then
        if [ -d ${GDB_NAME} ]; then
            rm -rf "${GDB_NAME}"
            rm -rf "$DIR/Build/$ARCH/$GDB_NAME"
        fi
        echo "Extracting GDB..."
        tar -xzf ${GDB_PKG}

        pushd ${GDB_NAME}
            if [ "$git_patch" = "1" ]; then
                git init > /dev/null
                git add . > /dev/null
                git commit -am "BASE" > /dev/null
                git am "${DIR}"/Patches/gdb/*.patch > /dev/null
            else
                for patch in "${DIR}"/Patches/gdb/*.patch; do
                    patch -p1 < "${patch}" > /dev/null
                done
            fi
            $MD5SUM "$DIR"/Patches/gdb/*.patch > .patch.applied
        popd
    fi

    patch_md5="$(${MD5SUM} "${DIR}"/Patches/binutils/*.patch)"

    if [ ! -d "${BINUTILS_NAME}" ] || [ "$(cat ${BINUTILS_NAME}/.patch.applied)" != "${patch_md5}" ]; then
        if [ -d ${BINUTILS_NAME} ]; then
            rm -rf "${BINUTILS_NAME}"
            rm -rf "${DIR}/Build/${ARCH}/${BINUTILS_NAME}"
        fi
        echo "Extracting binutils..."
        tar -xzf ${BINUTILS_PKG}

        pushd ${BINUTILS_NAME}
            if [ "${git_patch}" = "1" ]; then
                git init > /dev/null
                git add . > /dev/null
                git commit -am "BASE" > /dev/null
                git am "${DIR}"/Patches/binutils/*.patch > /dev/null
            else
                for patch in "${DIR}"/Patches/binutils/*.patch; do
                    patch -p1 < "${patch}" > /dev/null
                done
            fi
            ${MD5SUM} "${DIR}"/Patches/binutils/*.patch > .patch.applied
        popd
    else
        echo "Using existing binutils source directory"
    fi


    patch_md5="$(${MD5SUM} "${DIR}"/Patches/gcc/*.patch)"

    if [ ! -d "${GCC_NAME}" ] || [ "$(cat ${GCC_NAME}/.patch.applied)" != "${patch_md5}" ]; then
        if [ -d ${GCC_NAME} ]; then
            rm -rf "${GCC_NAME}"
            rm -rf "${DIR}/Build/${ARCH}/${GCC_NAME}"
        fi
        echo "Extracting gcc..."
        tar -xzf ${GCC_PKG}

        pushd ${GCC_NAME}
            if [ "${git_patch}" = "1" ]; then
                git init > /dev/null
                git add . > /dev/null
                git commit -am "BASE" > /dev/null
                git am --keep-non-patch "${DIR}"/Patches/gcc/*.patch > /dev/null
            else
                for patch in "${DIR}"/Patches/gcc/*.patch; do
                    patch -p1 < "${patch}" > /dev/null
                done
            fi
            ${MD5SUM} "${DIR}"/Patches/gcc/*.patch > .patch.applied

            if [ "${SYSTEM_NAME}" = "Darwin" ]; then
                ./contrib/download_prerequisites
            fi
        popd
    else
        echo "Using existing GCC source directory"
    fi
popd


# === COMPILE AND INSTALL ===

rm -rf "$PREFIX"
mkdir -p "$PREFIX"

if [ -z "$MAKEJOBS" ]; then
    MAKEJOBS=$($NPROC)
fi

mkdir -p "$DIR/Build/$ARCH"

pushd "$DIR/Build/$ARCH"
    unset PKG_CONFIG_LIBDIR # Just in case

    if [ $NEEDS_GDB -eq 1 ]; then
        rm -rf gdb
        mkdir -p gdb

        pushd gdb
            echo "XXX configure gdb"


            if [ "$SYSTEM_NAME" = "Darwin" ]; then
                buildstep "gdb/configure" "$DIR"/Tarballs/$GDB_NAME/configure --prefix="$PREFIX" \
                                                        --target="$TARGET" \
                                                        --with-sysroot="$SYSROOT" \
                                                        --enable-shared \
                                                        --disable-werror \
                                                        --with-libgmp-prefix="$(brew --prefix gmp)" \
                                                        --with-gmp="$(brew --prefix gmp)" \
                                                        --with-isl="$(brew --prefix isl)" \
                                                        --with-mpc="$(brew --prefix libmpc)" \
                                                        --with-mpfr="$(brew --prefix mpfr)" \
                                                        --disable-nls \
                                                        ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1
            else
                buildstep "gdb/configure" "$DIR"/Tarballs/$GDB_NAME/configure --prefix="$PREFIX" \
                                                        --target="$TARGET" \
                                                        --with-sysroot="$SYSROOT" \
                                                        --enable-shared \
                                                        --disable-werror \
                                                        --disable-nls \
                                                        ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1
            fi

            echo "XXX build gdb"
            buildstep "gdb/build" "$MAKE" MAKEINFO=true -j "$MAKEJOBS" || exit 1
            buildstep "gdb/install" "$MAKE" MAKEINFO=true install || exit 1
        popd
    fi

    rm -rf binutils
    mkdir -p binutils

    pushd binutils
        echo "XXX configure binutils"

        # We don't need the documentation that is being built, so
        # don't force people to install makeinfo just for that.
        export ac_cv_prog_MAKEINFO=true

        buildstep "binutils/configure" "$DIR"/Tarballs/$BINUTILS_NAME/configure --prefix="$PREFIX" \
                                                 --target="$TARGET" \
                                                 --with-sysroot="$SYSROOT" \
                                                 --enable-shared \
                                                 --disable-nls \
                                                 ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1
        if [ "$SYSTEM_NAME" = "Darwin" ]; then
            # under macOS generated makefiles are not resolving the "intl"
            # dependency properly to allow linking its own copy of
            # libintl when building with --enable-shared.
            buildstep "binutils/build" "$MAKE" -j "$MAKEJOBS" || true
            pushd intl
            buildstep "binutils/build" "$MAKE" all-yes
            popd
        fi
        echo "XXX build binutils"
        buildstep "binutils/build" "$MAKE" MAKEINFO=true -j "$MAKEJOBS" || exit 1
        buildstep "binutils/install" "$MAKE" MAKEINFO=true install || exit 1
    popd

    echo "XXX serenity libc headers"
    mkdir -p "$BUILD"
    pushd "$BUILD"
        mkdir -p Root/usr/include/
        SRC_ROOT=$($REALPATH "$DIR"/..)
        FILES=$(find \
            "$SRC_ROOT"/AK \
            "$SRC_ROOT"/Kernel/API \
            "$SRC_ROOT"/Kernel/Arch \
            "$SRC_ROOT"/Userland/Libraries/LibC \
            -name '*.h' -print)
        for header in $FILES; do
            target=$(echo "$header" | sed \
                -e "s|$SRC_ROOT/AK/|AK/|" \
                -e "s|$SRC_ROOT/Userland/Libraries/LibC||" \
                -e "s|$SRC_ROOT/Kernel/|Kernel/|")
            buildstep "system_headers" mkdir -p "$(dirname "Root/usr/include/$target")"
            buildstep "system_headers" $INSTALL "$header" "Root/usr/include/$target"
        done
        unset SRC_ROOT
    popd

    if [ "$SYSTEM_NAME" = "OpenBSD" ]; then
        perl -pi -e 's/-no-pie/-nopie/g' "$DIR/Tarballs/gcc-$GCC_VERSION/gcc/configure"
    fi

    rm -rf gcc
    mkdir -p gcc

    pushd gcc
        echo "XXX configure gcc and libgcc"
        buildstep "gcc/configure" "$DIR/Tarballs/gcc-$GCC_VERSION/configure" --prefix="$PREFIX" \
                                            --target="$TARGET" \
                                            --with-sysroot="$SYSROOT" \
                                            --disable-nls \
                                            --enable-shared \
                                            --enable-languages=c,c++ \
                                            --enable-default-pie \
                                            --enable-lto \
                                            --enable-threads=posix \
                                            --enable-initfini-array \
                                            --with-linker-hash-style=gnu \
                                            ${TRY_USE_LOCAL_TOOLCHAIN:+"--quiet"} || exit 1

        echo "XXX build gcc and libgcc"
        buildstep "gcc/build" "$MAKE" -j "$MAKEJOBS" all-gcc || exit 1
        buildstep "libgcc/build" "$MAKE" -j "$MAKEJOBS" all-target-libgcc || exit 1
        echo "XXX install gcc and libgcc"
        buildstep "gcc+libgcc/install" "$MAKE" install-gcc install-target-libgcc || exit 1

        echo "XXX build libstdc++"
        buildstep "libstdc++/build" "$MAKE" -j "$MAKEJOBS" all-target-libstdc++-v3 || exit 1
        echo "XXX install libstdc++"
        buildstep "libstdc++/install" "$MAKE" install-target-libstdc++-v3 || exit 1
    popd

popd

pushd "$DIR/Local/$ARCH/$ARCH-pc-serenity/bin"
    buildstep "mold_symlink" ln -s ../../../mold/bin/mold ld.mold
popd


# == SAVE TO CACHE ==

pushd "$DIR"
    if [ "${TRY_USE_LOCAL_TOOLCHAIN}" = "y" ] ; then
        echo "::endgroup::"
        echo "Building cache tar:"

        rm -f "${CACHED_TOOLCHAIN_ARCHIVE}"  # Just in case

        tar czf "${CACHED_TOOLCHAIN_ARCHIVE}" Local/

        echo "Cache (after):"
        ls -l Cache
    fi
popd