Add stacktrace (#674)

Related to #659 

Adds backtrace printing to SIGSEGV signal handling. Tries to print file
and line number with libbacktrace, falls back to using standard
backtrace when debug info is unavailable.

Before:
<img width="715" alt="Screenshot 2024-07-11 at 15 58 52"
src="https://github.com/urbit/vere/assets/22086553/3ba89d96-c50c-40a5-8fd4-f6e636b8aec3">

After (with dsyms):
<img width="715" alt="Screenshot 2024-07-24 at 17 11 56"
src="https://github.com/user-attachments/assets/b333516f-c36f-48ce-afd2-95de9f812843">

After (without dsyms):
<img width="715" alt="Screenshot 2024-07-24 at 17 14 31"
src="https://github.com/user-attachments/assets/59956429-2fc7-478e-a05d-1807db21b18f">

.github/workflows/shared.yml

@@ -56,6 +56,33 @@ jobs:
       #
       - uses: actions/checkout@v3
 
+      - name: Set swap space
+        if: ${{ matrix.target == 'linux-x86_64' || matrix.target == 'linux-aarch64'}}
+        run: |
+          echo "Memory and swap:"
+          free -h
+          echo
+          swapon --show
+          echo
+
+          export SWAP_FILE=$(swapon --show=NAME | tail -n 1)
+          if test -z "${SWAP_FILE}"; then
+              export SWAP_FILE=/swapfile
+          else
+              sudo swapoff -a
+              sudo rm "${SWAP_FILE}"
+          fi
+          sudo fallocate -l 10G "${SWAP_FILE}"
+          sudo chmod 600 "${SWAP_FILE}"
+          sudo mkswap "${SWAP_FILE}"
+          sudo swapon "${SWAP_FILE}"
+
+          echo "Memory and swap:"
+          free -h
+          echo
+          swapon --show
+          echo
+
       - name: chown /usr/local
         if: ${{ matrix.target == 'linux-x86_64' || matrix.target == 'linux-aarch64'}}
         run: |
@@ -64,7 +91,7 @@ jobs:
       - name: Set up build cache
         uses: actions/cache@v3
         with:
-          key: ${{ matrix.target }}-cache
+          key: ${{ matrix.target }}-cache-1
           path: |
             # # Cache bazel path on Linux.
             ~/.cache/bazel/_bazel_$(whoami)

BUILD.bazel

@@ -84,8 +84,8 @@ string_flag(
 # Version flag for gcc.
 string_flag(
     name = "gcc_version",
-    # musl-cross-make uses `gcc-9.4.0` by default.
-    build_setting_default = "9.4.0",
+    # musl-cross-make uses `gcc-11.4.0` by default.
+    build_setting_default = "11.4.0",
     visibility = ["//visibility:public"],
 )
 

WORKSPACE.bazel

@@ -233,6 +233,24 @@ versioned_http_file(
     version = "255fb1ca8206072f1d09425f0db61ecfe7ff5b17",
 )
 
+versioned_http_archive(
+    name = "libbacktrace",
+    build_file = "//bazel/third_party/libbacktrace:libbacktrace.BUILD",
+    sha256 = "609c17352ec38eaf5ff6618fcbfb38cd8fa0e94a15a0d9aa259df514bbf47fcd",
+    url = "https://github.com/ianlancetaylor/libbacktrace/archive/{version}.tar.gz",
+    strip_prefix = "libbacktrace-{version}",
+    version = "4ead348bb45f753121ca0bd44170ff8352d4c514",
+)
+
+versioned_http_archive(
+    name = "libunwind",
+    build_file = "//bazel/third_party/libunwind:libunwind.BUILD",
+    sha256 = "ddf0e32dd5fafe5283198d37e4bf9decf7ba1770b6e7e006c33e6df79e6a6157",
+    url = "https://github.com/libunwind/libunwind/releases/download/v1.8.1/libunwind-1.8.1.tar.gz",
+    strip_prefix = "libunwind-{version}",
+    version = "1.8.1",
+)
+
 versioned_http_archive(
     name = "lmdb",
     build_file = "//bazel/third_party/lmdb:lmdb.BUILD",

bazel/common_settings.bzl

@@ -13,20 +13,14 @@ string_flag = rule(
 
 def vere_library(copts = [], linkopts = [], **kwargs):
   native.cc_library(
-    copts = copts + select({
+    copts = copts + [
+        "-fno-omit-frame-pointer",
+    ] + select({
         "//:debug": ["-O0", "-g3", "-DC3DBG", "-fdebug-compilation-dir=."],
-        "//conditions:default": ["-O3"]
+        "//conditions:default": ["-O3", "-g"]
     }) + select({
         "//:lto": ['-flto'],
-        "//:thinlto": ['-flto=thin'],
         "//conditions:default": []
-    }) + select({
-        # Don't include source level debug info on macOS. See
-        # https://github.com/urbit/urbit/issues/5561 and
-        # https://github.com/urbit/vere/issues/131.
-        "//:debug": [],
-        "@platforms//os:linux": ["-g"],
-        "//conditions:default": [],
     }),
     linkopts = linkopts + ['-g'] + select({
         "//:lto": ['-flto'],
@@ -38,17 +32,14 @@ def vere_library(copts = [], linkopts = [], **kwargs):
 
 def vere_binary(copts = [], linkopts = [], **kwargs):
   native.cc_binary(
-    copts = copts + select({
+    copts = copts + [
+        "-fno-omit-frame-pointer",
+    ] + select({
         "//:debug": ["-O0", "-g3", "-DC3DBG", "-fdebug-compilation-dir=."],
-        "//conditions:default": ["-O3"]
+        "//conditions:default": ["-O3", "-g"]
     }) + select({
         "//:lto": ['-flto'],
-        "//:thinlto": ['-flto=thin'],
         "//conditions:default": []
-    }) + select({
-        "//:debug": [],
-        "@platforms//os:linux": ["-g"],
-        "//conditions:default": [],
     }),
     linkopts = linkopts + ['-g'] + select({
         "//:lto": ['-flto'],

bazel/third_party/expat/expat.BUILD

@@ -11,6 +11,12 @@ configure_make(
         "@platforms//os:macos": ["--jobs=`sysctl -n hw.logicalcpu`"],
         "//conditions:default": ["--jobs=`nproc`"],
     }),
+    configure_options = [
+    ] + select({
+        "@//:linux_aarch64": ["--host=aarch64-linux-musl"],
+        "@//:linux_x86_64": ["--host=x86_64-linux-musl"],
+        "//conditions:default": [],
+    }),
     copts = ["-O3"],
     lib_source = ":all",
     out_static_libs = ["libexpat.a"],

bazel/third_party/libbacktrace/libbacktrace.BUILD

@@ -0,0 +1,25 @@
+load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make")
+
+filegroup(
+    name = "all",
+    srcs = glob(["**"]),
+)
+
+configure_make(
+    name = "libbacktrace",
+    args = [
+    ] + select({
+        "@platforms//os:macos": ["--jobs=`sysctl -n hw.logicalcpu`"],
+        "//conditions:default": ["--jobs=`nproc`"],
+    }),
+    configure_options = [
+    ] + select({
+        "@//:linux_aarch64": ["--host=aarch64-linux-musl"],
+        "@//:linux_x86_64": ["--host=x86_64-linux-musl"],
+        "//conditions:default": [],
+    }),
+    copts = ["-O3"],
+    lib_source = ":all",
+    out_static_libs = ["libbacktrace.a"],
+    visibility = ["//visibility:public"],
+)

bazel/third_party/libunwind/libunwind.BUILD

@@ -0,0 +1,25 @@
+load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make")
+
+filegroup(
+    name = "all",
+    srcs = glob(["**"]),
+)
+
+configure_make(
+    name = "libunwind",
+    args = select({
+        "@platforms//os:macos": ["--jobs=`sysctl -n hw.logicalcpu`"],
+        "//conditions:default": ["--jobs=`nproc`"],
+    }),
+    configure_options = [
+        "--enable-debug-frame",
+    ] + select({
+        "@//:linux_aarch64": ["--host=aarch64-linux-musl"],
+        "@//:linux_x86_64": ["--host=x86_64-linux-musl"],
+        "//conditions:default": [],
+    }),
+    copts = ["-O3 -g"],
+    lib_source = ":all",
+    out_static_libs = ["libunwind.a"],
+    visibility = ["//visibility:public"],
+)

bazel/third_party/urcrypt/urcrypt.BUILD

@@ -15,7 +15,7 @@ configure_make(
     copts = [
         "-Wall",
         "-g",
-        "-O3",
+        "-O2",
     ],
     deps = [
         "@aes_siv",

bazel/toolchain/BUILD.bazel

@@ -335,9 +335,9 @@ toolchain(
 # so introduces a circular dependency during Bazel C/C++ toolchain resolution.
 
 # musl-cross-make builds musl-libc-compatible gcc toolchains from source.
-_musl_cross_make_version = "fe915821b652a7fa37b34a596f47d8e20bc72338"
+_musl_cross_make_version = "99f2cbc7e230f72bde3394be3ebd50497cb53e89"
 
-_musl_cross_make_archive = "https://github.com/richfelker/musl-cross-make/archive/{}.tar.gz".format(_musl_cross_make_version)
+_musl_cross_make_archive = "https://github.com/ripperi/musl-cross-make/archive/{}.tar.gz".format(_musl_cross_make_version)
 
 genrule(
     name = "install-aarch64-linux-musl-gcc",
@@ -349,6 +349,8 @@ genrule(
     echo '    tar -xf {}.tar.gz'                                                    >> $@
     echo '    archive=musl-cross-make-{}'                                           >> $@
     echo '    echo OUTPUT=$$aarch64_linux_musl_install > $$archive/config.mak'      >> $@
+    echo '    echo GCC_VER=11.4.0 >> $$archive/config.mak'                          >> $@
+    echo '    echo MUSL_VER=1.2.5 >> $$archive/config.mak'                          >> $@
     echo '    TARGET=aarch64-linux-musl make -s -C$$archive -j`nproc`'              >> $@
     echo '    sudo TARGET=aarch64-linux-musl make -s -C$$archive -j`nproc` install' >> $@
     echo '    sudo chown --recursive $$USER $$aarch64_linux_musl_install'           >> $@
@@ -382,6 +384,8 @@ genrule(
     echo '    tar -xf {}.tar.gz'                                                   >> $@
     echo '    archive=musl-cross-make-{}'                                          >> $@
     echo '    echo OUTPUT=$$x86_64_linux_musl_install > $$archive/config.mak'      >> $@
+    echo '    echo GCC_VER=11.4.0 >> $$archive/config.mak'                         >> $@
+    echo '    echo MUSL_VER=1.2.5 >> $$archive/config.mak'                         >> $@
     echo '    TARGET=x86_64-linux-musl make -s -C$$archive -j`nproc`'              >> $@
     echo '    sudo TARGET=x86_64-linux-musl make -s -C$$archive -j`nproc` install' >> $@
     echo '    sudo chown --recursive $$USER $$x86_64_linux_musl_install'           >> $@

pkg/noun/BUILD.bazel

@@ -34,15 +34,20 @@ vere_library(
         "//pkg/ent",
         "//pkg/ur",
         "@gmp",
+        "@libbacktrace",
         "@murmur3",
         "@openssl",
         "@pdjson",
         "@sigsegv",
         "@softfloat",
         "@urcrypt",
+        "@whereami",
     ] + select({
         "@platforms//os:macos": ["//pkg/noun/platform/darwin"],
-        "@platforms//os:linux": ["//pkg/noun/platform/linux"],
+        "@platforms//os:linux": [
+            "//pkg/noun/platform/linux",
+            "@libunwind",
+        ],
         "//conditions:default": [],
     }),
 )

pkg/noun/manage.c

@@ -5,11 +5,20 @@
 #include "pkg/noun/v3/manage.h"
 
 #include <ctype.h>
+#include <dlfcn.h>
 #include <errno.h>
+#if defined(U3_OS_osx)
+#include <execinfo.h>
+#endif
 #include <fcntl.h>
 #include <sys/stat.h>
+#if defined(U3_OS_linux)
+#define UNW_LOCAL_ONLY
+#include <libunwind.h>
+#endif
 
 #include "allocate.h"
+#include "backtrace.h"
 #include "events.h"
 #include "hashtable.h"
 #include "imprison.h"
@@ -24,6 +33,7 @@
 #include "trace.h"
 #include "urcrypt.h"
 #include "vortex.h"
+#include "whereami.h"
 #include "xtract.h"
 
 //  XX stack-overflow recovery should be gated by -a
@@ -728,6 +738,164 @@ u3m_dump(void)
 }
 #endif
 
+struct bt_cb_data {
+  c3_y  count;
+  c3_y  fail;
+  c3_c* pn_c;
+};
+
+static void
+err_cb(void* data, const char* msg, int errnum)
+{
+  struct bt_cb_data* bdata = (struct bt_cb_data *)data;
+  bdata->count++;
+
+  if ( bdata->count <= 1 ) {
+    /* u3l_log("Backtrace error %d: %s", errnum, msg); */
+    bdata->fail = 1;
+  }
+}
+
+static int
+bt_cb(void* data,
+      uintptr_t pc,
+      const char* filename,
+      int lineno,
+      const char* function)
+{
+  struct bt_cb_data* bdata = (struct bt_cb_data *)data;
+  bdata->count++;
+
+  Dl_info info = {};
+  c3_c*   fname_c = {0};
+
+  if ( dladdr((void *)pc, &info) ) {
+    for ( c3_w i_w = 0; info.dli_fname[i_w] != 0; i_w++ )
+      if ( info.dli_fname[i_w] == '/' ) {
+        fname_c = (c3_c*)&info.dli_fname[i_w + 1];
+      }
+  }
+
+  if ( bdata->count <= 100 ) {
+    c3_c* loc[128];
+    if (filename != 0) {
+      snprintf((c3_c*)loc, 128, "%s:%d", filename, lineno);
+    }
+    else {
+      snprintf((c3_c*)loc, 128, "%s", fname_c != 0 ? fname_c : "-");
+    }
+
+    c3_c* fn_c;
+    if (function != 0 || bdata->pn_c != 0) {
+      fn_c = (c3_c*)(function != 0 ? function : bdata->pn_c);
+    }
+    else {
+      fn_c = (c3_c*)(info.dli_sname != 0 ? info.dli_sname : "-");
+    }
+
+    fprintf(stderr, "%-3d %-35s %s\r\n", bdata->count - 1, fn_c, (c3_c *)loc);
+
+    bdata->pn_c = 0;
+    return 0;
+  }
+  else {
+    bdata->pn_c = 0;
+    return 1;
+  }
+}
+
+/* _self_path(): get binary self-path.
+ */
+static c3_y
+_self_path(c3_c *pat_c)
+{
+  c3_i len_i = 0;
+  c3_i pat_i;
+
+  if ( 0 < (len_i = wai_getExecutablePath(NULL, 0, &pat_i)) ) {
+    wai_getExecutablePath(pat_c, len_i, &pat_i);
+    pat_c[len_i] = 0;
+    return 0;
+  }
+
+  return 1;
+}
+
+void
+u3m_stacktrace()
+{
+  void* bt_state;
+  c3_i  ret_i;
+  struct bt_cb_data data = { 0, 0, 0 };
+  c3_c* self_path_c[4096] = {0};
+
+#if defined(U3_OS_osx)
+  fprintf(stderr, "Stacktrace:\r\n");
+
+  if ( _self_path((c3_c*)self_path_c) == 0 ) {
+    bt_state = backtrace_create_state((const c3_c*)self_path_c, 0, err_cb, 0);
+    ret_i = backtrace_full(bt_state, 0, bt_cb, err_cb, &data);
+    if (data.fail == 0) u3l_log("");
+  }
+  else {
+    data.fail = 1;
+  }
+
+  if ( data.fail == 1 ) {
+    void*  array[100];
+    c3_c** strings;
+    size_t size = backtrace(array, 100);
+
+    strings = backtrace_symbols(array, size);
+
+    if ( strings[0] == NULL ) {
+      fprintf(stderr, "Backtrace failed\r\n");
+    }
+    else {
+      for ( c3_i i = 0; i < size; i++ ) {
+        fprintf(stderr, "%s\r\n", strings[i]);
+      }
+      u3l_log("");
+    }
+
+    free(strings);
+  }
+#elif defined(U3_OS_linux)
+  /* TODO: Fix unwind not getting past signal trampoline on linux aarch64
+   */
+  fprintf(stderr, "Stacktrace:\r\n");
+
+  if ( _self_path((c3_c*)self_path_c) == 0 ) {
+    bt_state = backtrace_create_state((const c3_c*)self_path_c, 0, err_cb, 0);
+
+    unw_context_t context;
+    unw_cursor_t cursor;
+    unw_getcontext(&context);
+    unw_init_local(&cursor, &context);
+    unw_word_t pc, sp;
+
+    c3_c* pn_c[1024] = {0};
+    c3_w  offp_w = 0;
+
+    do {
+      unw_get_reg(&cursor, UNW_REG_IP, &pc);
+      unw_get_reg(&cursor, UNW_REG_SP, &sp);
+      if ( 0 == unw_get_proc_name(&cursor, pn_c, 1024, &offp_w) )
+        data.pn_c = pn_c;
+      ret_i = backtrace_pcinfo(bt_state, pc - 1, bt_cb, err_cb, &data);
+    } while (unw_step(&cursor) > 0);
+
+    if ( (data.count > 0) ) {
+      u3l_log("");
+    }
+  }
+  else {
+    data.fail = 1;
+    fprintf(stderr, "Backtrace failed\r\n");
+  }
+#endif
+}
+
 /* u3m_bail(): bail out.  Does not return.
 **
 **  Bail motes:
@@ -780,7 +948,8 @@ u3m_bail(u3_noun how)
   if ( &(u3H->rod_u) == u3R ) {
     //  XX set exit code
     //
-    fprintf(stderr, "home: bailing out\r\n");
+    fprintf(stderr, "home: bailing out\r\n\r\n");
+    u3m_stacktrace();
     abort();
   }
 
@@ -791,7 +960,8 @@ u3m_bail(u3_noun how)
     case c3__oops: {
       //  XX set exit code
       //
-      fprintf(stderr, "bailing out\r\n");
+      fprintf(stderr, "bailing out\r\n\r\n");
+      u3m_stacktrace();
       abort();
     }
   }
@@ -1771,7 +1941,7 @@ _cm_limits(void)
 }
 
 /* u3m_fault(): handle a memory event with libsigsegv protocol.
-*/
+ */
 c3_i
 u3m_fault(void* adr_v, c3_i ser_i)
 {
@@ -1787,7 +1957,8 @@ u3m_fault(void* adr_v, c3_i ser_i)
   //
   else if ( (adr_w < u3_Loom) || (adr_w >= (u3_Loom + u3C.wor_i)) ) {
     fprintf(stderr, "loom: external fault: %p (%p : %p)\r\n\r\n",
-                    adr_w, u3_Loom, u3_Loom + u3C.wor_i);
+            adr_w, u3_Loom, u3_Loom + u3C.wor_i);
+    u3m_stacktrace();
     u3_assert(0);
     return 0;
   }

pkg/vere/king.c

@@ -1437,7 +1437,7 @@ _king_copy_file(c3_c* src_c, c3_c* dst_c)
       do {
         //  XX fallback on any errors?
         //
-        if ( 0 > (sen_i = sendfile64(dst_i, src_i, &off_i, len_i)) ) {
+        if ( 0 > (sen_i = sendfile(dst_i, src_i, &off_i, len_i)) ) {
           err_i = errno;
           ret_i = -1;
           goto done3;