Lower the priority of DSO symbols than in-archive symbols

Assume both `foo.a` and `bar.so` define the same symbol `baz`.
If `baz`'s symbol visibility is hidden, it needs to be resolved within
the output file, i.e., from `foo.a`. However, previously, such symbol
was resolved to the one in `bar.so`.

To fix the problem, we'll lower the symbol priority for DSOs.

Fixes https://github.com/rui314/mold/issues/1342

src/input-files.cc

@@ -857,22 +857,27 @@ void ObjectFile<E>::parse(Context<E> &ctx) {
 //
 //  1. Strong defined symbol
 //  2. Weak defined symbol
-//  3. Strong defined symbol in a DSO/archive
-//  4. Weak Defined symbol in a DSO/archive
+//  3. Strong defined symbol in an archive
+//  4. Weak Defined symbol in an archive
 //  5. Common symbol
 //  6. Common symbol in an archive
-//  7. Unclaimed (nonexistent) symbol
+//  7. Strong defined symbol in a DSO
+//  8. Weak Defined symbol in a DSO
+//  9. Unclaimed (nonexistent) symbol
 //
 // Ties are broken by file priority.
 template <typename E>
 static u64 get_rank(InputFile<E> *file, const ElfSym<E> &esym, bool is_in_archive) {
   auto get_sym_rank = [&] {
+    if (file->is_dso)
+      return (esym.st_bind == STB_WEAK) ? 8 : 7;
+
     if (esym.is_common()) {
       assert(!file->is_dso);
       return is_in_archive ? 6 : 5;
     }
 
-    if (file->is_dso || is_in_archive)
+    if (is_in_archive)
       return (esym.st_bind == STB_WEAK) ? 4 : 3;
 
     if (esym.st_bind == STB_WEAK)
@@ -886,7 +891,7 @@ static u64 get_rank(InputFile<E> *file, const ElfSym<E> &esym, bool is_in_archiv
 template <typename E>
 static u64 get_rank(const Symbol<E> &sym) {
   if (!sym.file)
-    return 7 << 24;
+    return 9 << 24;
   return get_rank(sym.file, sym.esym(), !sym.file->is_alive);
 }
 

test/hidden-archive.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+. $(dirname $0)/common.inc
+
+cat <<EOF | $CC -c -o $t/a.o -fPIC -xc -
+void foo() {}
+EOF
+
+rm -f $t/b.a
+ar rcs $t/b.a $t/a.o
+
+cat <<EOF | $CC -shared -o $t/c.so -fPIC -xc -
+void foo() {}
+EOF
+
+cat <<EOF | $CC -o $t/d.o -fPIC -c -xc -
+__attribute__((visibility("hidden"))) void foo();
+int main() { foo(); }
+EOF
+
+$CC -B. -o $t/exe $t/d.o $t/c.so $t/b.a
+$QEMU $t/exe

test/link-order.sh

@@ -16,7 +16,7 @@ int main() {
 EOF
 
 $CC -B. -o $t/exe $t/b.o -Wl,--as-needed $t/libfoo.so $t/libfoo.a
-readelf --dynamic $t/exe | grep -q libfoo
+! readelf --dynamic $t/exe | grep -q libfoo || false
 
 $CC -B. -o $t/exe $t/b.o -Wl,--as-needed $t/libfoo.a $t/libfoo.so
 ! readelf --dynamic $t/exe | grep -q libfoo || false