#pragma once #ifndef _GNU_SOURCE #define _GNU_SOURCE #endif #include "elf.h" #include "xxHash/xxhash.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include typedef uint8_t u8; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; typedef int8_t i8; typedef int16_t i16; typedef int32_t i32; typedef int64_t i64; static constexpr i32 SECTOR_SIZE = 512; static constexpr i32 PAGE_SIZE = 4096; static constexpr i32 SHA256_SIZE = 32; template class InputFile; template class InputSection; template class MergedSection; template class ObjectFile; template class OutputChunk; template class OutputSection; template class SharedFile; template class Symbol; template struct Context; template void cleanup(); template std::ostream &operator<<(std::ostream &out, const Symbol &sym); // // Mergeable section fragments // template struct SectionFragment { SectionFragment(MergedSection *sec, std::string_view data) : output_section(*sec), data(data) {} SectionFragment(const SectionFragment &other) : output_section(other.output_section), data(other.data), offset(other.offset), alignment(other.alignment.load()), is_alive(other.is_alive.load()) {} inline u64 get_addr(Context &ctx) const; MergedSection &output_section; std::string_view data; u32 offset = -1; std::atomic_uint16_t alignment = 1; std::atomic_bool is_alive = false; }; template struct SectionFragmentRef { SectionFragment *frag = nullptr; i32 addend = 0; }; struct SymbolAux { i32 got_idx = -1; i32 gotplt_idx = -1; i32 gottp_idx = -1; i32 tlsgd_idx = -1; i32 tlsdesc_idx = -1; i32 plt_idx = -1; i32 pltgot_idx = -1; i32 dynsym_idx = -1; }; // // Interned string // inline u64 hash_string(std::string_view str) { return XXH3_64bits(str.data(), str.size()); } namespace tbb { template<> struct tbb_hash_compare { static size_t hash(const std::string_view &k) { return hash_string(k); } static bool equal(const std::string_view &k1, const std::string_view &k2) { return k1 == k2; } }; } template class ConcurrentMap { public: ValueT *insert(std::string_view key, const ValueT &val) { typename decltype(map)::const_accessor acc; map.insert(acc, std::make_pair(key, val)); return const_cast(&acc->second); } tbb::concurrent_hash_map map; }; // // input_sections.cc // enum { R_NONE = 1, R_ABS, R_DYN, R_BASEREL, R_PC, R_GOT, R_GOTOFF, R_GOTPC, R_GOTPCREL, R_TLSGD, R_TLSGD_RELAX_LE, R_TLSLD, R_TLSLD_RELAX_LE, R_DTPOFF, R_DTPOFF_RELAX, R_TPOFF, R_GOTTPOFF, R_GOTTPOFF_RELAX, R_GOTPC_TLSDESC, R_GOTPC_TLSDESC_RELAX_LE, R_SIZE, R_TLSDESC_CALL_RELAX, R_END, }; template struct EhReloc { EhReloc(Symbol &sym, u8 type, u32 offset, i64 addend) : sym(sym), type(type), offset(offset), addend(addend) { assert(this->offset == offset); assert(this->addend == addend); } Symbol &sym; u32 type : 8; u32 offset : 24; i32 addend; }; template inline bool operator==(const EhReloc &a, const EhReloc &b) { return std::tuple(&a.sym, a.type, a.offset, a.addend) == std::tuple(&b.sym, b.type, b.offset, b.addend); } template struct FdeRecord { FdeRecord(std::string_view contents, std::vector> &&rels, u32 cie_idx) : contents(contents), rels(std::move(rels)), cie_idx(cie_idx) { assert(this->cie_idx == cie_idx); } FdeRecord(const FdeRecord &&other) : contents(other.contents), rels(std::move(other.rels)), offset(other.offset), cie_idx(other.cie_idx), is_alive(other.is_alive.load()) {} std::string_view contents; std::vector> rels; u32 offset = -1; u16 cie_idx = -1; std::atomic_bool is_alive = true; }; template struct CieRecord { bool should_merge(const CieRecord &other) const; std::string_view contents; std::vector> rels; std::vector> fdes; // For .eh_frame u32 offset = -1; u32 leader_offset = -1; u32 fde_size = -1; // For .eh_frame_hdr u32 num_fdes = 0; u32 fde_idx = -1; // For ICF u32 icf_idx = -1; }; template class InputSection { public: InputSection(Context &ctx, ObjectFile &file, const ElfShdr &shdr, std::string_view name, i64 section_idx); void scan_relocations(Context &ctx); void report_undefined_symbols(); void copy_buf(Context &ctx); void apply_reloc_alloc(Context &ctx, u8 *base); void apply_reloc_nonalloc(Context &ctx, u8 *base); void kill(); inline i64 get_priority() const; inline u64 get_addr() const; inline i64 get_addend(const ElfRel &rel) const; inline std::span> get_rels(Context &ctx) const; ObjectFile &file; const ElfShdr &shdr; OutputSection *output_section = nullptr; std::string_view name; std::string_view contents; std::vector has_fragments; std::vector> rel_fragments; std::vector rel_types; std::span> fdes; u32 offset = -1; u32 section_idx = -1; u32 relsec_idx = -1; u32 reldyn_offset = 0; // For COMDAT de-duplication and garbage collection std::atomic_bool is_alive = true; // For garbage collection std::atomic_bool is_visited = false; // For ICF InputSection *leader = nullptr; u32 icf_idx = -1; bool icf_eligible = false; bool icf_leaf = false; bool is_ehframe = false; private: typedef enum { NONE, ERROR, COPYREL, PLT, DYNREL, BASEREL } Action; void dispatch(Context &ctx, Action table[3][4], u16 rel_type, i64 i); }; // // output_chunks.cc // template bool is_relro(Context &ctx, OutputChunk *chunk); template class OutputChunk { public: enum Kind : u8 { HEADER, REGULAR, SYNTHETIC }; virtual void copy_buf(Context &ctx) {} virtual void update_shdr(Context &ctx) {} std::string_view name; i64 shndx = 0; Kind kind; bool new_page = false; bool new_page_end = false; ElfShdr shdr = {}; protected: OutputChunk(Kind kind) : kind(kind) { shdr.sh_addralign = 1; } }; // ELF header template class OutputEhdr : public OutputChunk { public: OutputEhdr() : OutputChunk(this->HEADER) { this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_size = sizeof(ElfEhdr); } void copy_buf(Context &ctx) override; }; // Section header template class OutputShdr : public OutputChunk { public: OutputShdr() : OutputChunk(this->HEADER) { this->shdr.sh_flags = SHF_ALLOC; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; }; // Program header template class OutputPhdr : public OutputChunk { public: OutputPhdr() : OutputChunk(this->HEADER) { this->shdr.sh_flags = SHF_ALLOC; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; }; template class InterpSection : public OutputChunk { public: InterpSection() : OutputChunk(this->SYNTHETIC) { this->name = ".interp"; this->shdr.sh_type = SHT_PROGBITS; this->shdr.sh_flags = SHF_ALLOC; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; }; // Sections template class OutputSection : public OutputChunk { public: static OutputSection * get_instance(std::string_view name, u64 type, u64 flags); void copy_buf(Context &ctx) override; static inline std::vector instances; std::vector *> members; u32 idx; private: OutputSection(std::string_view name, u32 type, u64 flags); }; template class GotSection : public OutputChunk { public: GotSection() : OutputChunk(this->SYNTHETIC) { this->name = ".got"; this->shdr.sh_type = SHT_PROGBITS; this->shdr.sh_flags = SHF_ALLOC | SHF_WRITE; this->shdr.sh_addralign = E::got_size; } void add_got_symbol(Context &ctx, Symbol *sym); void add_gottp_symbol(Context &ctx, Symbol *sym); void add_tlsgd_symbol(Context &ctx, Symbol *sym); void add_tlsdesc_symbol(Context &ctx, Symbol *sym); void add_tlsld(Context &ctx); u64 get_tlsld_addr(Context &ctx) const { assert(tlsld_idx != -1); return this->shdr.sh_addr + tlsld_idx * E::got_size; } i64 get_reldyn_size(Context &ctx) const; void copy_buf(Context &ctx) override; std::vector *> got_syms; std::vector *> gottp_syms; std::vector *> tlsgd_syms; std::vector *> tlsdesc_syms; u32 tlsld_idx = -1; }; template class GotPltSection : public OutputChunk { public: GotPltSection() : OutputChunk(this->SYNTHETIC) { this->name = ".got.plt"; this->shdr.sh_type = SHT_PROGBITS; this->shdr.sh_flags = SHF_ALLOC | SHF_WRITE; this->shdr.sh_addralign = E::got_size; } void copy_buf(Context &ctx) override; }; template class PltSection : public OutputChunk { public: PltSection() : OutputChunk(this->SYNTHETIC) { this->name = ".plt"; this->shdr.sh_type = SHT_PROGBITS; this->shdr.sh_flags = SHF_ALLOC | SHF_EXECINSTR; this->shdr.sh_addralign = 16; } void add_symbol(Context &ctx, Symbol *sym); void copy_buf(Context &ctx) override; std::vector *> symbols; }; template class PltGotSection : public OutputChunk { public: PltGotSection() : OutputChunk(this->SYNTHETIC) { this->name = ".plt.got"; this->shdr.sh_type = SHT_PROGBITS; this->shdr.sh_flags = SHF_ALLOC | SHF_EXECINSTR; this->shdr.sh_addralign = 8; } void add_symbol(Context &ctx, Symbol *sym); void copy_buf(Context &ctx) override; std::vector *> symbols; }; template class RelPltSection : public OutputChunk { public: RelPltSection() : OutputChunk(this->SYNTHETIC) { this->name = (E::rel_type == SHT_REL) ? ".rel.plt" : ".rela.plt"; this->shdr.sh_type = E::rel_type; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_entsize = sizeof(ElfRel); this->shdr.sh_addralign = E::wordsize; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; }; template class RelDynSection : public OutputChunk { public: RelDynSection() : OutputChunk(this->SYNTHETIC) { this->name = (E::rel_type == SHT_REL) ? ".rel.dyn" : ".rela.dyn"; this->shdr.sh_type = E::rel_type; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_entsize = sizeof(ElfRel); this->shdr.sh_addralign = E::wordsize; } void update_shdr(Context &ctx) override; void sort(Context &ctx); }; template class StrtabSection : public OutputChunk { public: StrtabSection() : OutputChunk(this->SYNTHETIC) { this->name = ".strtab"; this->shdr.sh_type = SHT_STRTAB; this->shdr.sh_size = 1; } void update_shdr(Context &ctx) override; }; template class ShstrtabSection : public OutputChunk { public: ShstrtabSection() : OutputChunk(this->SYNTHETIC) { this->name = ".shstrtab"; this->shdr.sh_type = SHT_STRTAB; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; }; template class DynstrSection : public OutputChunk { public: DynstrSection() : OutputChunk(this->SYNTHETIC) { this->name = ".dynstr"; this->shdr.sh_type = SHT_STRTAB; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_size = 1; } i64 add_string(std::string_view str); i64 find_string(std::string_view str); void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; i64 dynsym_offset = -1; private: std::unordered_map strings; }; template class DynamicSection : public OutputChunk { public: DynamicSection() : OutputChunk(this->SYNTHETIC) { this->name = ".dynamic"; this->shdr.sh_type = SHT_DYNAMIC; this->shdr.sh_flags = SHF_ALLOC | SHF_WRITE; this->shdr.sh_addralign = E::wordsize; this->shdr.sh_entsize = sizeof(ElfDyn); } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; }; template class SymtabSection : public OutputChunk { public: SymtabSection() : OutputChunk(this->SYNTHETIC) { this->name = ".symtab"; this->shdr.sh_type = SHT_SYMTAB; this->shdr.sh_entsize = sizeof(ElfSym); this->shdr.sh_addralign = E::wordsize; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; }; template class DynsymSection : public OutputChunk { public: DynsymSection() : OutputChunk(this->SYNTHETIC) { this->name = ".dynsym"; this->shdr.sh_type = SHT_DYNSYM; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_entsize = sizeof(ElfSym); this->shdr.sh_addralign = E::wordsize; } void add_symbol(Context &ctx, Symbol *sym); void sort_symbols(Context &ctx); void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; std::vector *> symbols; }; template class HashSection : public OutputChunk { public: HashSection() : OutputChunk(this->SYNTHETIC) { this->name = ".hash"; this->shdr.sh_type = SHT_HASH; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_entsize = 4; this->shdr.sh_addralign = 4; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; }; template class GnuHashSection : public OutputChunk { public: GnuHashSection() : OutputChunk(this->SYNTHETIC) { this->name = ".gnu.hash"; this->shdr.sh_type = SHT_GNU_HASH; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_addralign = E::wordsize; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; static constexpr i64 LOAD_FACTOR = 8; static constexpr i64 HEADER_SIZE = 16; static constexpr i64 BLOOM_SHIFT = 26; static constexpr i64 ELFCLASS_BITS = E::wordsize * 8; u32 num_buckets = -1; u32 symoffset = -1; u32 num_bloom = 1; }; template class MergedSection : public OutputChunk { public: static MergedSection * get_instance(std::string_view name, u64 type, u64 flags); SectionFragment *insert(std::string_view data, i64 alignment); void assign_offsets(); void copy_buf(Context &ctx) override; static inline std::vector *> instances; private: using MapTy = tbb::concurrent_hash_map>; static constexpr i64 NUM_SHARDS = 64; MergedSection(std::string_view name, u64 flags, u32 type) : OutputChunk(this->SYNTHETIC) { this->name = name; this->shdr.sh_flags = flags; this->shdr.sh_type = type; } MapTy maps[NUM_SHARDS]; i64 shard_offsets[NUM_SHARDS + 1] = {}; std::atomic_uint16_t max_alignment; }; template class EhFrameSection : public OutputChunk { public: EhFrameSection() : OutputChunk(this->SYNTHETIC) { this->name = ".eh_frame"; this->shdr.sh_type = SHT_PROGBITS; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_addralign = E::wordsize; } void construct(Context &ctx); void copy_buf(Context &ctx) override; u64 get_addr(Context &ctx, const Symbol &sym); std::vector *> cies; u32 num_fdes = 0; }; template class EhFrameHdrSection : public OutputChunk { public: EhFrameHdrSection() : OutputChunk(this->SYNTHETIC) { this->name = ".eh_frame_hdr"; this->shdr.sh_type = SHT_PROGBITS; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_addralign = 4; this->shdr.sh_size = HEADER_SIZE; } static constexpr i64 HEADER_SIZE = 12; }; template class DynbssSection : public OutputChunk { public: DynbssSection(bool is_relro) : OutputChunk(this->SYNTHETIC) { this->name = is_relro ? ".dynbss.rel.ro" : ".dynbss"; this->shdr.sh_type = SHT_NOBITS; this->shdr.sh_flags = SHF_ALLOC | SHF_WRITE; this->shdr.sh_addralign = 64; } void add_symbol(Context &ctx, Symbol *sym); std::vector *> symbols; }; template class VersymSection : public OutputChunk { public: VersymSection() : OutputChunk(this->SYNTHETIC) { this->name = ".gnu.version"; this->shdr.sh_type = SHT_GNU_VERSYM; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_entsize = 2; this->shdr.sh_addralign = 2; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; std::vector contents; }; template class VerneedSection : public OutputChunk { public: VerneedSection() : OutputChunk(this->SYNTHETIC) { this->name = ".gnu.version_r"; this->shdr.sh_type = SHT_GNU_VERNEED; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_addralign = E::wordsize; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; std::vector contents; }; template class VerdefSection : public OutputChunk { public: VerdefSection() : OutputChunk(this->SYNTHETIC) { this->name = ".gnu.version_d"; this->shdr.sh_type = SHT_GNU_VERDEF; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_addralign = 8; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; std::vector contents; }; template class BuildIdSection : public OutputChunk { public: BuildIdSection() : OutputChunk(this->SYNTHETIC) { this->name = ".note.gnu.build-id"; this->shdr.sh_type = SHT_NOTE; this->shdr.sh_flags = SHF_ALLOC; this->shdr.sh_addralign = 4; this->shdr.sh_size = 1; } void update_shdr(Context &ctx) override; void copy_buf(Context &ctx) override; void write_buildid(Context &ctx, i64 filesize); static constexpr i64 HEADER_SIZE = 16; }; bool is_c_identifier(std::string_view name); template std::vector> create_phdr(Context &ctx); // // object_file.cc // struct ComdatGroup { ComdatGroup() = default; ComdatGroup(const ComdatGroup &other) : owner(other.owner.load()) {} std::atomic_uint32_t owner = -1; }; template class MemoryMappedFile { public: static MemoryMappedFile *open(std::string path); static MemoryMappedFile *must_open(Context &ctx, std::string path); MemoryMappedFile(std::string name, u8 *data, u64 size, u64 mtime = 0) : name(name), data_(data), size_(size), mtime(mtime) {} MemoryMappedFile() = delete; MemoryMappedFile *slice(std::string name, u64 start, u64 size); u8 *data(Context &ctx); i64 size() const { return size_; } std::string_view get_contents(Context &ctx) { return std::string_view((char *)data(ctx), size()); } std::string name; i64 mtime = 0; private: std::mutex mu; MemoryMappedFile *parent; std::atomic data_; i64 size_ = 0; }; template class InputFile { public: InputFile(Context &ctx, MemoryMappedFile *mb); InputFile() : name("") {} std::string_view get_string(Context &ctx, const ElfShdr &shdr); std::string_view get_string(Context &ctx, i64 idx); MemoryMappedFile *mb; std::span> elf_sections; std::vector *> symbols; std::string name; bool is_dso = false; u32 priority; std::atomic_bool is_alive = false; template std::span get_data(Context &ctx, const ElfShdr &shdr); template std::span get_data(Context &ctx, i64 idx); ElfShdr *find_section(i64 type); std::string_view shstrtab; }; template class ObjectFile : public InputFile { public: ObjectFile(Context &ctx, MemoryMappedFile *mb, std::string archive_name, bool is_in_lib); ObjectFile(Context &ctx); void parse(Context &ctx); void resolve_lazy_symbols(Context &ctx); void resolve_regular_symbols(Context &ctx); void mark_live_objects(Context &ctx, std::function *)> feeder); void convert_undefined_weak_symbols(Context &ctx); void resolve_comdat_groups(); void eliminate_duplicate_comdat_groups(); void claim_unresolved_symbols(); void scan_relocations(Context &ctx); void convert_common_symbols(Context &ctx); void compute_symtab(Context &ctx); void write_symtab(Context &ctx); inline i64 get_shndx(const ElfSym &esym); inline InputSection *get_section(const ElfSym &esym); inline std::span *> get_global_syms(); std::string archive_name; std::vector *> sections; std::span> elf_syms; i64 first_global = 0; const bool is_in_lib = false; std::vector> cies; std::vector symvers; std::vector *> fragments; std::vector> sym_fragments; bool exclude_libs = false; u64 num_dynrel = 0; u64 reldyn_offset = 0; u64 local_symtab_offset = 0; u64 global_symtab_offset = 0; u64 num_local_symtab = 0; u64 num_global_symtab = 0; u64 strtab_offset = 0; u64 strtab_size = 0; private: void initialize_sections(Context &ctx); void initialize_symbols(Context &ctx); void initialize_mergeable_sections(Context &ctx); void initialize_ehframe_sections(Context &ctx); void read_ehframe(Context &ctx, InputSection &isec); void maybe_override_symbol(Context &ctx, Symbol &sym, i64 symidx); void merge_visibility(Context &ctx, Symbol &sym, u8 visibility); std::vector>> comdat_groups; bool has_common_symbol; std::string_view symbol_strtab; const ElfShdr *symtab_sec; std::span symtab_shndx_sec; }; template class SharedFile : public InputFile { public: SharedFile(Context &ctx, MemoryMappedFile *mb); void parse(Context &ctx); void resolve_symbols(Context &ctx); std::vector *> find_aliases(Symbol *sym); bool is_readonly(Context &ctx, Symbol *sym); std::string_view soname; std::vector version_strings; std::vector *> undefs; std::vector *> elf_syms; private: std::string_view get_soname(Context &ctx); void maybe_override_symbol(Symbol &sym, const ElfSym &esym); std::vector read_verdef(Context &ctx); std::vector versyms; std::string_view symbol_strtab; const ElfShdr *symtab_sec; }; // // archive_file.cc // template std::vector *> read_fat_archive_members(Context &ctx, MemoryMappedFile *mb); template std::vector *> read_thin_archive_members(Context &ctx, MemoryMappedFile *mb); // // linker_script.cc // template void parse_linker_script(Context &ctx, MemoryMappedFile *mb); template void parse_version_script(Context &ctx, std::string path); template void parse_dynamic_list(Context &ctx, std::string path); // // output_file.cc // template class OutputFile { public: static OutputFile *open(Context &ctx, std::string path, u64 filesize); virtual void close(Context &ctx) = 0; u8 *buf; static inline char *tmpfile; protected: OutputFile(std::string path, u64 filesize) : path(path), filesize(filesize) {} std::string path; u64 filesize; }; // // filepath.cc // std::string path_dirname(std::string_view path); std::string path_basename(std::string_view path); std::string path_clean(std::string_view path); // // glob.cc // class GlobPattern { public: GlobPattern(std::string_view pat); bool match(std::string_view str) const; private: enum { EXACT, PREFIX, SUFFIX, GENERIC } kind; std::string_view pat; }; // // perf.cc // class Counter { public: Counter(std::string_view name, i64 value = 0) : name(name), values(value) { static std::mutex mu; std::lock_guard lock(mu); instances.push_back(this); } Counter &operator++(int) { if (enabled) values.local()++; return *this; } Counter &operator+=(int delta) { if (enabled) values.local() += delta; return *this; } static void print(); static inline bool enabled = false; private: i64 get_value(); std::string_view name; tbb::enumerable_thread_specific values; static inline std::vector instances; }; struct TimerRecord { TimerRecord(std::string name); void stop(); std::string name; i64 start; i64 end; i64 user; i64 sys; bool stopped = false; }; class Timer { public: Timer(std::string name); ~Timer(); void stop(); static void print(); private: static inline std::vector records; TimerRecord *record; }; // // gc_sections.cc // template void gc_sections(Context &ctx); // // icf.cc // template void icf_sections(Context &ctx); // // mapfile.cc // template void print_map(Context &ctx); // // subprocess.cc // inline char *socket_tmpfile; std::function fork_child(); template bool resume_daemon(Context &ctx, char **argv, i64 *code); template void daemonize(Context &ctx, char **argv, std::function *wait_for_client, std::function *on_complete); template [[noreturn]] void process_run_subcommand(Context &ctx, int argc, char **argv); // commandline.cc // template std::vector expand_response_files(Context &ctx, char **argv); bool read_flag(std::span &args, std::string name); template bool read_arg(Context &ctx, std::span &args, std::string_view &arg, std::string name); template void parse_nonpositional_args(Context &ctx, std::vector &remaining); // // passes.cc // template void apply_exclude_libs(Context &ctx); template void create_synthetic_sections(Context &ctx); template void set_file_priority(Context &ctx); template void resolve_obj_symbols(Context &ctx); template void resolve_dso_symbols(Context &ctx); template void eliminate_comdats(Context &ctx); template void convert_common_symbols(Context &ctx); template void add_comment_string(Context &ctx, std::string str); template void compute_merged_section_sizes(Context &ctx); template void bin_sections(Context &ctx); template void check_duplicate_symbols(Context &ctx); template std::vector *> collect_output_sections(Context &ctx); template void compute_section_sizes(Context &ctx); template void convert_undefined_weak_symbols(Context &ctx); template void scan_rels(Context &ctx); template void apply_version_script(Context &ctx); template void parse_symbol_version(Context &ctx); template void compute_import_export(Context &ctx); template void fill_verdef(Context &ctx); template void fill_verneed(Context &ctx); template void clear_padding(Context &ctx, i64 filesize); template i64 get_section_rank(Context &ctx, OutputChunk *chunk); template i64 set_osec_offsets(Context &ctx); template void fix_synthetic_symbols(Context &ctx); // // main.cc // struct BuildId { template i64 size(Context &ctx) const; enum { NONE, HEX, HASH, UUID } kind = NONE; std::vector value; i64 hash_size = 0; }; struct VersionPattern { std::string_view pattern; i16 ver_idx; bool is_extern_cpp; }; template struct Context { Context() = default; Context(const Context &) = delete; // Command-line arguments struct { BuildId build_id; bool Bsymbolic = false; bool Bsymbolic_functions = false; bool allow_multiple_definition = false; bool demangle = true; bool discard_all = false; bool discard_locals = false; bool eh_frame_hdr = true; bool export_dynamic = false; bool fatal_warnings = false; bool fork = true; bool gc_sections = false; bool hash_style_gnu = false; bool hash_style_sysv = true; bool icf = false; bool is_static = false; bool perf = false; bool pic = false; bool pie = false; bool preload = false; bool print_gc_sections = false; bool print_icf_sections = false; bool print_map = false; bool quick_exit = true; bool relax = true; bool shared = false; bool stats = false; bool strip_all = false; bool strip_debug = false; bool trace = false; bool warn_common = false; bool z_copyreloc = true; bool z_defs = false; bool z_delete = true; bool z_dlopen = true; bool z_execstack = false; bool z_now = false; bool z_relro = true; i16 default_version = VER_NDX_GLOBAL; std::vector version_definitions; std::vector version_patterns; i64 filler = -1; i64 thread_count = -1; std::string Map; std::string dynamic_linker; std::string entry = "_start"; std::string fini = "_fini"; std::string init = "_init"; std::string output; std::string rpaths; std::string soname; std::string sysroot; std::vector auxiliary; std::vector exclude_libs; std::vector filter; std::vector library_paths; std::vector trace_symbol; std::vector undefined; u64 image_base = 0x200000; } arg; void reset_reader_context(bool is_preloading) { as_needed = false; whole_archive = false; this->is_preloading = is_preloading; is_static = arg.is_static; visited.clear(); } // Reader context bool as_needed; bool whole_archive; bool is_preloading; bool is_static; std::unordered_set visited; tbb::task_group tg; bool has_error = false; // Symbol table ConcurrentMap> symbol_map; // Symbol auxiliary data std::vector symbol_aux; // Fully-expanded command line args std::vector cmdline_args; // Input files std::vector *> objs; std::vector *> dsos; ObjectFile *internal_obj = nullptr; // Output buffer u8 *buf = nullptr; std::vector *> chunks; std::atomic_bool has_gottp_rel = false; std::atomic_bool has_textrel = false; // Output chunks OutputEhdr *ehdr = nullptr; OutputShdr *shdr = nullptr; OutputPhdr *phdr = nullptr; InterpSection *interp = nullptr; GotSection *got = nullptr; GotPltSection *gotplt = nullptr; RelPltSection *relplt = nullptr; RelDynSection *reldyn = nullptr; DynamicSection *dynamic = nullptr; StrtabSection *strtab = nullptr; DynstrSection *dynstr = nullptr; HashSection *hash = nullptr; GnuHashSection *gnu_hash = nullptr; ShstrtabSection *shstrtab = nullptr; PltSection *plt = nullptr; PltGotSection *pltgot = nullptr; SymtabSection *symtab = nullptr; DynsymSection *dynsym = nullptr; EhFrameSection *eh_frame = nullptr; EhFrameHdrSection *eh_frame_hdr = nullptr; DynbssSection *dynbss = nullptr; DynbssSection *dynbss_relro = nullptr; VersymSection *versym = nullptr; VerneedSection *verneed = nullptr; VerdefSection *verdef = nullptr; BuildIdSection *buildid = nullptr; u64 tls_begin = -1; u64 tls_end = -1; // Linker-synthesized symbols Symbol *__bss_start = nullptr; Symbol *__ehdr_start = nullptr; Symbol *__rel_iplt_start = nullptr; Symbol *__rel_iplt_end = nullptr; Symbol *__init_array_start = nullptr; Symbol *__init_array_end = nullptr; Symbol *__fini_array_start = nullptr; Symbol *__fini_array_end = nullptr; Symbol *__preinit_array_start = nullptr; Symbol *__preinit_array_end = nullptr; Symbol *_DYNAMIC = nullptr; Symbol *_GLOBAL_OFFSET_TABLE_ = nullptr; Symbol *__GNU_EH_FRAME_HDR = nullptr; Symbol *_end = nullptr; Symbol *_etext = nullptr; Symbol *_edata = nullptr; Symbol *__executable_start = nullptr; }; template MemoryMappedFile *find_library(Context &ctx, std::string path); template void read_file(Context &ctx, MemoryMappedFile *mb); // // Symbol // enum { NEEDS_GOT = 1 << 0, NEEDS_PLT = 1 << 1, NEEDS_GOTTP = 1 << 2, NEEDS_TLSGD = 1 << 3, NEEDS_TLSLD = 1 << 4, NEEDS_COPYREL = 1 << 5, NEEDS_DYNSYM = 1 << 6, NEEDS_TLSDESC = 1 << 7, }; template class Symbol { public: Symbol() = default; Symbol(std::string_view name) : nameptr(name.data()), namelen(name.size()) {} Symbol(const Symbol &other) : Symbol(other.get_name()) {} static Symbol *intern(Context &ctx, std::string_view key, std::string_view name) { return ctx.symbol_map.insert(key, {name}); } static Symbol *intern(Context &ctx, std::string_view name) { return intern(ctx, name, name); } u64 get_addr(Context &ctx) const { if (SectionFragment *frag = get_frag()) { assert(frag->is_alive); return frag->get_addr(ctx) + value; } if (has_copyrel) { return copyrel_readonly ? ctx.dynbss_relro->shdr.sh_addr + value : ctx.dynbss->shdr.sh_addr + value; } if (has_plt(ctx) && esym().st_type == STT_GNU_IFUNC) return get_plt_addr(ctx); if (input_section) { if (input_section->is_ehframe) return ctx.eh_frame->get_addr(ctx, *this); if (!input_section->is_alive) { // The control can reach here if there's a relocation that refers // a local symbol belonging to a comdat group section. This is a // violation of the spec, as all relocations should use only global // symbols of comdat members. However, .eh_frame tends to have such // relocations. return 0; } return input_section->get_addr() + value; } if (has_plt(ctx)) return get_plt_addr(ctx); return value; } u64 get_got_addr(Context &ctx) const { return ctx.got->shdr.sh_addr + get_got_idx(ctx) * E::got_size; } u64 get_gotplt_addr(Context &ctx) const { assert(get_gotplt_idx(ctx) != -1); return ctx.gotplt->shdr.sh_addr + get_gotplt_idx(ctx) * E::got_size; } u64 get_gottp_addr(Context &ctx) const { assert(get_gottp_idx(ctx) != -1); return ctx.got->shdr.sh_addr + get_gottp_idx(ctx) * E::got_size; } u64 get_tlsgd_addr(Context &ctx) const { assert(get_tlsgd_idx(ctx) != -1); return ctx.got->shdr.sh_addr + get_tlsgd_idx(ctx) * E::got_size; } u64 get_tlsdesc_addr(Context &ctx) const { assert(get_tlsdesc_idx(ctx) != -1); return ctx.got->shdr.sh_addr + get_tlsdesc_idx(ctx) * E::got_size; } u64 get_plt_addr(Context &ctx) const { if (i32 idx = get_plt_idx(ctx); idx != -1) return ctx.plt->shdr.sh_addr + idx * E::plt_size; return ctx.pltgot->shdr.sh_addr + get_pltgot_idx(ctx) * E::pltgot_size; } void set_got_idx(Context &ctx, i32 idx) const { assert(aux_idx != -1); assert(ctx.symbol_aux[aux_idx].got_idx < 0); ctx.symbol_aux[aux_idx].got_idx = idx; } void set_gotplt_idx(Context &ctx, i32 idx) const { assert(aux_idx != -1); assert(ctx.symbol_aux[aux_idx].gotplt_idx < 0); ctx.symbol_aux[aux_idx].gotplt_idx = idx; } void set_gottp_idx(Context &ctx, i32 idx) const { assert(aux_idx != -1); assert(ctx.symbol_aux[aux_idx].gottp_idx < 0); ctx.symbol_aux[aux_idx].gottp_idx = idx; } void set_tlsgd_idx(Context &ctx, i32 idx) const { assert(aux_idx != -1); assert(ctx.symbol_aux[aux_idx].tlsgd_idx < 0); ctx.symbol_aux[aux_idx].tlsgd_idx = idx; } void set_tlsdesc_idx(Context &ctx, i32 idx) const { assert(aux_idx != -1); assert(ctx.symbol_aux[aux_idx].tlsdesc_idx < 0); ctx.symbol_aux[aux_idx].tlsdesc_idx = idx; } void set_plt_idx(Context &ctx, i32 idx) const { assert(aux_idx != -1); assert(ctx.symbol_aux[aux_idx].plt_idx < 0); ctx.symbol_aux[aux_idx].plt_idx = idx; } void set_pltgot_idx(Context &ctx, i32 idx) const { assert(aux_idx != -1); assert(ctx.symbol_aux[aux_idx].pltgot_idx < 0); ctx.symbol_aux[aux_idx].pltgot_idx = idx; } void set_dynsym_idx(Context &ctx, i32 idx) const { assert(aux_idx != -1); assert(ctx.symbol_aux[aux_idx].dynsym_idx < 0); ctx.symbol_aux[aux_idx].dynsym_idx = idx; } i32 get_got_idx(Context &ctx) const { return (aux_idx == -1) ? -1 : ctx.symbol_aux[aux_idx].got_idx; } i32 get_gotplt_idx(Context &ctx) const { return (aux_idx == -1) ? -1 : ctx.symbol_aux[aux_idx].gotplt_idx; } i32 get_gottp_idx(Context &ctx) const { return (aux_idx == -1) ? -1 : ctx.symbol_aux[aux_idx].gottp_idx; } i32 get_tlsgd_idx(Context &ctx) const { return (aux_idx == -1) ? -1 : ctx.symbol_aux[aux_idx].tlsgd_idx; } i32 get_tlsdesc_idx(Context &ctx) const { return (aux_idx == -1) ? -1 : ctx.symbol_aux[aux_idx].tlsdesc_idx; } i32 get_plt_idx(Context &ctx) const { return (aux_idx == -1) ? -1 : ctx.symbol_aux[aux_idx].plt_idx; } i32 get_pltgot_idx(Context &ctx) const { return (aux_idx == -1) ? -1 : ctx.symbol_aux[aux_idx].pltgot_idx; } i32 get_dynsym_idx(Context &ctx) const { return (aux_idx == -1) ? -1 : ctx.symbol_aux[aux_idx].dynsym_idx; } bool has_plt(Context &ctx) const { return get_plt_idx(ctx) != -1 || get_pltgot_idx(ctx) != -1; } bool has_got(Context &ctx) const { return get_got_idx(ctx) != -1; } bool is_alive() const { if (SectionFragment *frag = get_frag()) return frag->is_alive; if (input_section) return input_section->is_alive; return true; } bool is_absolute(Context &ctx) const { if (file == ctx.internal_obj) return false; if (file->is_dso) return esym().is_abs(); if (is_imported) return false; if (get_frag()) return false; return input_section == nullptr; } bool is_relative(Context &ctx) const { return !is_absolute(ctx); } bool is_undef() const { return esym().is_undef() && esym().st_bind != STB_WEAK; } bool is_undef_weak() const { return esym().is_undef() && esym().st_bind == STB_WEAK; } u32 get_type() const { if (esym().st_type == STT_GNU_IFUNC && file->is_dso) return STT_FUNC; return esym().st_type; } std::string_view get_version() const { if (file->is_dso) return ((SharedFile *)file)->version_strings[ver_idx]; return ""; } std::string_view get_demangled_name() const; void clear() { Symbol null; memcpy((char *)this, &null, sizeof(null)); } const ElfSym &esym() const { if (file->is_dso) return *((SharedFile *)file)->elf_syms[sym_idx]; return ((ObjectFile *)file)->elf_syms[sym_idx]; } SectionFragment *get_frag() const { if (!file || file->is_dso) return nullptr; return ((ObjectFile *)file)->sym_fragments[sym_idx].frag; } void set_name(std::string_view str) { nameptr = str.data(); namelen = str.size(); } std::string_view get_name() const { return {nameptr, (size_t)namelen}; } InputFile *file = nullptr; InputSection *input_section = nullptr; const char *nameptr = nullptr; u64 value = -1; i32 sym_idx = -1; i32 namelen = 0; i32 aux_idx = -1; u16 shndx = 0; u16 ver_idx = 0; std::atomic_uint8_t flags = 0; tbb::spin_mutex mu; std::atomic_uint8_t visibility = STV_DEFAULT; u8 is_lazy : 1 = false; u8 is_weak : 1 = false; u8 write_to_symtab : 1 = false; u8 traced : 1 = false; u8 has_copyrel : 1 = false; u8 copyrel_readonly : 1 = false; u8 is_imported : 1 = false; u8 is_exported : 1 = false; }; // // Error output // inline thread_local bool opt_demangle = false; template class SyncOut { public: SyncOut(Context &ctx, std::ostream &out = std::cout) : out(out) { opt_demangle = ctx.arg.demangle; } ~SyncOut() { std::lock_guard lock(mu); out << ss.str() << "\n"; } template SyncOut &operator<<(T &&val) { ss << std::forward(val); return *this; } static inline std::mutex mu; private: std::ostream &out; std::stringstream ss; }; template class Fatal { public: Fatal(Context &ctx) : out(ctx, std::cerr) {} [[noreturn]] ~Fatal() { out.~SyncOut(); cleanup(); _exit(1); } template Fatal &operator<<(T &&val) { out << std::forward(val); return *this; } private: SyncOut out; }; template class Error { public: Error(Context &ctx) : out(ctx, std::cerr) { ctx.has_error = true; } template Error &operator<<(T &&val) { out << std::forward(val); return *this; } static void checkpoint(Context &ctx) { if (!ctx.has_error) return; cleanup(); _exit(1); } private: SyncOut out; }; template class Warn { public: Warn(Context &ctx) : out(ctx, std::cerr) { if (ctx.arg.fatal_warnings) ctx.has_error = true; } template Warn &operator<<(T &&val) { out << std::forward(val); return *this; } private: SyncOut out; }; #define unreachable(ctx) \ do { \ Fatal(ctx) << "internal error at " << __FILE__ << ":" << __LINE__; \ } while (0) template std::ostream &operator<<(std::ostream &out, const InputFile &file); // // Inline objects and functions // template inline std::ostream & operator<<(std::ostream &out, const InputSection &isec) { out << isec.file << ":(" << isec.name << ")"; return out; } inline u64 align_to(u64 val, u64 align) { if (align == 0) return val; assert(__builtin_popcount(align) == 1); return (val + align - 1) & ~(align - 1); } inline u64 next_power_of_two(u64 val) { assert(val >> 63 == 0); if (val == 0 || val == 1) return 1; return (u64)1 << (64 - __builtin_clzl(val - 1)); } template inline u64 SectionFragment::get_addr(Context &ctx) const { return output_section.shdr.sh_addr + offset; } template inline u64 InputSection::get_addr() const { return output_section->shdr.sh_addr + offset; } template inline i64 InputSection::get_priority() const { return ((i64)file.priority << 32) | section_idx; } template <> inline i64 InputSection::get_addend(const ElfRel &rel) const { return rel.r_addend; } template <> inline i64 InputSection::get_addend(const ElfRel &rel) const { u8 *buf = (u8 *)contents.data(); return *(i32 *)(buf + rel.r_offset); } template inline std::span> InputSection::get_rels(Context &ctx) const { if (relsec_idx == -1) return {}; return file.template get_data>(ctx, file.elf_sections[relsec_idx]); } template inline i64 ObjectFile::get_shndx(const ElfSym &esym) { assert(&elf_syms[0] <= &esym); assert(&esym < &elf_syms[elf_syms.size()]); if (esym.st_shndx == SHN_XINDEX) return symtab_shndx_sec[&esym - &elf_syms[0]]; return esym.st_shndx; } template inline InputSection *ObjectFile::get_section(const ElfSym &esym) { return sections[get_shndx(esym)]; } template std::span *> ObjectFile::get_global_syms() { return std::span *>(this->symbols).subspan(first_global); } inline u32 elf_hash(std::string_view name) { u32 h = 0; for (u8 c : name) { h = (h << 4) + c; u32 g = h & 0xf0000000; if (g != 0) h ^= g >> 24; h &= ~g; } return h; } inline u32 djb_hash(std::string_view name) { u32 h = 5381; for (u8 c : name) h = (h << 5) + h + c; return h; } inline void write_string(u8 *buf, std::string_view str) { memcpy(buf, str.data(), str.size()); buf[str.size()] = '\0'; } template inline void write_vector(u8 *buf, const std::vector &vec) { memcpy(buf, vec.data(), vec.size() * sizeof(T)); } template inline void append(std::vector &vec1, std::vector vec2) { vec1.insert(vec1.end(), vec2.begin(), vec2.end()); } template inline std::vector flatten(std::vector> &vec) { std::vector ret; for (std::vector &v : vec) append(ret, v); return ret; } template inline void erase(std::vector &vec, U pred) { vec.erase(std::remove_if(vec.begin(), vec.end(), pred), vec.end()); } template inline void sort(T &vec, U less) { std::stable_sort(vec.begin(), vec.end(), less); }