rui314/mold
1
1
Fork 0
mirror of https://github.com/rui314/mold.git synced 2024-09-11 21:17:28 +03:00
Code Issues Projects Releases Wiki Activity

[ELF] Reimplement --repro

Browse Source 
`--repro` is a hidden command flag for debugging.

Previously, if the flag was given, mold would create a `.repro`
section in an output file with all input files as its contents.
The flaw of the design is that when mold fails to create an output
file, no .repro section would be created.

So I changed the behavior in this commit. Now, the tar file is
created as an independent file.
This commit is contained in:
Rui Ueyama 2022-02-01 13:47:37 +09:00
parent a380c1675f
commit e98aab7ea9
8 changed files with 114 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
elf/cmdline.cc
View File

@ -259,36 +259,6 @@ bool read_z_arg(Context<E> &ctx, std::span<std::string_view> &args,
return false;
}
template <typename E>
std::string create_response_file(Context<E> &ctx) {
std::string buf;
std::stringstream out;
std::string cwd = std::filesystem::current_path();
out << "-C " << cwd.substr(1) << "\n";
if (cwd != "/") {
out << "--chroot ..";
i64 depth = std::count(cwd.begin(), cwd.end(), '/');
for (i64 i = 1; i < depth; i++)
out << "/..";
out << "\n";
}
for (i64 i = 1; i < ctx.cmdline_args.size(); i++) {
std::string_view arg = ctx.cmdline_args[i];
if (arg == "-repro" || arg == "--repro") {
i++;
continue;
}
out << arg << "\n";
}
return out.str();
}
template <typename E>
static i64 parse_hex(Context<E> &ctx, std::string opt, std::string_view value) {
if (!value.starts_with("0x") && !value.starts_with("0X"))
@ -964,8 +934,6 @@ void parse_nonpositional_args(Context<E> &ctx,
std::string_view &arg, \
std::string name); \
\
template std::string create_response_file(Context<E> &ctx); \
\
template \
void parse_nonpositional_args(Context<E> &ctx, \
std::vector<std::string_view> &remaining)

4
elf/main.cc
View File

@ -524,6 +524,10 @@ static int elf_main(int argc, char **argv) {
// Beyond this point, no new symbols will be added to the result.
// Handle -repro
if (ctx.arg.repro)
write_repro_file(ctx);
// Make sure that all symbols have been resolved.
if (!ctx.arg.allow_multiple_definition)
check_duplicate_symbols(ctx);

20
elf/mold.h
View File

@ -870,21 +870,6 @@ private:
std::unique_ptr<ZlibCompressor> contents;
};
template <typename E>
class ReproSection : public Chunk<E> {
public:
ReproSection() : Chunk<E>(this->SYNTHETIC) {
this->name = ".repro";
this->shdr.sh_type = SHT_PROGBITS;
}
void update_shdr(Context<E> &ctx) override;
void copy_buf(Context<E> &ctx) override;
private:
std::unique_ptr<GzipCompressor> contents;
};
bool is_c_identifier(std::string_view name);
template <typename E>
@ -1211,9 +1196,6 @@ bool read_arg(Context<E> &ctx, std::span<std::string_view> &args,
std::string_view &arg,
std::string name);
template <typename E>
std::string create_response_file(Context<E> &ctx);
template <typename E>
void parse_nonpositional_args(Context<E> &ctx,
std::vector<std::string_view> &remaining);
@ -1232,6 +1214,7 @@ template <typename E> void compute_merged_section_sizes(Context<E> &);
template <typename E> void bin_sections(Context<E> &);
template <typename E> ObjectFile<E> *create_internal_file(Context<E> &);
template <typename E> void check_cet_errors(Context<E> &);
template <typename E> void write_repro_file(Context<E> &);
template <typename E> void check_duplicate_symbols(Context<E> &);
template <typename E> void sort_init_fini(Context<E> &);
template <typename E> std::vector<Chunk<E> *>
@ -1535,7 +1518,6 @@ struct Context {
std::unique_ptr<VerdefSection<E>> verdef;
std::unique_ptr<BuildIdSection<E>> buildid;
std::unique_ptr<NotePropertySection<E>> note_property;
std::unique_ptr<ReproSection<E>> repro;
// For --relocatable
std::vector<RChunk<E> *> r_chunks;

28
elf/output-chunks.cc
View File

@ -2022,33 +2022,6 @@ void GnuCompressedSection<E>::copy_buf(Context<E> &ctx) {
contents->write_to(base + 12);
}
template <typename E>
void ReproSection<E>::update_shdr(Context<E> &ctx) {
if (contents)
return;
TarFile tar("repro");
tar.append("response.txt", save_string(ctx, create_response_file(ctx)));
tar.append("version.txt", save_string(ctx, mold_version + "\n"));
std::unordered_set<std::string> seen;
for (std::unique_ptr<MappedFile<Context<E>>> &mf : ctx.mf_pool) {
std::string path = to_abs_path(mf->name);
if (seen.insert(path).second)
tar.append(path, mf->get_contents());
}
std::vector<u8> buf(tar.size());
tar.write_to(&buf[0]);
contents.reset(new GzipCompressor({(char *)&buf[0], buf.size()}));
this->shdr.sh_size = contents->size();
}
template <typename E>
void ReproSection<E>::copy_buf(Context<E> &ctx) {
contents->write_to(ctx.buf + this->shdr.sh_offset);
}
#define INSTANTIATE(E) \
template class Chunk<E>; \
template class OutputEhdr<E>; \
@ -2082,7 +2055,6 @@ void ReproSection<E>::copy_buf(Context<E> &ctx) {
template class NotePropertySection<E>; \
template class GabiCompressedSection<E>; \
template class GnuCompressedSection<E>; \
template class ReproSection<E>; \
template i64 BuildId::size(Context<E> &) const; \
template bool is_relro(Context<E> &, Chunk<E> *); \
template bool separate_page(Context<E> &, Chunk<E> *, Chunk<E> *); \

49
elf/passes.cc
View File

@ -73,9 +73,6 @@ void create_synthetic_sections(Context<E> &ctx) {
add(ctx.versym = std::make_unique<VersymSection<E>>());
add(ctx.verneed = std::make_unique<VerneedSection<E>>());
add(ctx.note_property = std::make_unique<NotePropertySection<E>>());
if (ctx.arg.repro)
add(ctx.repro = std::make_unique<ReproSection<E>>());
}
template <typename E>
@ -396,6 +393,51 @@ void check_cet_errors(Context<E> &ctx) {
}
}
template <typename E>
static std::string create_response_file(Context<E> &ctx) {
std::string buf;
std::stringstream out;
std::string cwd = std::filesystem::current_path();
out << "-C " << cwd.substr(1) << "\n";
if (cwd != "/") {
out << "--chroot ..";
i64 depth = std::count(cwd.begin(), cwd.end(), '/');
for (i64 i = 1; i < depth; i++)
out << "/..";
out << "\n";
}
for (std::string_view arg : std::span(ctx.cmdline_args).subspan(1))
if (arg != "-repro" && arg != "--repro")
out << arg << "\n";
return out.str();
}
template <typename E>
void write_repro_file(Context<E> &ctx) {
std::string path = ctx.arg.output + ".repro.tar";
std::unique_ptr<TarWriter> tar =
TarWriter::open(path, filepath(ctx.arg.output).filename().string() + ".repro");
if (!tar)
Fatal(ctx) << "cannot open " << path << ": " << errno_string();
tar->append("response.txt", save_string(ctx, create_response_file(ctx)));
tar->append("version.txt", save_string(ctx, mold_version + "\n"));
std::unordered_set<std::string> seen;
for (std::unique_ptr<MappedFile<Context<E>>> &mf : ctx.mf_pool) {
if (!mf->parent) {
std::string path = to_abs_path(mf->name);
if (seen.insert(path).second)
tar->append(path, mf->get_contents());
}
}
}
template <typename E>
void check_duplicate_symbols(Context<E> &ctx) {
Timer t(ctx, "check_duplicate_symbols");
@ -1113,6 +1155,7 @@ void compress_debug_sections(Context<E> &ctx) {
template void bin_sections(Context<E> &); \
template ObjectFile<E> *create_internal_file(Context<E> &); \
template void check_cet_errors(Context<E> &); \
template void write_repro_file(Context<E> &); \
template void check_duplicate_symbols(Context<E> &); \
template void sort_init_fini(Context<E> &); \
template std::vector<Chunk<E> *> collect_output_sections(Context<E> &); \

22
mold.h
View File

@ -582,24 +582,24 @@ private:
// TarFile is a class to create a tar file.
//
// If you pass `--reproduce=repro.tar` to mold, mold collects all
// input files and put them into `repro.tar`, so that it is easy to
// If you pass `--repro` to mold, mold collects all input files and
// put them into `<output-file-path>.repro.tar`, so that it is easy to
// run the same command with the same command line arguments.
class TarFile {
class TarWriter {
public:
TarFile(std::string basedir) : basedir(basedir) {}
static std::unique_ptr<TarWriter>
open(std::string output_path, std::string basedir);
~TarWriter();
void append(std::string path, std::string_view data);
void write_to(u8 *buf);
i64 size() const { return size_; }
private:
static constexpr i64 BLOCK_SIZE = 512;
std::string encode_path(std::string path);
TarWriter(FILE *out, std::string basedir) : out(out), basedir(basedir) {}
FILE *out = nullptr;
std::string basedir;
std::vector<std::pair<std::string, std::string_view>> contents;
i64 size_ = BLOCK_SIZE * 2;
};
//
@ -635,8 +635,6 @@ MappedFile<C> *MappedFile<C>::open(C &ctx, std::string path) {
MappedFile *mf = new MappedFile;
mf->name = path;
ctx.mf_pool.push_back(std::unique_ptr<MappedFile>(mf));
if (path.starts_with('/') && !ctx.arg.chroot.empty())
path = ctx.arg.chroot + "/" + path_clean(path);
@ -644,6 +642,8 @@ MappedFile<C> *MappedFile<C>::open(C &ctx, std::string path) {
if (fd == -1)
return nullptr;
ctx.mf_pool.push_back(std::unique_ptr<MappedFile>(mf));
struct stat st;
if (fstat(fd, &st) == -1)
Fatal(ctx) << path << ": fstat failed: " << errno_string();

79
tar.cc
View File

@ -11,7 +11,11 @@ namespace mold {
//
// For simplicity, we always emit a PAX header even for a short filename.
struct UstarHeader {
void flush() {
UstarHeader() {
memset(this, 0, sizeof(*this));
}
void finalize() {
memset(checksum, ' ', sizeof(checksum));
memcpy(magic, "ustar", 5);
memcpy(version, "00", 2);
@ -43,7 +47,7 @@ struct UstarHeader {
char pad[12];
};
std::string TarFile::encode_path(std::string path) {
static std::string encode_path(std::string basedir, std::string path) {
path = path_clean(basedir + "/" + path);
// Construct a string which contains something like
@ -55,50 +59,51 @@ std::string TarFile::encode_path(std::string path) {
return std::to_string(total) + " path=" + path + "\n";
}
void TarFile::append(std::string path, std::string_view data) {
contents.push_back({path, data});
size_ += BLOCK_SIZE * 2;
size_ += align_to(encode_path(path).size(), BLOCK_SIZE);
size_ += align_to(data.size(), BLOCK_SIZE);
std::unique_ptr<TarWriter>
TarWriter::open(std::string output_path, std::string basedir) {
FILE *out = fopen(output_path.c_str(), "w");
if (!out)
return nullptr;
return std::unique_ptr<TarWriter>(new TarWriter(out, basedir));
}
void TarFile::write_to(u8 *buf) {
u8 *start = buf;
memset(buf, 0, size_);
TarWriter::~TarWriter() {
fclose(out);
}
for (i64 i = 0; i < contents.size(); i++) {
assert(buf - start <= size_);
void TarWriter::append(std::string path, std::string_view data) {
// Write PAX header
static_assert(sizeof(UstarHeader) == BLOCK_SIZE);
UstarHeader pax;
const std::string &path = contents[i].first;
std::string_view data = contents[i].second;
std::string attr = encode_path(basedir, path);
sprintf(pax.size, "%011zo", attr.size());
pax.typeflag[0] = 'x';
pax.finalize();
fwrite(&pax, sizeof(pax), 1, out);
// Write PAX header
static_assert(sizeof(UstarHeader) == BLOCK_SIZE);
UstarHeader &pax = *(UstarHeader *)buf;
buf += BLOCK_SIZE;
// Write pathname
fwrite(attr.data(), attr.size(), 1, out);
fseek(out, align_to(ftell(out), BLOCK_SIZE), SEEK_SET);
std::string attr = encode_path(path);
sprintf(pax.size, "%011zo", attr.size());
pax.typeflag[0] = 'x';
pax.flush();
// Write Ustar header
UstarHeader ustar;
memcpy(ustar.mode, "0000664", 8);
sprintf(ustar.size, "%011zo", data.size());
ustar.finalize();
fwrite(&ustar, sizeof(ustar), 1, out);
// Write pathname
memcpy(buf, attr.data(), attr.size());
buf += align_to(attr.size(), BLOCK_SIZE);
// Write file contents
fwrite(data.data(), data.size(), 1, out);
fseek(out, align_to(ftell(out), BLOCK_SIZE), SEEK_SET);
// Write Ustar header
UstarHeader &ustar = *(UstarHeader *)buf;
buf += BLOCK_SIZE;
// A tar file must ends with two empty blocks, so write such
// terminator and seek back.
u8 terminator[BLOCK_SIZE * 2] = {};
fwrite(&terminator, BLOCK_SIZE * 2, 1, out);
fseek(out, -BLOCK_SIZE * 2, SEEK_END);
memcpy(ustar.mode, "0000664", 8);
sprintf(ustar.size, "%011zo", data.size());
ustar.flush();
// Write file contents
memcpy(buf, data.data(), data.size());
buf += align_to(data.size(), BLOCK_SIZE);
}
assert(ftell(out) % BLOCK_SIZE == 0);
}
} // namespace mold

21
test/elf/repro.sh
View File

@ -19,23 +19,22 @@ int main() {
}
EOF
$CC -B. -o $t/exe $t/a.o
! readelf --sections $t/exe | fgrep -q .repro || false
rm -rf $t/exe.repro $t/exe.repro.tar
$CC -B. -o $t/exe $t/a.o
! [ -f $t/exe.repro.tar ] || false
$CC -B. -o $t/exe $t/a.o -Wl,-repro
objcopy --dump-section .repro=$t/repro.tar.gz $t/exe
tar -C $t -xzf $t/repro.tar.gz
fgrep -q /a.o $t/repro/response.txt
fgrep -q mold $t/repro/version.txt
tar -C $t -xf $t/exe.repro.tar
fgrep -q /a.o $t/exe.repro/response.txt
fgrep -q mold $t/exe.repro/version.txt
rm -rf $t/exe.repro $t/exe.repro.tar
MOLD_REPRO=1 $CC -B. -o $t/exe $t/a.o
objcopy --dump-section .repro=$t/repro.tar.gz $t/exe
tar -C $t -xzf $t/repro.tar.gz
fgrep -q /a.o $t/repro/response.txt
fgrep -q mold $t/repro/version.txt
tar -C $t -xf $t/exe.repro.tar
fgrep -q /a.o $t/exe.repro/response.txt
fgrep -q mold $t/exe.repro/version.txt
echo OK