roc-lang/roc
1
1
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2024-09-17 13:47:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Enable rebuilding hosts into a dynamic executable

Browse Source
This commit is contained in:
Brendan Hansknecht 2021-09-14 21:59:15 -07:00
parent 7297e969bd
commit e96291e9a7
5 changed files with 260 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -3728,8 +3728,10 @@ dependencies = [
"object 0.26.2",
"roc_build",
"roc_collections",
"roc_mono",
"serde",
"target-lexicon",
"tempfile",
]
[[package]]

8
cli/src/build.rs
View File

@ -284,13 +284,19 @@ fn spawn_rebuild_thread(
let rebuild_host_start = SystemTime::now();
if surgically_link {
roc_linker::build_and_preprocess_host(
opt_level,
&thread_local_target,
host_input_path.as_path(),
exported_symbols,
)
.unwrap();
} else {
rebuild_host(opt_level, &thread_local_target, host_input_path.as_path());
rebuild_host(
opt_level,
&thread_local_target,
host_input_path.as_path(),
None,
);
}
let rebuild_host_end = rebuild_host_start.elapsed().unwrap();
rebuild_host_end.as_millis()

293
compiler/build/src/link.rs
View File

@ -84,30 +84,35 @@ pub fn build_zig_host_native(
zig_str_path: &str,
target: &str,
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output {
let mut command = Command::new("zig");
command
.env_clear()
.env("PATH", env_path)
.env("HOME", env_home)
.args(&[
"build-obj",
zig_host_src,
emit_bin,
"--pkg-begin",
"str",
zig_str_path,
"--pkg-end",
// include the zig runtime
"-fcompiler-rt",
// include libc
"--library",
"c",
"-fPIC",
// cross-compile?
"-target",
target,
]);
.env("HOME", env_home);
if let Some(shared_lib_path) = shared_lib_path {
command.args(&["build-exe", shared_lib_path.to_str().unwrap()]);
} else {
command.arg("build-obj");
}
command.args(&[
zig_host_src,
emit_bin,
"--pkg-begin",
"str",
zig_str_path,
"--pkg-end",
// include the zig runtime
"-fcompiler-rt",
// include libc
"--library",
"c",
"-fPIC",
// cross-compile?
"-target",
target,
]);
if matches!(opt_level, OptLevel::Optimize) {
command.args(&["-O", "ReleaseSafe"]);
}
@ -123,6 +128,7 @@ pub fn build_zig_host_native(
zig_str_path: &str,
_target: &str,
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output {
use serde_json::Value;
@ -168,25 +174,29 @@ pub fn build_zig_host_native(
command
.env_clear()
.env("PATH", &env_path)
.env("HOME", &env_home)
.args(&[
"build-obj",
zig_host_src,
emit_bin,
"--pkg-begin",
"str",
zig_str_path,
"--pkg-end",
// include the zig runtime
"--pkg-begin",
"compiler_rt",
zig_compiler_rt_path.to_str().unwrap(),
"--pkg-end",
// include libc
"--library",
"c",
"-fPIC",
]);
.env("HOME", &env_home);
if let Some(shared_lib_path) = shared_lib_path {
command.args(&["build-exe", shared_lib_path.to_str().unwrap()]);
} else {
command.arg("build-obj");
}
command.args(&[
zig_host_src,
emit_bin,
"--pkg-begin",
"str",
zig_str_path,
"--pkg-end",
// include the zig runtime
"--pkg-begin",
"compiler_rt",
zig_compiler_rt_path.to_str().unwrap(),
"--pkg-end",
// include libc
"--library",
"c",
"-fPIC",
]);
if matches!(opt_level, OptLevel::Optimize) {
command.args(&["-O", "ReleaseSafe"]);
}
@ -200,7 +210,11 @@ pub fn build_zig_host_wasm32(
zig_host_src: &str,
zig_str_path: &str,
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output {
if shared_lib_path.is_some() {
unimplemented!("Linking a shared library to wasm not yet implemented");
}
// NOTE currently just to get compiler warnings if the host code is invalid.
// the produced artifact is not used
//
@ -239,14 +253,56 @@ pub fn build_zig_host_wasm32(
command.output().unwrap()
}
pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path) {
pub fn build_c_host_native(
env_path: &str,
env_home: &str,
dest: &str,
sources: &[&str],
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output {
let mut command = Command::new("clang");
command
.env_clear()
.env("PATH", &env_path)
.env("HOME", &env_home)
.args(sources)
.args(&["-fPIC", "-o", dest]);
if let Some(shared_lib_path) = shared_lib_path {
command.args(&[
shared_lib_path.to_str().unwrap(),
"-lm",
"-lpthread",
"-ldl",
"-lrt",
"-lutil",
]);
} else {
command.arg("-c");
}
if matches!(opt_level, OptLevel::Optimize) {
command.arg("-O2");
}
command.output().unwrap()
}
pub fn rebuild_host(
opt_level: OptLevel,
target: &Triple,
host_input_path: &Path,
shared_lib_path: Option<&Path>,
) {
let c_host_src = host_input_path.with_file_name("host.c");
let c_host_dest = host_input_path.with_file_name("c_host.o");
let zig_host_src = host_input_path.with_file_name("host.zig");
let rust_host_src = host_input_path.with_file_name("host.rs");
let rust_host_dest = host_input_path.with_file_name("rust_host.o");
let cargo_host_src = host_input_path.with_file_name("Cargo.toml");
let host_dest_native = host_input_path.with_file_name("host.o");
let host_dest_native = host_input_path.with_file_name(if shared_lib_path.is_some() {
"dynhost"
} else {
"host.o"
});
let host_dest_wasm = host_input_path.with_file_name("host.bc");
let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string());
@ -273,6 +329,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
zig_host_src.to_str().unwrap(),
zig_str_path.to_str().unwrap(),
opt_level,
shared_lib_path,
)
}
Architecture::X86_64 => {
@ -285,6 +342,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
zig_str_path.to_str().unwrap(),
"native",
opt_level,
shared_lib_path,
)
}
Architecture::X86_32(_) => {
@ -297,31 +355,14 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
zig_str_path.to_str().unwrap(),
"i386-linux-musl",
opt_level,
shared_lib_path,
)
}
_ => panic!("Unsupported architecture {:?}", target.architecture),
};
validate_output("host.zig", "zig", output)
} else {
// Compile host.c
let mut command = Command::new("clang");
command.env_clear().env("PATH", &env_path).args(&[
"-fPIC",
"-c",
c_host_src.to_str().unwrap(),
"-o",
c_host_dest.to_str().unwrap(),
]);
if matches!(opt_level, OptLevel::Optimize) {
command.arg("-O2");
}
let output = command.output().unwrap();
validate_output("host.c", "clang", output);
}
if cargo_host_src.exists() {
} else if cargo_host_src.exists() {
// Compile and link Cargo.toml, if it exists
let cargo_dir = host_input_path.parent().unwrap();
let libhost_dir =
@ -332,6 +373,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
} else {
"debug"
});
let libhost = libhost_dir.join("libhost.a");
let mut command = Command::new("cargo");
command.arg("build").current_dir(cargo_dir);
@ -342,22 +384,54 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
validate_output("src/lib.rs", "cargo build", output);
let output = Command::new("ld")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-r",
"-L",
libhost_dir.to_str().unwrap(),
c_host_dest.to_str().unwrap(),
"-lhost",
"-o",
// Cargo hosts depend on a c wrapper for the api. Compile host.c as well.
if shared_lib_path.is_some() {
// If compiling to executable, let c deal with linking as well.
let output = build_c_host_native(
&env_path,
&env_home,
host_dest_native.to_str().unwrap(),
])
.output()
.unwrap();
&[c_host_src.to_str().unwrap(), libhost.to_str().unwrap()],
opt_level,
shared_lib_path,
);
validate_output("host.c", "clang", output);
} else {
let output = build_c_host_native(
&env_path,
&env_home,
c_host_dest.to_str().unwrap(),
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
);
validate_output("host.c", "clang", output);
validate_output("c_host.o", "ld", output);
let output = Command::new("ld")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-r",
"-L",
libhost_dir.to_str().unwrap(),
c_host_dest.to_str().unwrap(),
"-lhost",
"-o",
host_dest_native.to_str().unwrap(),
])
.output()
.unwrap();
validate_output("c_host.o", "ld", output);
// Clean up c_host.o
let output = Command::new("rm")
.env_clear()
.args(&["-f", c_host_dest.to_str().unwrap()])
.output()
.unwrap();
validate_output("rust_host.o", "rm", output);
}
} else if rust_host_src.exists() {
// Compile and link host.rs, if it exists
let mut command = Command::new("rustc");
@ -373,22 +447,49 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
validate_output("host.rs", "rustc", output);
let output = Command::new("ld")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-r",
c_host_dest.to_str().unwrap(),
rust_host_dest.to_str().unwrap(),
"-o",
// Rust hosts depend on a c wrapper for the api. Compile host.c as well.
if shared_lib_path.is_some() {
// If compiling to executable, let c deal with linking as well.
let output = build_c_host_native(
&env_path,
&env_home,
host_dest_native.to_str().unwrap(),
])
.output()
.unwrap();
&[
c_host_src.to_str().unwrap(),
rust_host_dest.to_str().unwrap(),
],
opt_level,
shared_lib_path,
);
validate_output("host.c", "clang", output);
} else {
let output = build_c_host_native(
&env_path,
&env_home,
c_host_dest.to_str().unwrap(),
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
);
validate_output("rust_host.o", "ld", output);
validate_output("host.c", "clang", output);
let output = Command::new("ld")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-r",
c_host_dest.to_str().unwrap(),
rust_host_dest.to_str().unwrap(),
"-o",
host_dest_native.to_str().unwrap(),
])
.output()
.unwrap();
// Clean up rust_host.o
validate_output("rust_host.o", "ld", output);
}
// Clean up rust_host.o and c_host.o
let output = Command::new("rm")
.env_clear()
.args(&[
@ -400,15 +501,17 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
.unwrap();
validate_output("rust_host.o", "rm", output);
} else if c_host_dest.exists() {
// Clean up c_host.o
let output = Command::new("mv")
.env_clear()
.args(&[c_host_dest, host_dest_native])
.output()
.unwrap();
validate_output("c_host.o", "mv", output);
} else if c_host_src.exists() {
// Compile host.c, if it exists
let output = build_c_host_native(
&env_path,
&env_home,
host_dest_native.to_str().unwrap(),
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
);
validate_output("host.c", "clang", output);
}
}

2
linker/Cargo.toml
View File

@ -18,6 +18,7 @@ test = false
bench = false
[dependencies]
roc_mono = { path = "../compiler/mono" }
roc_build = { path = "../compiler/build", default-features = false }
roc_collections = { path = "../compiler/collections" }
bumpalo = { version = "3.6", features = ["collections"] }
@ -29,3 +30,4 @@ object = { version = "0.26", features = ["read"] }
serde = { version = "1.0", features = ["derive"] }
bincode = "1.3"
target-lexicon = "0.12.2"
tempfile = "3.1.0"

67
linker/src/lib.rs
View File

@ -8,8 +8,9 @@ use object::{
Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex,
Symbol, SymbolIndex, SymbolSection,
};
use roc_build::link::LinkType;
use roc_build::link::{rebuild_host, LinkType};
use roc_collections::all::MutMap;
use roc_mono::ir::OptLevel;
use std::cmp::Ordering;
use std::convert::TryFrom;
use std::ffi::CStr;
@ -21,6 +22,7 @@ use std::os::raw::c_char;
use std::path::Path;
use std::time::{Duration, SystemTime};
use target_lexicon::Triple;
use tempfile::{Builder, NamedTempFile};
mod metadata;
@ -132,27 +134,47 @@ pub fn supported(link_type: &LinkType, target: &Triple) -> bool {
}
pub fn build_and_preprocess_host(
opt_level: OptLevel,
target: &Triple,
host_input_path: &Path,
exposed_to_host: Vec<String>,
) -> io::Result<()> {
let lib = generate_dynamic_lib(exposed_to_host)?;
rebuild_host(opt_level, target, host_input_path, Some(&lib.path()));
Ok(())
}
fn generate_dynamic_lib(exposed_to_host: Vec<String>) -> io::Result<()> {
Ok(())
fn generate_dynamic_lib(exposed_to_host: Vec<String>) -> io::Result<NamedTempFile> {
for sym in exposed_to_host {
println!("{}", sym);
}
let dummy_lib_file = Builder::new().prefix("roc_lib").suffix(".so").tempfile()?;
Ok(dummy_lib_file)
}
pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
preprocess_impl(
&matches.value_of(EXEC).unwrap(),
&matches.value_of(METADATA).unwrap(),
&matches.value_of(OUT).unwrap(),
&matches.value_of(SHARED_LIB).unwrap(),
matches.is_present(FLAG_VERBOSE),
matches.is_present(FLAG_TIME),
)
}
// TODO: Most of this file is a mess of giant functions just to check if things work.
// Clean it all up and refactor nicely.
pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
let verbose = matches.is_present(FLAG_VERBOSE);
let time = matches.is_present(FLAG_TIME);
fn preprocess_impl(
exec_filename: &str,
metadata_filename: &str,
out_filename: &str,
shared_lib_filename: &str,
verbose: bool,
time: bool,
) -> io::Result<i32> {
let total_start = SystemTime::now();
let exec_parsing_start = total_start;
let exec_file = fs::File::open(&matches.value_of(EXEC).unwrap())?;
let exec_file = fs::File::open(exec_filename)?;
let exec_mmap = unsafe { Mmap::map(&exec_file)? };
let exec_data = &*exec_mmap;
let exec_obj = match object::File::parse(exec_data) {
@ -489,7 +511,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
}
};
let shared_lib_name = Path::new(matches.value_of(SHARED_LIB).unwrap())
let shared_lib_name = Path::new(shared_lib_filename)
.file_name()
.unwrap()
.to_str()
@ -623,7 +645,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
.write(true)
.create(true)
.truncate(true)
.open(&matches.value_of(OUT).unwrap())?;
.open(out_filename)?;
out_file.set_len(md.exec_len)?;
let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? };
@ -884,7 +906,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
}
let saving_metadata_start = SystemTime::now();
let output = fs::File::create(&matches.value_of(METADATA).unwrap())?;
let output = fs::File::create(metadata_filename)?;
let output = BufWriter::new(output);
if let Err(err) = serialize_into(output, &md) {
println!("Failed to serialize metadata: {}", err);
@ -929,12 +951,25 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
}
pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
let verbose = matches.is_present(FLAG_VERBOSE);
let time = matches.is_present(FLAG_TIME);
surgery_impl(
&matches.value_of(APP).unwrap(),
&matches.value_of(METADATA).unwrap(),
&matches.value_of(OUT).unwrap(),
matches.is_present(FLAG_VERBOSE),
matches.is_present(FLAG_TIME),
)
}
fn surgery_impl(
app_filename: &str,
metadata_filename: &str,
out_filename: &str,
verbose: bool,
time: bool,
) -> io::Result<i32> {
let total_start = SystemTime::now();
let loading_metadata_start = total_start;
let input = fs::File::open(&matches.value_of(METADATA).unwrap())?;
let input = fs::File::open(metadata_filename)?;
let input = BufReader::new(input);
let md: metadata::Metadata = match deserialize_from(input) {
Ok(data) => data,
@ -946,7 +981,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
let loading_metadata_duration = loading_metadata_start.elapsed().unwrap();
let app_parsing_start = SystemTime::now();
let app_file = fs::File::open(&matches.value_of(APP).unwrap())?;
let app_file = fs::File::open(app_filename)?;
let app_mmap = unsafe { Mmap::map(&app_file)? };
let app_data = &*app_mmap;
let app_obj = match object::File::parse(app_data) {
@ -962,7 +997,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
let exec_file = fs::OpenOptions::new()
.read(true)
.write(true)
.open(&matches.value_of(OUT).unwrap())?;
.open(out_filename)?;
let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint;
exec_file.set_len(max_out_len)?;