roc-lang/roc
1
1
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2024-09-19 06:40:20 +03:00
Code Issues Packages Projects Releases Wiki Activity

Enable rebuilding hosts into a dynamic executable

Browse Source
This commit is contained in:
Brendan Hansknecht 2021-09-14 21:59:15 -07:00
parent 7297e969bd
commit e96291e9a7
5 changed files with 260 additions and 112 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -3728,8 +3728,10 @@ dependencies = [
"object 0.26.2", "object 0.26.2",
"roc_build", "roc_build",
"roc_collections", "roc_collections",
"roc_mono",
"serde", "serde",
"target-lexicon", "target-lexicon",
"tempfile",
] ]
[[package]] [[package]]

8
cli/src/build.rs
View File

@ -284,13 +284,19 @@ fn spawn_rebuild_thread(
let rebuild_host_start = SystemTime::now(); let rebuild_host_start = SystemTime::now();
if surgically_link { if surgically_link {
roc_linker::build_and_preprocess_host( roc_linker::build_and_preprocess_host(
opt_level,
&thread_local_target, &thread_local_target,
host_input_path.as_path(), host_input_path.as_path(),
exported_symbols, exported_symbols,
) )
.unwrap(); .unwrap();
} else { } else {
rebuild_host(opt_level, &thread_local_target, host_input_path.as_path()); rebuild_host(
opt_level,
&thread_local_target,
host_input_path.as_path(),
None,
);
} }
let rebuild_host_end = rebuild_host_start.elapsed().unwrap(); let rebuild_host_end = rebuild_host_start.elapsed().unwrap();
rebuild_host_end.as_millis() rebuild_host_end.as_millis()

293
compiler/build/src/link.rs
View File

@ -84,30 +84,35 @@ pub fn build_zig_host_native(
zig_str_path: &str, zig_str_path: &str,
target: &str, target: &str,
opt_level: OptLevel, opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output { ) -> Output {
let mut command = Command::new("zig"); let mut command = Command::new("zig");
command command
.env_clear() .env_clear()
.env("PATH", env_path) .env("PATH", env_path)
.env("HOME", env_home) .env("HOME", env_home);
.args(&[ if let Some(shared_lib_path) = shared_lib_path {
"build-obj", command.args(&["build-exe", shared_lib_path.to_str().unwrap()]);
zig_host_src, } else {
emit_bin, command.arg("build-obj");
"--pkg-begin", }
"str", command.args(&[
zig_str_path, zig_host_src,
"--pkg-end", emit_bin,
// include the zig runtime "--pkg-begin",
"-fcompiler-rt", "str",
// include libc zig_str_path,
"--library", "--pkg-end",
"c", // include the zig runtime
"-fPIC", "-fcompiler-rt",
// cross-compile? // include libc
"-target", "--library",
target, "c",
]); "-fPIC",
// cross-compile?
"-target",
target,
]);
if matches!(opt_level, OptLevel::Optimize) { if matches!(opt_level, OptLevel::Optimize) {
command.args(&["-O", "ReleaseSafe"]); command.args(&["-O", "ReleaseSafe"]);
} }
@ -123,6 +128,7 @@ pub fn build_zig_host_native(
zig_str_path: &str, zig_str_path: &str,
_target: &str, _target: &str,
opt_level: OptLevel, opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output { ) -> Output {
use serde_json::Value; use serde_json::Value;
@ -168,25 +174,29 @@ pub fn build_zig_host_native(
command command
.env_clear() .env_clear()
.env("PATH", &env_path) .env("PATH", &env_path)
.env("HOME", &env_home) .env("HOME", &env_home);
.args(&[ if let Some(shared_lib_path) = shared_lib_path {
"build-obj", command.args(&["build-exe", shared_lib_path.to_str().unwrap()]);
zig_host_src, } else {
emit_bin, command.arg("build-obj");
"--pkg-begin", }
"str", command.args(&[
zig_str_path, zig_host_src,
"--pkg-end", emit_bin,
// include the zig runtime "--pkg-begin",
"--pkg-begin", "str",
"compiler_rt", zig_str_path,
zig_compiler_rt_path.to_str().unwrap(), "--pkg-end",
"--pkg-end", // include the zig runtime
// include libc "--pkg-begin",
"--library", "compiler_rt",
"c", zig_compiler_rt_path.to_str().unwrap(),
"-fPIC", "--pkg-end",
]); // include libc
"--library",
"c",
"-fPIC",
]);
if matches!(opt_level, OptLevel::Optimize) { if matches!(opt_level, OptLevel::Optimize) {
command.args(&["-O", "ReleaseSafe"]); command.args(&["-O", "ReleaseSafe"]);
} }
@ -200,7 +210,11 @@ pub fn build_zig_host_wasm32(
zig_host_src: &str, zig_host_src: &str,
zig_str_path: &str, zig_str_path: &str,
opt_level: OptLevel, opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output { ) -> Output {
if shared_lib_path.is_some() {
unimplemented!("Linking a shared library to wasm not yet implemented");
}
// NOTE currently just to get compiler warnings if the host code is invalid. // NOTE currently just to get compiler warnings if the host code is invalid.
// the produced artifact is not used // the produced artifact is not used
// //
@ -239,14 +253,56 @@ pub fn build_zig_host_wasm32(
command.output().unwrap() command.output().unwrap()
} }
pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path) { pub fn build_c_host_native(
env_path: &str,
env_home: &str,
dest: &str,
sources: &[&str],
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output {
let mut command = Command::new("clang");
command
.env_clear()
.env("PATH", &env_path)
.env("HOME", &env_home)
.args(sources)
.args(&["-fPIC", "-o", dest]);
if let Some(shared_lib_path) = shared_lib_path {
command.args(&[
shared_lib_path.to_str().unwrap(),
"-lm",
"-lpthread",
"-ldl",
"-lrt",
"-lutil",
]);
} else {
command.arg("-c");
}
if matches!(opt_level, OptLevel::Optimize) {
command.arg("-O2");
}
command.output().unwrap()
}
pub fn rebuild_host(
opt_level: OptLevel,
target: &Triple,
host_input_path: &Path,
shared_lib_path: Option<&Path>,
) {
let c_host_src = host_input_path.with_file_name("host.c"); let c_host_src = host_input_path.with_file_name("host.c");
let c_host_dest = host_input_path.with_file_name("c_host.o"); let c_host_dest = host_input_path.with_file_name("c_host.o");
let zig_host_src = host_input_path.with_file_name("host.zig"); let zig_host_src = host_input_path.with_file_name("host.zig");
let rust_host_src = host_input_path.with_file_name("host.rs"); let rust_host_src = host_input_path.with_file_name("host.rs");
let rust_host_dest = host_input_path.with_file_name("rust_host.o"); let rust_host_dest = host_input_path.with_file_name("rust_host.o");
let cargo_host_src = host_input_path.with_file_name("Cargo.toml"); let cargo_host_src = host_input_path.with_file_name("Cargo.toml");
let host_dest_native = host_input_path.with_file_name("host.o"); let host_dest_native = host_input_path.with_file_name(if shared_lib_path.is_some() {
"dynhost"
} else {
"host.o"
});
let host_dest_wasm = host_input_path.with_file_name("host.bc"); let host_dest_wasm = host_input_path.with_file_name("host.bc");
let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string()); let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string());
@ -273,6 +329,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
zig_host_src.to_str().unwrap(), zig_host_src.to_str().unwrap(),
zig_str_path.to_str().unwrap(), zig_str_path.to_str().unwrap(),
opt_level, opt_level,
shared_lib_path,
) )
} }
Architecture::X86_64 => { Architecture::X86_64 => {
@ -285,6 +342,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
zig_str_path.to_str().unwrap(), zig_str_path.to_str().unwrap(),
"native", "native",
opt_level, opt_level,
shared_lib_path,
) )
} }
Architecture::X86_32(_) => { Architecture::X86_32(_) => {
@ -297,31 +355,14 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
zig_str_path.to_str().unwrap(), zig_str_path.to_str().unwrap(),
"i386-linux-musl", "i386-linux-musl",
opt_level, opt_level,
shared_lib_path,
) )
} }
_ => panic!("Unsupported architecture {:?}", target.architecture), _ => panic!("Unsupported architecture {:?}", target.architecture),
}; };
validate_output("host.zig", "zig", output) validate_output("host.zig", "zig", output)
} else { } else if cargo_host_src.exists() {
// Compile host.c
let mut command = Command::new("clang");
command.env_clear().env("PATH", &env_path).args(&[
"-fPIC",
"-c",
c_host_src.to_str().unwrap(),
"-o",
c_host_dest.to_str().unwrap(),
]);
if matches!(opt_level, OptLevel::Optimize) {
command.arg("-O2");
}
let output = command.output().unwrap();
validate_output("host.c", "clang", output);
}
if cargo_host_src.exists() {
// Compile and link Cargo.toml, if it exists // Compile and link Cargo.toml, if it exists
let cargo_dir = host_input_path.parent().unwrap(); let cargo_dir = host_input_path.parent().unwrap();
let libhost_dir = let libhost_dir =
@ -332,6 +373,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
} else { } else {
"debug" "debug"
}); });
let libhost = libhost_dir.join("libhost.a");
let mut command = Command::new("cargo"); let mut command = Command::new("cargo");
command.arg("build").current_dir(cargo_dir); command.arg("build").current_dir(cargo_dir);
@ -342,22 +384,54 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
validate_output("src/lib.rs", "cargo build", output); validate_output("src/lib.rs", "cargo build", output);
let output = Command::new("ld") // Cargo hosts depend on a c wrapper for the api. Compile host.c as well.
.env_clear() if shared_lib_path.is_some() {
.env("PATH", &env_path) // If compiling to executable, let c deal with linking as well.
.args(&[ let output = build_c_host_native(
"-r", &env_path,
"-L", &env_home,
libhost_dir.to_str().unwrap(),
c_host_dest.to_str().unwrap(),
"-lhost",
"-o",
host_dest_native.to_str().unwrap(), host_dest_native.to_str().unwrap(),
]) &[c_host_src.to_str().unwrap(), libhost.to_str().unwrap()],
.output() opt_level,
.unwrap(); shared_lib_path,
);
validate_output("host.c", "clang", output);
} else {
let output = build_c_host_native(
&env_path,
&env_home,
c_host_dest.to_str().unwrap(),
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
);
validate_output("host.c", "clang", output);
validate_output("c_host.o", "ld", output); let output = Command::new("ld")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-r",
"-L",
libhost_dir.to_str().unwrap(),
c_host_dest.to_str().unwrap(),
"-lhost",
"-o",
host_dest_native.to_str().unwrap(),
])
.output()
.unwrap();
validate_output("c_host.o", "ld", output);
// Clean up c_host.o
let output = Command::new("rm")
.env_clear()
.args(&["-f", c_host_dest.to_str().unwrap()])
.output()
.unwrap();
validate_output("rust_host.o", "rm", output);
}
} else if rust_host_src.exists() { } else if rust_host_src.exists() {
// Compile and link host.rs, if it exists // Compile and link host.rs, if it exists
let mut command = Command::new("rustc"); let mut command = Command::new("rustc");
@ -373,22 +447,49 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
validate_output("host.rs", "rustc", output); validate_output("host.rs", "rustc", output);
let output = Command::new("ld") // Rust hosts depend on a c wrapper for the api. Compile host.c as well.
.env_clear() if shared_lib_path.is_some() {
.env("PATH", &env_path) // If compiling to executable, let c deal with linking as well.
.args(&[ let output = build_c_host_native(
"-r", &env_path,
c_host_dest.to_str().unwrap(), &env_home,
rust_host_dest.to_str().unwrap(),
"-o",
host_dest_native.to_str().unwrap(), host_dest_native.to_str().unwrap(),
]) &[
.output() c_host_src.to_str().unwrap(),
.unwrap(); rust_host_dest.to_str().unwrap(),
],
opt_level,
shared_lib_path,
);
validate_output("host.c", "clang", output);
} else {
let output = build_c_host_native(
&env_path,
&env_home,
c_host_dest.to_str().unwrap(),
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
);
validate_output("rust_host.o", "ld", output); validate_output("host.c", "clang", output);
let output = Command::new("ld")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-r",
c_host_dest.to_str().unwrap(),
rust_host_dest.to_str().unwrap(),
"-o",
host_dest_native.to_str().unwrap(),
])
.output()
.unwrap();
// Clean up rust_host.o validate_output("rust_host.o", "ld", output);
}
// Clean up rust_host.o and c_host.o
let output = Command::new("rm") let output = Command::new("rm")
.env_clear() .env_clear()
.args(&[ .args(&[
@ -400,15 +501,17 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
.unwrap(); .unwrap();
validate_output("rust_host.o", "rm", output); validate_output("rust_host.o", "rm", output);
} else if c_host_dest.exists() { } else if c_host_src.exists() {
// Clean up c_host.o // Compile host.c, if it exists
let output = Command::new("mv") let output = build_c_host_native(
.env_clear() &env_path,
.args(&[c_host_dest, host_dest_native]) &env_home,
.output() host_dest_native.to_str().unwrap(),
.unwrap(); &[c_host_src.to_str().unwrap()],
opt_level,
validate_output("c_host.o", "mv", output); shared_lib_path,
);
validate_output("host.c", "clang", output);
} }
} }

2
linker/Cargo.toml
View File

@ -18,6 +18,7 @@ test = false
bench = false bench = false
[dependencies] [dependencies]
roc_mono = { path = "../compiler/mono" }
roc_build = { path = "../compiler/build", default-features = false } roc_build = { path = "../compiler/build", default-features = false }
roc_collections = { path = "../compiler/collections" } roc_collections = { path = "../compiler/collections" }
bumpalo = { version = "3.6", features = ["collections"] } bumpalo = { version = "3.6", features = ["collections"] }
@ -29,3 +30,4 @@ object = { version = "0.26", features = ["read"] }
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
bincode = "1.3" bincode = "1.3"
target-lexicon = "0.12.2" target-lexicon = "0.12.2"
tempfile = "3.1.0"

67
linker/src/lib.rs
View File

@ -8,8 +8,9 @@ use object::{
Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex, Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex,
Symbol, SymbolIndex, SymbolSection, Symbol, SymbolIndex, SymbolSection,
}; };
use roc_build::link::LinkType; use roc_build::link::{rebuild_host, LinkType};
use roc_collections::all::MutMap; use roc_collections::all::MutMap;
use roc_mono::ir::OptLevel;
use std::cmp::Ordering; use std::cmp::Ordering;
use std::convert::TryFrom; use std::convert::TryFrom;
use std::ffi::CStr; use std::ffi::CStr;
@ -21,6 +22,7 @@ use std::os::raw::c_char;
use std::path::Path; use std::path::Path;
use std::time::{Duration, SystemTime}; use std::time::{Duration, SystemTime};
use target_lexicon::Triple; use target_lexicon::Triple;
use tempfile::{Builder, NamedTempFile};
mod metadata; mod metadata;
@ -132,27 +134,47 @@ pub fn supported(link_type: &LinkType, target: &Triple) -> bool {
} }
pub fn build_and_preprocess_host( pub fn build_and_preprocess_host(
opt_level: OptLevel,
target: &Triple, target: &Triple,
host_input_path: &Path, host_input_path: &Path,
exposed_to_host: Vec<String>, exposed_to_host: Vec<String>,
) -> io::Result<()> { ) -> io::Result<()> {
let lib = generate_dynamic_lib(exposed_to_host)?; let lib = generate_dynamic_lib(exposed_to_host)?;
rebuild_host(opt_level, target, host_input_path, Some(&lib.path()));
Ok(()) Ok(())
} }
fn generate_dynamic_lib(exposed_to_host: Vec<String>) -> io::Result<()> { fn generate_dynamic_lib(exposed_to_host: Vec<String>) -> io::Result<NamedTempFile> {
Ok(()) for sym in exposed_to_host {
println!("{}", sym);
}
let dummy_lib_file = Builder::new().prefix("roc_lib").suffix(".so").tempfile()?;
Ok(dummy_lib_file)
} }
pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
preprocess_impl(
&matches.value_of(EXEC).unwrap(),
&matches.value_of(METADATA).unwrap(),
&matches.value_of(OUT).unwrap(),
&matches.value_of(SHARED_LIB).unwrap(),
matches.is_present(FLAG_VERBOSE),
matches.is_present(FLAG_TIME),
)
}
// TODO: Most of this file is a mess of giant functions just to check if things work. // TODO: Most of this file is a mess of giant functions just to check if things work.
// Clean it all up and refactor nicely. // Clean it all up and refactor nicely.
pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> { fn preprocess_impl(
let verbose = matches.is_present(FLAG_VERBOSE); exec_filename: &str,
let time = matches.is_present(FLAG_TIME); metadata_filename: &str,
out_filename: &str,
shared_lib_filename: &str,
verbose: bool,
time: bool,
) -> io::Result<i32> {
let total_start = SystemTime::now(); let total_start = SystemTime::now();
let exec_parsing_start = total_start; let exec_parsing_start = total_start;
let exec_file = fs::File::open(&matches.value_of(EXEC).unwrap())?; let exec_file = fs::File::open(exec_filename)?;
let exec_mmap = unsafe { Mmap::map(&exec_file)? }; let exec_mmap = unsafe { Mmap::map(&exec_file)? };
let exec_data = &*exec_mmap; let exec_data = &*exec_mmap;
let exec_obj = match object::File::parse(exec_data) { let exec_obj = match object::File::parse(exec_data) {
@ -489,7 +511,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
} }
}; };
let shared_lib_name = Path::new(matches.value_of(SHARED_LIB).unwrap()) let shared_lib_name = Path::new(shared_lib_filename)
.file_name() .file_name()
.unwrap() .unwrap()
.to_str() .to_str()
@ -623,7 +645,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
.write(true) .write(true)
.create(true) .create(true)
.truncate(true) .truncate(true)
.open(&matches.value_of(OUT).unwrap())?; .open(out_filename)?;
out_file.set_len(md.exec_len)?; out_file.set_len(md.exec_len)?;
let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? }; let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? };
@ -884,7 +906,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
} }
let saving_metadata_start = SystemTime::now(); let saving_metadata_start = SystemTime::now();
let output = fs::File::create(&matches.value_of(METADATA).unwrap())?; let output = fs::File::create(metadata_filename)?;
let output = BufWriter::new(output); let output = BufWriter::new(output);
if let Err(err) = serialize_into(output, &md) { if let Err(err) = serialize_into(output, &md) {
println!("Failed to serialize metadata: {}", err); println!("Failed to serialize metadata: {}", err);
@ -929,12 +951,25 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
} }
pub fn surgery(matches: &ArgMatches) -> io::Result<i32> { pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
let verbose = matches.is_present(FLAG_VERBOSE); surgery_impl(
let time = matches.is_present(FLAG_TIME); &matches.value_of(APP).unwrap(),
&matches.value_of(METADATA).unwrap(),
&matches.value_of(OUT).unwrap(),
matches.is_present(FLAG_VERBOSE),
matches.is_present(FLAG_TIME),
)
}
fn surgery_impl(
app_filename: &str,
metadata_filename: &str,
out_filename: &str,
verbose: bool,
time: bool,
) -> io::Result<i32> {
let total_start = SystemTime::now(); let total_start = SystemTime::now();
let loading_metadata_start = total_start; let loading_metadata_start = total_start;
let input = fs::File::open(&matches.value_of(METADATA).unwrap())?; let input = fs::File::open(metadata_filename)?;
let input = BufReader::new(input); let input = BufReader::new(input);
let md: metadata::Metadata = match deserialize_from(input) { let md: metadata::Metadata = match deserialize_from(input) {
Ok(data) => data, Ok(data) => data,
@ -946,7 +981,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
let loading_metadata_duration = loading_metadata_start.elapsed().unwrap(); let loading_metadata_duration = loading_metadata_start.elapsed().unwrap();
let app_parsing_start = SystemTime::now(); let app_parsing_start = SystemTime::now();
let app_file = fs::File::open(&matches.value_of(APP).unwrap())?; let app_file = fs::File::open(app_filename)?;
let app_mmap = unsafe { Mmap::map(&app_file)? }; let app_mmap = unsafe { Mmap::map(&app_file)? };
let app_data = &*app_mmap; let app_data = &*app_mmap;
let app_obj = match object::File::parse(app_data) { let app_obj = match object::File::parse(app_data) {
@ -962,7 +997,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
let exec_file = fs::OpenOptions::new() let exec_file = fs::OpenOptions::new()
.read(true) .read(true)
.write(true) .write(true)
.open(&matches.value_of(OUT).unwrap())?; .open(out_filename)?;
let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint; let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint;
exec_file.set_len(max_out_len)?; exec_file.set_len(max_out_len)?;