roc-lang/roc
1
1
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2024-11-13 09:49:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/trunk' into hello-web

Browse Source
This commit is contained in:
Folkert 2021-09-25 14:59:27 +02:00
commit 45d3438b3d
17 changed files with 728 additions and 266 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
Cargo.lock generated
View File

@ -2517,7 +2517,9 @@ version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2"
dependencies = [
"crc32fast",
"flate2",
"indexmap",
"memchr",
]
@ -3485,6 +3487,7 @@ dependencies = [
"roc_editor",
"roc_fmt",
"roc_gen_llvm",
"roc_linker",
"roc_load",
"roc_module",
"roc_mono",
@ -3727,8 +3730,12 @@ dependencies = [
"iced-x86",
"memmap2 0.3.1",
"object 0.26.2",
"roc_build",
"roc_collections",
"roc_mono",
"serde",
"target-lexicon",
"tempfile",
]
[[package]]

1
cli/Cargo.toml
View File

@ -57,6 +57,7 @@ roc_build = { path = "../compiler/build", default-features = false }
roc_fmt = { path = "../compiler/fmt" }
roc_reporting = { path = "../compiler/reporting" }
roc_editor = { path = "../editor", optional = true }
roc_linker = { path = "../linker" }
# TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it
clap = { git = "https://github.com/rtfeldman/clap", branch = "master" }
const_format = "0.2"

154
cli/src/build.rs
View File

@ -53,6 +53,8 @@ pub fn build_file<'a>(
emit_debug_info: bool,
emit_timings: bool,
link_type: LinkType,
surgically_link: bool,
precompiled: bool,
) -> Result<BuiltFile, LoadingProblem<'a>> {
let compilation_start = SystemTime::now();
let ptr_bytes = target.pointer_width().unwrap().bytes() as u32;
@ -85,7 +87,39 @@ pub fn build_file<'a>(
let host_extension = if emit_wasm { "zig" } else { "o" };
let app_extension = if emit_wasm { "bc" } else { "o" };
let cwd = roc_file_path.parent().unwrap();
let mut binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows
if emit_wasm {
binary_path.set_extension("wasm");
}
let mut host_input_path = PathBuf::from(cwd);
let path_to_platform = loaded.platform_path.clone();
host_input_path.push(&*path_to_platform);
host_input_path.push("host");
host_input_path.set_extension(host_extension);
// TODO this should probably be moved before load_and_monomorphize.
// To do this we will need to preprocess files just for their exported symbols.
// Also, we should no longer need to do this once we have platforms on
// a package repository, as we can then get precompiled hosts from there.
let rebuild_thread = spawn_rebuild_thread(
opt_level,
surgically_link,
precompiled,
host_input_path.clone(),
binary_path.clone(),
target,
loaded
.exposed_to_host
.keys()
.map(|x| x.as_str(&loaded.interns).to_string())
.collect(),
);
// TODO try to move as much of this linking as possible to the precompiled
// host, to minimize the amount of host-application linking required.
let app_o_file = Builder::new()
.prefix("roc_app")
.suffix(&format!(".{}", app_extension))
@ -142,13 +176,6 @@ pub fn build_file<'a>(
program::report_problems(&mut loaded);
let loaded = loaded;
let cwd = roc_file_path.parent().unwrap();
let mut binary_path = cwd.join(&*loaded.output_path); // TODO should join ".exe" on Windows
if emit_wasm {
binary_path.set_extension("wasm");
}
let code_gen_timing = match opt_level {
OptLevel::Normal | OptLevel::Optimize => program::gen_from_mono_module_llvm(
arena,
@ -196,44 +223,45 @@ pub fn build_file<'a>(
);
}
// Step 2: link the precompiled host and compiled app
let mut host_input_path = PathBuf::from(cwd);
host_input_path.push(&*path_to_platform);
host_input_path.push("host");
host_input_path.set_extension(host_extension);
// TODO we should no longer need to do this once we have platforms on
// a package repository, as we can then get precompiled hosts from there.
let rebuild_host_start = SystemTime::now();
rebuild_host(target, host_input_path.as_path());
let rebuild_host_end = rebuild_host_start.elapsed().unwrap();
if emit_timings {
let rebuild_duration = rebuild_thread.join().unwrap();
if emit_timings && !precompiled {
println!(
"Finished rebuilding the host in {} ms\n",
rebuild_host_end.as_millis()
"Finished rebuilding and preprocessing the host in {} ms\n",
rebuild_duration
);
}
// TODO try to move as much of this linking as possible to the precompiled
// host, to minimize the amount of host-application linking required.
// Step 2: link the precompiled host and compiled app
let link_start = SystemTime::now();
let (mut child, binary_path) = // TODO use lld
link(
target,
binary_path,
&[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()],
link_type
)
.map_err(|_| {
todo!("gracefully handle `rustc` failing to spawn.");
let outcome = if surgically_link {
roc_linker::link_preprocessed_host(target, &host_input_path, app_o_file, &binary_path)
.map_err(|_| {
todo!("gracefully handle failing to surgically link");
})?;
BuildOutcome::NoProblems
} else {
let (mut child, _) = // TODO use lld
link(
target,
binary_path.clone(),
&[host_input_path.as_path().to_str().unwrap(), app_o_file.to_str().unwrap()],
link_type
)
.map_err(|_| {
todo!("gracefully handle `ld` failing to spawn.");
})?;
let exit_status = child.wait().map_err(|_| {
todo!("gracefully handle error after `ld` spawned");
})?;
let cmd_result = child.wait().map_err(|_| {
todo!("gracefully handle error after `rustc` spawned");
});
// TODO change this to report whether there were errors or warnings!
if exit_status.success() {
BuildOutcome::NoProblems
} else {
BuildOutcome::Errors
}
};
let linking_time = link_start.elapsed().unwrap();
if emit_timings {
@ -242,16 +270,6 @@ pub fn build_file<'a>(
let total_time = compilation_start.elapsed().unwrap();
// If the cmd errored out, return the Err.
let exit_status = cmd_result?;
// TODO change this to report whether there were errors or warnings!
let outcome = if exit_status.success() {
BuildOutcome::NoProblems
} else {
BuildOutcome::Errors
};
Ok(BuiltFile {
binary_path,
outcome,
@ -259,6 +277,46 @@ pub fn build_file<'a>(
})
}
fn spawn_rebuild_thread(
opt_level: OptLevel,
surgically_link: bool,
precompiled: bool,
host_input_path: PathBuf,
binary_path: PathBuf,
target: &Triple,
exported_symbols: Vec<String>,
) -> std::thread::JoinHandle<u128> {
let thread_local_target = target.clone();
std::thread::spawn(move || {
let rebuild_host_start = SystemTime::now();
if !precompiled {
if surgically_link {
roc_linker::build_and_preprocess_host(
opt_level,
&thread_local_target,
host_input_path.as_path(),
exported_symbols,
)
.unwrap();
} else {
rebuild_host(
opt_level,
&thread_local_target,
host_input_path.as_path(),
None,
);
}
}
if surgically_link {
// Copy preprocessed host to executable location.
let prehost = host_input_path.with_file_name("preprocessedhost");
std::fs::copy(prehost, binary_path.as_path()).unwrap();
}
let rebuild_host_end = rebuild_host_start.elapsed().unwrap();
rebuild_host_end.as_millis()
})
}
#[allow(clippy::too_many_arguments)]
pub fn check_file(
arena: &Bump,

36
cli/src/lib.rs
View File

@ -34,6 +34,8 @@ pub const FLAG_OPTIMIZE: &str = "optimize";
pub const FLAG_LIB: &str = "lib";
pub const FLAG_BACKEND: &str = "backend";
pub const FLAG_TIME: &str = "time";
pub const FLAG_LINK: &str = "roc-linker";
pub const FLAG_PRECOMPILED: &str = "precompiled-host";
pub const ROC_FILE: &str = "ROC_FILE";
pub const BACKEND: &str = "BACKEND";
pub const DIRECTORY_OR_FILES: &str = "DIRECTORY_OR_FILES";
@ -89,6 +91,18 @@ pub fn build_app<'a>() -> App<'a> {
.help("Prints detailed compilation time information.")
.required(false),
)
.arg(
Arg::with_name(FLAG_LINK)
.long(FLAG_LINK)
.help("Uses the roc linker instead of the system linker.")
.required(false),
)
.arg(
Arg::with_name(FLAG_PRECOMPILED)
.long(FLAG_PRECOMPILED)
.help("Assumes the host has been precompiled and skips recompiling the host.")
.required(false),
)
)
.subcommand(App::new(CMD_RUN)
.about("DEPRECATED - now use `roc [FILE]` instead of `roc run [FILE]`")
@ -177,6 +191,18 @@ pub fn build_app<'a>() -> App<'a> {
.help("Prints detailed compilation time information.")
.required(false),
)
.arg(
Arg::with_name(FLAG_LINK)
.long(FLAG_LINK)
.help("Uses the roc linker instead of the system linker.")
.required(false),
)
.arg(
Arg::with_name(FLAG_PRECOMPILED)
.long(FLAG_PRECOMPILED)
.help("Assumes the host has been precompiled and skips recompiling the host.")
.required(false),
)
.arg(
Arg::with_name(FLAG_BACKEND)
.long(FLAG_BACKEND)
@ -261,6 +287,14 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result<i32> {
} else {
LinkType::Executable
};
let surgically_link = matches.is_present(FLAG_LINK);
let precompiled = matches.is_present(FLAG_PRECOMPILED);
if surgically_link && !roc_linker::supported(&link_type, &target) {
panic!(
"Link type, {:?}, with target, {}, not supported by roc linker",
link_type, target
);
}
let path = Path::new(filename);
@ -293,6 +327,8 @@ pub fn build(matches: &ArgMatches, config: BuildConfig) -> io::Result<i32> {
emit_debug_info,
emit_timings,
link_type,
surgically_link,
precompiled,
);
match res_binary_path {

380
compiler/build/src/link.rs
View File

@ -76,6 +76,7 @@ fn find_wasi_libc_path() -> PathBuf {
}
#[cfg(not(target_os = "macos"))]
#[allow(clippy::too_many_arguments)]
pub fn build_zig_host_native(
env_path: &str,
env_home: &str,
@ -83,36 +84,44 @@ pub fn build_zig_host_native(
zig_host_src: &str,
zig_str_path: &str,
target: &str,
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output {
Command::new("zig")
let mut command = Command::new("zig");
command
.env_clear()
.env("PATH", env_path)
.env("HOME", env_home)
.args(&[
"build-obj",
zig_host_src,
emit_bin,
"--pkg-begin",
"str",
zig_str_path,
"--pkg-end",
// include the zig runtime
"-fcompiler-rt",
// include libc
"--library",
"c",
"-fPIC",
"-O",
"ReleaseSafe",
// cross-compile?
"-target",
target,
])
.output()
.unwrap()
.env("HOME", env_home);
if let Some(shared_lib_path) = shared_lib_path {
command.args(&["build-exe", "-fPIE", shared_lib_path.to_str().unwrap()]);
} else {
command.args(&["build-obj", "-fPIC"]);
}
command.args(&[
zig_host_src,
emit_bin,
"--pkg-begin",
"str",
zig_str_path,
"--pkg-end",
// include the zig runtime
"-fcompiler-rt",
// include libc
"--library",
"c",
"--strip",
// cross-compile?
"-target",
target,
]);
if matches!(opt_level, OptLevel::Optimize) {
command.args(&["-O", "ReleaseSafe"]);
}
command.output().unwrap()
}
#[cfg(target_os = "macos")]
#[allow(clippy::too_many_arguments)]
pub fn build_zig_host_native(
env_path: &str,
env_home: &str,
@ -120,6 +129,8 @@ pub fn build_zig_host_native(
zig_host_src: &str,
zig_str_path: &str,
_target: &str,
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output {
use serde_json::Value;
@ -161,32 +172,37 @@ pub fn build_zig_host_native(
zig_compiler_rt_path.push("special");
zig_compiler_rt_path.push("compiler_rt.zig");
Command::new("zig")
let mut command = Command::new("zig");
command
.env_clear()
.env("PATH", &env_path)
.env("HOME", &env_home)
.args(&[
"build-obj",
zig_host_src,
emit_bin,
"--pkg-begin",
"str",
zig_str_path,
"--pkg-end",
// include the zig runtime
"--pkg-begin",
"compiler_rt",
zig_compiler_rt_path.to_str().unwrap(),
"--pkg-end",
// include libc
"--library",
"c",
"-fPIC",
"-O",
"ReleaseSafe",
])
.output()
.unwrap()
.env("HOME", &env_home);
if let Some(shared_lib_path) = shared_lib_path {
command.args(&["build-exe", "-fPIE", shared_lib_path.to_str().unwrap()]);
} else {
command.args(&["build-obj", "-fPIC"]);
}
command.args(&[
zig_host_src,
emit_bin,
"--pkg-begin",
"str",
zig_str_path,
"--pkg-end",
// include the zig runtime
"--pkg-begin",
"compiler_rt",
zig_compiler_rt_path.to_str().unwrap(),
"--pkg-end",
// include libc
"--library",
"c",
"--strip",
]);
if matches!(opt_level, OptLevel::Optimize) {
command.args(&["-O", "ReleaseSafe"]);
}
command.output().unwrap()
}
pub fn build_zig_host_wasm32(
@ -195,7 +211,12 @@ pub fn build_zig_host_wasm32(
emit_bin: &str,
zig_host_src: &str,
zig_str_path: &str,
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output {
if shared_lib_path.is_some() {
unimplemented!("Linking a shared library to wasm not yet implemented");
}
// NOTE currently just to get compiler warnings if the host code is invalid.
// the produced artifact is not used
//
@ -204,7 +225,8 @@ pub fn build_zig_host_wasm32(
// we'd like to compile with `-target wasm32-wasi` but that is blocked on
//
// https://github.com/ziglang/zig/issues/9414
Command::new("zig")
let mut command = Command::new("zig");
command
.env_clear()
.env("PATH", env_path)
.env("HOME", env_home)
@ -226,21 +248,66 @@ pub fn build_zig_host_wasm32(
// "wasm32-wasi",
// "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll",
"-fPIC",
"-O",
"ReleaseSafe",
])
.output()
.unwrap()
"--strip",
]);
if matches!(opt_level, OptLevel::Optimize) {
command.args(&["-O", "ReleaseSafe"]);
}
command.output().unwrap()
}
pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
pub fn build_c_host_native(
env_path: &str,
env_home: &str,
dest: &str,
sources: &[&str],
opt_level: OptLevel,
shared_lib_path: Option<&Path>,
) -> Output {
let mut command = Command::new("clang");
command
.env_clear()
.env("PATH", &env_path)
.env("HOME", &env_home)
.args(sources)
.args(&["-o", dest]);
if let Some(shared_lib_path) = shared_lib_path {
command.args(&[
shared_lib_path.to_str().unwrap(),
"-fPIE",
"-pie",
"-lm",
"-lpthread",
"-ldl",
"-lrt",
"-lutil",
]);
} else {
command.args(&["-fPIC", "-c"]);
}
if matches!(opt_level, OptLevel::Optimize) {
command.arg("-O2");
}
command.output().unwrap()
}
pub fn rebuild_host(
opt_level: OptLevel,
target: &Triple,
host_input_path: &Path,
shared_lib_path: Option<&Path>,
) {
let c_host_src = host_input_path.with_file_name("host.c");
let c_host_dest = host_input_path.with_file_name("c_host.o");
let zig_host_src = host_input_path.with_file_name("host.zig");
let rust_host_src = host_input_path.with_file_name("host.rs");
let rust_host_dest = host_input_path.with_file_name("rust_host.o");
let cargo_host_src = host_input_path.with_file_name("Cargo.toml");
let host_dest_native = host_input_path.with_file_name("host.o");
let host_dest_native = host_input_path.with_file_name(if shared_lib_path.is_some() {
"dynhost"
} else {
"host.o"
});
let host_dest_wasm = host_input_path.with_file_name("host.bc");
let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string());
@ -266,6 +333,8 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
&emit_bin,
zig_host_src.to_str().unwrap(),
zig_str_path.to_str().unwrap(),
opt_level,
shared_lib_path,
)
}
Architecture::X86_64 => {
@ -277,6 +346,8 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
zig_host_src.to_str().unwrap(),
zig_str_path.to_str().unwrap(),
"native",
opt_level,
shared_lib_path,
)
}
Architecture::X86_32(_) => {
@ -288,89 +359,142 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
zig_host_src.to_str().unwrap(),
zig_str_path.to_str().unwrap(),
"i386-linux-musl",
opt_level,
shared_lib_path,
)
}
_ => panic!("Unsupported architecture {:?}", target.architecture),
};
validate_output("host.zig", "zig", output)
} else {
// Compile host.c
let output = Command::new("clang")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-O2",
"-fPIC",
"-c",
c_host_src.to_str().unwrap(),
"-o",
c_host_dest.to_str().unwrap(),
])
.output()
.unwrap();
validate_output("host.c", "clang", output);
}
if cargo_host_src.exists() {
} else if cargo_host_src.exists() {
// Compile and link Cargo.toml, if it exists
let cargo_dir = host_input_path.parent().unwrap();
let libhost_dir = cargo_dir.join("target").join("release");
let libhost_dir =
cargo_dir
.join("target")
.join(if matches!(opt_level, OptLevel::Optimize) {
"release"
} else {
"debug"
});
let libhost = libhost_dir.join("libhost.a");
let output = Command::new("cargo")
.args(&["build", "--release"])
.current_dir(cargo_dir)
.output()
.unwrap();
let mut command = Command::new("cargo");
command.arg("build").current_dir(cargo_dir);
if matches!(opt_level, OptLevel::Optimize) {
command.arg("--release");
}
let output = command.output().unwrap();
validate_output("src/lib.rs", "cargo build --release", output);
validate_output("src/lib.rs", "cargo build", output);
let output = Command::new("ld")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-r",
"-L",
libhost_dir.to_str().unwrap(),
c_host_dest.to_str().unwrap(),
"-lhost",
"-o",
// Cargo hosts depend on a c wrapper for the api. Compile host.c as well.
if shared_lib_path.is_some() {
// If compiling to executable, let c deal with linking as well.
let output = build_c_host_native(
&env_path,
&env_home,
host_dest_native.to_str().unwrap(),
])
.output()
.unwrap();
&[c_host_src.to_str().unwrap(), libhost.to_str().unwrap()],
opt_level,
shared_lib_path,
);
validate_output("host.c", "clang", output);
} else {
let output = build_c_host_native(
&env_path,
&env_home,
c_host_dest.to_str().unwrap(),
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
);
validate_output("host.c", "clang", output);
validate_output("c_host.o", "ld", output);
let output = Command::new("ld")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-r",
"-L",
libhost_dir.to_str().unwrap(),
c_host_dest.to_str().unwrap(),
"-lhost",
"-o",
host_dest_native.to_str().unwrap(),
])
.output()
.unwrap();
validate_output("c_host.o", "ld", output);
// Clean up c_host.o
let output = Command::new("rm")
.env_clear()
.args(&["-f", c_host_dest.to_str().unwrap()])
.output()
.unwrap();
validate_output("rust_host.o", "rm", output);
}
} else if rust_host_src.exists() {
// Compile and link host.rs, if it exists
let output = Command::new("rustc")
.args(&[
rust_host_src.to_str().unwrap(),
"-o",
rust_host_dest.to_str().unwrap(),
])
.output()
.unwrap();
let mut command = Command::new("rustc");
command.args(&[
rust_host_src.to_str().unwrap(),
"-o",
rust_host_dest.to_str().unwrap(),
]);
if matches!(opt_level, OptLevel::Optimize) {
command.arg("-O");
}
let output = command.output().unwrap();
validate_output("host.rs", "rustc", output);
let output = Command::new("ld")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-r",
c_host_dest.to_str().unwrap(),
rust_host_dest.to_str().unwrap(),
"-o",
// Rust hosts depend on a c wrapper for the api. Compile host.c as well.
if shared_lib_path.is_some() {
// If compiling to executable, let c deal with linking as well.
let output = build_c_host_native(
&env_path,
&env_home,
host_dest_native.to_str().unwrap(),
])
.output()
.unwrap();
&[
c_host_src.to_str().unwrap(),
rust_host_dest.to_str().unwrap(),
],
opt_level,
shared_lib_path,
);
validate_output("host.c", "clang", output);
} else {
let output = build_c_host_native(
&env_path,
&env_home,
c_host_dest.to_str().unwrap(),
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
);
validate_output("rust_host.o", "ld", output);
validate_output("host.c", "clang", output);
let output = Command::new("ld")
.env_clear()
.env("PATH", &env_path)
.args(&[
"-r",
c_host_dest.to_str().unwrap(),
rust_host_dest.to_str().unwrap(),
"-o",
host_dest_native.to_str().unwrap(),
])
.output()
.unwrap();
// Clean up rust_host.o
validate_output("rust_host.o", "ld", output);
}
// Clean up rust_host.o and c_host.o
let output = Command::new("rm")
.env_clear()
.args(&[
@ -382,15 +506,17 @@ pub fn rebuild_host(target: &Triple, host_input_path: &Path) {
.unwrap();
validate_output("rust_host.o", "rm", output);
} else if c_host_dest.exists() {
// Clean up c_host.o
let output = Command::new("mv")
.env_clear()
.args(&[c_host_dest, host_dest_native])
.output()
.unwrap();
validate_output("c_host.o", "mv", output);
} else if c_host_src.exists() {
// Compile host.c, if it exists
let output = build_c_host_native(
&env_path,
&env_home,
host_dest_native.to_str().unwrap(),
&[c_host_src.to_str().unwrap()],
opt_level,
shared_lib_path,
);
validate_output("host.c", "clang", output);
}
}

4
examples/.gitignore vendored
View File

@ -4,3 +4,7 @@ app
libhost.a
roc_app.ll
roc_app.bc
dynhost
preprocessedhost
metadata
libapp.so

11
examples/benchmarks/platform/host.zig
View File

@ -29,10 +29,13 @@ extern fn roc__mainForHost_1_Fx_caller(*const u8, [*]u8, [*]u8) void;
extern fn roc__mainForHost_1_Fx_size() i64;
extern fn roc__mainForHost_1_Fx_result_size() i64;
const Align = 2 * @alignOf(usize);
extern fn malloc(size: usize) callconv(.C) ?*align(Align) c_void;
extern fn realloc(c_ptr: [*]align(Align) u8, size: usize) callconv(.C) ?*c_void;
extern fn free(c_ptr: [*]align(Align) u8) callconv(.C) void;
extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void;
extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void;
const DEBUG: bool = false;
@ -74,6 +77,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void {
std.process.exit(0);
}
export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{
return memcpy(dst, src, size);
}
export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{
return memset(dst, value, size);
}
const Unit = extern struct {};
pub export fn main() callconv(.C) u8 {

10
examples/effect/thing/platform-dir/host.zig
View File

@ -32,6 +32,8 @@ extern fn roc__mainForHost_1_Fx_result_size() i64;
extern fn malloc(size: usize) callconv(.C) ?*c_void;
extern fn realloc(c_ptr: [*]align(@alignOf(u128)) u8, size: usize) callconv(.C) ?*c_void;
extern fn free(c_ptr: [*]align(@alignOf(u128)) u8) callconv(.C) void;
extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void;
extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void;
export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void {
return malloc(size);
@ -52,6 +54,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void {
std.process.exit(0);
}
export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{
return memcpy(dst, src, size);
}
export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{
return memset(dst, value, size);
}
const Unit = extern struct {};
pub export fn main() u8 {

2
examples/fib/.gitignore vendored
View File

@ -1,2 +1,2 @@
add
fib
fib

9
examples/hello-rust/platform/host.c
View File

@ -1,7 +1,12 @@
#include <stdio.h>
#include <string.h>
extern int rust_main();
int main() {
return rust_main();
int main() { return rust_main(); }
void *roc_memcpy(void *dest, const void *src, size_t n) {
return memcpy(dest, src, n);
}
void *roc_memset(void *str, int c, size_t n) { return memset(str, c, n); }

108
examples/hello-world/platform/host.c
View File

@ -1,81 +1,81 @@
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <errno.h>
#include <string.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
void* roc_alloc(size_t size, unsigned int alignment) {
return malloc(size);
void* roc_alloc(size_t size, unsigned int alignment) { return malloc(size); }
void* roc_realloc(void* ptr, size_t old_size, size_t new_size,
unsigned int alignment) {
return realloc(ptr, new_size);
}
void* roc_realloc(void* ptr, size_t old_size, size_t new_size, unsigned int alignment) {
return realloc(ptr, new_size);
}
void roc_dealloc(void* ptr, unsigned int alignment) {
free(ptr);
}
void roc_dealloc(void* ptr, unsigned int alignment) { free(ptr); }
void roc_panic(void* ptr, unsigned int alignment) {
char* msg = (char *)ptr;
fprintf(stderr, "Application crashed with message\n\n %s\n\nShutting down\n", msg);
exit(0);
char* msg = (char*)ptr;
fprintf(stderr,
"Application crashed with message\n\n %s\n\nShutting down\n", msg);
exit(0);
}
void* roc_memcpy(void* dest, const void* src, size_t n) {
return memcpy(dest, src, n);
}
void* roc_memset(void* str, int c, size_t n) { return memset(str, c, n); }
struct RocStr {
char* bytes;
size_t len;
char* bytes;
size_t len;
};
bool is_small_str(struct RocStr str) {
return ((ssize_t)str.len) < 0;
}
bool is_small_str(struct RocStr str) { return ((ssize_t)str.len) < 0; }
// Determine the length of the string, taking into
// account the small string optimization
size_t roc_str_len(struct RocStr str) {
char* bytes = (char*)&str;
char last_byte = bytes[sizeof(str) - 1];
char last_byte_xored = last_byte ^ 0b10000000;
size_t small_len = (size_t)(last_byte_xored);
size_t big_len = str.len;
char* bytes = (char*)&str;
char last_byte = bytes[sizeof(str) - 1];
char last_byte_xored = last_byte ^ 0b10000000;
size_t small_len = (size_t)(last_byte_xored);
size_t big_len = str.len;
// Avoid branch misprediction costs by always
// determining both small_len and big_len,
// so this compiles to a cmov instruction.
if (is_small_str(str)) {
return small_len;
} else {
return big_len;
}
// Avoid branch misprediction costs by always
// determining both small_len and big_len,
// so this compiles to a cmov instruction.
if (is_small_str(str)) {
return small_len;
} else {
return big_len;
}
}
extern struct RocStr roc__mainForHost_1_exposed();
int main() {
// Call Roc to populate call_result
struct RocStr call_result = roc__mainForHost_1_exposed();
struct RocStr str = roc__mainForHost_1_exposed();
// Determine str_len and the str_bytes pointer,
// taking into account the small string optimization.
struct RocStr str = call_result;
size_t str_len = roc_str_len(str);
char* str_bytes;
// Determine str_len and the str_bytes pointer,
// taking into account the small string optimization.
size_t str_len = roc_str_len(str);
char* str_bytes;
if (is_small_str(str)) {
str_bytes = (char*)&str;
} else {
str_bytes = str.bytes;
}
if (is_small_str(str)) {
str_bytes = (char*)&str;
} else {
str_bytes = str.bytes;
}
// Write to stdout
if (write(1, str_bytes, str_len) >= 0) {
// Writing succeeded!
return 0;
} else {
printf("Error writing to stdout: %s\n", strerror(errno));
// Write to stdout
if (write(1, str_bytes, str_len) >= 0) {
// Writing succeeded!
return 0;
} else {
printf("Error writing to stdout: %s\n", strerror(errno));
return 1;
}
return 1;
}
}

10
examples/hello-zig/platform/host.zig
View File

@ -23,6 +23,8 @@ const Align = extern struct { a: usize, b: usize };
extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void;
extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void;
extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void;
extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void;
extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void;
export fn roc_alloc(size: usize, alignment: u32) callconv(.C) ?*c_void {
_ = alignment;
@ -51,6 +53,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void {
std.process.exit(0);
}
export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{
return memcpy(dst, src, size);
}
export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{
return memset(dst, value, size);
}
const mem = std.mem;
const Allocator = mem.Allocator;

10
examples/quicksort/platform/host.zig
View File

@ -26,6 +26,8 @@ const Align = extern struct { a: usize, b: usize };
extern fn malloc(size: usize) callconv(.C) ?*align(@alignOf(Align)) c_void;
extern fn realloc(c_ptr: [*]align(@alignOf(Align)) u8, size: usize) callconv(.C) ?*c_void;
extern fn free(c_ptr: [*]align(@alignOf(Align)) u8) callconv(.C) void;
extern fn memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void;
extern fn memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void;
const DEBUG: bool = false;
@ -67,6 +69,14 @@ export fn roc_panic(c_ptr: *c_void, tag_id: u32) callconv(.C) void {
std.process.exit(0);
}
export fn roc_memcpy(dst: [*]u8, src: [*]u8, size: usize) callconv(.C) void{
return memcpy(dst, src, size);
}
export fn roc_memset(dst: [*]u8, value: i32, size: usize) callconv(.C) void{
return memset(dst, value, size);
}
// warning! the array is currently stack-allocated so don't make this too big
const NUM_NUMS = 100;

6
linker/Cargo.toml
View File

@ -18,12 +18,16 @@ test = false
bench = false
[dependencies]
roc_mono = { path = "../compiler/mono" }
roc_build = { path = "../compiler/build", default-features = false }
roc_collections = { path = "../compiler/collections" }
bumpalo = { version = "3.6", features = ["collections"] }
# TODO switch to clap 3.0.0 once it's out. Tried adding clap = "~3.0.0-beta.1" and cargo wouldn't accept it
clap = { git = "https://github.com/rtfeldman/clap", branch = "master" }
iced-x86 = "1.14"
memmap2 = "0.3"
object = { version = "0.26", features = ["read"] }
object = { version = "0.26", features = ["read", "write"] }
serde = { version = "1.0", features = ["derive"] }
bincode = "1.3"
target-lexicon = "0.12.2"
tempfile = "3.1.0"

21
linker/README.md
View File

@ -29,13 +29,16 @@ This linker is run in 2 phases: preprocessing and surigical linking.
1. Surgically update all call locations in the platform
1. Surgically update call information in the application (also dealing with other relocations for builtins)
## TODO for merging with compiler flow
## TODO (In a lightly prioritized order)
1. Add new compiler flag to hide this all behind.
1. Get compiler to generate dummy shared libraries with Roc exported symbols defined.
1. Modify host linking to generate dynamic executable that links against the dummy lib.
1. Call the preprocessor on the dynamic executable host.
1. Call the surgical linker on the emitted roc object file and the preprocessed host.
1. Enjoy!
1. Extract preprocessing generation to run earlier, maybe in parallel with the main compiler until we have full precompiled hosts.
1. Maybe add a roc command to generate the dummy lib to be used by platform authors.
- Run CLI tests and/or benchmarks with the Roc Linker.
- Test with an executable completely generated by Cargo (It will hopefully work out of the box like zig).
- Add Macho support
- Honestly should be almost exactly the same code.
This means we likely need to do a lot of refactoring to minimize the duplicate code.
The fun of almost but not quite the same.
- Add PE support
- As a prereq, we need roc building on Windows (I'm not sure it does currently).
- Definitely a solid bit different than elf, but hopefully after refactoring for Macho, won't be that crazy to add.
- Look at enabling completely in memory linking that could be used with `roc run` and/or `roc repl`
- Add a feature to the compiler to make this linker optional.

218
linker/src/lib.rs
View File

@ -2,13 +2,16 @@ use bincode::{deserialize_from, serialize_into};
use clap::{App, AppSettings, Arg, ArgMatches};
use iced_x86::{Decoder, DecoderOptions, Instruction, OpCodeOperandKind, OpKind};
use memmap2::{Mmap, MmapMut};
use object::write;
use object::{elf, endian};
use object::{
Architecture, BinaryFormat, CompressedFileRange, CompressionFormat, LittleEndian, NativeEndian,
Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex,
Symbol, SymbolIndex, SymbolSection,
Architecture, BinaryFormat, CompressedFileRange, CompressionFormat, Endianness, LittleEndian,
NativeEndian, Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section,
SectionIndex, Symbol, SymbolFlags, SymbolIndex, SymbolKind, SymbolScope, SymbolSection,
};
use roc_build::link::{rebuild_host, LinkType};
use roc_collections::all::MutMap;
use roc_mono::ir::OptLevel;
use std::cmp::Ordering;
use std::convert::TryFrom;
use std::ffi::CStr;
@ -17,8 +20,12 @@ use std::io;
use std::io::{BufReader, BufWriter};
use std::mem;
use std::os::raw::c_char;
use std::os::unix::fs::PermissionsExt;
use std::path::Path;
use std::process::Command;
use std::time::{Duration, SystemTime};
use target_lexicon::Triple;
use tempfile::Builder;
mod metadata;
@ -122,15 +129,149 @@ pub fn build_app<'a>() -> App<'a> {
)
}
pub fn supported(link_type: &LinkType, target: &Triple) -> bool {
link_type == &LinkType::Executable
&& target.architecture == target_lexicon::Architecture::X86_64
&& target.operating_system == target_lexicon::OperatingSystem::Linux
&& target.binary_format == target_lexicon::BinaryFormat::Elf
}
pub fn build_and_preprocess_host(
opt_level: OptLevel,
target: &Triple,
host_input_path: &Path,
exposed_to_host: Vec<String>,
) -> io::Result<()> {
let dummy_lib = host_input_path.with_file_name("libapp.so");
generate_dynamic_lib(target, exposed_to_host, &dummy_lib)?;
rebuild_host(opt_level, target, host_input_path, Some(&dummy_lib));
let dynhost = host_input_path.with_file_name("dynhost");
let metadata = host_input_path.with_file_name("metadata");
let prehost = host_input_path.with_file_name("preprocessedhost");
if preprocess_impl(
dynhost.to_str().unwrap(),
metadata.to_str().unwrap(),
prehost.to_str().unwrap(),
dummy_lib.to_str().unwrap(),
false,
false,
)? != 0
{
panic!("Failed to preprocess host");
}
Ok(())
}
pub fn link_preprocessed_host(
_target: &Triple,
host_input_path: &Path,
roc_app_obj: &Path,
binary_path: &Path,
) -> io::Result<()> {
let metadata = host_input_path.with_file_name("metadata");
if surgery_impl(
roc_app_obj.to_str().unwrap(),
metadata.to_str().unwrap(),
binary_path.to_str().unwrap(),
false,
false,
)? != 0
{
panic!("Failed to surgically link host");
}
Ok(())
}
fn generate_dynamic_lib(
_target: &Triple,
exposed_to_host: Vec<String>,
dummy_lib_path: &Path,
) -> io::Result<()> {
let dummy_obj_file = Builder::new().prefix("roc_lib").suffix(".o").tempfile()?;
let dummy_obj_file = dummy_obj_file.path();
// TODO deal with other architectures here.
let mut out_object =
write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little);
let text_section = out_object.section_id(write::StandardSection::Text);
for sym in exposed_to_host {
// TODO properly generate this list.
for name in &[
format!("roc__{}_1_exposed", sym),
format!("roc__{}_1_Fx_caller", sym),
format!("roc__{}_1_Fx_size", sym),
format!("roc__{}_1_Fx_result_size", sym),
format!("roc__{}_size", sym),
] {
out_object.add_symbol(write::Symbol {
name: name.as_bytes().to_vec(),
value: 0,
size: 0,
kind: SymbolKind::Text,
scope: SymbolScope::Dynamic,
weak: false,
section: write::SymbolSection::Section(text_section),
flags: SymbolFlags::None,
});
}
}
std::fs::write(
&dummy_obj_file,
out_object.write().expect("failed to build output object"),
)
.expect("failed to write object to file");
let output = Command::new("ld")
.args(&[
"-shared",
"-soname",
dummy_lib_path.file_name().unwrap().to_str().unwrap(),
dummy_obj_file.to_str().unwrap(),
"-o",
dummy_lib_path.to_str().unwrap(),
])
.output()
.unwrap();
if !output.status.success() {
match std::str::from_utf8(&output.stderr) {
Ok(stderr) => panic!(
"Failed to link dummy shared library - stderr of the `ld` command was:\n{}",
stderr
),
Err(utf8_err) => panic!(
"Failed to link dummy shared library - stderr of the `ld` command was invalid utf8 ({:?})",
utf8_err
),
}
}
Ok(())
}
pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
preprocess_impl(
matches.value_of(EXEC).unwrap(),
matches.value_of(METADATA).unwrap(),
matches.value_of(OUT).unwrap(),
matches.value_of(SHARED_LIB).unwrap(),
matches.is_present(FLAG_VERBOSE),
matches.is_present(FLAG_TIME),
)
}
// TODO: Most of this file is a mess of giant functions just to check if things work.
// Clean it all up and refactor nicely.
pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
let verbose = matches.is_present(FLAG_VERBOSE);
let time = matches.is_present(FLAG_TIME);
fn preprocess_impl(
exec_filename: &str,
metadata_filename: &str,
out_filename: &str,
shared_lib_filename: &str,
verbose: bool,
time: bool,
) -> io::Result<i32> {
let total_start = SystemTime::now();
let exec_parsing_start = total_start;
let exec_file = fs::File::open(&matches.value_of(EXEC).unwrap())?;
let exec_file = fs::File::open(exec_filename)?;
let exec_mmap = unsafe { Mmap::map(&exec_file)? };
let exec_data = &*exec_mmap;
let exec_obj = match object::File::parse(exec_data) {
@ -226,6 +367,9 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
println!("PLT File Offset: {:+x}", plt_offset);
}
// TODO: it looks like we may need to support global data host relocations.
// Rust host look to be using them by default instead of the plt.
// I think this is due to first linking into a static lib and then linking to the c wrapper.
let plt_relocs = (match exec_obj.dynamic_relocations() {
Some(relocs) => relocs,
None => {
@ -410,6 +554,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
|| inst.is_jmp_far_indirect()
|| inst.is_jmp_near_indirect())
&& !indirect_warning_given
&& verbose
{
indirect_warning_given = true;
println!();
@ -467,7 +612,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
}
};
let shared_lib_name = Path::new(matches.value_of(SHARED_LIB).unwrap())
let shared_lib_name = Path::new(shared_lib_filename)
.file_name()
.unwrap()
.to_str()
@ -494,7 +639,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
) as usize;
let c_buf: *const c_char = dynstr_data[dynstr_off..].as_ptr() as *const i8;
let c_str = unsafe { CStr::from_ptr(c_buf) }.to_str().unwrap();
if c_str == shared_lib_name {
if Path::new(c_str).file_name().unwrap().to_str().unwrap() == shared_lib_name {
shared_lib_index = Some(dyn_lib_index);
if verbose {
println!(
@ -601,7 +746,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
.write(true)
.create(true)
.truncate(true)
.open(&matches.value_of(OUT).unwrap())?;
.open(out_filename)?;
out_file.set_len(md.exec_len)?;
let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? };
@ -862,16 +1007,22 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
}
let saving_metadata_start = SystemTime::now();
let output = fs::File::create(&matches.value_of(METADATA).unwrap())?;
let output = BufWriter::new(output);
if let Err(err) = serialize_into(output, &md) {
println!("Failed to serialize metadata: {}", err);
return Ok(-1);
};
// This block ensure that the metadata is fully written and timed before continuing.
{
let output = fs::File::create(metadata_filename)?;
let output = BufWriter::new(output);
if let Err(err) = serialize_into(output, &md) {
println!("Failed to serialize metadata: {}", err);
return Ok(-1);
};
}
let saving_metadata_duration = saving_metadata_start.elapsed().unwrap();
let flushing_data_start = SystemTime::now();
out_mmap.flush()?;
// Also drop files to to ensure data is fully written here.
drop(out_mmap);
drop(out_file);
let flushing_data_duration = flushing_data_start.elapsed().unwrap();
let total_duration = total_start.elapsed().unwrap();
@ -907,12 +1058,25 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
}
pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
let verbose = matches.is_present(FLAG_VERBOSE);
let time = matches.is_present(FLAG_TIME);
surgery_impl(
matches.value_of(APP).unwrap(),
matches.value_of(METADATA).unwrap(),
matches.value_of(OUT).unwrap(),
matches.is_present(FLAG_VERBOSE),
matches.is_present(FLAG_TIME),
)
}
fn surgery_impl(
app_filename: &str,
metadata_filename: &str,
out_filename: &str,
verbose: bool,
time: bool,
) -> io::Result<i32> {
let total_start = SystemTime::now();
let loading_metadata_start = total_start;
let input = fs::File::open(&matches.value_of(METADATA).unwrap())?;
let input = fs::File::open(metadata_filename)?;
let input = BufReader::new(input);
let md: metadata::Metadata = match deserialize_from(input) {
Ok(data) => data,
@ -924,7 +1088,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
let loading_metadata_duration = loading_metadata_start.elapsed().unwrap();
let app_parsing_start = SystemTime::now();
let app_file = fs::File::open(&matches.value_of(APP).unwrap())?;
let app_file = fs::File::open(app_filename)?;
let app_mmap = unsafe { Mmap::map(&app_file)? };
let app_data = &*app_mmap;
let app_obj = match object::File::parse(app_data) {
@ -940,7 +1104,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
let exec_file = fs::OpenOptions::new()
.read(true)
.write(true)
.open(&matches.value_of(OUT).unwrap())?;
.open(out_filename)?;
let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint;
exec_file.set_len(max_out_len)?;
@ -1378,9 +1542,17 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
let flushing_data_start = SystemTime::now();
exec_mmap.flush()?;
// Also drop files to to ensure data is fully written here.
drop(exec_mmap);
exec_file.set_len(offset as u64 + 1)?;
drop(exec_file);
let flushing_data_duration = flushing_data_start.elapsed().unwrap();
exec_file.set_len(offset as u64 + 1)?;
// Make sure the final executable has permision to execute.
let mut perms = fs::metadata(out_filename)?.permissions();
perms.set_mode(perms.mode() | 0o111);
fs::set_permissions(out_filename, perms)?;
let total_duration = total_start.elapsed().unwrap();
if verbose || time {

7
linker/tests/fib/.gitignore vendored
View File

@ -3,4 +3,9 @@ fib
zig-cache
zig-out
*.o
*.o
dynhost
preprocessedhost
metadata
libapp.so