Enable rebuilding hosts into a dynamic executable

Cargo.lock

@@ -3728,8 +3728,10 @@ dependencies = [
  "object 0.26.2",
  "roc_build",
  "roc_collections",
+ "roc_mono",
  "serde",
  "target-lexicon",
+ "tempfile",
 ]
 
 [[package]]

cli/src/build.rs

@@ -284,13 +284,19 @@ fn spawn_rebuild_thread(
         let rebuild_host_start = SystemTime::now();
         if surgically_link {
             roc_linker::build_and_preprocess_host(
+                opt_level,
                 &thread_local_target,
                 host_input_path.as_path(),
                 exported_symbols,
             )
             .unwrap();
         } else {
-            rebuild_host(opt_level, &thread_local_target, host_input_path.as_path());
+            rebuild_host(
+                opt_level,
+                &thread_local_target,
+                host_input_path.as_path(),
+                None,
+            );
         }
         let rebuild_host_end = rebuild_host_start.elapsed().unwrap();
         rebuild_host_end.as_millis()

compiler/build/src/link.rs

@@ -84,14 +84,19 @@ pub fn build_zig_host_native(
     zig_str_path: &str,
     target: &str,
     opt_level: OptLevel,
+    shared_lib_path: Option<&Path>,
 ) -> Output {
     let mut command = Command::new("zig");
     command
         .env_clear()
         .env("PATH", env_path)
-        .env("HOME", env_home)
+        .env("HOME", env_home);
-        .args(&[
+    if let Some(shared_lib_path) = shared_lib_path {
-            "build-obj",
+        command.args(&["build-exe", shared_lib_path.to_str().unwrap()]);
+    } else {
+        command.arg("build-obj");
+    }
+    command.args(&[
         zig_host_src,
         emit_bin,
         "--pkg-begin",
@@ -123,6 +128,7 @@ pub fn build_zig_host_native(
     zig_str_path: &str,
     _target: &str,
     opt_level: OptLevel,
+    shared_lib_path: Option<&Path>,
 ) -> Output {
     use serde_json::Value;
 
@@ -168,9 +174,13 @@ pub fn build_zig_host_native(
     command
         .env_clear()
         .env("PATH", &env_path)
-        .env("HOME", &env_home)
+        .env("HOME", &env_home);
-        .args(&[
+    if let Some(shared_lib_path) = shared_lib_path {
-            "build-obj",
+        command.args(&["build-exe", shared_lib_path.to_str().unwrap()]);
+    } else {
+        command.arg("build-obj");
+    }
+    command.args(&[
         zig_host_src,
         emit_bin,
         "--pkg-begin",
@@ -200,7 +210,11 @@ pub fn build_zig_host_wasm32(
     zig_host_src: &str,
     zig_str_path: &str,
     opt_level: OptLevel,
+    shared_lib_path: Option<&Path>,
 ) -> Output {
+    if shared_lib_path.is_some() {
+        unimplemented!("Linking a shared library to wasm not yet implemented");
+    }
     // NOTE currently just to get compiler warnings if the host code is invalid.
     // the produced artifact is not used
     //
@@ -239,14 +253,56 @@ pub fn build_zig_host_wasm32(
     command.output().unwrap()
 }
 
-pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path) {
+pub fn build_c_host_native(
+    env_path: &str,
+    env_home: &str,
+    dest: &str,
+    sources: &[&str],
+    opt_level: OptLevel,
+    shared_lib_path: Option<&Path>,
+) -> Output {
+    let mut command = Command::new("clang");
+    command
+        .env_clear()
+        .env("PATH", &env_path)
+        .env("HOME", &env_home)
+        .args(sources)
+        .args(&["-fPIC", "-o", dest]);
+    if let Some(shared_lib_path) = shared_lib_path {
+        command.args(&[
+            shared_lib_path.to_str().unwrap(),
+            "-lm",
+            "-lpthread",
+            "-ldl",
+            "-lrt",
+            "-lutil",
+        ]);
+    } else {
+        command.arg("-c");
+    }
+    if matches!(opt_level, OptLevel::Optimize) {
+        command.arg("-O2");
+    }
+    command.output().unwrap()
+}
+
+pub fn rebuild_host(
+    opt_level: OptLevel,
+    target: &Triple,
+    host_input_path: &Path,
+    shared_lib_path: Option<&Path>,
+) {
     let c_host_src = host_input_path.with_file_name("host.c");
     let c_host_dest = host_input_path.with_file_name("c_host.o");
     let zig_host_src = host_input_path.with_file_name("host.zig");
     let rust_host_src = host_input_path.with_file_name("host.rs");
     let rust_host_dest = host_input_path.with_file_name("rust_host.o");
     let cargo_host_src = host_input_path.with_file_name("Cargo.toml");
-    let host_dest_native = host_input_path.with_file_name("host.o");
+    let host_dest_native = host_input_path.with_file_name(if shared_lib_path.is_some() {
+        "dynhost"
+    } else {
+        "host.o"
+    });
     let host_dest_wasm = host_input_path.with_file_name("host.bc");
 
     let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string());
@@ -273,6 +329,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
                     zig_host_src.to_str().unwrap(),
                     zig_str_path.to_str().unwrap(),
                     opt_level,
+                    shared_lib_path,
                 )
             }
             Architecture::X86_64 => {
@@ -285,6 +342,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
                     zig_str_path.to_str().unwrap(),
                     "native",
                     opt_level,
+                    shared_lib_path,
                 )
             }
             Architecture::X86_32(_) => {
@@ -297,31 +355,14 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
                     zig_str_path.to_str().unwrap(),
                     "i386-linux-musl",
                     opt_level,
+                    shared_lib_path,
                 )
             }
             _ => panic!("Unsupported architecture {:?}", target.architecture),
         };
 
         validate_output("host.zig", "zig", output)
-    } else {
+    } else if cargo_host_src.exists() {
-        // Compile host.c
-        let mut command = Command::new("clang");
-        command.env_clear().env("PATH", &env_path).args(&[
-            "-fPIC",
-            "-c",
-            c_host_src.to_str().unwrap(),
-            "-o",
-            c_host_dest.to_str().unwrap(),
-        ]);
-        if matches!(opt_level, OptLevel::Optimize) {
-            command.arg("-O2");
-        }
-        let output = command.output().unwrap();
-
-        validate_output("host.c", "clang", output);
-    }
-
-    if cargo_host_src.exists() {
         // Compile and link Cargo.toml, if it exists
         let cargo_dir = host_input_path.parent().unwrap();
         let libhost_dir =
@@ -332,6 +373,7 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
                 } else {
                     "debug"
                 });
+        let libhost = libhost_dir.join("libhost.a");
 
         let mut command = Command::new("cargo");
         command.arg("build").current_dir(cargo_dir);
@@ -342,6 +384,29 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
 
         validate_output("src/lib.rs", "cargo build", output);
 
+        // Cargo hosts depend on a c wrapper for the api. Compile host.c as well.
+        if shared_lib_path.is_some() {
+            // If compiling to executable, let c deal with linking as well.
+            let output = build_c_host_native(
+                &env_path,
+                &env_home,
+                host_dest_native.to_str().unwrap(),
+                &[c_host_src.to_str().unwrap(), libhost.to_str().unwrap()],
+                opt_level,
+                shared_lib_path,
+            );
+            validate_output("host.c", "clang", output);
+        } else {
+            let output = build_c_host_native(
+                &env_path,
+                &env_home,
+                c_host_dest.to_str().unwrap(),
+                &[c_host_src.to_str().unwrap()],
+                opt_level,
+                shared_lib_path,
+            );
+            validate_output("host.c", "clang", output);
+
             let output = Command::new("ld")
                 .env_clear()
                 .env("PATH", &env_path)
@@ -356,8 +421,17 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
                 ])
                 .output()
                 .unwrap();
-
             validate_output("c_host.o", "ld", output);
+
+            // Clean up c_host.o
+            let output = Command::new("rm")
+                .env_clear()
+                .args(&["-f", c_host_dest.to_str().unwrap()])
+                .output()
+                .unwrap();
+
+            validate_output("rust_host.o", "rm", output);
+        }
     } else if rust_host_src.exists() {
         // Compile and link host.rs, if it exists
         let mut command = Command::new("rustc");
@@ -373,6 +447,32 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
 
         validate_output("host.rs", "rustc", output);
 
+        // Rust hosts depend on a c wrapper for the api. Compile host.c as well.
+        if shared_lib_path.is_some() {
+            // If compiling to executable, let c deal with linking as well.
+            let output = build_c_host_native(
+                &env_path,
+                &env_home,
+                host_dest_native.to_str().unwrap(),
+                &[
+                    c_host_src.to_str().unwrap(),
+                    rust_host_dest.to_str().unwrap(),
+                ],
+                opt_level,
+                shared_lib_path,
+            );
+            validate_output("host.c", "clang", output);
+        } else {
+            let output = build_c_host_native(
+                &env_path,
+                &env_home,
+                c_host_dest.to_str().unwrap(),
+                &[c_host_src.to_str().unwrap()],
+                opt_level,
+                shared_lib_path,
+            );
+
+            validate_output("host.c", "clang", output);
             let output = Command::new("ld")
                 .env_clear()
                 .env("PATH", &env_path)
@@ -387,8 +487,9 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
                 .unwrap();
 
             validate_output("rust_host.o", "ld", output);
+        }
 
-        // Clean up rust_host.o
+        // Clean up rust_host.o and c_host.o
         let output = Command::new("rm")
             .env_clear()
             .args(&[
@@ -400,15 +501,17 @@ pub fn rebuild_host(opt_level: OptLevel, target: &Triple, host_input_path: &Path
             .unwrap();
 
         validate_output("rust_host.o", "rm", output);
-    } else if c_host_dest.exists() {
+    } else if c_host_src.exists() {
-        // Clean up c_host.o
+        // Compile host.c, if it exists
-        let output = Command::new("mv")
+        let output = build_c_host_native(
-            .env_clear()
+            &env_path,
-            .args(&[c_host_dest, host_dest_native])
+            &env_home,
-            .output()
+            host_dest_native.to_str().unwrap(),
-            .unwrap();
+            &[c_host_src.to_str().unwrap()],
-
+            opt_level,
-        validate_output("c_host.o", "mv", output);
+            shared_lib_path,
+        );
+        validate_output("host.c", "clang", output);
     }
 }
 

linker/Cargo.toml

@@ -18,6 +18,7 @@ test = false
 bench = false
 
 [dependencies]
+roc_mono = { path = "../compiler/mono" }
 roc_build = { path = "../compiler/build", default-features = false }
 roc_collections = { path = "../compiler/collections" }
 bumpalo = { version = "3.6", features = ["collections"] }
@@ -29,3 +30,4 @@ object = { version = "0.26", features = ["read"] }
 serde = { version = "1.0", features = ["derive"] }
 bincode = "1.3"
 target-lexicon = "0.12.2"
+tempfile = "3.1.0"

linker/src/lib.rs

@@ -8,8 +8,9 @@ use object::{
     Object, ObjectSection, ObjectSymbol, RelocationKind, RelocationTarget, Section, SectionIndex,
     Symbol, SymbolIndex, SymbolSection,
 };
-use roc_build::link::LinkType;
+use roc_build::link::{rebuild_host, LinkType};
 use roc_collections::all::MutMap;
+use roc_mono::ir::OptLevel;
 use std::cmp::Ordering;
 use std::convert::TryFrom;
 use std::ffi::CStr;
@@ -21,6 +22,7 @@ use std::os::raw::c_char;
 use std::path::Path;
 use std::time::{Duration, SystemTime};
 use target_lexicon::Triple;
+use tempfile::{Builder, NamedTempFile};
 
 mod metadata;
 
@@ -132,27 +134,47 @@ pub fn supported(link_type: &LinkType, target: &Triple) -> bool {
 }
 
 pub fn build_and_preprocess_host(
+    opt_level: OptLevel,
     target: &Triple,
     host_input_path: &Path,
     exposed_to_host: Vec<String>,
 ) -> io::Result<()> {
     let lib = generate_dynamic_lib(exposed_to_host)?;
+    rebuild_host(opt_level, target, host_input_path, Some(&lib.path()));
     Ok(())
 }
 
-fn generate_dynamic_lib(exposed_to_host: Vec<String>) -> io::Result<()> {
+fn generate_dynamic_lib(exposed_to_host: Vec<String>) -> io::Result<NamedTempFile> {
-    Ok(())
+    for sym in exposed_to_host {
+        println!("{}", sym);
+    }
+    let dummy_lib_file = Builder::new().prefix("roc_lib").suffix(".so").tempfile()?;
+    Ok(dummy_lib_file)
 }
 
+pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
+    preprocess_impl(
+        &matches.value_of(EXEC).unwrap(),
+        &matches.value_of(METADATA).unwrap(),
+        &matches.value_of(OUT).unwrap(),
+        &matches.value_of(SHARED_LIB).unwrap(),
+        matches.is_present(FLAG_VERBOSE),
+        matches.is_present(FLAG_TIME),
+    )
+}
 // TODO: Most of this file is a mess of giant functions just to check if things work.
 // Clean it all up and refactor nicely.
-pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
+fn preprocess_impl(
-    let verbose = matches.is_present(FLAG_VERBOSE);
+    exec_filename: &str,
-    let time = matches.is_present(FLAG_TIME);
+    metadata_filename: &str,
-
+    out_filename: &str,
+    shared_lib_filename: &str,
+    verbose: bool,
+    time: bool,
+) -> io::Result<i32> {
     let total_start = SystemTime::now();
     let exec_parsing_start = total_start;
-    let exec_file = fs::File::open(&matches.value_of(EXEC).unwrap())?;
+    let exec_file = fs::File::open(exec_filename)?;
     let exec_mmap = unsafe { Mmap::map(&exec_file)? };
     let exec_data = &*exec_mmap;
     let exec_obj = match object::File::parse(exec_data) {
@@ -489,7 +511,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
         }
     };
 
-    let shared_lib_name = Path::new(matches.value_of(SHARED_LIB).unwrap())
+    let shared_lib_name = Path::new(shared_lib_filename)
         .file_name()
         .unwrap()
         .to_str()
@@ -623,7 +645,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
         .write(true)
         .create(true)
         .truncate(true)
-        .open(&matches.value_of(OUT).unwrap())?;
+        .open(out_filename)?;
     out_file.set_len(md.exec_len)?;
     let mut out_mmap = unsafe { MmapMut::map_mut(&out_file)? };
 
@@ -884,7 +906,7 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
     }
 
     let saving_metadata_start = SystemTime::now();
-    let output = fs::File::create(&matches.value_of(METADATA).unwrap())?;
+    let output = fs::File::create(metadata_filename)?;
     let output = BufWriter::new(output);
     if let Err(err) = serialize_into(output, &md) {
         println!("Failed to serialize metadata: {}", err);
@@ -929,12 +951,25 @@ pub fn preprocess(matches: &ArgMatches) -> io::Result<i32> {
 }
 
 pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
-    let verbose = matches.is_present(FLAG_VERBOSE);
+    surgery_impl(
-    let time = matches.is_present(FLAG_TIME);
+        &matches.value_of(APP).unwrap(),
+        &matches.value_of(METADATA).unwrap(),
+        &matches.value_of(OUT).unwrap(),
+        matches.is_present(FLAG_VERBOSE),
+        matches.is_present(FLAG_TIME),
+    )
+}
 
+fn surgery_impl(
+    app_filename: &str,
+    metadata_filename: &str,
+    out_filename: &str,
+    verbose: bool,
+    time: bool,
+) -> io::Result<i32> {
     let total_start = SystemTime::now();
     let loading_metadata_start = total_start;
-    let input = fs::File::open(&matches.value_of(METADATA).unwrap())?;
+    let input = fs::File::open(metadata_filename)?;
     let input = BufReader::new(input);
     let md: metadata::Metadata = match deserialize_from(input) {
         Ok(data) => data,
@@ -946,7 +981,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
     let loading_metadata_duration = loading_metadata_start.elapsed().unwrap();
 
     let app_parsing_start = SystemTime::now();
-    let app_file = fs::File::open(&matches.value_of(APP).unwrap())?;
+    let app_file = fs::File::open(app_filename)?;
     let app_mmap = unsafe { Mmap::map(&app_file)? };
     let app_data = &*app_mmap;
     let app_obj = match object::File::parse(app_data) {
@@ -962,7 +997,7 @@ pub fn surgery(matches: &ArgMatches) -> io::Result<i32> {
     let exec_file = fs::OpenOptions::new()
         .read(true)
         .write(true)
-        .open(&matches.value_of(OUT).unwrap())?;
+        .open(out_filename)?;
 
     let max_out_len = md.exec_len + app_data.len() as u64 + md.load_align_constraint;
     exec_file.set_len(max_out_len)?;