Get a basic dylib linking started

Cargo.lock

@@ -2435,6 +2435,7 @@ dependencies = [
  "pretty_assertions",
  "quickcheck",
  "quickcheck_macros",
+ "roc_build",
  "roc_builtins",
  "roc_can",
  "roc_collections",
@@ -2452,6 +2453,7 @@ dependencies = [
  "roc_unify",
  "roc_uniq",
  "target-lexicon",
+ "tempfile",
  "tokio",
 ]
 

cli/src/build.rs

@@ -1,6 +1,6 @@
 use bumpalo::Bump;
 use roc_build::{
-    link::{link, LinkType},
+    link::{link, rebuild_host, LinkType},
     program,
 };
 use roc_collections::all::MutMap;
@@ -22,7 +22,7 @@ fn report_timing(buf: &mut String, label: &str, duration: Duration) {
 pub fn build_file(
     target: &Triple,
     src_dir: PathBuf,
-    filename: PathBuf,
+    roc_file_path: PathBuf,
     opt_level: OptLevel,
 ) -> Result<PathBuf, LoadingProblem> {
     let compilation_start = SystemTime::now();
@@ -38,12 +38,12 @@ pub fn build_file(
     };
     let loaded = roc_load::file::load_and_monomorphize(
         &arena,
-        filename.clone(),
+        roc_file_path.clone(),
         stdlib,
         src_dir.as_path(),
         subs_by_module,
     )?;
-    let dest_filename = filename.with_file_name("roc_app.o");
+    let app_o_file = roc_file_path.with_file_name("roc_app.o");
     let buf = &mut String::with_capacity(1024);
 
     for (module_id, module_timing) in loaded.timings.iter() {
@@ -72,12 +72,14 @@ pub fn build_file(
     program::gen_from_mono_module(
         &arena,
         loaded,
-        filename,
+        roc_file_path,
         Triple::host(),
-        &dest_filename,
+        &app_o_file,
         opt_level,
     );
 
+    println!("\nSuccess! 🎉\n\n\t➡ {}\n", app_o_file.display());
+
     let compilation_end = compilation_start.elapsed().unwrap();
 
     println!(
@@ -85,35 +87,38 @@ pub fn build_file(
         compilation_end.as_millis()
     );
 
-    let cwd = dest_filename.parent().unwrap();
+    let cwd = app_o_file.parent().unwrap();
 
     // Step 2: link the precompiled host and compiled app
     let host_input_path = cwd.join("platform").join("host.o");
     let binary_path = cwd.join("app"); // TODO should be app.exe on Windows
 
+    // TODO we should no longer need to do this once we have platforms on
+    // a package repository, as we can then get precompiled hosts from there.
+    rebuild_host(host_input_path.as_path());
+
     // TODO try to move as much of this linking as possible to the precompiled
     // host, to minimize the amount of host-application linking required.
-    let cmd_result =  // TODO use lld
+    let (mut child, binary_path) =  // TODO use lld
         link(
             target,
-            binary_path.as_path(),
-            host_input_path.as_path(),
-            dest_filename.as_path(),
+            binary_path,
+            &[host_input_path.as_path().to_str().unwrap(), app_o_file.as_path().to_str().unwrap()],
             // LinkType::Executable,
             LinkType::Dylib,
         )
         .map_err(|_| {
             todo!("gracefully handle `rustc` failing to spawn.");
-        })?
-        .wait()
-        .map_err(|_| {
-            todo!("gracefully handle error after `rustc` spawned");
-        });
+        })?;
+
+    let cmd_result = child.wait().map_err(|_| {
+        todo!("gracefully handle error after `rustc` spawned");
+    });
 
     // Clean up the leftover .o file from the Roc, if possible.
     // (If cleaning it up fails, that's fine. No need to take action.)
-    // TODO compile the dest_filename to a tmpdir, as an extra precaution.
-    let _ = fs::remove_file(dest_filename);
+    // TODO compile the app_o_file to a tmpdir, as an extra precaution.
+    let _ = fs::remove_file(app_o_file);
 
     // If the cmd errored out, return the Err.
     cmd_result?;

compiler/build/src/link.rs

@@ -1,6 +1,6 @@
 use crate::target::arch_str;
 use std::io;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::{Child, Command};
 use target_lexicon::{Architecture, OperatingSystem, Triple};
 
@@ -10,45 +10,29 @@ pub enum LinkType {
     Dylib,
 }
 
+/// input_paths can include the host as well as the app. e.g. &["host.o", "roc_app.o"]
 pub fn link(
     target: &Triple,
-    binary_path: &Path,
-    host_input_path: &Path,
-    dest_filename: &Path,
+    output_path: PathBuf,
+    input_paths: &[&str],
     link_type: LinkType,
-) -> io::Result<Child> {
-    // TODO we should no longer need to do this once we have platforms on
-    // a package repository, as we can then get precompiled hosts from there.
-    rebuild_host(host_input_path);
-
+) -> io::Result<(Child, PathBuf)> {
     match target {
         Triple {
             architecture: Architecture::X86_64,
             operating_system: OperatingSystem::Linux,
             ..
-        } => link_linux(
-            target,
-            binary_path,
-            host_input_path,
-            dest_filename,
-            link_type,
-        ),
+        } => link_linux(target, output_path, input_paths, link_type),
         Triple {
             architecture: Architecture::X86_64,
             operating_system: OperatingSystem::Darwin,
             ..
-        } => link_macos(
-            target,
-            binary_path,
-            host_input_path,
-            dest_filename,
-            link_type,
-        ),
+        } => link_macos(target, output_path, input_paths, link_type),
         _ => panic!("TODO gracefully handle unsupported target: {:?}", target),
     }
 }
 
-fn rebuild_host(host_input_path: &Path) {
+pub fn rebuild_host(host_input_path: &Path) {
     let c_host_src = host_input_path.with_file_name("host.c");
     let c_host_dest = host_input_path.with_file_name("c_host.o");
     let rust_host_src = host_input_path.with_file_name("host.rs");
@@ -137,21 +121,33 @@ fn rebuild_host(host_input_path: &Path) {
 
 fn link_linux(
     target: &Triple,
-    binary_path: &Path,
-    host_input_path: &Path,
-    dest_filename: &Path,
+    output_path: PathBuf,
+    input_paths: &[&str],
     link_type: LinkType,
-) -> io::Result<Child> {
-    let base_args = match link_type {
-        LinkType::Executable => Vec::new(),
-        // TODO: find a way to avoid using a vec! here - should theoretically be
-        // able to do this somehow using &[] but the borrow checker isn't having it.
-        //
-        // TODO: do we need to add a version number on to this? e.g. ".1"
-        //
-        // See https://software.intel.com/content/www/us/en/develop/articles/create-a-unix-including-linux-shared-library.html
-        // TODO: do we even need the -soname argument?
-        LinkType::Dylib => vec!["-shared", "-soname", binary_path.to_str().unwrap()],
+) -> io::Result<(Child, PathBuf)> {
+    let mut soname;
+    let (base_args, output_path) = match link_type {
+        LinkType::Executable => (Vec::new(), output_path),
+        LinkType::Dylib => {
+            // TODO: do we acually need the version number on this?
+            // Do we even need the "-soname" argument?
+            //
+            // See https://software.intel.com/content/www/us/en/develop/articles/create-a-unix-including-linux-shared-library.html
+
+            soname = output_path.clone();
+            soname.set_extension("so.1");
+
+            let mut output_path = output_path;
+
+            output_path.set_extension("so.1.0");
+
+            (
+                // TODO: find a way to avoid using a vec! here - should theoretically be
+                // able to do this somehow using &[] but the borrow checker isn't having it.
+                vec!["-shared", "-soname", soname.as_path().to_str().unwrap()],
+                output_path,
+            )
+        }
     };
 
     let libcrt_path = if Path::new("/usr/lib/x86_64-linux-gnu").exists() {
@@ -170,77 +166,88 @@ fn link_linux(
 
     // NOTE: order of arguments to `ld` matters here!
     // The `-l` flags should go after the `.o` arguments
-    Command::new("ld")
-        // Don't allow LD_ env vars to affect this
-        .env_clear()
-        .args(&base_args)
-        .args(&[
-            "-arch",
-            arch_str(target),
-            libcrt_path.join("crti.o").to_str().unwrap(),
-            libcrt_path.join("crtn.o").to_str().unwrap(),
-            libcrt_path.join("Scrt1.o").to_str().unwrap(),
-            "-dynamic-linker",
-            "/lib64/ld-linux-x86-64.so.2",
-            // Inputs
-            host_input_path.to_str().unwrap(), // host.o
-            dest_filename.to_str().unwrap(),   // app.o
-            // Libraries - see https://github.com/rtfeldman/roc/pull/554#discussion_r496365925
-            // for discussion and further references
-            "-lc",
-            "-lm",
-            "-lpthread",
-            "-ldl",
-            "-lrt",
-            "-lutil",
-            "-lc_nonshared",
-            "-lc++",
-            "-lunwind",
-            libgcc_path.to_str().unwrap(),
-            // Output
-            "-o",
-            binary_path.to_str().unwrap(), // app
-        ])
-        .spawn()
+    Ok((
+        Command::new("ld")
+            // Don't allow LD_ env vars to affect this
+            .env_clear()
+            .args(&base_args)
+            .args(&[
+                "-arch",
+                arch_str(target),
+                libcrt_path.join("crti.o").to_str().unwrap(),
+                libcrt_path.join("crtn.o").to_str().unwrap(),
+                libcrt_path.join("Scrt1.o").to_str().unwrap(),
+                "-dynamic-linker",
+                "/lib64/ld-linux-x86-64.so.2",
+            ])
+            .args(input_paths)
+            .args(&[
+                // Libraries - see https://github.com/rtfeldman/roc/pull/554#discussion_r496365925
+                // for discussion and further references
+                "-lc",
+                "-lm",
+                "-lpthread",
+                "-ldl",
+                "-lrt",
+                "-lutil",
+                "-lc_nonshared",
+                "-lc++",
+                "-lunwind",
+                libgcc_path.to_str().unwrap(),
+                // Output
+                "-o",
+                output_path.as_path().to_str().unwrap(), // app (or app.so or app.dylib etc.)
+            ])
+            .spawn()?,
+        output_path,
+    ))
 }
 
 fn link_macos(
     target: &Triple,
-    binary_path: &Path,
-    host_input_path: &Path,
-    dest_filename: &Path,
+    output_path: PathBuf,
+    input_paths: &[&str],
     link_type: LinkType,
-) -> io::Result<Child> {
-    let link_type_arg = match link_type {
-        LinkType::Executable => "-execute",
-        LinkType::Dylib => "-dylib",
+) -> io::Result<(Child, PathBuf)> {
+    let (link_type_arg, output_path) = match link_type {
+        LinkType::Executable => ("-execute", output_path),
+        LinkType::Dylib => {
+            let mut output_path = output_path;
+
+            output_path.set_extension("dylib");
+
+            ("-dylib", output_path)
+        }
     };
 
-    // NOTE: order of arguments to `ld` matters here!
-    // The `-l` flags should go after the `.o` arguments
-    Command::new("ld")
-        // Don't allow LD_ env vars to affect this
-        .env_clear()
-        .args(&[
-            link_type_arg,
-            "-arch",
-            target.architecture.to_string().as_str(),
-            // Inputs
-            host_input_path.to_str().unwrap(), // host.o
-            dest_filename.to_str().unwrap(),   // roc_app.o
-            // Libraries - see https://github.com/rtfeldman/roc/pull/554#discussion_r496392274
-            // for discussion and further references
-            "-lSystem",
-            "-lresolv",
-            "-lpthread",
-            // "-lrt", // TODO shouldn't we need this?
-            // "-lc_nonshared", // TODO shouldn't we need this?
-            // "-lgcc", // TODO will eventually need compiler_rt from gcc or something - see https://github.com/rtfeldman/roc/pull/554#discussion_r496370840
-            // "-lunwind", // TODO will eventually need this, see https://github.com/rtfeldman/roc/pull/554#discussion_r496370840
-            "-lc++", // TODO shouldn't we need this?
-            // Output
-            "-o",
-            binary_path.to_str().unwrap(), // app
-        ])
-        .spawn()
+    Ok((
+        // NOTE: order of arguments to `ld` matters here!
+        // The `-l` flags should go after the `.o` arguments
+        Command::new("ld")
+            // Don't allow LD_ env vars to affect this
+            .env_clear()
+            .args(&[
+                link_type_arg,
+                "-arch",
+                target.architecture.to_string().as_str(),
+            ])
+            .args(input_paths)
+            .args(&[
+                // Libraries - see https://github.com/rtfeldman/roc/pull/554#discussion_r496392274
+                // for discussion and further references
+                "-lSystem",
+                "-lresolv",
+                "-lpthread",
+                // "-lrt", // TODO shouldn't we need this?
+                // "-lc_nonshared", // TODO shouldn't we need this?
+                // "-lgcc", // TODO will eventually need compiler_rt from gcc or something - see https://github.com/rtfeldman/roc/pull/554#discussion_r496370840
+                // "-lunwind", // TODO will eventually need this, see https://github.com/rtfeldman/roc/pull/554#discussion_r496370840
+                "-lc++", // TODO shouldn't we need this?
+                // Output
+                "-o",
+                output_path.to_str().unwrap(), // app
+            ])
+            .spawn()?,
+        output_path,
+    ))
 }

compiler/build/src/program.rs

@@ -16,9 +16,9 @@ use target_lexicon::Triple;
 pub fn gen_from_mono_module(
     arena: &Bump,
     loaded: MonomorphizedModule,
-    filename: PathBuf,
+    file_path: PathBuf,
     target: Triple,
-    dest_filename: &Path,
+    app_o_file: &Path,
     opt_level: OptLevel,
 ) {
     use roc_reporting::report::{can_problem, type_problem, RocDocAllocator, DEFAULT_PALETTE};
@@ -32,7 +32,7 @@ pub fn gen_from_mono_module(
     let alloc = RocDocAllocator::new(&src_lines, home, &loaded.interns);
 
     for problem in loaded.can_problems.into_iter() {
-        let report = can_problem(&alloc, filename.clone(), problem);
+        let report = can_problem(&alloc, file_path.clone(), problem);
         let mut buf = String::new();
 
         report.render_color_terminal(&mut buf, &alloc, &palette);
@@ -41,7 +41,7 @@ pub fn gen_from_mono_module(
     }
 
     for problem in loaded.type_problems.into_iter() {
-        let report = type_problem(&alloc, filename.clone(), problem);
+        let report = type_problem(&alloc, file_path.clone(), problem);
         let mut buf = String::new();
 
         report.render_color_terminal(&mut buf, &alloc, &palette);
@@ -132,8 +132,6 @@ pub fn gen_from_mono_module(
     let target_machine = target::target_machine(&target, opt, reloc, model).unwrap();
 
     target_machine
-        .write_to_file(&env.module, FileType::Object, &dest_filename)
+        .write_to_file(&env.module, FileType::Object, &app_o_file)
         .expect("Writing .o file failed");
-
-    println!("\nSuccess! 🎉\n\n\t➡ {}\n", dest_filename.display());
 }

compiler/gen/Cargo.toml

@@ -40,6 +40,7 @@ inlinable_string = "0.1"
 # This way, GitHub Actions works and nobody's builds get broken.
 inkwell = { git = "https://github.com/rtfeldman/inkwell", tag = "llvm10-0.release2" }
 target-lexicon = "0.10"
+tempfile = "3.1.0"
 libloading = "0.6"
 
 [dev-dependencies]
@@ -47,6 +48,8 @@ roc_can = { path = "../can" }
 roc_parse = { path = "../parse" }
 roc_load = { path = "../load" }
 roc_reporting = { path = "../reporting" }
+roc_build = { path = "../build" }
+roc_std = { path = "../../roc_std" }
 pretty_assertions = "0.5.1"
 maplit = "1.0.1"
 indoc = "0.3.3"
@@ -55,4 +58,3 @@ quickcheck_macros = "0.8"
 tokio = { version = "0.2", features = ["blocking", "fs", "sync", "rt-threaded"] }
 bumpalo = { version = "3.2", features = ["collections"] }
 libc = "0.2"
-roc_std = { path = "../../roc_std" }

compiler/gen/tests/gen_num.rs

@@ -788,4 +788,16 @@ mod gen_num {
             f64
         );
     }
+
+    #[test]
+    fn blah() {
+        let lib =
+            libloading::Library::new("/Users/rtfeldman/code/roc/examples/hello-world/app.dylib")
+                .unwrap();
+
+        let func: libloading::Symbol<unsafe extern "C" fn() -> &'static str> =
+            unsafe { lib.get(b"main_1").unwrap() };
+
+        println!("Roc says: {}", unsafe { func() });
+    }
 }

compiler/gen/tests/helpers/eval.rs

@@ -1,5 +1,11 @@
 use libloading::Library;
+use roc_build::{
+    link::{link, LinkType},
+    program::gen_from_mono_module,
+};
 use roc_collections::all::{MutMap, MutSet};
+use target_lexicon::Triple;
+use tempfile::tempdir;
 
 fn promote_expr_to_module(src: &str) -> String {
     let mut buffer = String::from("app Test provides [ main ] imports []\n\nmain =\n");
@@ -21,7 +27,6 @@ pub fn helper<'a>(
     leak: bool,
     context: &'a inkwell::context::Context,
 ) -> (&'static str, Vec<roc_problem::can::Problem>, Library) {
-    use inkwell::OptimizationLevel;
     use roc_gen::llvm::build::{build_proc, build_proc_header, Scope};
     use std::path::{Path, PathBuf};
 
@@ -151,120 +156,56 @@ pub fn helper<'a>(
         }
     }
 
-    let module = roc_gen::llvm::build::module_from_builtins(context, "app");
-    let builder = context.create_builder();
     let opt_level = if cfg!(debug_assertions) {
         roc_gen::llvm::build::OptLevel::Normal
     } else {
         roc_gen::llvm::build::OptLevel::Optimize
     };
 
-    let module = arena.alloc(module);
-    let (module_pass, function_pass) =
-        roc_gen::llvm::build::construct_optimization_passes(module, opt_level);
-
-    // TODO: use build/program to generate a .o file in a tempdir
-    // TODO: use link.rs to link the .o into .dylib/.so
-
-    let path = "";
-    let lib = Library::new(path).expect("Error loading compiled dylib for test");
-
-    // Compile and add all the Procs before adding main
-    let env = roc_gen::llvm::build::Env {
-        arena: &arena,
-        builder: &builder,
-        context,
+    let dir = tempdir().unwrap();
+    let filename = PathBuf::from("Test.roc");
+    let file_path = dir.path().join(filename.clone());
+    let full_file_path = PathBuf::from(file_path.clone());
+    let monomorphized_module = MonomorphizedModule {
+        module_id: home,
+        can_problems: Vec::new(),
+        type_problems: Vec::new(),
+        mono_problems: Vec::new(),
+        procedures,
         interns,
-        module,
-        ptr_bytes,
-        leak,
-        // important! we don't want any procedures to get the C calling convention
-        exposed_to_host: MutSet::default(),
+        exposed_to_host,
+        src: loaded.src,
+        subs: loaded.subs,
+        timings: loaded.timings,
     };
 
-    let mut layout_ids = roc_gen::layout_id::LayoutIds::default();
-    let mut headers = Vec::with_capacity(procedures.len());
-
-    // Add all the Proc headers to the module.
-    // We have to do this in a separate pass first,
-    // because their bodies may reference each other.
-    let mut scope = Scope::default();
-    for ((symbol, layout), proc) in procedures.drain() {
-        let fn_val = build_proc_header(&env, &mut layout_ids, symbol, &layout, &proc);
-
-        if proc.args.is_empty() {
-            // this is a 0-argument thunk, i.e. a top-level constant definition
-            // it must be in-scope everywhere in the module!
-            scope.insert_top_level_thunk(symbol, layout, fn_val);
-        }
-
-        headers.push((proc, fn_val));
-    }
-
-    // Build each proc using its header info.
-    for (proc, fn_val) in headers {
-        let mut current_scope = scope.clone();
-
-        // only have top-level thunks for this proc's module in scope
-        // this retain is not needed for correctness, but will cause less confusion when debugging
-        let home = proc.name.module_id();
-        current_scope.retain_top_level_thunks_for_module(home);
-
-        build_proc(&env, &mut layout_ids, scope.clone(), proc, fn_val);
-
-        if fn_val.verify(true) {
-            function_pass.run_on(&fn_val);
-        } else {
-            use roc_builtins::std::Mode;
-
-            let mode = match stdlib_mode {
-                Mode::Uniqueness => "OPTIMIZED",
-                Mode::Standard => "NON-OPTIMIZED",
-            };
-
-            eprintln!(
-                "\n\nFunction {:?} failed LLVM verification in {} build. Its content was:\n",
-                fn_val.get_name().to_str().unwrap(),
-                mode,
-            );
-
-            fn_val.print_to_stderr();
-            // module.print_to_stderr();
-
-            panic!(
-                "The preceding code was from {:?}, which failed LLVM verification in {} build.",
-                fn_val.get_name().to_str().unwrap(),
-                mode,
-            );
-        }
-    }
-    let (main_fn_name, main_fn) = roc_gen::llvm::build::promote_to_main_function(
-        &env,
-        &mut layout_ids,
-        main_fn_symbol,
-        &main_fn_layout,
+    // Generate app.o
+    gen_from_mono_module(
+        arena,
+        monomorphized_module,
+        filename,
+        Triple::host(),
+        &full_file_path,
+        opt_level,
     );
 
-    // Uncomment this to see the module's un-optimized LLVM instruction output:
-    // env.module.print_to_stderr();
+    // Link app.o into a dylib - e.g. app.so or app.dylib
+    let (mut child, dylib_path) = link(
+        &Triple::host(),
+        full_file_path.clone(),
+        &[full_file_path.to_str().unwrap()],
+        LinkType::Dylib,
+    )
+    .unwrap();
 
-    if main_fn.verify(true) {
-        function_pass.run_on(&main_fn);
-    } else {
-        panic!("Main function {} failed LLVM verification in NON-OPTIMIZED build. Uncomment things nearby to see more details.", main_fn_name);
-    }
+    child.wait().unwrap();
 
-    module_pass.run_on(env.module);
+    // Load the dylib
+    let path = dylib_path.as_path().to_str().unwrap();
+    let lib = Library::new(path).expect("Error loading compiled dylib for test");
 
-    // Verify the module
-    if let Err(errors) = env.module.verify() {
-        panic!("Errors defining module: {:?}", errors);
-    }
-
-    // Uncomment this to see the module's optimized LLVM instruction output:
-    // env.module.print_to_stderr();
-
-    (main_fn_name, errors, lib)
+    todo!("TODO promote a main function");
+    // (main_fn_name, errors, lib)
 }
 
 // TODO this is almost all code duplication with assert_llvm_evals_to
@@ -311,7 +252,7 @@ macro_rules! assert_llvm_evals_to {
         let context = Context::create();
         let stdlib = roc_builtins::std::standard_stdlib();
 
-        let (main_fn_name, errors, li) =
+        let (main_fn_name, errors, lib) =
             $crate::helpers::eval::helper(&arena, $src, stdlib, $leak, &context);
 
         let transform = |success| {

compiler/solve/Cargo.toml

@@ -25,7 +25,7 @@ roc_solve = { path = "../solve" }
 pretty_assertions = "0.5.1"
 maplit = "1.0.1"
 indoc = "0.3.3"
-tempfile = "3.0.1"
+tempfile = "3.1.0"
 quickcheck = "0.8"
 quickcheck_macros = "0.8"
 bumpalo = { version = "3.2", features = ["collections"] }