roc-lang/roc
1
1
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2024-09-22 08:17:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

repl: refactor LLVM-specific code under an optional Cargo feature

Browse Source
This commit is contained in:
Brian Carroll 2022-02-01 10:19:25 +00:00
parent 33e6afe83c
commit bbe82fcf25
4 changed files with 244 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
repl_eval/Cargo.toml
View File

@ -5,18 +5,22 @@ version = "0.1.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
bumpalo = {version = "3.8.0", features = ["collections"]}
inkwell = {path = "../vendor/inkwell"}# TODO
libloading = "0.7.1" # TODO
target-lexicon = "0.12.2"
[features]
default = ["llvm"]
llvm = ["inkwell", "libloading", "roc_gen_llvm", "roc_build/llvm"]
roc_build = {path = "../compiler/build", default-features = false, features = ["llvm"]}# TODO
[dependencies]
bumpalo = {version = "3.8.0", features = ["collections"]}
inkwell = {path = "../vendor/inkwell", optional = true}
libloading = {version = "0.7.1", optional = true}
target-lexicon = "0.12.2"
roc_build = {path = "../compiler/build", default-features = false}
roc_builtins = {path = "../compiler/builtins"}
roc_can = {path = "../compiler/can"}
roc_collections = {path = "../compiler/collections"}
roc_fmt = {path = "../compiler/fmt"}
roc_gen_llvm = {path = "../compiler/gen_llvm"}# TODO
roc_gen_llvm = {path = "../compiler/gen_llvm", optional = true}
roc_load = {path = "../compiler/load"}
roc_module = {path = "../compiler/module"}
roc_mono = {path = "../compiler/mono"}

35
repl_eval/src/eval.rs
View File

@ -1,9 +1,9 @@
use bumpalo::collections::Vec;
use bumpalo::Bump;
use libloading::Library;
use std::cmp::{max_by_key, min_by_key};
use roc_builtins::bitcode::{FloatWidth, IntWidth};
use roc_collections::all::MutMap;
use roc_gen_llvm::{run_jit_function, run_jit_function_dynamic_type};
use roc_module::called_via::CalledVia;
use roc_module::ident::TagName;
use roc_module::symbol::{Interns, ModuleId, Symbol};
@ -15,7 +15,12 @@ use roc_parse::ast::{AssignedField, Collection, Expr, StrLiteral};
use roc_region::all::{Loc, Region};
use roc_target::TargetInfo;
use roc_types::subs::{Content, FlatType, GetSubsSlice, RecordFields, Subs, UnionTags, Variable};
use std::cmp::{max_by_key, min_by_key};
#[cfg(feature = "llvm")]
use roc_gen_llvm::{run_jit_function, run_jit_function_dynamic_type};
#[cfg(feature = "llvm")]
type AppExecutable = libloading::Library;
use super::app_memory::AppMemory;
@ -43,7 +48,7 @@ pub enum ToAstProblem {
#[allow(clippy::too_many_arguments)]
pub unsafe fn jit_to_ast<'a, M: AppMemory>(
arena: &'a Bump,
lib: Library,
app: AppExecutable,
main_fn_name: &str,
layout: ProcLayout<'a>,
content: &'a Content,
@ -68,7 +73,7 @@ pub unsafe fn jit_to_ast<'a, M: AppMemory>(
result,
} => {
// this is a thunk
jit_to_ast_help(&env, lib, main_fn_name, &result, content)
jit_to_ast_help(&env, app, main_fn_name, &result, content)
}
_ => Err(ToAstProblem::FunctionLayout),
}
@ -261,7 +266,7 @@ const OPAQUE_FUNCTION: Expr = Expr::Var {
fn jit_to_ast_help<'a, M: AppMemory>(
env: &Env<'a, 'a, M>,
lib: Library,
app: AppExecutable,
main_fn_name: &str,
layout: &Layout<'a>,
content: &'a Content,
@ -269,7 +274,7 @@ fn jit_to_ast_help<'a, M: AppMemory>(
let (newtype_containers, content) = unroll_newtypes(env, content);
let content = unroll_aliases(env, content);
let result = match layout {
Layout::Builtin(Builtin::Bool) => Ok(run_jit_function!(lib, main_fn_name, bool, |num| {
Layout::Builtin(Builtin::Bool) => Ok(run_jit_function!(app, main_fn_name, bool, |num| {
bool_to_ast(env, num, content)
})),
Layout::Builtin(Builtin::Int(int_width)) => {
@ -277,7 +282,7 @@ fn jit_to_ast_help<'a, M: AppMemory>(
macro_rules! helper {
($ty:ty) => {
run_jit_function!(lib, main_fn_name, $ty, |num| num_to_ast(
run_jit_function!(app, main_fn_name, $ty, |num| num_to_ast(
env,
number_literal_to_ast(env.arena, num),
content
@ -288,7 +293,7 @@ fn jit_to_ast_help<'a, M: AppMemory>(
let result = match int_width {
U8 | I8 => {
// NOTE: this is does not handle 8-bit numbers yet
run_jit_function!(lib, main_fn_name, u8, |num| byte_to_ast(env, num, content))
run_jit_function!(app, main_fn_name, u8, |num| byte_to_ast(env, num, content))
}
U16 => helper!(u16),
U32 => helper!(u32),
@ -307,7 +312,7 @@ fn jit_to_ast_help<'a, M: AppMemory>(
macro_rules! helper {
($ty:ty) => {
run_jit_function!(lib, main_fn_name, $ty, |num| num_to_ast(
run_jit_function!(app, main_fn_name, $ty, |num| num_to_ast(
env,
number_literal_to_ast(env.arena, num),
content
@ -324,13 +329,13 @@ fn jit_to_ast_help<'a, M: AppMemory>(
Ok(result)
}
Layout::Builtin(Builtin::Str) => Ok(run_jit_function!(
lib,
app,
main_fn_name,
&'static str,
|string: &'static str| { str_to_ast(env.arena, env.arena.alloc(string)) }
)),
Layout::Builtin(Builtin::List(elem_layout)) => Ok(run_jit_function!(
lib,
app,
main_fn_name,
(usize, usize),
|(addr, len): (usize, usize)| { list_to_ast(env, addr, len, elem_layout, content) }
@ -391,7 +396,7 @@ fn jit_to_ast_help<'a, M: AppMemory>(
let result_stack_size = layout.stack_size(env.target_info);
run_jit_function_dynamic_type!(
lib,
app,
main_fn_name,
result_stack_size as usize,
|bytes_addr: usize| { struct_addr_to_ast(bytes_addr as usize) }
@ -400,7 +405,7 @@ fn jit_to_ast_help<'a, M: AppMemory>(
Layout::Union(UnionLayout::NonRecursive(_)) => {
let size = layout.stack_size(env.target_info);
Ok(run_jit_function_dynamic_type!(
lib,
app,
main_fn_name,
size as usize,
|addr: usize| {
@ -414,7 +419,7 @@ fn jit_to_ast_help<'a, M: AppMemory>(
| Layout::Union(UnionLayout::NullableWrapped { .. }) => {
let size = layout.stack_size(env.target_info);
Ok(run_jit_function_dynamic_type!(
lib,
app,
main_fn_name,
size as usize,
|addr: usize| {

384
repl_eval/src/gen.rs
View File

@ -1,23 +1,30 @@
use crate::app_memory::AppMemoryInternal;
use crate::eval;
use bumpalo::Bump;
use inkwell::context::Context; // TODO
use inkwell::module::Linkage; // TODO
use roc_build::{link::module_to_dylib, program::FunctionIterator};
use std::path::{Path, PathBuf};
use std::str::from_utf8_unchecked;
use target_lexicon::Triple;
use roc_can::builtins::builtin_defs_map;
use roc_collections::all::{MutMap, MutSet};
use roc_fmt::annotation::Formattable;
use roc_fmt::annotation::{Newlines, Parens};
use roc_gen_llvm::llvm::externs::add_default_roc_externs;
use roc_load::file::LoadingProblem;
use roc_load::file::MonomorphizedModule;
use roc_mono::ir::OptLevel;
use roc_parse::ast::Expr;
use roc_parse::parser::SyntaxError;
use roc_region::all::LineInfo;
use roc_target::TargetInfo;
use roc_types::pretty_print::{content_to_string, name_all_type_vars};
use std::path::{Path, PathBuf};
use std::str::from_utf8_unchecked;
use target_lexicon::Triple;
#[cfg(feature = "llvm")]
use {
inkwell::context::Context, inkwell::module::Linkage, roc_build::link::module_to_dylib,
roc_build::program::FunctionIterator,
};
use crate::app_memory::AppMemoryInternal;
use crate::eval::{self, ToAstProblem};
pub enum ReplOutput {
Problems(Vec<String>),
@ -29,29 +36,201 @@ pub fn gen_and_eval<'a>(
target: Triple,
opt_level: OptLevel,
) -> Result<ReplOutput, SyntaxError<'a>> {
if cfg!(feature = "llvm") {
gen_and_eval_llvm(src, target, opt_level)
} else {
todo!("REPL must be compiled with LLVM feature for now")
}
}
pub fn gen_and_eval_llvm<'a>(
src: &[u8],
target: Triple,
opt_level: OptLevel,
) -> Result<ReplOutput, SyntaxError<'a>> {
let arena = Bump::new();
let target_info = TargetInfo::from(&target);
let loaded = match compile_to_mono(&arena, src, target_info) {
Ok(x) => x,
Err(prob_strings) => {
return Ok(ReplOutput::Problems(prob_strings));
}
};
let MonomorphizedModule {
procedures,
entry_point,
interns,
exposed_to_host,
mut subs,
module_id: home,
..
} = loaded;
let context = Context::create();
let builder = context.create_builder();
let module = arena.alloc(roc_gen_llvm::llvm::build::module_from_builtins(
&target, &context, "",
));
// mark our zig-defined builtins as internal
for function in FunctionIterator::from_module(module) {
let name = function.get_name().to_str().unwrap();
if name.starts_with("roc_builtins") {
function.set_linkage(Linkage::Internal);
}
}
debug_assert_eq!(exposed_to_host.values.len(), 1);
let (main_fn_symbol, main_fn_var) = exposed_to_host.values.iter().next().unwrap();
let main_fn_symbol = *main_fn_symbol;
let main_fn_var = *main_fn_var;
// pretty-print the expr type string for later.
name_all_type_vars(main_fn_var, &mut subs);
let content = subs.get_content_without_compacting(main_fn_var);
let expr_type_str = content_to_string(content, &subs, home, &interns);
let (_, main_fn_layout) = match procedures.keys().find(|(s, _)| *s == main_fn_symbol) {
Some(layout) => *layout,
None => {
return Ok(ReplOutput::NoProblems {
expr: "<function>".to_string(),
expr_type: expr_type_str,
});
}
};
let module = arena.alloc(module);
let (module_pass, function_pass) =
roc_gen_llvm::llvm::build::construct_optimization_passes(module, opt_level);
let (dibuilder, compile_unit) = roc_gen_llvm::llvm::build::Env::new_debug_info(module);
// Compile and add all the Procs before adding main
let env = roc_gen_llvm::llvm::build::Env {
arena: &arena,
builder: &builder,
dibuilder: &dibuilder,
compile_unit: &compile_unit,
context: &context,
interns,
module,
target_info,
is_gen_test: true, // so roc_panic is generated
// important! we don't want any procedures to get the C calling convention
exposed_to_host: MutSet::default(),
};
// Add roc_alloc, roc_realloc, and roc_dealloc, since the repl has no
// platform to provide them.
add_default_roc_externs(&env);
let (main_fn_name, main_fn) = roc_gen_llvm::llvm::build::build_procedures_return_main(
&env,
opt_level,
procedures,
entry_point,
);
env.dibuilder.finalize();
// we don't use the debug info, and it causes weird errors.
module.strip_debug_info();
// Uncomment this to see the module's un-optimized LLVM instruction output:
// env.module.print_to_stderr();
if main_fn.verify(true) {
function_pass.run_on(&main_fn);
} else {
panic!("Main function {} failed LLVM verification in build. Uncomment things nearby to see more details.", main_fn_name);
}
module_pass.run_on(env.module);
// Uncomment this to see the module's optimized LLVM instruction output:
// env.module.print_to_stderr();
// Verify the module
if let Err(errors) = env.module.verify() {
panic!(
"Errors defining module:\n{}\n\nUncomment things nearby to see more details.",
errors.to_string()
);
}
let lib = module_to_dylib(env.module, &target, opt_level)
.expect("Error loading compiled dylib for test");
let res_answer = unsafe {
eval::jit_to_ast(
&arena,
lib,
main_fn_name,
main_fn_layout,
content,
&env.interns,
home,
&subs,
target_info,
&AppMemoryInternal,
)
};
let formatted = format_answer(&arena, res_answer, expr_type_str);
Ok(formatted)
}
fn format_answer(
arena: &Bump,
res_answer: Result<Expr, ToAstProblem>,
expr_type_str: String,
) -> ReplOutput {
let mut expr = roc_fmt::Buf::new_in(arena);
use eval::ToAstProblem::*;
match res_answer {
Ok(answer) => {
answer.format_with_options(&mut expr, Parens::NotNeeded, Newlines::Yes, 0);
}
Err(FunctionLayout) => {
expr.indent(0);
expr.push_str("<function>");
}
}
ReplOutput::NoProblems {
expr: expr.into_bump_str().to_string(),
expr_type: expr_type_str,
}
}
fn compile_to_mono<'a>(
arena: &'a Bump,
src: &[u8],
target_info: TargetInfo,
) -> Result<MonomorphizedModule<'a>, Vec<String>> {
use roc_reporting::report::{
can_problem, mono_problem, type_problem, RocDocAllocator, DEFAULT_PALETTE,
};
let arena = Bump::new();
// SAFETY: we've already verified that this is valid UTF-8 during parsing.
let src_str: &str = unsafe { from_utf8_unchecked(src) };
let stdlib = roc_builtins::std::standard_stdlib();
let stdlib = arena.alloc(roc_builtins::std::standard_stdlib());
let filename = PathBuf::from("REPL.roc");
let src_dir = Path::new("fake/test/path");
let module_src = promote_expr_to_module(src_str);
let target_info = TargetInfo::from(&target);
let module_src = arena.alloc(promote_expr_to_module(src_str));
let exposed_types = MutMap::default();
let loaded = roc_load::file::load_and_monomorphize_from_str(
&arena,
arena,
filename,
&module_src,
&stdlib,
module_src,
stdlib,
src_dir,
exposed_types,
target_info,
@ -61,46 +240,43 @@ pub fn gen_and_eval<'a>(
let mut loaded = match loaded {
Ok(v) => v,
Err(LoadingProblem::FormattedReport(report)) => {
return Ok(ReplOutput::Problems(vec![report]));
return Err(vec![report]);
}
Err(e) => {
panic!("error while loading module: {:?}", e)
}
};
use roc_load::file::MonomorphizedModule;
let MonomorphizedModule {
procedures,
entry_point,
interns,
exposed_to_host,
mut subs,
module_id: home,
sources,
can_problems,
type_problems,
mono_problems,
..
} = loaded;
} = &mut loaded;
let mut lines = Vec::new();
for (home, (module_path, src)) in sources {
let can_problems = loaded.can_problems.remove(&home).unwrap_or_default();
let type_problems = loaded.type_problems.remove(&home).unwrap_or_default();
let mono_problems = loaded.mono_problems.remove(&home).unwrap_or_default();
for (home, (module_path, src)) in sources.iter() {
let can_probs = can_problems.remove(home).unwrap_or_default();
let type_probs = type_problems.remove(home).unwrap_or_default();
let mono_probs = mono_problems.remove(home).unwrap_or_default();
let error_count = can_problems.len() + type_problems.len() + mono_problems.len();
let error_count = can_probs.len() + type_probs.len() + mono_probs.len();
if error_count == 0 {
continue;
}
let line_info = LineInfo::new(&module_src);
let line_info = LineInfo::new(module_src);
let src_lines: Vec<&str> = src.split('\n').collect();
let palette = DEFAULT_PALETTE;
// Report parsing and canonicalization problems
let alloc = RocDocAllocator::new(&src_lines, home, &interns);
let alloc = RocDocAllocator::new(&src_lines, *home, interns);
for problem in can_problems.into_iter() {
for problem in can_probs.into_iter() {
let report = can_problem(&alloc, &line_info, module_path.clone(), problem);
let mut buf = String::new();
@ -109,7 +285,7 @@ pub fn gen_and_eval<'a>(
lines.push(buf);
}
for problem in type_problems {
for problem in type_probs {
if let Some(report) = type_problem(&alloc, &line_info, module_path.clone(), problem) {
let mut buf = String::new();
@ -119,7 +295,7 @@ pub fn gen_and_eval<'a>(
}
}
for problem in mono_problems {
for problem in mono_probs {
let report = mono_problem(&alloc, &line_info, module_path.clone(), problem);
let mut buf = String::new();
@ -130,143 +306,9 @@ pub fn gen_and_eval<'a>(
}
if !lines.is_empty() {
Ok(ReplOutput::Problems(lines))
Err(lines)
} else {
let context = Context::create();
let builder = context.create_builder();
let module = arena.alloc(roc_gen_llvm::llvm::build::module_from_builtins(
&target, &context, "",
));
// mark our zig-defined builtins as internal
for function in FunctionIterator::from_module(module) {
let name = function.get_name().to_str().unwrap();
if name.starts_with("roc_builtins") {
function.set_linkage(Linkage::Internal);
}
}
debug_assert_eq!(exposed_to_host.values.len(), 1);
let (main_fn_symbol, main_fn_var) = exposed_to_host.values.iter().next().unwrap();
let main_fn_symbol = *main_fn_symbol;
let main_fn_var = *main_fn_var;
// pretty-print the expr type string for later.
name_all_type_vars(main_fn_var, &mut subs);
let content = subs.get_content_without_compacting(main_fn_var);
let expr_type_str = content_to_string(content, &subs, home, &interns);
let (_, main_fn_layout) = match procedures.keys().find(|(s, _)| *s == main_fn_symbol) {
Some(layout) => *layout,
None => {
return Ok(ReplOutput::NoProblems {
expr: "<function>".to_string(),
expr_type: expr_type_str,
});
}
};
/*--------------------------------------------------------------------
START OF LLVM-SPECIFIC STUFF
--------------------------------------------------------------------*/
let module = arena.alloc(module);
let (module_pass, function_pass) =
roc_gen_llvm::llvm::build::construct_optimization_passes(module, opt_level);
let (dibuilder, compile_unit) = roc_gen_llvm::llvm::build::Env::new_debug_info(module);
// Compile and add all the Procs before adding main
let env = roc_gen_llvm::llvm::build::Env {
arena: &arena,
builder: &builder,
dibuilder: &dibuilder,
compile_unit: &compile_unit,
context: &context,
interns,
module,
target_info,
is_gen_test: true, // so roc_panic is generated
// important! we don't want any procedures to get the C calling convention
exposed_to_host: MutSet::default(),
};
// Add roc_alloc, roc_realloc, and roc_dealloc, since the repl has no
// platform to provide them.
add_default_roc_externs(&env);
let (main_fn_name, main_fn) = roc_gen_llvm::llvm::build::build_procedures_return_main(
&env,
opt_level,
procedures,
entry_point,
);
env.dibuilder.finalize();
// we don't use the debug info, and it causes weird errors.
module.strip_debug_info();
// Uncomment this to see the module's un-optimized LLVM instruction output:
// env.module.print_to_stderr();
if main_fn.verify(true) {
function_pass.run_on(&main_fn);
} else {
panic!("Main function {} failed LLVM verification in build. Uncomment things nearby to see more details.", main_fn_name);
}
module_pass.run_on(env.module);
// Uncomment this to see the module's optimized LLVM instruction output:
// env.module.print_to_stderr();
// Verify the module
if let Err(errors) = env.module.verify() {
panic!(
"Errors defining module:\n{}\n\nUncomment things nearby to see more details.",
errors.to_string()
);
}
let lib = module_to_dylib(env.module, &target, opt_level)
.expect("Error loading compiled dylib for test");
/*--------------------------------------------------------------------
END OF LLVM-SPECIFIC STUFF
--------------------------------------------------------------------*/
let res_answer = unsafe {
eval::jit_to_ast(
&arena,
lib,
main_fn_name,
main_fn_layout,
content,
&env.interns,
home,
&subs,
target_info,
&AppMemoryInternal,
)
};
let mut expr = roc_fmt::Buf::new_in(&arena);
use eval::ToAstProblem::*;
match res_answer {
Ok(answer) => {
answer.format_with_options(&mut expr, Parens::NotNeeded, Newlines::Yes, 0);
}
Err(FunctionLayout) => {
expr.indent(0);
expr.push_str("<function>");
}
}
Ok(ReplOutput::NoProblems {
expr: expr.into_bump_str().to_string(),
expr_type: expr_type_str,
})
Ok(loaded)
}
}

1
repl_eval/src/lib.rs
View File

@ -1,4 +1,3 @@
mod app_memory;
// mod debug; TODO: Is this in the right place? Seems to be specifically for solve.rs
mod eval;
pub mod gen;