Merge remote-tracking branch 'origin/trunk' into gen_wasm_join

This commit is contained in:
Folkert 2021-09-10 13:16:46 +02:00
commit daf0a0e02b
12 changed files with 144 additions and 366 deletions

View File

@ -11,7 +11,7 @@ env:
jobs:
build-fmt-clippy-test:
name: fmt, clippy, test --release
runs-on: [self-hosted]
runs-on: [self-hosted, i5-4690K]
timeout-minutes: 90
env:
FORCE_COLOR: 1

32
Cargo.lock generated
View File

@ -1949,9 +1949,9 @@ dependencies = [
[[package]]
name = "js-sys"
version = "0.3.53"
version = "0.3.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4bf49d50e2961077d9c99f4b7997d770a1114f087c3c2e0069b36c13fc2979d"
checksum = "1866b355d9c878e5e607473cbe3f63282c0b7aad2db1dbebf55076c686918254"
dependencies = [
"wasm-bindgen",
]
@ -4120,9 +4120,9 @@ dependencies = [
[[package]]
name = "sha2"
version = "0.9.6"
version = "0.9.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9204c41a1597a8c5af23c82d1c921cb01ec0a4c59e07a9c7306062829a3903f3"
checksum = "91e36fa7752016a6c4483706d634fd82c48860dd2df17a0cfaaebc714f23b2dd"
dependencies = [
"block-buffer 0.9.0",
"cfg-if 1.0.0",
@ -4785,9 +4785,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasm-bindgen"
version = "0.2.76"
version = "0.2.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ce9b1b516211d33767048e5d47fa2a381ed8b76fc48d2ce4aa39877f9f183e0"
checksum = "5e68338db6becec24d3c7977b5bf8a48be992c934b5d07177e3931f5dc9b076c"
dependencies = [
"cfg-if 1.0.0",
"wasm-bindgen-macro",
@ -4795,9 +4795,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.76"
version = "0.2.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfe8dc78e2326ba5f845f4b5bf548401604fa20b1dd1d365fb73b6c1d6364041"
checksum = "f34c405b4f0658583dba0c1c7c9b694f3cac32655db463b56c254a1c75269523"
dependencies = [
"bumpalo",
"lazy_static",
@ -4810,9 +4810,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.26"
version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95fded345a6559c2cfee778d562300c581f7d4ff3edb9b0d230d69800d213972"
checksum = "a87d738d4abc4cf22f6eb142f5b9a81301331ee3c767f2fef2fda4e325492060"
dependencies = [
"cfg-if 1.0.0",
"js-sys",
@ -4822,9 +4822,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.76"
version = "0.2.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44468aa53335841d9d6b6c023eaab07c0cd4bddbcfdee3e2bb1e8d2cb8069fef"
checksum = "b9d5a6580be83b19dc570a8f9c324251687ab2184e57086f71625feb57ec77c8"
dependencies = [
"quote 1.0.9",
"wasm-bindgen-macro-support",
@ -4832,9 +4832,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.76"
version = "0.2.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0195807922713af1e67dc66132c7328206ed9766af3858164fb583eedc25fbad"
checksum = "e3775a030dc6f5a0afd8a84981a21cc92a781eb429acef9ecce476d0c9113e92"
dependencies = [
"proc-macro2 1.0.29",
"quote 1.0.9",
@ -4845,9 +4845,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.76"
version = "0.2.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acdb075a845574a1fa5f09fd77e43f7747599301ea3417a9fbffdeedfc1f4a29"
checksum = "c279e376c7a8e8752a8f1eaa35b7b0bee6bb9fb0cdacfa97cc3f1f289c87e2b4"
[[package]]
name = "wasmer"

View File

@ -327,7 +327,7 @@ mod cli_run {
// TODO fix QuicksortApp and then remove this!
match benchmark.filename {
"QuicksortApp.roc" | "TestBase64.roc" => {
"QuicksortApp.roc" => {
eprintln!("WARNING: skipping testing benchmark {} because the test is broken right now!", benchmark.filename);
return;
}
@ -608,6 +608,14 @@ fn run_with_wasmer(wasm_path: &std::path::Path, stdin: &[&str]) -> String {
use std::io::Write;
use wasmer::{Instance, Module, Store};
// std::process::Command::new("cp")
// .args(&[
// wasm_path.to_str().unwrap(),
// "/home/folkertdev/roc/wasm/nqueens.wasm",
// ])
// .output()
// .unwrap();
let store = Store::default();
let module = Module::from_file(&store, &wasm_path).unwrap();

View File

@ -606,26 +606,27 @@ fn link_wasm32(
) -> io::Result<(Child, PathBuf)> {
let zig_str_path = find_zig_str_path();
let child =
Command::new("/home/folkertdev/Downloads/zig-linux-x86_64-0.9.0-dev.848+d5ef5da59/zig")
// .env_clear()
// .env("PATH", &env_path)
.args(&["build-exe"])
.args(input_paths)
.args([
&format!("-femit-bin={}", output_path.to_str().unwrap()),
// include libc
"-lc",
"-target",
"wasm32-wasi",
"--pkg-begin",
"str",
zig_str_path.to_str().unwrap(),
"--pkg-end",
// useful for debugging
// "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll",
])
.spawn()?;
let child = Command::new("zig9")
// .env_clear()
// .env("PATH", &env_path)
.args(&["build-exe"])
.args(input_paths)
.args([
&format!("-femit-bin={}", output_path.to_str().unwrap()),
// include libc
"-lc",
"-target",
"wasm32-wasi-musl",
"--pkg-begin",
"str",
zig_str_path.to_str().unwrap(),
"--pkg-end",
"--strip",
// "-O", "ReleaseSmall",
// useful for debugging
// "-femit-llvm-ir=/home/folkertdev/roc/roc/examples/benchmarks/platform/host.ll",
])
.spawn()?;
Ok((child, output_path))
}

View File

@ -1150,8 +1150,8 @@ fn strToBytes(arg: RocStr) RocList {
}
const FromUtf8Result = extern struct {
byte_index: usize,
string: RocStr,
byte_index: usize,
is_ok: bool,
problem_code: Utf8ByteProblem,
};

View File

@ -2189,8 +2189,15 @@ fn list_literal<'a, 'ctx, 'env>(
let global = {
let mut global_elements = Vec::with_capacity_in(list_length, env.arena);
// insert NULL bytes for the refcount
// these elements are (dropped again if the list contains non-constants)
// Add zero bytes that represent the refcount
//
// - if all elements are const, then we store the whole list as a constant.
// It then needs a refcount before the first element.
// - but if the list is not all constants, then we will just copy the constant values,
// and we do not need that refcount at the start
//
// In the latter case, we won't store the zeros in the globals
// (we slice them off again below)
for _ in 0..zero_elements {
global_elements.push(element_type.const_zero());
}

View File

@ -1,5 +1,5 @@
use crate::llvm::bitcode::{call_bitcode_fn, call_void_bitcode_fn};
use crate::llvm::build::{complex_bitcast, Env, Scope};
use crate::llvm::build::{complex_bitcast, struct_from_fields, Env, Scope};
use crate::llvm::build_list::{allocate_list, call_bitcode_fn_returns_list, store_list};
use inkwell::builder::Builder;
use inkwell::values::{BasicValueEnum, FunctionValue, IntValue, PointerValue, StructValue};
@ -260,6 +260,73 @@ pub fn str_to_utf8<'a, 'ctx, 'env>(
call_bitcode_fn_returns_list(env, &[string], bitcode::STR_TO_UTF8)
}
fn decode_from_utf8_result<'a, 'ctx, 'env>(
env: &Env<'a, 'ctx, 'env>,
pointer: PointerValue<'ctx>,
) -> StructValue<'ctx> {
let builder = env.builder;
let ctx = env.context;
let fields = match env.ptr_bytes {
8 => [
env.ptr_int().into(),
super::convert::zig_str_type(env).into(),
env.context.bool_type().into(),
ctx.i8_type().into(),
],
4 => [
super::convert::zig_str_type(env).into(),
env.ptr_int().into(),
env.context.bool_type().into(),
ctx.i8_type().into(),
],
_ => unreachable!(),
};
let record_type = env.context.struct_type(&fields, false);
match env.ptr_bytes {
8 => {
let zig_struct = builder
.build_load(pointer, "load_utf8_validate_bytes_result")
.into_struct_value();
let string = builder
.build_extract_value(zig_struct, 0, "string")
.unwrap();
let byte_index = builder
.build_extract_value(zig_struct, 1, "byte_index")
.unwrap();
let is_ok = builder.build_extract_value(zig_struct, 2, "is_ok").unwrap();
let problem_code = builder
.build_extract_value(zig_struct, 3, "problem_code")
.unwrap();
let values = [byte_index, string, is_ok, problem_code];
struct_from_fields(env, record_type, values.iter().copied().enumerate())
}
4 => {
let result_ptr_cast = env
.builder
.build_bitcast(
pointer,
record_type.ptr_type(AddressSpace::Generic),
"to_unnamed",
)
.into_pointer_value();
builder
.build_load(result_ptr_cast, "load_utf8_validate_bytes_result")
.into_struct_value()
}
_ => unreachable!(),
}
}
/// Str.fromUtf8 : List U8, { count : Nat, start : Nat } -> { a : Bool, b : Str, c : Nat, d : I8 }
pub fn str_from_utf8_range<'a, 'ctx, 'env>(
env: &Env<'a, 'ctx, 'env>,
@ -268,7 +335,6 @@ pub fn str_from_utf8_range<'a, 'ctx, 'env>(
count_and_start: StructValue<'ctx>,
) -> BasicValueEnum<'ctx> {
let builder = env.builder;
let ctx = env.context;
let result_type = env.module.get_struct_type("str.FromUtf8Result").unwrap();
let result_ptr = builder.build_alloca(result_type, "alloca_utf8_validate_bytes_result");
@ -293,26 +359,7 @@ pub fn str_from_utf8_range<'a, 'ctx, 'env>(
bitcode::STR_FROM_UTF8_RANGE,
);
let record_type = env.context.struct_type(
&[
env.ptr_int().into(),
super::convert::zig_str_type(env).into(),
env.context.bool_type().into(),
ctx.i8_type().into(),
],
false,
);
let result_ptr_cast = env
.builder
.build_bitcast(
result_ptr,
record_type.ptr_type(AddressSpace::Generic),
"to_unnamed",
)
.into_pointer_value();
builder.build_load(result_ptr_cast, "load_utf8_validate_bytes_result")
decode_from_utf8_result(env, result_ptr).into()
}
/// Str.fromUtf8 : List U8 -> { a : Bool, b : Str, c : Nat, d : I8 }
@ -322,7 +369,6 @@ pub fn str_from_utf8<'a, 'ctx, 'env>(
original_wrapper: StructValue<'ctx>,
) -> BasicValueEnum<'ctx> {
let builder = env.builder;
let ctx = env.context;
let result_type = env.module.get_struct_type("str.FromUtf8Result").unwrap();
let result_ptr = builder.build_alloca(result_type, "alloca_utf8_validate_bytes_result");
@ -341,26 +387,7 @@ pub fn str_from_utf8<'a, 'ctx, 'env>(
bitcode::STR_FROM_UTF8,
);
let record_type = env.context.struct_type(
&[
env.ptr_int().into(),
super::convert::zig_str_type(env).into(),
env.context.bool_type().into(),
ctx.i8_type().into(),
],
false,
);
let result_ptr_cast = env
.builder
.build_bitcast(
result_ptr,
record_type.ptr_type(AddressSpace::Generic),
"to_unnamed",
)
.into_pointer_value();
builder.build_load(result_ptr_cast, "load_utf8_validate_bytes_result")
decode_from_utf8_result(env, result_ptr).into()
}
/// Str.fromInt : Int -> Str

View File

@ -139,7 +139,7 @@ where
let is_gen_test = true;
let instance =
crate::helpers::eval_simple::helper_wasm(&arena, src, stdlib, is_gen_test, ignore_problems);
crate::helpers::eval::helper_wasm(&arena, src, stdlib, is_gen_test, ignore_problems);
let main_function = instance.exports.get_function("#UserApp_main_1").unwrap();
@ -166,8 +166,7 @@ where
#[macro_export]
macro_rules! assert_wasm_evals_to {
($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems:expr) => {
match $crate::helpers::eval_simple::assert_wasm_evals_to_help::<$ty>($src, $ignore_problems)
{
match $crate::helpers::eval::assert_wasm_evals_to_help::<$ty>($src, $ignore_problems) {
Err(msg) => println!("{:?}", msg),
Ok(actual) => {
#[allow(clippy::bool_assert_comparison)]
@ -177,13 +176,7 @@ macro_rules! assert_wasm_evals_to {
};
($src:expr, $expected:expr, $ty:ty) => {
$crate::assert_wasm_evals_to!(
$src,
$expected,
$ty,
$crate::helpers::eval_simple::identity,
false
);
$crate::assert_wasm_evals_to!($src, $expected, $ty, $crate::helpers::eval::identity, false);
};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
@ -194,7 +187,7 @@ macro_rules! assert_wasm_evals_to {
#[macro_export]
macro_rules! assert_evals_to {
($src:expr, $expected:expr, $ty:ty) => {{
assert_evals_to!($src, $expected, $ty, $crate::helpers::eval_simple::identity);
assert_evals_to!($src, $expected, $ty, $crate::helpers::eval::identity);
}};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
// Same as above, except with an additional transformation argument.

View File

@ -1,264 +0,0 @@
use roc_can::builtins::builtin_defs_map;
use roc_collections::all::{MutMap, MutSet};
use roc_gen_wasm::from_wasm32_memory::FromWasm32Memory;
fn promote_expr_to_module(src: &str) -> String {
let mut buffer = String::from("app \"test\" provides [ main ] to \"./platform\"\n\nmain =\n");
for line in src.lines() {
// indent the body!
buffer.push_str(" ");
buffer.push_str(line);
buffer.push('\n');
}
buffer
}
#[allow(dead_code)]
pub fn helper_wasm<'a>(
arena: &'a bumpalo::Bump,
src: &str,
stdlib: &'a roc_builtins::std::StdLib,
_is_gen_test: bool,
_ignore_problems: bool,
) -> wasmer::Instance {
use std::path::{Path, PathBuf};
let filename = PathBuf::from("Test.roc");
let src_dir = Path::new("fake/test/path");
let module_src;
let temp;
if src.starts_with("app") {
// this is already a module
module_src = src;
} else {
// this is an expression, promote it to a module
temp = promote_expr_to_module(src);
module_src = &temp;
}
let exposed_types = MutMap::default();
let loaded = roc_load::file::load_and_monomorphize_from_str(
arena,
filename,
module_src,
stdlib,
src_dir,
exposed_types,
8,
builtin_defs_map,
);
let loaded = loaded.expect("failed to load module");
use roc_load::file::MonomorphizedModule;
let MonomorphizedModule {
procedures: top_procedures,
interns,
exposed_to_host,
..
} = loaded;
let mut procedures = MutMap::default();
for (key, proc) in top_procedures {
procedures.insert(key, proc);
}
// You can comment and uncomment this block out to get more useful information
// while you're working on the wasm backend!
// {
// println!("=========== Procedures ==========");
// println!("{:?}", procedures);
// println!("=================================\n");
// println!("=========== Interns ==========");
// println!("{:?}", interns);
// println!("=================================\n");
// println!("=========== Exposed ==========");
// println!("{:?}", exposed_to_host);
// println!("=================================\n");
// }
let exposed_to_host = exposed_to_host.keys().copied().collect::<MutSet<_>>();
let env = gen_wasm::Env {
arena,
interns,
exposed_to_host,
};
let module_bytes = gen_wasm::build_module(&env, procedures).unwrap();
// for debugging (e.g. with wasm2wat)
if true {
use std::io::Write;
let mut file =
std::fs::File::create("/home/brian/Documents/roc/compiler/gen_wasm/debug.wasm")
.unwrap();
file.write_all(&module_bytes).unwrap();
}
// now, do wasmer stuff
use wasmer::{Function, Instance, Module, Store};
let store = Store::default();
// let module = Module::from_file(&store, &test_wasm_path).unwrap();
let module = Module::from_binary(&store, &module_bytes).unwrap();
// First, we create the `WasiEnv`
use wasmer_wasi::WasiState;
let mut wasi_env = WasiState::new("hello")
// .args(&["world"])
// .env("KEY", "Value")
.finalize()
.unwrap();
// Then, we get the import object related to our WASI
// and attach it to the Wasm instance.
let mut import_object = wasi_env
.import_object(&module)
.unwrap_or_else(|_| wasmer::imports!());
{
let mut exts = wasmer::Exports::new();
let main_function = Function::new_native(&store, fake_wasm_main_function);
let ext = wasmer::Extern::Function(main_function);
exts.insert("main", ext);
let main_function = Function::new_native(&store, wasm_roc_panic);
let ext = wasmer::Extern::Function(main_function);
exts.insert("roc_panic", ext);
import_object.register("env", exts);
}
Instance::new(&module, &import_object).unwrap()
}
#[allow(dead_code)]
fn wasm_roc_panic(address: u32, tag_id: u32) {
match tag_id {
0 => {
let mut string = "";
MEMORY.with(|f| {
let memory = f.borrow().unwrap();
let ptr: wasmer::WasmPtr<u8, wasmer::Array> = wasmer::WasmPtr::new(address);
let width = 100;
let c_ptr = (ptr.deref(memory, 0, width)).unwrap();
use libc::c_char;
use std::ffi::CStr;
let slice = unsafe { CStr::from_ptr(c_ptr as *const _ as *const c_char) };
string = slice.to_str().unwrap();
});
panic!("Roc failed with message: {:?}", string)
}
_ => todo!(),
}
}
use std::cell::RefCell;
thread_local! {
pub static MEMORY: RefCell<Option<&'static wasmer::Memory>> = RefCell::new(None);
}
#[allow(dead_code)]
fn fake_wasm_main_function(_: u32, _: u32) -> u32 {
panic!("wasm entered the main function; this should never happen!")
}
#[allow(dead_code)]
pub fn assert_wasm_evals_to_help<T>(src: &str, ignore_problems: bool) -> Result<T, String>
where
T: FromWasm32Memory,
{
let arena = bumpalo::Bump::new();
// NOTE the stdlib must be in the arena; just taking a reference will segfault
let stdlib = arena.alloc(roc_builtins::std::standard_stdlib());
let is_gen_test = true;
let instance =
crate::helpers::eval_full::helper_wasm(&arena, src, stdlib, is_gen_test, ignore_problems);
let memory = instance.exports.get_memory("memory").unwrap();
crate::helpers::eval_full::MEMORY.with(|f| {
*f.borrow_mut() = Some(unsafe { std::mem::transmute(memory) });
});
let test_wrapper = instance.exports.get_function("test_wrapper").unwrap();
match test_wrapper.call(&[]) {
Err(e) => Err(format!("{:?}", e)),
Ok(result) => {
let address = match result[0] {
wasmer::Value::I32(a) => a,
_ => panic!(),
};
let output = <T as crate::helpers::eval_full::FromWasm32Memory>::decode(
memory,
// skip the RocCallResult tag id
address as u32 + 8,
);
Ok(output)
}
}
}
#[macro_export]
macro_rules! assert_wasm_evals_to {
($src:expr, $expected:expr, $ty:ty, $transform:expr, $ignore_problems:expr) => {
match $crate::helpers::eval_full::assert_wasm_evals_to_help::<$ty>($src, $ignore_problems) {
Err(msg) => println!("{:?}", msg),
Ok(actual) => {
#[allow(clippy::bool_assert_comparison)]
assert_eq!($transform(actual), $expected)
}
}
};
($src:expr, $expected:expr, $ty:ty) => {
$crate::assert_wasm_evals_to!(
$src,
$expected,
$ty,
$crate::helpers::eval_full::identity,
false
);
};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
$crate::assert_wasm_evals_to!($src, $expected, $ty, $transform, false);
};
}
#[macro_export]
macro_rules! assert_evals_to {
($src:expr, $expected:expr, $ty:ty) => {{
assert_evals_to!($src, $expected, $ty, $crate::helpers::eval_full::identity);
}};
($src:expr, $expected:expr, $ty:ty, $transform:expr) => {
// Same as above, except with an additional transformation argument.
{
$crate::assert_wasm_evals_to!($src, $expected, $ty, $transform, false);
}
};
}
#[allow(dead_code)]
pub fn identity<T>(value: T) -> T {
value
}

View File

@ -1,7 +1,7 @@
extern crate bumpalo;
#[macro_use]
pub mod eval_simple;
pub mod eval;
/// Used in the with_larger_debug_stack() function, for tests that otherwise
/// run out of stack space in debug builds (but don't in --release builds)

View File

@ -1151,10 +1151,15 @@ impl<'a> Builtin<'a> {
Float64 => align_of::<f64>() as u32,
Float32 => align_of::<f32>() as u32,
Float16 => align_of::<i16>() as u32,
Str | EmptyStr => pointer_size,
Dict(_, _) | EmptyDict => pointer_size,
Set(_) | EmptySet => pointer_size,
List(_) | EmptyList => pointer_size,
// we often treat these as i128 (64-bit systems)
// or i64 (32-bit systems).
//
// In webassembly, For that to be safe
// they must be aligned to allow such access
List(_) | EmptyList => pointer_size.max(8),
Str | EmptyStr => pointer_size.max(8),
}
}
@ -1240,9 +1245,10 @@ impl<'a> Builtin<'a> {
Builtin::Str => pointer_size,
Builtin::Dict(k, v) => k
.alignment_bytes(pointer_size)
.max(v.alignment_bytes(pointer_size)),
Builtin::Set(k) => k.alignment_bytes(pointer_size),
Builtin::List(e) => e.alignment_bytes(pointer_size),
.max(v.alignment_bytes(pointer_size))
.max(pointer_size),
Builtin::Set(k) => k.alignment_bytes(pointer_size).max(pointer_size),
Builtin::List(e) => e.alignment_bytes(pointer_size).max(pointer_size),
Builtin::EmptyStr | Builtin::EmptyList | Builtin::EmptyDict | Builtin::EmptySet => {
unreachable!("not heap-allocated")
}

View File

@ -366,7 +366,7 @@ pub fn helper_wasm<'a>(
use std::process::Command;
Command::new("/home/folkertdev/Downloads/zig-linux-x86_64-0.9.0-dev.848+d5ef5da59/zig")
Command::new("zig9")
.current_dir(dir_path)
.args(&[
"wasm-ld",