roc-lang/roc
1
1
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2024-09-20 07:17:50 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #701 from rtfeldman/improved-offset

Browse Source 
Improved offset
This commit is contained in:
Richard Feldman 2020-11-17 22:48:15 -05:00 committed by GitHub
commit 63f930440b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 312 additions and 212 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
compiler/gen/src/llvm/build.rs
View File

@ -9,8 +9,8 @@ use crate::llvm::convert::{
basic_type_from_layout, block_of_memory, collection, get_fn_type, get_ptr_type, ptr_int,
};
use crate::llvm::refcounting::{
decrement_refcount_layout, increment_refcount_layout, list_get_refcount_ptr,
refcount_is_one_comparison,
decrement_refcount_layout, increment_refcount_layout, refcount_is_one_comparison,
PointerToRefcount,
};
use bumpalo::collections::Vec;
use bumpalo::Bump;
@ -989,57 +989,54 @@ pub fn allocate_with_refcount<'a, 'ctx, 'env>(
// bytes per element
let bytes_len = len_type.const_int(value_bytes, false);
let offset = crate::llvm::refcounting::refcount_offset(env, layout);
let extra_bytes = layout.alignment_bytes(env.ptr_bytes);
let extra_bytes_intvalue = len_type.const_int(extra_bytes as u64, false);
let ptr = {
let len = bytes_len;
let len =
builder.build_int_add(len, len_type.const_int(offset, false), "add_refcount_space");
let len = builder.build_int_add(len, extra_bytes_intvalue, "add_alignment_space");
env.builder
.build_array_malloc(ctx.i8_type(), len, "create_list_ptr")
.build_array_malloc(ctx.i8_type(), len, "create_ptr")
.unwrap()
// TODO check if malloc returned null; if so, runtime error for OOM!
};
// We must return a pointer to the first element:
let ptr_bytes = env.ptr_bytes;
let int_type = ptr_int(ctx, ptr_bytes);
let ptr_as_int = builder.build_ptr_to_int(ptr, int_type, "allocate_refcount_pti");
let incremented = builder.build_int_add(
ptr_as_int,
ctx.i64_type().const_int(offset, false),
"increment_list_ptr",
);
let data_ptr = {
let int_type = ptr_int(ctx, env.ptr_bytes);
let ptr_as_int = builder.build_ptr_to_int(ptr, int_type, "calculate_data_ptr_pti");
let incremented = builder.build_int_add(
ptr_as_int,
extra_bytes_intvalue,
"calculate_data_ptr_increment",
);
let ptr_type = get_ptr_type(&value_type, AddressSpace::Generic);
let list_element_ptr = builder.build_int_to_ptr(incremented, ptr_type, "allocate_refcount_itp");
let ptr_type = get_ptr_type(&value_type, AddressSpace::Generic);
builder.build_int_to_ptr(incremented, ptr_type, "calculate_data_ptr")
};
// subtract ptr_size, to access the refcount
let refcount_ptr = builder.build_int_sub(
incremented,
ctx.i64_type().const_int(env.ptr_bytes as u64, false),
"refcount_ptr",
);
let refcount_ptr = match extra_bytes {
n if n == env.ptr_bytes => {
// the malloced pointer is the same as the refcounted pointer
unsafe { PointerToRefcount::from_ptr(env, ptr) }
}
n if n == 2 * env.ptr_bytes => {
// the refcount is stored just before the start of the actual data
// but in this case (because of alignment) not at the start of the malloced buffer
PointerToRefcount::from_ptr_to_data(env, data_ptr)
}
n => unreachable!("invalid extra_bytes {}", n),
};
let refcount_ptr = builder.build_int_to_ptr(
refcount_ptr,
int_type.ptr_type(AddressSpace::Generic),
"make ptr",
);
// the refcount of a new allocation is initially 1
// we assume that the allocation is indeed used (dead variables are eliminated)
builder.build_store(
refcount_ptr,
crate::llvm::refcounting::refcount_1(ctx, env.ptr_bytes),
);
let rc1 = crate::llvm::refcounting::refcount_1(ctx, env.ptr_bytes);
refcount_ptr.set_refcount(env, rc1);
// store the value in the pointer
builder.build_store(list_element_ptr, value);
builder.build_store(data_ptr, value);
list_element_ptr
data_ptr
}
fn list_literal<'a, 'ctx, 'env>(
@ -2654,12 +2651,8 @@ where
let ret_type = basic_type_from_layout(env.arena, ctx, list_layout, env.ptr_bytes);
let refcount_ptr = list_get_refcount_ptr(env, list_layout, original_wrapper);
let refcount = env
.builder
.build_load(refcount_ptr, "get_refcount")
.into_int_value();
let refcount_ptr = PointerToRefcount::from_list_wrapper(env, original_wrapper);
let refcount = refcount_ptr.get_refcount(env);
let comparison = refcount_is_one_comparison(env, refcount);

415
compiler/gen/src/llvm/refcounting.rs
View File

@ -5,7 +5,6 @@ use crate::llvm::build::{
use crate::llvm::build_list::list_len;
use crate::llvm::convert::{basic_type_from_layout, block_of_memory, ptr_int};
use bumpalo::collections::Vec;
use inkwell::basic_block::BasicBlock;
use inkwell::context::Context;
use inkwell::module::Linkage;
use inkwell::values::{BasicValueEnum, FunctionValue, IntValue, PointerValue, StructValue};
@ -28,6 +27,243 @@ pub fn refcount_1(ctx: &Context, ptr_bytes: u32) -> IntValue<'_> {
}
}
pub struct PointerToRefcount<'ctx> {
value: PointerValue<'ctx>,
}
impl<'ctx> PointerToRefcount<'ctx> {
/// # Safety
///
/// the invariant is that the given pointer really points to the refcount,
/// not the data, and only is the start of the malloced buffer if the alignment
/// works out that way.
pub unsafe fn from_ptr<'a, 'env>(env: &Env<'a, 'ctx, 'env>, ptr: PointerValue<'ctx>) -> Self {
// must make sure it's a pointer to usize
let refcount_type = ptr_int(env.context, env.ptr_bytes);
let value = cast_basic_basic(
env.builder,
ptr.into(),
refcount_type.ptr_type(AddressSpace::Generic).into(),
)
.into_pointer_value();
Self { value }
}
pub fn from_ptr_to_data<'a, 'env>(
env: &Env<'a, 'ctx, 'env>,
data_ptr: PointerValue<'ctx>,
) -> Self {
// pointer to usize
let refcount_type = ptr_int(env.context, env.ptr_bytes);
let ptr_as_int =
cast_basic_basic(env.builder, data_ptr.into(), refcount_type.into()).into_int_value();
// subtract offset, to access the refcount
let refcount_ptr_as_int = env.builder.build_int_sub(
ptr_as_int,
refcount_type.const_int(env.ptr_bytes as u64, false),
"make_refcount_ptr",
);
let refcount_ptr = env.builder.build_int_to_ptr(
refcount_ptr_as_int,
refcount_type.ptr_type(AddressSpace::Generic),
"get_refcount_ptr",
);
Self {
value: refcount_ptr,
}
}
pub fn from_list_wrapper(env: &Env<'_, 'ctx, '_>, list_wrapper: StructValue<'ctx>) -> Self {
let ptr_as_int = env
.builder
.build_extract_value(list_wrapper, Builtin::WRAPPER_PTR, "read_list_ptr")
.unwrap()
.into_int_value();
let ptr = env.builder.build_int_to_ptr(
ptr_as_int,
env.context.i64_type().ptr_type(AddressSpace::Generic),
"list_int_to_ptr",
);
Self::from_ptr_to_data(env, ptr)
}
pub fn get_refcount<'a, 'env>(&self, env: &Env<'a, 'ctx, 'env>) -> IntValue<'ctx> {
env.builder
.build_load(self.value, "get_refcount")
.into_int_value()
}
pub fn set_refcount<'a, 'env>(&self, env: &Env<'a, 'ctx, 'env>, refcount: IntValue<'ctx>) {
env.builder.build_store(self.value, refcount);
}
fn increment<'a, 'env>(&self, env: &Env<'a, 'ctx, 'env>) {
let refcount = self.get_refcount(env);
let builder = env.builder;
let refcount_type = ptr_int(env.context, env.ptr_bytes);
let max = builder.build_int_compare(
IntPredicate::EQ,
refcount,
refcount_type.const_int(REFCOUNT_MAX as u64, false),
"refcount_max_check",
);
let incremented = builder.build_int_add(
refcount,
refcount_type.const_int(1 as u64, false),
"increment_refcount",
);
let new_refcount = builder
.build_select(max, refcount, incremented, "select_refcount")
.into_int_value();
self.set_refcount(env, new_refcount);
}
fn decrement<'a, 'env>(&self, env: &Env<'a, 'ctx, 'env>, layout: &Layout<'a>) {
let context = env.context;
let block = env.builder.get_insert_block().expect("to be in a function");
let alignment = layout.alignment_bytes(env.ptr_bytes);
let fn_name = &format!("decrement_refcounted_ptr_{}", alignment);
let function = match env.module.get_function(fn_name) {
Some(function_value) => function_value,
None => {
// inc and dec return void
let fn_type = context.void_type().fn_type(
&[context.i64_type().ptr_type(AddressSpace::Generic).into()],
false,
);
let function_value =
env.module
.add_function(fn_name, fn_type, Some(Linkage::Private));
// Because it's an internal-only function, it should use the fast calling convention.
function_value.set_call_conventions(FAST_CALL_CONV);
Self::_build_decrement_function_body(env, function_value, alignment);
function_value
}
};
let refcount_ptr = self.value;
env.builder.position_at_end(block);
let call = env
.builder
.build_call(function, &[refcount_ptr.into()], fn_name);
call.set_call_convention(FAST_CALL_CONV);
}
fn _build_decrement_function_body<'a, 'env>(
env: &Env<'a, 'ctx, 'env>,
parent: FunctionValue<'ctx>,
extra_bytes: u32,
) {
let builder = env.builder;
let ctx = env.context;
let refcount_type = ptr_int(ctx, env.ptr_bytes);
let entry = ctx.append_basic_block(parent, "entry");
builder.position_at_end(entry);
let refcount_ptr = {
let raw_refcount_ptr = parent.get_nth_param(0).unwrap();
debug_assert!(raw_refcount_ptr.is_pointer_value());
Self {
value: raw_refcount_ptr.into_pointer_value(),
}
};
let refcount = refcount_ptr.get_refcount(env);
let add_with_overflow = env
.call_intrinsic(
LLVM_SADD_WITH_OVERFLOW_I64,
&[
refcount.into(),
refcount_type.const_int((-1 as i64) as u64, true).into(),
],
)
.into_struct_value();
let has_overflowed = builder
.build_extract_value(add_with_overflow, 1, "has_overflowed")
.unwrap();
let has_overflowed_comparison = builder.build_int_compare(
IntPredicate::EQ,
has_overflowed.into_int_value(),
ctx.bool_type().const_int(1 as u64, false),
"has_overflowed",
);
// build blocks
let then_block = ctx.append_basic_block(parent, "then");
let else_block = ctx.append_basic_block(parent, "else");
// TODO what would be most optimial for the branch predictor
//
// are most refcounts 1 most of the time? or not?
builder.build_conditional_branch(has_overflowed_comparison, then_block, else_block);
// build then block
{
builder.position_at_end(then_block);
if !env.leak {
match extra_bytes {
n if env.ptr_bytes == n => {
// the refcount ptr is also the ptr to the malloced region
builder.build_free(refcount_ptr.value);
}
n if 2 * env.ptr_bytes == n => {
// we need to step back another ptr_bytes to get the malloced ptr
let malloced = Self::from_ptr_to_data(env, refcount_ptr.value);
builder.build_free(malloced.value);
}
n => unreachable!("invalid extra_bytes {:?}", n),
}
}
builder.build_return(None);
}
// build else block
{
builder.position_at_end(else_block);
let max = builder.build_int_compare(
IntPredicate::EQ,
refcount,
refcount_type.const_int(REFCOUNT_MAX as u64, false),
"refcount_max_check",
);
let decremented = builder
.build_extract_value(add_with_overflow, 0, "decrement_refcount")
.unwrap()
.into_int_value();
let selected = builder.build_select(max, refcount, decremented, "select_refcount");
refcount_ptr.set_refcount(env, selected.into_int_value());
builder.build_return(None);
}
}
}
pub fn decrement_refcount_struct<'a, 'ctx, 'env>(
env: &Env<'a, 'ctx, 'env>,
parent: FunctionValue<'ctx>,
@ -332,8 +568,8 @@ fn build_inc_list_help<'a, 'ctx, 'env>(
builder.position_at_end(increment_block);
let refcount_ptr = list_get_refcount_ptr(env, layout, original_wrapper);
increment_refcount_help(env, refcount_ptr);
let refcount_ptr = PointerToRefcount::from_list_wrapper(env, original_wrapper);
refcount_ptr.increment(env);
builder.build_unconditional_branch(cont_block);
@ -410,7 +646,7 @@ fn build_dec_list_help<'a, 'ctx, 'env>(
let parent = fn_val;
// the block we'll always jump to when we're done
let cont_block = ctx.append_basic_block(parent, "after_decrement_block");
let cont_block = ctx.append_basic_block(parent, "after_decrement_block_build_dec_list_help");
let decrement_block = ctx.append_basic_block(parent, "decrement_block");
// currently, an empty list has a null-pointer in its length is 0
@ -431,9 +667,12 @@ fn build_dec_list_help<'a, 'ctx, 'env>(
builder.build_conditional_branch(is_non_empty, decrement_block, cont_block);
builder.position_at_end(decrement_block);
let refcount_ptr = list_get_refcount_ptr(env, layout, original_wrapper);
let refcount_ptr = PointerToRefcount::from_list_wrapper(env, original_wrapper);
refcount_ptr.decrement(env, layout);
decrement_refcount_help(env, parent, refcount_ptr, cont_block);
env.builder.build_unconditional_branch(cont_block);
builder.position_at_end(cont_block);
// this function returns void
builder.build_return(None);
@ -527,8 +766,9 @@ fn build_inc_str_help<'a, 'ctx, 'env>(
builder.build_conditional_branch(is_big_and_non_empty, decrement_block, cont_block);
builder.position_at_end(decrement_block);
let refcount_ptr = list_get_refcount_ptr(env, layout, str_wrapper);
increment_refcount_help(env, refcount_ptr);
let refcount_ptr = PointerToRefcount::from_list_wrapper(env, str_wrapper);
refcount_ptr.increment(env);
builder.build_unconditional_branch(cont_block);
builder.position_at_end(cont_block);
@ -619,150 +859,23 @@ fn build_dec_str_help<'a, 'ctx, 'env>(
);
// the block we'll always jump to when we're done
let cont_block = ctx.append_basic_block(parent, "after_decrement_block");
let cont_block = ctx.append_basic_block(parent, "after_decrement_block_build_dec_str_help");
let decrement_block = ctx.append_basic_block(parent, "decrement_block");
builder.build_conditional_branch(is_big_and_non_empty, decrement_block, cont_block);
builder.position_at_end(decrement_block);
let refcount_ptr = list_get_refcount_ptr(env, layout, str_wrapper);
decrement_refcount_help(env, parent, refcount_ptr, cont_block);
let refcount_ptr = PointerToRefcount::from_list_wrapper(env, str_wrapper);
refcount_ptr.decrement(env, layout);
builder.build_unconditional_branch(cont_block);
builder.position_at_end(cont_block);
// this function returns void
builder.build_return(None);
}
fn increment_refcount_ptr<'a, 'ctx, 'env>(
env: &Env<'a, 'ctx, 'env>,
layout: &Layout<'a>,
field_ptr: PointerValue<'ctx>,
) {
let refcount_ptr = get_refcount_ptr(env, layout, field_ptr);
increment_refcount_help(env, refcount_ptr);
}
fn increment_refcount_help<'a, 'ctx, 'env>(
env: &Env<'a, 'ctx, 'env>,
refcount_ptr: PointerValue<'ctx>,
) {
let builder = env.builder;
let ctx = env.context;
let refcount_type = ptr_int(ctx, env.ptr_bytes);
let refcount = env
.builder
.build_load(refcount_ptr, "get_refcount")
.into_int_value();
let max = builder.build_int_compare(
IntPredicate::EQ,
refcount,
refcount_type.const_int(REFCOUNT_MAX as u64, false),
"refcount_max_check",
);
let incremented = builder.build_int_add(
refcount,
refcount_type.const_int(1 as u64, false),
"increment_refcount",
);
let selected = builder.build_select(max, refcount, incremented, "select_refcount");
builder.build_store(refcount_ptr, selected);
}
fn decrement_refcount_ptr<'a, 'ctx, 'env>(
env: &Env<'a, 'ctx, 'env>,
parent: FunctionValue<'ctx>,
layout: &Layout<'a>,
field_ptr: PointerValue<'ctx>,
) {
let ctx = env.context;
// the block we'll always jump to when we're done
let cont_block = ctx.append_basic_block(parent, "after_decrement_block");
let refcount_ptr = get_refcount_ptr(env, layout, field_ptr);
decrement_refcount_help(env, parent, refcount_ptr, cont_block);
}
fn decrement_refcount_help<'a, 'ctx, 'env>(
env: &Env<'a, 'ctx, 'env>,
parent: FunctionValue<'ctx>,
refcount_ptr: PointerValue<'ctx>,
cont_block: BasicBlock,
) {
let builder = env.builder;
let ctx = env.context;
let refcount_type = ptr_int(ctx, env.ptr_bytes);
let refcount = env
.builder
.build_load(refcount_ptr, "get_refcount")
.into_int_value();
let add_with_overflow = env
.call_intrinsic(
LLVM_SADD_WITH_OVERFLOW_I64,
&[
refcount.into(),
refcount_type.const_int((-1 as i64) as u64, true).into(),
],
)
.into_struct_value();
let has_overflowed = builder
.build_extract_value(add_with_overflow, 1, "has_overflowed")
.unwrap();
let has_overflowed_comparison = builder.build_int_compare(
IntPredicate::EQ,
has_overflowed.into_int_value(),
ctx.bool_type().const_int(1 as u64, false),
"has_overflowed",
);
// build blocks
let then_block = ctx.append_basic_block(parent, "then");
let else_block = ctx.append_basic_block(parent, "else");
// TODO what would be most optimial for the branch predictor
//
// are most refcounts 1 most of the time? or not?
builder.build_conditional_branch(has_overflowed_comparison, then_block, else_block);
// build then block
{
builder.position_at_end(then_block);
if !env.leak {
builder.build_free(refcount_ptr);
}
builder.build_unconditional_branch(cont_block);
}
// build else block
{
builder.position_at_end(else_block);
let max = builder.build_int_compare(
IntPredicate::EQ,
refcount,
refcount_type.const_int(REFCOUNT_MAX as u64, false),
"refcount_max_check",
);
let decremented = builder
.build_extract_value(add_with_overflow, 0, "decrement_refcount")
.unwrap()
.into_int_value();
let selected = builder.build_select(max, refcount, decremented, "select_refcount");
builder.build_store(refcount_ptr, selected);
builder.build_unconditional_branch(cont_block);
}
// emit merge block
builder.position_at_end(cont_block);
}
/// Build an increment or decrement function for a specific layout
pub fn build_header<'a, 'ctx, 'env>(
env: &Env<'a, 'ctx, 'env>,
@ -926,7 +1039,8 @@ pub fn build_dec_union_help<'a, 'ctx, 'env>(
// TODO do this decrement before the recursive call?
// Then the recursive call is potentially TCE'd
decrement_refcount_ptr(env, parent, &layout, recursive_field_ptr);
let refcount_ptr = PointerToRefcount::from_ptr_to_data(env, recursive_field_ptr);
refcount_ptr.decrement(env, &layout);
} else if field_layout.contains_refcounted() {
let field_ptr = env
.builder
@ -1130,7 +1244,8 @@ pub fn build_inc_union_help<'a, 'ctx, 'env>(
// TODO do this decrement before the recursive call?
// Then the recursive call is potentially TCE'd
increment_refcount_ptr(env, &layout, recursive_field_ptr);
let refcount_ptr = PointerToRefcount::from_ptr_to_data(env, recursive_field_ptr);
refcount_ptr.increment(env);
} else if field_layout.contains_refcounted() {
let field_ptr = env
.builder
@ -1184,18 +1299,6 @@ pub fn list_get_refcount_ptr<'a, 'ctx, 'env>(
get_refcount_ptr_help(env, layout, ptr_as_int)
}
fn get_refcount_ptr<'a, 'ctx, 'env>(
env: &Env<'a, 'ctx, 'env>,
layout: &Layout<'a>,
ptr: PointerValue<'ctx>,
) -> PointerValue<'ctx> {
let refcount_type = ptr_int(env.context, env.ptr_bytes);
let ptr_as_int =
cast_basic_basic(env.builder, ptr.into(), refcount_type.into()).into_int_value();
get_refcount_ptr_help(env, layout, ptr_as_int)
}
pub fn refcount_offset<'a, 'ctx, 'env>(env: &Env<'a, 'ctx, 'env>, layout: &Layout<'a>) -> u64 {
let value_bytes = layout.stack_size(env.ptr_bytes) as u64;

32
compiler/mono/src/layout.rs
View File

@ -432,26 +432,28 @@ impl<'a> Layout<'a> {
}
}
pub fn alignment_bytes(&self) -> u32 {
pub fn alignment_bytes(&self, pointer_size: u32) -> u32 {
match self {
Layout::Struct(fields) => fields
.iter()
.map(|x| x.alignment_bytes())
.map(|x| x.alignment_bytes(pointer_size))
.max()
.unwrap_or(0),
Layout::Union(tags) | Layout::RecursiveUnion(tags) => tags
.iter()
.map(|x| x.iter())
.flatten()
.map(|x| x.alignment_bytes())
.map(|x| x.alignment_bytes(pointer_size))
.max()
.unwrap_or(0),
Layout::Builtin(builtin) => builtin.alignment_bytes(),
Layout::Builtin(builtin) => builtin.alignment_bytes(pointer_size),
Layout::PhantomEmptyStruct => 0,
Layout::RecursivePointer => std::mem::align_of::<*const u8>() as u32,
Layout::FunctionPointer(_, _) => std::mem::align_of::<*const u8>() as u32,
Layout::Pointer(_) => std::mem::align_of::<*const u8>() as u32,
Layout::Closure(_, _, _) => todo!(),
Layout::RecursivePointer => pointer_size,
Layout::FunctionPointer(_, _) => pointer_size,
Layout::Pointer(_) => pointer_size,
Layout::Closure(_, captured, _) => {
pointer_size.max(captured.layout.alignment_bytes(pointer_size))
}
}
}
@ -656,10 +658,13 @@ impl<'a> Builtin<'a> {
}
}
pub fn alignment_bytes(&self) -> u32 {
pub fn alignment_bytes(&self, pointer_size: u32) -> u32 {
use std::mem::align_of;
use Builtin::*;
// for our data structures, what counts is the alignment of the `( ptr, len )` tuple, and
// since both of those are one pointer size, the alignment of that structure is a pointer
// size
match self {
Int128 => align_of::<i128>() as u32,
Int64 => align_of::<i64>() as u32,
@ -671,11 +676,10 @@ impl<'a> Builtin<'a> {
Float64 => align_of::<f64>() as u32,
Float32 => align_of::<f32>() as u32,
Float16 => align_of::<i16>() as u32,
Str | EmptyStr => align_of::<char>() as u32,
Map(_, _) | EmptyMap => todo!(),
Set(_) | EmptySet => todo!(),
EmptyList => 0,
List(_, element_layout) => element_layout.alignment_bytes(),
Str | EmptyStr => pointer_size,
Map(_, _) | EmptyMap => pointer_size,
Set(_) | EmptySet => pointer_size,
List(_, _) | EmptyList => pointer_size,
}
}