roc-lang/roc
1
1
Fork 0
mirror of https://github.com/roc-lang/roc.git synced 2024-09-22 08:17:40 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4302 from roc-lang/serialize-ability-store

Browse Source 
Serialize the abilities store and solved implementations for builtins
This commit is contained in:
Ayaz 2022-10-12 16:37:16 -05:00 committed by GitHub
commit b2e7fd91ab
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 1391 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Cargo.lock generated
View File

@ -3380,6 +3380,7 @@ dependencies = [
"roc_parse",
"roc_problem",
"roc_region",
"roc_serialize",
"roc_types",
"static_assertions",
]
@ -4027,6 +4028,13 @@ dependencies = [
"ven_pretty",
]
[[package]]
name = "roc_serialize"
version = "0.0.1"
dependencies = [
"roc_collections",
]
[[package]]
name = "roc_solve"
version = "0.0.1"
@ -4117,6 +4125,7 @@ dependencies = [
"roc_error_macros",
"roc_module",
"roc_region",
"roc_serialize",
"static_assertions",
]

1
Cargo.toml
View File

@ -32,6 +32,7 @@ members = [
"crates/compiler/test_gen",
"crates/compiler/roc_target",
"crates/compiler/debug_flags",
"crates/compiler/serialize",
"crates/vendor/inkwell",
"crates/vendor/pathfinding",
"crates/vendor/pretty",

1
crates/compiler/can/Cargo.toml
View File

@ -14,6 +14,7 @@ roc_module = { path = "../module" }
roc_parse = { path = "../parse" }
roc_problem = { path = "../problem" }
roc_types = { path = "../types" }
roc_serialize = { path = "../serialize" }
bumpalo = { version = "3.11.0", features = ["collections"] }
static_assertions = "1.1.0"
bitvec = "1"

725
crates/compiler/can/src/abilities.rs
View File

@ -9,6 +9,14 @@ use roc_types::{
types::{MemberImpl, Type},
};
/// During type solving and monomorphization, a module must know how its imported ability
/// implementations are resolved - are they derived, or have a concrete implementation?
///
/// Unfortunately we cannot keep this information opaque, as it's important for properly
/// restoring specialization lambda sets. As such, we need to export implementation information,
/// which is the job of this structure.
pub type ResolvedImplementations = VecMap<ImplKey, ResolvedImpl>;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MemberVariables {
pub able_vars: Vec<Variable>,
@ -20,7 +28,7 @@ pub struct MemberVariables {
/// The member and its signature is defined locally, in the module the store is created for.
/// We need to instantiate and introduce this during solving.
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub struct ResolvedMemberType(Variable);
/// Member type information that needs to be resolved from imports.
@ -48,7 +56,7 @@ impl ResolvePhase for Pending {
type MemberType = PendingMemberType;
}
#[derive(Default, Debug, Clone, Copy)]
#[derive(Default, Debug, Clone, Copy, PartialEq)]
pub struct Resolved;
impl ResolvePhase for Resolved {
type MemberType = ResolvedMemberType;
@ -57,7 +65,7 @@ impl ResolvePhase for Resolved {
/// Stores information about an ability member definition, including the parent ability, the
/// defining type, and what type variables need to be instantiated with instances of the ability.
// TODO: SoA and put me in an arena
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub struct AbilityMemberData<Phase: ResolvePhase> {
pub parent_ability: Symbol,
pub region: Region,
@ -82,7 +90,7 @@ impl AbilityMemberData<Resolved> {
pub type SpecializationLambdaSets = VecMap<u8, Variable>;
/// A particular specialization of an ability member.
#[derive(Debug, Clone)]
#[derive(Debug, Clone, PartialEq)]
pub struct MemberSpecializationInfo<Phase: ResolvePhase> {
_phase: std::marker::PhantomData<Phase>,
pub symbol: Symbol,
@ -100,6 +108,7 @@ impl MemberSpecializationInfo<Resolved> {
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
#[repr(transparent)]
pub struct SpecializationId(NonZeroU32);
static_assertions::assert_eq_size!(SpecializationId, Option<SpecializationId>);
@ -469,8 +478,19 @@ impl IAbilitiesStore<Resolved> {
pub fn get_resolved(&self, id: SpecializationId) -> Option<Symbol> {
self.resolved_specializations.get(&id).copied()
}
pub fn serialize(&self, writer: &mut impl std::io::Write) -> std::io::Result<usize> {
serialize::serialize(self, writer)
}
pub fn deserialize(bytes: &[u8]) -> (Self, usize) {
serialize::deserialize(bytes)
}
}
pub use serialize::deserialize_solved_implementations;
pub use serialize::serialize_solved_implementations;
impl IAbilitiesStore<Pending> {
pub fn import_implementation(&mut self, impl_key: ImplKey, resolved_impl: &ResolvedImpl) {
let member_impl = match resolved_impl {
@ -660,3 +680,700 @@ impl IAbilitiesStore<Pending> {
}
}
}
mod serialize {
use roc_collections::{MutMap, VecMap};
use roc_module::symbol::Symbol;
use roc_region::all::Region;
use roc_serialize::bytes;
use roc_types::{
subs::{SubsSlice, Variable},
types::MemberImpl,
};
use super::{
AbilitiesStore, AbilityMemberData, ImplKey, MemberSpecializationInfo, Resolved,
ResolvedImpl, ResolvedImplementations, ResolvedMemberType, SpecializationId,
};
use std::io::{self, Write};
#[repr(C)]
#[derive(Clone, Copy, Debug)]
struct Header {
members_of_ability: u64,
specialization_to_root: u64,
ability_members: u64,
declared_implementations: u64,
specializations: u64,
next_specialization_id: u64,
resolved_specializations: u64,
}
impl Header {
fn from_store(store: &AbilitiesStore) -> Self {
let AbilitiesStore {
members_of_ability,
specialization_to_root,
ability_members,
declared_implementations,
specializations,
next_specialization_id,
resolved_specializations,
} = store;
Self {
members_of_ability: members_of_ability.len() as _,
specialization_to_root: specialization_to_root.len() as _,
ability_members: ability_members.len() as _,
declared_implementations: declared_implementations.len() as _,
specializations: specializations.len() as _,
next_specialization_id: next_specialization_id.get() as _,
resolved_specializations: resolved_specializations.len() as _,
}
}
fn to_array(self) -> [u8; std::mem::size_of::<Self>()] {
// Safety: With repr(c) all fields are in order and properly aligned without padding.
unsafe { std::mem::transmute(self) }
}
fn from_array(array: [u8; std::mem::size_of::<Self>()]) -> Self {
// Safety: With repr(c) all fields are in order and properly aligned without padding.
unsafe { std::mem::transmute(array) }
}
}
pub(super) fn serialize(store: &AbilitiesStore, writer: &mut impl Write) -> io::Result<usize> {
let header = Header::from_store(store).to_array();
let written = header.len();
writer.write_all(&header)?;
let AbilitiesStore {
members_of_ability,
specialization_to_root,
ability_members,
declared_implementations,
specializations,
next_specialization_id: _, // written in the header
resolved_specializations,
} = store;
let written = serialize_members_of_ability(members_of_ability, writer, written)?;
let written = serialize_specializations_to_root(specialization_to_root, writer, written)?;
let written = serialize_ability_members(ability_members, writer, written)?;
let written =
serialize_declared_implementations(declared_implementations, writer, written)?;
let written = serialize_specializations(specializations, writer, written)?;
let written =
serialize_resolved_specializations(resolved_specializations, writer, written)?;
Ok(written)
}
pub(super) fn deserialize(bytes: &[u8]) -> (AbilitiesStore, usize) {
let mut offset = 0;
let header_slice = &bytes[..std::mem::size_of::<Header>()];
offset += header_slice.len();
let header = Header::from_array(header_slice.try_into().unwrap());
let (members_of_ability, offset) =
deserialize_members_of_ability(bytes, header.members_of_ability as _, offset);
let (specialization_to_root, offset) =
deserialize_specialization_to_root(bytes, header.specialization_to_root as _, offset);
let (ability_members, offset) =
deserialize_ability_members(bytes, header.ability_members as _, offset);
let (declared_implementations, offset) = deserialize_declared_implementations(
bytes,
header.declared_implementations as _,
offset,
);
let (specializations, offset) =
deserialize_specializations(bytes, header.specializations as _, offset);
let (resolved_specializations, offset) = deserialize_resolved_specializations(
bytes,
header.resolved_specializations as _,
offset,
);
(
AbilitiesStore {
members_of_ability,
specialization_to_root,
ability_members,
declared_implementations,
specializations,
next_specialization_id: (header.next_specialization_id as u32).try_into().unwrap(),
resolved_specializations,
},
offset,
)
}
fn serialize_members_of_ability(
members_of_ability: &MutMap<Symbol, Vec<Symbol>>,
writer: &mut impl Write,
written: usize,
) -> io::Result<usize> {
bytes::serialize_map(
members_of_ability,
bytes::serialize_slice,
bytes::serialize_slice_of_slices,
writer,
written,
)
}
fn deserialize_members_of_ability(
bytes: &[u8],
length: usize,
offset: usize,
) -> (MutMap<Symbol, Vec<Symbol>>, usize) {
bytes::deserialize_map(
bytes,
bytes::deserialize_vec,
bytes::deserialize_slice_of_slices,
length,
offset,
)
}
#[derive(Clone, Copy)]
#[repr(C)]
struct SerImplKey(Symbol, Symbol);
impl From<&ImplKey> for SerImplKey {
fn from(k: &ImplKey) -> Self {
Self(k.opaque, k.ability_member)
}
}
impl From<&SerImplKey> for ImplKey {
fn from(k: &SerImplKey) -> Self {
Self {
opaque: k.0,
ability_member: k.1,
}
}
}
fn serialize_specializations_to_root(
specialization_to_root: &MutMap<Symbol, ImplKey>,
writer: &mut impl Write,
written: usize,
) -> io::Result<usize> {
bytes::serialize_map(
specialization_to_root,
bytes::serialize_slice,
|keys, writer, written| {
bytes::serialize_slice(
&keys.iter().map(SerImplKey::from).collect::<Vec<_>>(),
writer,
written,
)
},
writer,
written,
)
}
fn deserialize_specialization_to_root(
bytes: &[u8],
length: usize,
offset: usize,
) -> (MutMap<Symbol, ImplKey>, usize) {
bytes::deserialize_map(
bytes,
bytes::deserialize_vec,
|bytes, length, offset| {
let (slice, offset) = bytes::deserialize_slice::<SerImplKey>(bytes, length, offset);
(slice.iter().map(ImplKey::from).collect(), offset)
},
length,
offset,
)
}
#[derive(Clone, Copy)]
#[repr(C)]
struct SerMemberData(Symbol, Region, Variable);
impl From<&AbilityMemberData<Resolved>> for SerMemberData {
fn from(k: &AbilityMemberData<Resolved>) -> Self {
Self(k.parent_ability, k.region, k.typ.0)
}
}
impl From<&SerMemberData> for AbilityMemberData<Resolved> {
fn from(k: &SerMemberData) -> Self {
Self {
parent_ability: k.0,
region: k.1,
typ: ResolvedMemberType(k.2),
}
}
}
fn serialize_ability_members(
ability_members: &MutMap<Symbol, AbilityMemberData<Resolved>>,
writer: &mut impl Write,
written: usize,
) -> io::Result<usize> {
bytes::serialize_map(
ability_members,
bytes::serialize_slice,
|keys, writer, written| {
bytes::serialize_slice(
&keys.iter().map(SerMemberData::from).collect::<Vec<_>>(),
writer,
written,
)
},
writer,
written,
)
}
fn deserialize_ability_members(
bytes: &[u8],
length: usize,
offset: usize,
) -> (MutMap<Symbol, AbilityMemberData<Resolved>>, usize) {
bytes::deserialize_map(
bytes,
bytes::deserialize_vec,
|bytes, length, offset| {
let (slice, offset) =
bytes::deserialize_slice::<SerMemberData>(bytes, length, offset);
(slice.iter().map(AbilityMemberData::from).collect(), offset)
},
length,
offset,
)
}
#[derive(Clone, Copy)]
#[repr(C)]
enum SerMemberImpl {
Impl(Symbol),
Derived,
Error,
}
impl From<&MemberImpl> for SerMemberImpl {
fn from(k: &MemberImpl) -> Self {
match k {
MemberImpl::Impl(s) => Self::Impl(*s),
MemberImpl::Derived => Self::Derived,
MemberImpl::Error => Self::Error,
}
}
}
impl From<&SerMemberImpl> for MemberImpl {
fn from(k: &SerMemberImpl) -> Self {
match k {
SerMemberImpl::Impl(s) => Self::Impl(*s),
SerMemberImpl::Derived => Self::Derived,
SerMemberImpl::Error => Self::Error,
}
}
}
fn serialize_declared_implementations(
declared_implementations: &MutMap<ImplKey, MemberImpl>,
writer: &mut impl Write,
written: usize,
) -> io::Result<usize> {
bytes::serialize_map(
declared_implementations,
bytes::serialize_slice,
|keys, writer, written| {
bytes::serialize_slice(
&keys.iter().map(SerMemberImpl::from).collect::<Vec<_>>(),
writer,
written,
)
},
writer,
written,
)
}
fn deserialize_declared_implementations(
bytes: &[u8],
length: usize,
offset: usize,
) -> (MutMap<ImplKey, MemberImpl>, usize) {
bytes::deserialize_map(
bytes,
bytes::deserialize_vec,
|bytes, length, offset| {
let (slice, offset) =
bytes::deserialize_slice::<SerMemberImpl>(bytes, length, offset);
(slice.iter().map(MemberImpl::from).collect(), offset)
},
length,
offset,
)
}
#[derive(Clone, Copy)]
#[repr(C)]
struct SerMemberSpecInfo(Symbol, SubsSlice<u8>, SubsSlice<Variable>);
fn serialize_specializations(
specializations: &MutMap<Symbol, MemberSpecializationInfo<Resolved>>,
writer: &mut impl Write,
written: usize,
) -> io::Result<usize> {
bytes::serialize_map(
specializations,
bytes::serialize_slice,
|spec_info, writer, written| {
let mut spec_lambda_sets_regions: Vec<u8> = Vec::new();
let mut spec_lambda_sets_vars: Vec<Variable> = Vec::new();
let mut ser_member_spec_infos: Vec<SerMemberSpecInfo> = Vec::new();
for MemberSpecializationInfo {
_phase: _,
symbol,
specialization_lambda_sets,
} in spec_info
{
let regions = SubsSlice::extend_new(
&mut spec_lambda_sets_regions,
specialization_lambda_sets.keys().copied(),
);
let vars = SubsSlice::extend_new(
&mut spec_lambda_sets_vars,
specialization_lambda_sets.values().copied(),
);
ser_member_spec_infos.push(SerMemberSpecInfo(*symbol, regions, vars));
}
let written = bytes::serialize_slice(&ser_member_spec_infos, writer, written)?;
let written = bytes::serialize_slice(&spec_lambda_sets_regions, writer, written)?;
let written = bytes::serialize_slice(&spec_lambda_sets_vars, writer, written)?;
Ok(written)
},
writer,
written,
)
}
fn deserialize_specializations(
bytes: &[u8],
length: usize,
offset: usize,
) -> (MutMap<Symbol, MemberSpecializationInfo<Resolved>>, usize) {
bytes::deserialize_map(
bytes,
bytes::deserialize_vec,
|bytes, length, offset| {
let (serialized_slices, offset) =
bytes::deserialize_slice::<SerMemberSpecInfo>(bytes, length, offset);
let (regions_slice, offset) = {
let total_items = serialized_slices.iter().map(|s| s.1.len()).sum();
bytes::deserialize_slice::<u8>(bytes, total_items, offset)
};
let (vars_slice, offset) = {
let total_items = serialized_slices.iter().map(|s| s.2.len()).sum();
bytes::deserialize_slice::<Variable>(bytes, total_items, offset)
};
let mut spec_infos: Vec<MemberSpecializationInfo<Resolved>> =
Vec::with_capacity(length);
for SerMemberSpecInfo(symbol, regions, vars) in serialized_slices {
let regions = regions_slice[regions.indices()].to_vec();
let lset_vars = vars_slice[vars.indices()].to_vec();
let spec_info = MemberSpecializationInfo::new(*symbol, unsafe {
VecMap::zip(regions, lset_vars)
});
spec_infos.push(spec_info)
}
(spec_infos, offset)
},
length,
offset,
)
}
fn serialize_resolved_specializations(
resolved_specializations: &MutMap<SpecializationId, Symbol>,
writer: &mut impl Write,
written: usize,
) -> io::Result<usize> {
bytes::serialize_map(
resolved_specializations,
bytes::serialize_slice,
bytes::serialize_slice,
writer,
written,
)
}
fn deserialize_resolved_specializations(
bytes: &[u8],
length: usize,
offset: usize,
) -> (MutMap<SpecializationId, Symbol>, usize) {
bytes::deserialize_map(
bytes,
bytes::deserialize_vec,
bytes::deserialize_vec,
length,
offset,
)
}
#[derive(Copy, Clone)]
#[repr(C)]
enum SerResolvedImpl {
Impl(SerMemberSpecInfo),
Derived,
Error,
}
impl SerResolvedImpl {
fn num_regions(&self) -> usize {
match self {
SerResolvedImpl::Impl(spec) => spec.1.len(),
SerResolvedImpl::Derived => 0,
SerResolvedImpl::Error => 0,
}
}
}
pub fn serialize_solved_implementations(
solved_impls: &ResolvedImplementations,
writer: &mut impl std::io::Write,
) -> std::io::Result<usize> {
let len = solved_impls.len() as u64;
let written = bytes::serialize_slice(&[len], writer, 0)?;
bytes::serialize_vec_map(
solved_impls,
|keys, writer, written| {
bytes::serialize_slice(
&keys.iter().map(SerImplKey::from).collect::<Vec<_>>(),
writer,
written,
)
},
|resolved_impls, writer, written| {
let mut spec_lambda_sets_regions: Vec<u8> = Vec::new();
let mut spec_lambda_sets_vars: Vec<Variable> = Vec::new();
let mut ser_resolved_impls: Vec<SerResolvedImpl> = Vec::new();
for resolved_impl in resolved_impls {
let ser_resolved_impl = match resolved_impl {
ResolvedImpl::Impl(MemberSpecializationInfo {
_phase: _,
symbol,
specialization_lambda_sets,
}) => {
let regions = SubsSlice::extend_new(
&mut spec_lambda_sets_regions,
specialization_lambda_sets.keys().copied(),
);
let vars = SubsSlice::extend_new(
&mut spec_lambda_sets_vars,
specialization_lambda_sets.values().copied(),
);
SerResolvedImpl::Impl(SerMemberSpecInfo(*symbol, regions, vars))
}
ResolvedImpl::Derived => SerResolvedImpl::Derived,
ResolvedImpl::Error => SerResolvedImpl::Error,
};
ser_resolved_impls.push(ser_resolved_impl);
}
let written = bytes::serialize_slice(&ser_resolved_impls, writer, written)?;
let written = bytes::serialize_slice(&spec_lambda_sets_regions, writer, written)?;
let written = bytes::serialize_slice(&spec_lambda_sets_vars, writer, written)?;
Ok(written)
},
writer,
written,
)
}
pub fn deserialize_solved_implementations(bytes: &[u8]) -> (ResolvedImplementations, usize) {
let (len_slice, offset) = bytes::deserialize_slice::<u64>(bytes, 1, 0);
let length = len_slice[0];
bytes::deserialize_vec_map(
bytes,
|bytes, length, offset| {
let (slice, offset) = bytes::deserialize_slice::<SerImplKey>(bytes, length, offset);
(slice.iter().map(ImplKey::from).collect(), offset)
},
|bytes, length, offset| {
let (serialized_slices, offset) =
bytes::deserialize_slice::<SerResolvedImpl>(bytes, length, offset);
let total_num_regions = serialized_slices.iter().map(|s| s.num_regions()).sum();
let (regions_slice, offset) =
bytes::deserialize_slice::<u8>(bytes, total_num_regions, offset);
let (vars_slice, offset) =
{ bytes::deserialize_slice::<Variable>(bytes, total_num_regions, offset) };
let mut all_resolved: Vec<ResolvedImpl> = Vec::with_capacity(length);
for ser_resolved in serialized_slices {
let resolved = match ser_resolved {
SerResolvedImpl::Impl(SerMemberSpecInfo(symbol, regions, vars)) => {
let regions = regions_slice[regions.indices()].to_vec();
let lset_vars = vars_slice[vars.indices()].to_vec();
let spec_info = MemberSpecializationInfo::new(*symbol, unsafe {
VecMap::zip(regions, lset_vars)
});
ResolvedImpl::Impl(spec_info)
}
SerResolvedImpl::Derived => ResolvedImpl::Derived,
SerResolvedImpl::Error => ResolvedImpl::Error,
};
all_resolved.push(resolved);
}
(all_resolved, offset)
},
length as _,
offset,
)
}
}
#[cfg(test)]
mod test {
use roc_collections::VecMap;
use roc_module::symbol::Symbol;
use roc_region::all::Region;
use roc_types::{subs::Variable, types::MemberImpl};
use super::{
AbilitiesStore, AbilityMemberData, ImplKey, MemberSpecializationInfo, ResolvedMemberType,
};
#[test]
fn serde_abilities_store() {
let store = {
let mut store = AbilitiesStore::default();
store.register_ability(
Symbol::ARG_1,
[
(
Symbol::ARG_2,
AbilityMemberData {
parent_ability: Symbol::ARG_1,
region: Region::zero(),
typ: ResolvedMemberType(Variable::BOOL),
},
),
(
Symbol::ARG_3,
AbilityMemberData {
parent_ability: Symbol::ARG_1,
region: Region::zero(),
typ: ResolvedMemberType(Variable::BOOL),
},
),
],
);
store.register_ability(
Symbol::ARG_4,
[(
Symbol::ARG_5,
AbilityMemberData {
parent_ability: Symbol::ARG_4,
region: Region::zero(),
typ: ResolvedMemberType(Variable::BOOL),
},
)],
);
store.register_declared_implementations(
Symbol::ATTR_ATTR,
[
(Symbol::ARG_2, MemberImpl::Impl(Symbol::ATTR_INVALID)),
(Symbol::ARG_3, MemberImpl::Impl(Symbol::ARG_CLOSURE)),
],
);
store.register_declared_implementations(
Symbol::ATTR_ATTR,
[(Symbol::ARG_5, MemberImpl::Derived)],
);
store
.mark_implementation(
ImplKey {
opaque: Symbol::ATTR_ATTR,
ability_member: Symbol::ARG_2,
},
Ok(MemberSpecializationInfo::new(Symbol::UNDERSCORE, {
let mut map = VecMap::default();
map.insert(1, Variable::BOOL);
map.insert(2, Variable::U8);
map
})),
)
.unwrap();
store
.mark_implementation(
ImplKey {
opaque: Symbol::ATTR_ATTR,
ability_member: Symbol::ARG_3,
},
Ok(MemberSpecializationInfo::new(Symbol::UNDERSCORE, {
let mut map = VecMap::default();
map.insert(1, Variable::BOOL);
map.insert(2, Variable::U8);
map.insert(3, Variable::U32);
map.insert(4, Variable::U64);
map
})),
)
.unwrap();
let spec_id1 = store.fresh_specialization_id();
let spec_id2 = store.fresh_specialization_id();
store.insert_resolved(spec_id1, Symbol::ARG_2);
store.insert_resolved(spec_id2, Symbol::ARG_3);
store
};
let mut bytes = Vec::new();
let written = store.serialize(&mut bytes).unwrap();
assert_eq!(bytes.len(), written);
let AbilitiesStore {
members_of_ability,
specialization_to_root,
ability_members,
declared_implementations,
specializations,
next_specialization_id,
resolved_specializations,
} = store;
let (de_store, offset) = AbilitiesStore::deserialize(&bytes);
assert_eq!(bytes.len(), offset);
assert_eq!(members_of_ability, de_store.members_of_ability);
assert_eq!(specialization_to_root, de_store.specialization_to_root);
assert_eq!(ability_members, de_store.ability_members);
assert_eq!(declared_implementations, de_store.declared_implementations);
assert_eq!(specializations, de_store.specializations);
assert_eq!(next_specialization_id, de_store.next_specialization_id);
assert_eq!(resolved_specializations, de_store.resolved_specializations);
}
}

54
crates/compiler/can/src/module.rs
View File

@ -1,4 +1,4 @@
use crate::abilities::{ImplKey, PendingAbilitiesStore, ResolvedImpl};
use crate::abilities::{AbilitiesStore, ImplKey, PendingAbilitiesStore, ResolvedImpl};
use crate::annotation::canonicalize_annotation;
use crate::def::{canonicalize_defs, Def};
use crate::effect_module::HostedGeneratedFunctions;
@ -17,7 +17,7 @@ use roc_parse::header::HeaderFor;
use roc_parse::pattern::PatternType;
use roc_problem::can::{Problem, RuntimeError};
use roc_region::all::{Loc, Region};
use roc_types::subs::{ExposedTypesStorageSubs, VarStore, Variable};
use roc_types::subs::{ExposedTypesStorageSubs, Subs, VarStore, Variable};
use roc_types::types::{Alias, AliasKind, AliasVar, Type};
/// The types of all exposed values/functions of a collection of modules
@ -1179,3 +1179,53 @@ fn fix_values_captured_in_closure_expr(
OpaqueWrapFunction(_) => {}
}
}
/// Type state for a single module.
#[derive(Debug)]
pub struct TypeState {
pub subs: Subs,
pub exposed_vars_by_symbol: Vec<(Symbol, Variable)>,
pub abilities: AbilitiesStore,
pub solved_implementations: ResolvedImplementations,
}
impl TypeState {
pub fn serialize(&self, writer: &mut impl std::io::Write) -> std::io::Result<usize> {
let Self {
subs,
exposed_vars_by_symbol,
abilities,
solved_implementations,
} = self;
let written_subs = subs.serialize(exposed_vars_by_symbol, writer)?;
let written_ab = abilities.serialize(writer)?;
let written_solved_impls =
crate::abilities::serialize_solved_implementations(solved_implementations, writer)?;
Ok(written_subs + written_ab + written_solved_impls)
}
pub fn deserialize(bytes: &[u8]) -> (Self, usize) {
let ((subs, exposed_vars_by_symbol), len_subs) = Subs::deserialize(bytes);
let bytes = &bytes[len_subs..];
let (abilities, len_abilities) = AbilitiesStore::deserialize(bytes);
let bytes = &bytes[len_abilities..];
let (solved_implementations, len_solved_impls) =
crate::abilities::deserialize_solved_implementations(bytes);
let total_offset = len_subs + len_abilities + len_solved_impls;
(
Self {
subs,
exposed_vars_by_symbol: exposed_vars_by_symbol.to_vec(),
abilities,
solved_implementations,
},
total_offset,
)
}
}

41
crates/compiler/collections/src/vec_map.rs
View File

@ -25,6 +25,14 @@ impl<K, V> VecMap<K, V> {
(k, v)
}
pub fn unzip(self) -> (Vec<K>, Vec<V>) {
(self.keys, self.values)
}
pub fn unzip_slices(&self) -> (&[K], &[V]) {
(&self.keys, &self.values)
}
}
impl<K: PartialEq, V> VecMap<K, V> {
@ -114,14 +122,6 @@ impl<K: PartialEq, V> VecMap<K, V> {
self.values.truncate(len);
}
pub fn unzip(self) -> (Vec<K>, Vec<V>) {
(self.keys, self.values)
}
pub fn unzip_slices(&self) -> (&[K], &[V]) {
(&self.keys, &self.values)
}
/// # Safety
///
/// keys and values must have the same length, and there must not
@ -250,6 +250,31 @@ where
}
}
impl<K, V> PartialEq for VecMap<K, V>
where
K: PartialEq,
V: PartialEq,
{
fn eq(&self, other: &Self) -> bool {
if self.len() != other.len() {
return false;
}
for (k, v) in self.iter() {
match other.get(k) {
Some(v1) => {
if v != v1 {
return false;
}
}
None => return false,
}
}
true
}
}
#[cfg(test)]
mod test_drain_filter {
use crate::VecMap;

16
crates/compiler/collections/src/vec_set.rs
View File

@ -1,4 +1,4 @@
use std::iter::FromIterator;
use std::{borrow::Borrow, iter::FromIterator};
#[derive(Clone, Debug, PartialEq)]
pub struct VecSet<T> {
@ -131,3 +131,17 @@ impl<T> IntoIterator for VecSet<T> {
self.elements.into_iter()
}
}
impl<T> Borrow<[T]> for VecSet<T> {
fn borrow(&self) -> &[T] {
&self.elements
}
}
impl<T> From<Vec<T>> for VecSet<T> {
fn from(elements: Vec<T>) -> Self {
// Not totally safe, but good enough for our purposes - also, duplicates in the VecSet are
// fine, just inefficient.
Self { elements }
}
}

1
crates/compiler/load/Cargo.toml
View File

@ -20,6 +20,7 @@ roc_builtins = { path = "../builtins" }
roc_module = { path = "../module" }
roc_reporting = { path = "../../reporting" }
roc_target = { path = "../roc_target" }
roc_can = { path = "../can" }
bumpalo = { version = "3.11.0", features = ["collections"] }
[target.'cfg(not(windows))'.build-dependencies]

26
crates/compiler/load/build.rs
View File

@ -11,6 +11,7 @@ const SKIP_SUBS_CACHE: bool = {
}
};
// IFTTT: crates/compiler/load/src/lib.rs
const MODULES: &[(ModuleId, &str)] = &[
(ModuleId::BOOL, "Bool.roc"),
(ModuleId::RESULT, "Result.roc"),
@ -46,26 +47,27 @@ fn write_subs_for_module(module_id: ModuleId, filename: &str) {
#[cfg(not(windows))]
if SKIP_SUBS_CACHE {
write_subs_for_module_dummy(&output_path)
write_types_for_module_dummy(&output_path)
} else {
write_subs_for_module_real(module_id, filename, &output_path)
write_types_for_module_real(module_id, filename, &output_path)
}
#[cfg(windows)]
{
let _ = SKIP_SUBS_CACHE;
let _ = module_id;
write_subs_for_module_dummy(&output_path)
write_types_for_module_dummy(&output_path)
}
}
fn write_subs_for_module_dummy(output_path: &Path) {
fn write_types_for_module_dummy(output_path: &Path) {
// write out a dummy file
std::fs::write(output_path, &[]).unwrap();
}
#[cfg(not(windows))]
fn write_subs_for_module_real(module_id: ModuleId, filename: &str, output_path: &Path) {
fn write_types_for_module_real(module_id: ModuleId, filename: &str, output_path: &Path) {
use roc_can::module::TypeState;
use roc_load_internal::file::{LoadingProblem, Threading};
let arena = Bump::new();
@ -94,9 +96,19 @@ fn write_subs_for_module_real(module_id: ModuleId, filename: &str, output_path:
}
};
let subs = module.solved.inner();
let subs = module.solved.into_inner();
let exposed_vars_by_symbol: Vec<_> = module.exposed_to_host.into_iter().collect();
let abilities = module.abilities_store;
let solved_implementations = module.resolved_implementations;
let mut file = std::fs::File::create(&output_path).unwrap();
subs.serialize(&exposed_vars_by_symbol, &mut file).unwrap();
let type_state = TypeState {
subs,
exposed_vars_by_symbol,
abilities,
solved_implementations,
};
type_state.serialize(&mut file).unwrap();
}

31
crates/compiler/load/src/lib.rs
View File

@ -1,10 +1,9 @@
use bumpalo::Bump;
use roc_can::module::ExposedByModule;
use roc_can::module::{ExposedByModule, TypeState};
use roc_collections::all::MutMap;
use roc_module::symbol::{ModuleId, Symbol};
use roc_module::symbol::ModuleId;
use roc_reporting::report::RenderTarget;
use roc_target::TargetInfo;
use roc_types::subs::{Subs, Variable};
use std::path::PathBuf;
const SKIP_SUBS_CACHE: bool = {
@ -27,9 +26,9 @@ fn load<'a>(
exposed_types: ExposedByModule,
load_config: LoadConfig,
) -> Result<LoadResult<'a>, LoadingProblem<'a>> {
let cached_subs = read_cached_subs();
let cached_types = read_cached_types();
roc_load_internal::file::load(arena, load_start, exposed_types, cached_subs, load_config)
roc_load_internal::file::load(arena, load_start, exposed_types, cached_types, load_config)
}
/// Load using only a single thread; used when compiling to webassembly
@ -41,7 +40,7 @@ pub fn load_single_threaded<'a>(
render: RenderTarget,
exec_mode: ExecutionMode,
) -> Result<LoadResult<'a>, LoadingProblem<'a>> {
let cached_subs = read_cached_subs();
let cached_subs = read_cached_types();
roc_load_internal::file::load_single_threaded(
arena,
@ -157,22 +156,27 @@ pub fn load_and_typecheck_str<'a>(
}
}
// IFTTT: crates/compiler/load/build.rs
const BOOL: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Bool.dat")) as &[_];
const RESULT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Result.dat")) as &[_];
const NUM: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Num.dat")) as &[_];
const LIST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/List.dat")) as &[_];
const STR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Str.dat")) as &[_];
const DICT: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Dict.dat")) as &[_];
const SET: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Set.dat")) as &[_];
const BOX: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Box.dat")) as &[_];
const NUM: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Num.dat")) as &[_];
const ENCODE: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Encode.dat")) as &[_];
const DECODE: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Decode.dat")) as &[_];
const HASH: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/Hash.dat")) as &[_];
fn deserialize_help(bytes: &[u8]) -> (Subs, Vec<(Symbol, Variable)>) {
let (subs, slice) = Subs::deserialize(bytes);
fn deserialize_help(bytes: &[u8]) -> TypeState {
let (state, _offset) = TypeState::deserialize(bytes);
debug_assert_eq!(bytes.len(), _offset);
(subs, slice.to_vec())
state
}
fn read_cached_subs() -> MutMap<ModuleId, (Subs, Vec<(Symbol, Variable)>)> {
fn read_cached_types() -> MutMap<ModuleId, TypeState> {
let mut output = MutMap::default();
// Wasm seems to re-order definitions between build time and runtime, but only in release mode.
@ -188,6 +192,11 @@ fn read_cached_subs() -> MutMap<ModuleId, (Subs, Vec<(Symbol, Variable)>)> {
output.insert(ModuleId::SET, deserialize_help(SET));
output.insert(ModuleId::BOX, deserialize_help(BOX));
output.insert(ModuleId::ENCODE, deserialize_help(ENCODE));
output.insert(ModuleId::DECODE, deserialize_help(DECODE));
output.insert(ModuleId::HASH, deserialize_help(HASH));
}
output

61
crates/compiler/load_internal/src/file.rs
View File

@ -11,7 +11,7 @@ use roc_can::expr::Declarations;
use roc_can::expr::PendingDerives;
use roc_can::module::{
canonicalize_module_defs, ExposedByModule, ExposedForModule, ExposedModuleTypes, Module,
ResolvedImplementations,
ResolvedImplementations, TypeState,
};
use roc_collections::{default_hasher, BumpMap, MutMap, MutSet, VecMap, VecSet};
use roc_constrain::module::constrain_module;
@ -353,7 +353,7 @@ fn start_phase<'a>(
let skip_constraint_gen = {
// Give this its own scope to make sure that the Guard from the lock() is dropped
// immediately after contains_key returns
state.cached_subs.lock().contains_key(&module_id)
state.cached_types.lock().contains_key(&module_id)
};
BuildTask::CanonicalizeAndConstrain {
@ -398,7 +398,7 @@ fn start_phase<'a>(
&state.exposed_types,
dep_idents,
declarations,
state.cached_subs.clone(),
state.cached_types.clone(),
derived_module,
)
}
@ -927,13 +927,13 @@ struct State<'a> {
make_specializations_pass: MakeSpecializationsPass,
// cached subs (used for builtin modules, could include packages in the future too)
cached_subs: CachedSubs,
// cached types (used for builtin modules, could include packages in the future too)
cached_types: CachedTypeState,
layout_interner: Arc<GlobalInterner<'a, Layout<'a>>>,
}
type CachedSubs = Arc<Mutex<MutMap<ModuleId, (Subs, Vec<(Symbol, Variable)>)>>>;
type CachedTypeState = Arc<Mutex<MutMap<ModuleId, TypeState>>>;
impl<'a> State<'a> {
fn goal_phase(&self) -> Phase {
@ -947,7 +947,7 @@ impl<'a> State<'a> {
exposed_types: ExposedByModule,
arc_modules: Arc<Mutex<PackageModuleIds<'a>>>,
ident_ids_by_module: SharedIdentIdsByModule,
cached_subs: MutMap<ModuleId, (Subs, Vec<(Symbol, Variable)>)>,
cached_types: MutMap<ModuleId, TypeState>,
render: RenderTarget,
number_of_workers: usize,
exec_mode: ExecutionMode,
@ -978,7 +978,7 @@ impl<'a> State<'a> {
exposed_symbols_by_module: MutMap::default(),
timings: MutMap::default(),
layout_caches: std::vec::Vec::with_capacity(number_of_workers),
cached_subs: Arc::new(Mutex::new(cached_subs)),
cached_types: Arc::new(Mutex::new(cached_types)),
render,
exec_mode,
make_specializations_pass: MakeSpecializationsPass::Pass(1),
@ -1091,7 +1091,7 @@ enum BuildTask<'a> {
var_store: VarStore,
declarations: Declarations,
dep_idents: IdentIdsByModule,
cached_subs: CachedSubs,
cached_subs: CachedTypeState,
derived_module: SharedDerivedModule,
},
BuildPendingSpecializations {
@ -1456,7 +1456,7 @@ pub fn load<'a>(
arena: &'a Bump,
load_start: LoadStart<'a>,
exposed_types: ExposedByModule,
cached_subs: MutMap<ModuleId, (Subs, Vec<(Symbol, Variable)>)>,
cached_types: MutMap<ModuleId, TypeState>,
load_config: LoadConfig,
) -> Result<LoadResult<'a>, LoadingProblem<'a>> {
enum Threads {
@ -1489,7 +1489,7 @@ pub fn load<'a>(
load_start,
exposed_types,
load_config.target_info,
cached_subs,
cached_types,
load_config.render,
load_config.exec_mode,
),
@ -1498,7 +1498,7 @@ pub fn load<'a>(
load_start,
exposed_types,
load_config.target_info,
cached_subs,
cached_types,
load_config.render,
threads,
load_config.exec_mode,
@ -1513,7 +1513,7 @@ pub fn load_single_threaded<'a>(
load_start: LoadStart<'a>,
exposed_types: ExposedByModule,
target_info: TargetInfo,
cached_subs: MutMap<ModuleId, (Subs, Vec<(Symbol, Variable)>)>,
cached_types: MutMap<ModuleId, TypeState>,
render: RenderTarget,
exec_mode: ExecutionMode,
) -> Result<LoadResult<'a>, LoadingProblem<'a>> {
@ -1539,7 +1539,7 @@ pub fn load_single_threaded<'a>(
exposed_types,
arc_modules,
ident_ids_by_module,
cached_subs,
cached_types,
render,
number_of_workers,
exec_mode,
@ -1761,7 +1761,7 @@ fn load_multi_threaded<'a>(
load_start: LoadStart<'a>,
exposed_types: ExposedByModule,
target_info: TargetInfo,
cached_subs: MutMap<ModuleId, (Subs, Vec<(Symbol, Variable)>)>,
cached_types: MutMap<ModuleId, TypeState>,
render: RenderTarget,
available_threads: usize,
exec_mode: ExecutionMode,
@ -1803,7 +1803,7 @@ fn load_multi_threaded<'a>(
exposed_types,
arc_modules,
ident_ids_by_module,
cached_subs,
cached_types,
render,
num_workers,
exec_mode,
@ -4192,7 +4192,7 @@ impl<'a> BuildTask<'a> {
exposed_types: &ExposedByModule,
dep_idents: IdentIdsByModule,
declarations: Declarations,
cached_subs: CachedSubs,
cached_subs: CachedTypeState,
derived_module: SharedDerivedModule,
) -> Self {
let exposed_by_module = exposed_types.retain_modules(imported_modules.keys());
@ -4499,7 +4499,7 @@ fn run_solve<'a>(
var_store: VarStore,
decls: Declarations,
dep_idents: IdentIdsByModule,
cached_subs: CachedSubs,
cached_types: CachedTypeState,
derived_module: SharedDerivedModule,
) -> Msg<'a> {
let solve_start = Instant::now();
@ -4515,7 +4515,7 @@ fn run_solve<'a>(
let (solved_subs, solved_implementations, exposed_vars_by_symbol, problems, abilities_store) = {
if module_id.is_builtin() {
match cached_subs.lock().remove(&module_id) {
match cached_types.lock().remove(&module_id) {
None => run_solve_solve(
exposed_for_module,
constraints,
@ -4525,17 +4525,18 @@ fn run_solve<'a>(
module,
derived_module,
),
Some((subs, exposed_vars_by_symbol)) => {
(
Solved(subs),
// TODO(abilities) cache abilities for builtins
VecMap::default(),
exposed_vars_by_symbol.to_vec(),
vec![],
// TODO(abilities) cache abilities for builtins
AbilitiesStore::default(),
)
}
Some(TypeState {
subs,
exposed_vars_by_symbol,
abilities,
solved_implementations,
}) => (
Solved(subs),
solved_implementations,
exposed_vars_by_symbol,
vec![],
abilities,
),
}
} else {
run_solve_solve(

17
crates/compiler/load_internal/src/lib.rs
View File

@ -1,9 +1,26 @@
#![warn(clippy::dbg_macro)]
// See github.com/roc-lang/roc/issues/800 for discussion of the large_enum_variant check.
#![allow(clippy::large_enum_variant)]
use roc_module::symbol::ModuleId;
pub mod docs;
pub mod file;
mod work;
#[cfg(target_family = "wasm")]
mod wasm_instant;
pub const BUILTIN_MODULES: &[(ModuleId, &str)] = &[
(ModuleId::BOOL, "Bool"),
(ModuleId::RESULT, "Result"),
(ModuleId::NUM, "Num"),
(ModuleId::LIST, "List"),
(ModuleId::STR, "Str"),
(ModuleId::DICT, "Dict"),
(ModuleId::SET, "Set"),
(ModuleId::BOX, "Box"),
(ModuleId::ENCODE, "Encode"),
(ModuleId::DECODE, "Decode"),
(ModuleId::HASH, "Hash"),
(ModuleId::JSON, "Json"),
];

9
crates/compiler/serialize/Cargo.toml Normal file
View File

@ -0,0 +1,9 @@
[package]
name = "roc_serialize"
version = "0.0.1"
authors = ["The Roc Contributors"]
license = "UPL-1.0"
edition = "2021"
[dependencies]
roc_collections = { path = "../collections" }

381
crates/compiler/serialize/src/bytes.rs Normal file
View File

@ -0,0 +1,381 @@
use std::{
borrow::Borrow,
io::{self, Write},
};
use roc_collections::{MutMap, VecMap};
pub fn serialize_slice<T: Copy>(
slice: &[T],
writer: &mut impl Write,
written: usize,
) -> io::Result<usize> {
let alignment = std::mem::align_of::<T>();
let padding_bytes = round_to_multiple_of(written, alignment) - written;
for _ in 0..padding_bytes {
writer.write_all(&[0])?;
}
let bytes_slice = unsafe { slice_as_bytes(slice) };
writer.write_all(bytes_slice)?;
Ok(written + padding_bytes + bytes_slice.len())
}
pub fn deserialize_slice<T: Copy>(bytes: &[u8], length: usize, mut offset: usize) -> (&[T], usize) {
let alignment = std::mem::align_of::<T>();
let size = std::mem::size_of::<T>();
offset = round_to_multiple_of(offset, alignment);
let byte_length = length * size;
let byte_slice = &bytes[offset..][..byte_length];
let slice = unsafe { std::slice::from_raw_parts(byte_slice.as_ptr() as *const T, length) };
(slice, offset + byte_length)
}
pub fn deserialize_vec<T: Clone + Copy>(
bytes: &[u8],
length: usize,
offset: usize,
) -> (Vec<T>, usize) {
let (slice, offset) = deserialize_slice(bytes, length, offset);
(slice.to_vec(), offset)
}
#[derive(Copy, Clone)]
struct VecSlice<T> {
pub start: u32,
pub length: u16,
_marker: std::marker::PhantomData<T>,
}
impl<T> VecSlice<T> {
const fn len(&self) -> usize {
self.length as usize
}
fn indices(&self) -> std::ops::Range<usize> {
self.start as usize..(self.start as usize + self.length as usize)
}
fn extend_new(vec: &mut Vec<T>, it: impl IntoIterator<Item = T>) -> Self {
let start = vec.len();
vec.extend(it);
let end = vec.len();
Self {
start: start as u32,
length: (end - start) as u16,
_marker: Default::default(),
}
}
}
pub fn serialize_slice_of_slices<'a, T, U>(
slice_of_slices: &[U],
writer: &mut impl Write,
written: usize,
) -> io::Result<usize>
where
T: 'a + Copy,
U: 'a + Borrow<[T]> + Sized,
{
let mut item_buf: Vec<T> = Vec::new();
let mut serialized_slices: Vec<VecSlice<T>> = Vec::new();
for slice in slice_of_slices {
let slice = VecSlice::extend_new(&mut item_buf, slice.borrow().iter().copied());
serialized_slices.push(slice);
}
let written = serialize_slice(&serialized_slices, writer, written)?;
serialize_slice(&item_buf, writer, written)
}
pub fn deserialize_slice_of_slices<T, Container>(
bytes: &[u8],
length: usize,
offset: usize,
) -> (Vec<Container>, usize)
where
T: Copy,
Container: From<Vec<T>>,
{
let (serialized_slices, offset) = deserialize_slice::<VecSlice<T>>(bytes, length, offset);
let (vars_slice, offset) = {
let total_items = serialized_slices.iter().map(|s| s.len()).sum();
deserialize_slice::<T>(bytes, total_items, offset)
};
let mut slice_of_slices = Vec::with_capacity(length);
for slice in serialized_slices {
let deserialized_slice = &vars_slice[slice.indices()];
slice_of_slices.push(deserialized_slice.to_vec().into())
}
(slice_of_slices, offset)
}
pub fn serialize_map<K: Clone, V: Clone, W: Write>(
map: &MutMap<K, V>,
ser_keys: fn(&[K], &mut W, usize) -> io::Result<usize>,
ser_values: fn(&[V], &mut W, usize) -> io::Result<usize>,
writer: &mut W,
written: usize,
) -> io::Result<usize> {
let keys = map.keys().cloned().collect::<Vec<_>>();
let values = map.values().cloned().collect::<Vec<_>>();
let written = ser_keys(keys.as_slice(), writer, written)?;
let written = ser_values(values.as_slice(), writer, written)?;
Ok(written)
}
#[allow(clippy::type_complexity)]
pub fn deserialize_map<K, V>(
bytes: &[u8],
deser_keys: fn(&[u8], usize, usize) -> (Vec<K>, usize),
deser_values: fn(&[u8], usize, usize) -> (Vec<V>, usize),
length: usize,
offset: usize,
) -> (MutMap<K, V>, usize)
where
K: Clone + std::hash::Hash + Eq,
V: Clone,
{
let (keys, offset) = deser_keys(bytes, length, offset);
let (values, offset) = deser_values(bytes, length, offset);
(
MutMap::from_iter((keys.iter().cloned()).zip(values.iter().cloned())),
offset,
)
}
pub fn serialize_vec_map<K, V, W: Write>(
map: &VecMap<K, V>,
ser_keys: fn(&[K], &mut W, usize) -> io::Result<usize>,
ser_values: fn(&[V], &mut W, usize) -> io::Result<usize>,
writer: &mut W,
written: usize,
) -> io::Result<usize> {
let (keys, values) = map.unzip_slices();
let written = ser_keys(keys, writer, written)?;
let written = ser_values(values, writer, written)?;
Ok(written)
}
#[allow(clippy::type_complexity)]
pub fn deserialize_vec_map<K, V>(
bytes: &[u8],
deser_keys: fn(&[u8], usize, usize) -> (Vec<K>, usize),
deser_values: fn(&[u8], usize, usize) -> (Vec<V>, usize),
length: usize,
offset: usize,
) -> (VecMap<K, V>, usize)
where
K: PartialEq,
{
let (keys, offset) = deser_keys(bytes, length, offset);
let (values, offset) = deser_values(bytes, length, offset);
(unsafe { VecMap::zip(keys, values) }, offset)
}
unsafe fn slice_as_bytes<T>(slice: &[T]) -> &[u8] {
let ptr = slice.as_ptr();
let byte_length = std::mem::size_of::<T>() * slice.len();
std::slice::from_raw_parts(ptr as *const u8, byte_length)
}
fn round_to_multiple_of(value: usize, base: usize) -> usize {
(value + (base - 1)) / base * base
}
#[cfg(test)]
mod test {
use roc_collections::{MutMap, VecMap, VecSet};
use super::{
deserialize_map, deserialize_slice, deserialize_slice_of_slices, deserialize_vec,
deserialize_vec_map, serialize_map, serialize_slice, serialize_slice_of_slices,
serialize_vec_map,
};
#[test]
fn serde_empty_slice() {
let mut buf = vec![];
serialize_slice(&[] as &[u8], &mut buf, 0).unwrap();
assert!(buf.is_empty());
let (out, size) = deserialize_slice::<u8>(&buf, 0, 0);
assert!(out.is_empty());
assert_eq!(size, 0);
}
#[test]
fn serde_slice() {
let input: &[u64] = &[15u64, 23, 37, 89];
let mut buf = vec![];
serialize_slice(input, &mut buf, 0).unwrap();
assert!(!buf.is_empty());
let (out, size) = deserialize_slice::<u64>(&buf, 4, 0);
assert_eq!(out, input);
assert_eq!(size, 4 * 8);
}
#[test]
fn serde_vec() {
let input: &[u64] = &[15u64, 23, 37, 89];
let mut buf = vec![];
serialize_slice(input, &mut buf, 0).unwrap();
assert!(!buf.is_empty());
let (out, size) = deserialize_vec::<u64>(&buf, 4, 0);
assert_eq!(out, input);
assert_eq!(size, buf.len());
}
#[test]
fn serde_empty_slice_of_slices() {
let input: &[&[u64]] = &[];
let mut buf = vec![];
serialize_slice_of_slices(input, &mut buf, 0).unwrap();
assert!(buf.is_empty());
let (out, size) = deserialize_slice_of_slices::<u64, Vec<_>>(&buf, 0, 0);
assert!(out.is_empty());
assert_eq!(size, 0);
}
#[test]
fn serde_slice_of_slices() {
let input: &[&[u64]] = &[&[15, 23, 47], &[61, 72], &[85, 91]];
let mut buf = vec![];
serialize_slice_of_slices(input, &mut buf, 0).unwrap();
assert!(!buf.is_empty());
let (out, size) = deserialize_slice_of_slices::<u64, Vec<_>>(&buf, 3, 0);
assert_eq!(out, input);
assert_eq!(size, buf.len());
}
#[test]
fn serde_slice_of_slices_into_vec_set() {
let input: &[&[u64]] = &[&[15, 23, 47], &[61, 72], &[85, 91]];
let mut buf = vec![];
serialize_slice_of_slices(input, &mut buf, 0).unwrap();
assert!(!buf.is_empty());
let (out, size) = deserialize_slice_of_slices::<u64, VecSet<_>>(&buf, 3, 0);
assert_eq!(size, buf.len());
let mut out = out.into_iter();
assert_eq!(out.next().unwrap().into_vec(), &[15, 23, 47]);
assert_eq!(out.next().unwrap().into_vec(), &[61, 72]);
assert_eq!(out.next().unwrap().into_vec(), &[85, 91]);
assert!(out.next().is_none());
}
#[test]
fn serde_empty_map() {
let input: MutMap<u64, u64> = Default::default();
let mut buf = vec![];
serialize_map(&input, serialize_slice, serialize_slice, &mut buf, 0).unwrap();
assert!(buf.is_empty());
let (out, size) = deserialize_map::<u64, u64>(&buf, deserialize_vec, deserialize_vec, 0, 0);
assert!(out.is_empty());
assert_eq!(size, 0);
}
#[test]
fn serde_map() {
let mut input: MutMap<u64, Vec<u64>> = Default::default();
input.insert(51, vec![15, 23, 37]);
input.insert(39, vec![17, 91, 43]);
input.insert(82, vec![90, 35, 76]);
let mut buf = vec![];
serialize_map(
&input,
serialize_slice,
serialize_slice_of_slices,
&mut buf,
0,
)
.unwrap();
assert!(!buf.is_empty());
let (out, size) = deserialize_map::<u64, Vec<u64>>(
&buf,
deserialize_vec,
deserialize_slice_of_slices,
3,
0,
);
assert_eq!(out, input);
assert_eq!(size, buf.len());
}
#[test]
fn serde_empty_vec_map() {
let input: VecMap<u64, u64> = Default::default();
let mut buf = vec![];
serialize_vec_map(&input, serialize_slice, serialize_slice, &mut buf, 0).unwrap();
assert!(buf.is_empty());
let (out, size) =
deserialize_vec_map::<u64, u64>(&buf, deserialize_vec, deserialize_vec, 0, 0);
assert!(out.is_empty());
assert_eq!(size, 0);
}
#[test]
fn serde_vec_map() {
let mut input: VecMap<u64, Vec<u64>> = Default::default();
input.insert(51, vec![15, 23, 37]);
input.insert(39, vec![17, 91, 43]);
input.insert(82, vec![90, 35, 76]);
let mut buf = vec![];
serialize_vec_map(
&input,
serialize_slice,
serialize_slice_of_slices,
&mut buf,
0,
)
.unwrap();
assert!(!buf.is_empty());
let (out, size) = deserialize_vec_map::<u64, Vec<u64>>(
&buf,
deserialize_vec,
deserialize_slice_of_slices,
3,
0,
);
assert_eq!(out.unzip_slices(), input.unzip_slices());
assert_eq!(size, buf.len());
}
}

1
crates/compiler/serialize/src/lib.rs Normal file
View File

@ -0,0 +1 @@
pub mod bytes;

1
crates/compiler/types/Cargo.toml
View File

@ -11,5 +11,6 @@ roc_region = { path = "../region" }
roc_module = { path = "../module" }
roc_error_macros = {path="../../error_macros"}
roc_debug_flags = {path="../debug_flags"}
roc_serialize = {path="../serialize"}
bumpalo = { version = "3.11.0", features = ["collections"] }
static_assertions = "1.1.0"

144
crates/compiler/types/src/subs.rs
View File

@ -77,6 +77,7 @@ struct SubsHeader {
record_fields: u64,
variable_slices: u64,
unspecialized_lambda_sets: u64,
uls_of_var: u64,
exposed_vars_by_symbol: u64,
}
@ -95,6 +96,7 @@ impl SubsHeader {
record_fields: subs.record_fields.len() as u64,
variable_slices: subs.variable_slices.len() as u64,
unspecialized_lambda_sets: subs.unspecialized_lambda_sets.len() as u64,
uls_of_var: subs.uls_of_var.len() as u64,
exposed_vars_by_symbol: exposed_vars_by_symbol as u64,
}
}
@ -110,19 +112,11 @@ impl SubsHeader {
}
}
unsafe fn slice_as_bytes<T>(slice: &[T]) -> &[u8] {
let ptr = slice.as_ptr();
let byte_length = std::mem::size_of::<T>() * slice.len();
unsafe { std::slice::from_raw_parts(ptr as *const u8, byte_length) }
}
fn round_to_multiple_of(value: usize, base: usize) -> usize {
(value + (base - 1)) / base * base
}
#[derive(Clone, Copy)]
struct SerializedTagName(SubsSlice<u8>);
use roc_serialize::bytes;
impl Subs {
pub fn serialize(
&self,
@ -137,14 +131,15 @@ impl Subs {
written = self.utable.serialize(writer, written)?;
written = Self::serialize_slice(&self.variables, writer, written)?;
written = bytes::serialize_slice(&self.variables, writer, written)?;
written = Self::serialize_tag_names(&self.tag_names, writer, written)?;
written = Self::serialize_slice(&self.closure_names, writer, written)?;
written = bytes::serialize_slice(&self.closure_names, writer, written)?;
written = Self::serialize_field_names(&self.field_names, writer, written)?;
written = Self::serialize_slice(&self.record_fields, writer, written)?;
written = Self::serialize_slice(&self.variable_slices, writer, written)?;
written = Self::serialize_slice(&self.unspecialized_lambda_sets, writer, written)?;
written = Self::serialize_slice(exposed_vars_by_symbol, writer, written)?;
written = bytes::serialize_slice(&self.record_fields, writer, written)?;
written = bytes::serialize_slice(&self.variable_slices, writer, written)?;
written = bytes::serialize_slice(&self.unspecialized_lambda_sets, writer, written)?;
written = Self::serialize_uls_of_var(&self.uls_of_var, writer, written)?;
written = bytes::serialize_slice(exposed_vars_by_symbol, writer, written)?;
Ok(written)
}
@ -164,9 +159,9 @@ impl Subs {
slices.push(slice);
}
let written = Self::serialize_slice(&slices, writer, written)?;
let written = bytes::serialize_slice(&slices, writer, written)?;
Self::serialize_slice(&buf, writer, written)
bytes::serialize_slice(&buf, writer, written)
}
/// Global tag names can be heap-allocated
@ -185,30 +180,39 @@ impl Subs {
slices.push(serialized);
}
let written = Self::serialize_slice(&slices, writer, written)?;
let written = bytes::serialize_slice(&slices, writer, written)?;
Self::serialize_slice(&buf, writer, written)
bytes::serialize_slice(&buf, writer, written)
}
pub(crate) fn serialize_slice<T>(
slice: &[T],
fn serialize_uls_of_var(
uls_of_vars: &UlsOfVar,
writer: &mut impl std::io::Write,
written: usize,
) -> std::io::Result<usize> {
let alignment = std::mem::align_of::<T>();
let padding_bytes = round_to_multiple_of(written, alignment) - written;
for _ in 0..padding_bytes {
writer.write_all(&[0])?;
}
let bytes_slice = unsafe { slice_as_bytes(slice) };
writer.write_all(bytes_slice)?;
Ok(written + padding_bytes + bytes_slice.len())
bytes::serialize_vec_map(
&uls_of_vars.0,
bytes::serialize_slice,
bytes::serialize_slice_of_slices,
writer,
written,
)
}
pub fn deserialize(bytes: &[u8]) -> (Self, &[(Symbol, Variable)]) {
fn deserialize_uls_of_var(bytes: &[u8], length: usize, offset: usize) -> (UlsOfVar, usize) {
let (vec_map, offset) = bytes::deserialize_vec_map(
bytes,
bytes::deserialize_vec,
bytes::deserialize_slice_of_slices,
length,
offset,
);
(UlsOfVar(vec_map), offset)
}
#[allow(clippy::type_complexity)]
pub fn deserialize(bytes: &[u8]) -> ((Self, &[(Symbol, Variable)]), usize) {
let mut offset = 0;
let header_slice = &bytes[..std::mem::size_of::<SubsHeader>()];
offset += header_slice.len();
@ -216,37 +220,43 @@ impl Subs {
let (utable, offset) = UnificationTable::deserialize(bytes, header.utable as usize, offset);
let (variables, offset) = Self::deserialize_slice(bytes, header.variables as usize, offset);
let (variables, offset) =
bytes::deserialize_slice(bytes, header.variables as usize, offset);
let (tag_names, offset) =
Self::deserialize_tag_names(bytes, header.tag_names as usize, offset);
let (closure_names, offset) =
Self::deserialize_slice(bytes, header.closure_names as usize, offset);
bytes::deserialize_slice(bytes, header.closure_names as usize, offset);
let (field_names, offset) =
Self::deserialize_field_names(bytes, header.field_names as usize, offset);
let (record_fields, offset) =
Self::deserialize_slice(bytes, header.record_fields as usize, offset);
bytes::deserialize_slice(bytes, header.record_fields as usize, offset);
let (variable_slices, offset) =
Self::deserialize_slice(bytes, header.variable_slices as usize, offset);
bytes::deserialize_slice(bytes, header.variable_slices as usize, offset);
let (unspecialized_lambda_sets, offset) =
Self::deserialize_slice(bytes, header.unspecialized_lambda_sets as usize, offset);
let (exposed_vars_by_symbol, _) =
Self::deserialize_slice(bytes, header.exposed_vars_by_symbol as usize, offset);
bytes::deserialize_slice(bytes, header.unspecialized_lambda_sets as usize, offset);
let (uls_of_var, offset) =
Self::deserialize_uls_of_var(bytes, header.uls_of_var as usize, offset);
let (exposed_vars_by_symbol, offset) =
bytes::deserialize_slice(bytes, header.exposed_vars_by_symbol as usize, offset);
(
Self {
utable,
variables: variables.to_vec(),
tag_names: tag_names.to_vec(),
closure_names: closure_names.to_vec(),
field_names,
record_fields: record_fields.to_vec(),
variable_slices: variable_slices.to_vec(),
unspecialized_lambda_sets: unspecialized_lambda_sets.to_vec(),
tag_name_cache: Default::default(),
problems: Default::default(),
uls_of_var: Default::default(),
},
exposed_vars_by_symbol,
(
Self {
utable,
variables: variables.to_vec(),
tag_names: tag_names.to_vec(),
closure_names: closure_names.to_vec(),
field_names,
record_fields: record_fields.to_vec(),
variable_slices: variable_slices.to_vec(),
unspecialized_lambda_sets: unspecialized_lambda_sets.to_vec(),
tag_name_cache: Default::default(),
problems: Default::default(),
uls_of_var,
},
exposed_vars_by_symbol,
),
offset,
)
}
@ -255,7 +265,7 @@ impl Subs {
length: usize,
offset: usize,
) -> (Vec<Lowercase>, usize) {
let (slices, mut offset) = Self::deserialize_slice::<SubsSlice<u8>>(bytes, length, offset);
let (slices, mut offset) = bytes::deserialize_slice::<SubsSlice<u8>>(bytes, length, offset);
let string_slice = &bytes[offset..];
@ -273,7 +283,7 @@ impl Subs {
fn deserialize_tag_names(bytes: &[u8], length: usize, offset: usize) -> (Vec<TagName>, usize) {
let (slices, mut offset) =
Self::deserialize_slice::<SerializedTagName>(bytes, length, offset);
bytes::deserialize_slice::<SerializedTagName>(bytes, length, offset);
let string_slice = &bytes[offset..];
@ -290,24 +300,6 @@ impl Subs {
(tag_names, offset)
}
pub(crate) fn deserialize_slice<T>(
bytes: &[u8],
length: usize,
mut offset: usize,
) -> (&[T], usize) {
let alignment = std::mem::align_of::<T>();
let size = std::mem::size_of::<T>();
offset = round_to_multiple_of(offset, alignment);
let byte_length = length * size;
let byte_slice = &bytes[offset..][..byte_length];
let slice = unsafe { std::slice::from_raw_parts(byte_slice.as_ptr() as *const T, length) };
(slice, offset + byte_length)
}
}
/// Mapping of variables to [Content::LambdaSet]s containing unspecialized lambda sets depending on

25
crates/compiler/types/src/unification_table.rs
View File

@ -1,6 +1,7 @@
use std::hint::unreachable_unchecked;
use crate::subs::{Content, Descriptor, Mark, OptVariable, Rank, Variable, VariableSubsSlice};
use roc_serialize::bytes;
#[derive(Clone, Default)]
pub struct UnificationTable {
@ -376,9 +377,7 @@ impl UnificationTable {
writer: &mut impl std::io::Write,
mut written: usize,
) -> std::io::Result<usize> {
use crate::subs::Subs;
written = Subs::serialize_slice(&self.contents, writer, written)?;
written = bytes::serialize_slice(&self.contents, writer, written)?;
let mut ranks = Vec::new();
let mut marks = Vec::new();
@ -406,22 +405,20 @@ impl UnificationTable {
}
}
written = Subs::serialize_slice(&ranks, writer, written)?;
written = Subs::serialize_slice(&marks, writer, written)?;
written = Subs::serialize_slice(&copies, writer, written)?;
written = Subs::serialize_slice(&redirects, writer, written)?;
written = bytes::serialize_slice(&ranks, writer, written)?;
written = bytes::serialize_slice(&marks, writer, written)?;
written = bytes::serialize_slice(&copies, writer, written)?;
written = bytes::serialize_slice(&redirects, writer, written)?;
Ok(written)
}
pub(crate) fn deserialize(bytes: &[u8], length: usize, offset: usize) -> (Self, usize) {
use crate::subs::Subs;
let (contents, offset) = Subs::deserialize_slice::<Content>(bytes, length, offset);
let (ranks, offset) = Subs::deserialize_slice::<Rank>(bytes, length, offset);
let (marks, offset) = Subs::deserialize_slice::<Mark>(bytes, length, offset);
let (copies, offset) = Subs::deserialize_slice::<OptVariable>(bytes, length, offset);
let (redirects, offset) = Subs::deserialize_slice::<OptVariable>(bytes, length, offset);
let (contents, offset) = bytes::deserialize_slice::<Content>(bytes, length, offset);
let (ranks, offset) = bytes::deserialize_slice::<Rank>(bytes, length, offset);
let (marks, offset) = bytes::deserialize_slice::<Mark>(bytes, length, offset);
let (copies, offset) = bytes::deserialize_slice::<OptVariable>(bytes, length, offset);
let (redirects, offset) = bytes::deserialize_slice::<OptVariable>(bytes, length, offset);
let mut metadata = Vec::with_capacity(ranks.len());