urbit/ares
1
1
Fork 0
mirror of https://github.com/urbit/ares.git synced 2024-11-26 09:57:56 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #53 from urbit/barter-simsum/philip-pma-changes

Browse Source 
philip's pma changes inter alia
This commit is contained in:
barter-simsum 2023-05-19 11:10:53 -04:00 committed by GitHub
commit a07d4d9e74
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 609 additions and 486 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
rust/ares/Cargo.lock generated
View File

@ -15,6 +15,7 @@ dependencies = [
"ares_macros",
"assert_no_alloc",
"bitvec",
"cc",
"criterion",
"either",
"ibig",
@ -88,6 +89,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.0.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
[[package]]
name = "cfg-if"
version = "1.0.0"

3
rust/ares/Cargo.toml
View File

@ -24,6 +24,9 @@ static_assertions = "1.1.0"
ibig = "0.3.6"
assert_no_alloc = "1.1.2"
[build-dependencies]
cc = "1.0.79"
[[bin]]
name = "ares"
path = "src/main.rs"

11
rust/ares/build.rs Normal file
View File

@ -0,0 +1,11 @@
fn main() {
println!("cargo:rerun-if-changed=build.rs");
println!("cargo:rerun-if-changed=./src/pma");
cc::Build::new()
.file("./src/pma/malloc.c")
.file("./src/pma/includes/checksum.c")
// .opt_level(3)
.opt_level(0)
.flag("-g3")
.compile("pma_malloc");
}

1
rust/ares/src/main.rs
View File

@ -28,6 +28,7 @@ fn main() -> io::Result<()> {
ares::serialization::use_gdb();
ares::snapshot::use_gdb();
ares::snapshot::double_jam::use_gdb();
ares::snapshot::pma::use_gdb();
}
if filename == "serf" {

75
rust/ares/src/mem.rs
View File

@ -1,5 +1,6 @@
use crate::assert_acyclic;
use crate::noun::{Atom, Cell, CellMemory, IndirectAtom, Noun, NounAllocator};
use crate::snapshot::pma::{pma_in_arena, pma_malloc};
use either::Either::{self, Left, Right};
use ibig::Stack;
use libc::{c_void, memcmp};
@ -585,6 +586,80 @@ impl NockStack {
assert_acyclic!(*noun);
}
/** Copy out to the PMA
*
* See copy_east/west for inline comments
*/
pub unsafe fn copy_pma(&mut self, noun: &mut Noun) {
assert!(self.polarity == Polarity::West);
let work_start = self.stack_pointer;
self.stack_pointer = self.stack_pointer.add(2);
*(self.stack_pointer.sub(2) as *mut Noun) = *noun;
*(self.stack_pointer.sub(1) as *mut *mut Noun) = noun as *mut Noun;
loop {
if self.stack_pointer == work_start {
break;
}
let next_noun = *(self.stack_pointer.sub(2) as *const Noun);
let next_dest = *(self.stack_pointer.sub(1) as *const *mut Noun);
self.stack_pointer = self.stack_pointer.sub(2);
match next_noun.as_either_direct_allocated() {
Either::Left(_direct) => {
*next_dest = next_noun;
}
Either::Right(allocated) => match allocated.forwarding_pointer() {
Option::Some(new_allocated) => {
*next_dest = new_allocated.as_noun();
}
Option::None => {
if pma_in_arena(allocated.to_raw_pointer()) {
*next_dest = allocated.as_noun();
} else {
match allocated.as_either() {
Either::Left(mut indirect) => {
let new_indirect_alloc =
pma_malloc(indirect_raw_size(indirect));
copy_nonoverlapping(
indirect.to_raw_pointer(),
new_indirect_alloc,
indirect_raw_size(indirect),
);
indirect.set_forwarding_pointer(new_indirect_alloc);
*next_dest = IndirectAtom::from_raw_pointer(new_indirect_alloc)
.as_noun();
}
Either::Right(mut cell) => {
let new_cell_alloc: *mut CellMemory =
pma_malloc(word_size_of::<CellMemory>());
(*new_cell_alloc).metadata = (*cell.to_raw_pointer()).metadata;
*(self.stack_pointer as *mut Noun) = cell.tail();
*(self.stack_pointer.add(1) as *mut *mut Noun) =
&mut (*new_cell_alloc).tail;
*(self.stack_pointer.add(2) as *mut Noun) = cell.head();
*(self.stack_pointer.add(3) as *mut *mut Noun) =
&mut (*new_cell_alloc).head;
self.stack_pointer = self.stack_pointer.add(4);
cell.set_forwarding_pointer(new_cell_alloc);
*next_dest = Cell::from_raw_pointer(new_cell_alloc).as_noun();
}
}
}
}
},
}
}
assert_acyclic!(*noun);
}
pub fn frame_size(&self) -> usize {
match self.polarity {
Polarity::East => self.frame_pointer as usize - self.stack_pointer as usize,

3
rust/ares/src/pma/.gitignore vendored
View File

@ -1,3 +0,0 @@
bin/
dep/
obj/

3
rust/ares/src/pma/includes/checksum.c
View File

@ -51,7 +51,6 @@ static uint32_t crc_tab32[256];
* a byte string that is passed to the function together with a parameter
* indicating the length.
*/
uint32_t crc_32( const unsigned char *input_str, size_t num_bytes ) {
uint32_t crc;
@ -132,4 +131,4 @@ static void init_crc32_tab( void ) {
crc_tab32_init = true;
} /* init_crc32_tab */
} /* init_crc32_tab */

114
rust/ares/src/pma/makefile
View File

@ -1,114 +0,0 @@
#==============================================================================
# MACROS
#==============================================================================
CC := gcc
CSTD := -std=c11
LIB_CFLAGS := -D_GNU_SOURCE
DEV_CFLAGS := -Wall -Wextra -Wpedantic -Wformat=2 -Wno-unused-parameter \
-Wshadow -Wwrite-strings -Wstrict-prototypes \
-Wold-style-definition -Wredundant-decls -Wnested-externs \
-Wmissing-include-dirs -Og
# Core sources
SRC_DIR := ./
OBJ_DIR := obj/
DEP_DIR := dep/
BIN_DIR := bin/
SOURCES := $(shell ls $(SRC_DIR)*.c)
OBJECTS := $(subst $(SRC_DIR),$(OBJ_DIR),$(subst .c,.o,$(SOURCES)))
DEPFILES := $(subst $(SRC_DIR),$(DEP_DIR),$(subst .c,.d,$(SOURCES)))
# Additional sources
INC_DIR := includes/
INC_SRC_DIR := ${addprefix ${SRC_DIR},${INC_DIR}}
INC_OBJ_DIR := ${addprefix ${OBJ_DIR},${INC_DIR}}
INC_DEP_DIR := ${addprefix ${DEP_DIR},${INC_DIR}}
INC_SOURCES := $(shell ls $(INC_SRC_DIR)*.c)
INC_OBJECTS := $(subst $(INC_SRC_DIR),$(INC_OBJ_DIR),$(subst .c,.o,$(INC_SOURCES)))
INC_DEPFILES := $(subst $(INC_SRC_DIR),$(INC_DEP_DIR),$(subst .c,.d,$(INC_SOURCES)))
# Tests
CMD_NAME := sanity
TEST_DIR := ${BIN_DIR}test/
TST_SRC_DIR := $(addprefix $(SRC_DIR),"test/")
TEST_SRC := $(addprefix $(TST_SRC_DIR),$(addsuffix .c,$(CMD_NAME)))
TEST_CMD := $(addprefix $(BIN_DIR),$(CMD_NAME))
#==============================================================================
# TARGETS
#==============================================================================
#
# DEFAULT TARGET
#
default : help
#
# VISIBLE TARGETS
#
# Helpful rule which lists all other rules and encourages documentation
#
# target: help - Display all targets in makefile
#
help :
@egrep "^# target:" makefile
# Run sanity check
#
# target: sane - Run sanity check
sane : $(TEST_CMD)
@$(TEST_CMD) $(TEST_DIR)
# Clean up files produced by the makefile. Any invocation should execute, regardless of file modification date, hence
# dependency on FRC.
#
# target: clean - Remove all files produced by this makefile
clean : FRC
@rm -rf $(BIN_DIR) $(OBJ_DIR) $(DEP_DIR)
#
# HIDDEN TARGETS
#
# Force build of dependency and object files to import additional makefile targets
#
-include $(DEPFILES) ${INC_DEPFILES}
# Link executable binary for sanity check
#
$(TEST_CMD) : $(OBJECTS) ${INC_OBJECTS}
@mkdir -p $(BIN_DIR)
$(CC) $(TEST_SRC) $^ -o $@ $(CSTD) $(LIB_CFLAGS) $(DEV_CFLAGS)
# Compile all source files, but do not link. As a side effect, compile a dependency file for each source file.
#
# Dependency files are a common makefile feature used to speed up builds by auto-generating granular makefile targets.
# These files minimize the number of targets that need to be recomputed when source files are modified and can lead to
# massive build-time improvements.
#
# For more information, see the "-M" option documentation in the GCC man page, as well as this paper:
# https://web.archive.org/web/20150319074420/http://aegis.sourceforge.net/auug97.pdf
#
$(addprefix $(DEP_DIR),%.d) : $(addprefix $(SRC_DIR),%.c)
@mkdir -p $(OBJ_DIR)
@mkdir -p $(DEP_DIR)
$(CC) -MD -MP -MF $@ -MT '$@ $(subst $(DEP_DIR),$(OBJ_DIR),$(@:.d=.o))' \
$< -c -o $(subst $(DEP_DIR),$(OBJ_DIR),$(@:.d=.o)) $(CSTD) $(LIB_CFLAGS) $(DEV_CFLAGS)
# Same as above, but for additional dependencies.
$(addprefix $(INC_DEP_DIR),%.d) : $(addprefix $(INC_SRC_DIR),%.c)
@mkdir -p $(INC_OBJ_DIR)
@mkdir -p $(INC_DEP_DIR)
$(CC) -MD -MP -MF $@ -MT '$@ $(subst $(INC_DEP_DIR),$(INC_OBJ_DIR),$(@:.d=.o))' \
$< -c -o $(subst $(INC_DEP_DIR),$(INC_OBJ_DIR),$(@:.d=.o)) $(CSTD) $(LIB_CFLAGS) $(DEV_CFLAGS)
# Special pseudo target which always needs to be recomputed. Forces full rebuild of target every time when used as a
# component.
FRC :

605
rust/ares/src/pma/malloc.c
View File

File diff suppressed because it is too large Load Diff

40
rust/ares/src/pma/malloc.h
View File

@ -4,6 +4,7 @@
#pragma once
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
@ -11,6 +12,16 @@
// PROTOTYPES
//==============================================================================
/**
* Struct returned from pma_load()
*/
typedef struct PMARootState PMARootState;
struct PMARootState {
uint64_t epoch; // Epoch ID of the most recently processed event
uint64_t event; // ID of the most recently processed event
uint64_t root; // Root after most recent event
};
/**
* Initialize a brand new PMA environment and event snapshot
*
@ -32,7 +43,7 @@ pma_init(const char *path);
* @return 0 success
* @return -1 failure; errno set to error code
*/
int
PMARootState
pma_load(const char *path);
/**
@ -46,7 +57,7 @@ pma_load(const char *path);
* @return -1 failure; errno set to error code
*/
int
pma_close(uint64_t epoch, uint64_t event);
pma_close(uint64_t epoch, uint64_t event, uint64_t root);
/**
* Allocate a new block of memory in the PMA
@ -81,4 +92,27 @@ pma_free(void *address);
* @return -1 failure; errno set to error code
*/
int
pma_sync(uint64_t epoch, uint64_t event);
pma_sync(uint64_t epoch, uint64_t event, uint64_t root);
/**
* True if the address is in the PMA
*/
bool
pma_in_arena(void *address);
/*
bp(X) where X is false will raise a SIGTRAP. If the process is being run
inside a debugger, this can be caught and ignored. It's equivalent to a
breakpoint. If run without a debugger, it will dump core, like an assert
*/
#if defined(__i386__) || defined(__x86_64__)
#define bp(x) do { if(!(x)) __asm__ volatile("int $3"); } while (0)
#elif defined(__thumb__)
#define bp(x) do { if(!(x)) __asm__ volatile(".inst 0xde01"); } while (0)
#elif defined(__aarch64__)
#define bp(x) do { if(!(x)) __asm__ volatile(".inst 0xd4200000"); } while (0)
#elif defined(__arm__)
#define bp(x) do { if(!(x)) __asm__ volatile(".inst 0xe7f001f0"); } while (0)
#else
STATIC_ASSERT(0, "debugger break instruction unimplemented");
#endif

73
rust/ares/src/pma/test/sanity.c
View File

@ -1,73 +0,0 @@
#include <errno.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include "../malloc.h"
//==============================================================================
// Functions
//==============================================================================
int
main(int argc, char** argv) {
void *ptr_1;
void *ptr_2;
void *ptr_3;
void *ptr_4;
void *ptr_5;
void *ptr_6;
void *ptr_7;
void *ptr_8;
void *ptr_9;
void *ptr_10;
void *ptr_11;
if (pma_init(argv[1])) {
fprintf(stderr, "init not sane:\n");
goto test_error;
};
ptr_1 = pma_malloc(8);
ptr_2 = pma_malloc(16);
ptr_3 = pma_malloc(32);
ptr_4 = pma_malloc(64);
ptr_5 = pma_malloc(128);
ptr_6 = pma_malloc(256);
ptr_7 = pma_malloc(512);
ptr_8 = pma_malloc(1024);
ptr_9 = pma_malloc(2048);
ptr_10 = pma_malloc(4096);
ptr_11 = pma_malloc(8192);
if (pma_sync(1UL, 1UL)) {
fprintf(stderr, "sync not sane:\n");
goto test_error;
};
pma_free(ptr_1);
pma_free(ptr_2);
pma_free(ptr_3);
pma_free(ptr_4);
pma_free(ptr_5);
pma_free(ptr_6);
pma_free(ptr_7);
pma_free(ptr_8);
pma_free(ptr_9);
pma_free(ptr_10);
pma_free(ptr_11);
if (pma_close(1UL, 2UL)) {
fprintf(stderr, "sync not sane:\n");
goto test_error;
};
printf("sane\n");
return 0;
test_error:
fprintf(stderr, "%s\n", strerror(errno));
return -1;
}

2
rust/ares/src/serf.rs
View File

@ -33,6 +33,8 @@ pub fn serf() -> io::Result<()> {
snap_path.push(".urb");
snap_path.push("chk");
create_dir_all(&snap_path)?;
// TODO: switch to Pma when ready
// let snap = &mut snapshot::pma::Pma::new(snap_path);
let snap = &mut snapshot::double_jam::DoubleJam::new(snap_path);
let stack = &mut NockStack::new(96 << 10 << 10, 0);

1
rust/ares/src/snapshot.rs
View File

@ -2,6 +2,7 @@ use crate::mem::NockStack;
use crate::noun::Noun;
pub mod double_jam;
pub mod pma;
crate::gdb!();

157
rust/ares/src/snapshot/pma.rs Normal file
View File

@ -0,0 +1,157 @@
use super::Snapshot;
use crate::mem::NockStack;
use crate::mug::mug_u32;
use crate::noun::{Noun, D};
use libc::{c_char, c_int, c_void, size_t};
use std::ffi::CString;
use std::path::{Path, PathBuf};
crate::gdb!();
mod raw {
use super::*;
#[repr(C)]
pub struct RootState {
pub epoch: u64,
pub event: u64,
pub root: u64,
}
#[link(name = "pma_malloc", kind = "static")]
extern "C" {
pub(super) fn pma_init(path: *const c_char) -> c_int;
pub(super) fn pma_load(path: *const c_char) -> RootState;
pub(super) fn pma_close(epoch: u64, event: u64, root: u64) -> c_int;
pub(super) fn pma_malloc(size: size_t) -> *mut c_void;
pub(super) fn pma_free(ptr: *mut c_void) -> c_int;
pub(super) fn pma_sync(epoch: u64, event: u64, root: u64) -> c_int;
pub(super) fn pma_in_arena(ptr: *const c_void) -> bool;
}
}
unsafe fn pma_init<P: AsRef<Path>>(path: P) -> i32 {
let path = CString::new(path.as_ref().to_str().unwrap()).unwrap();
raw::pma_init(path.as_ptr())
}
unsafe fn pma_load<P: AsRef<Path>>(path: P) -> (u64, u64, Noun) {
let path = CString::new(path.as_ref().to_str().unwrap()).unwrap();
let rs = raw::pma_load(path.as_ptr());
(rs.epoch, rs.event, Noun::from_raw(rs.root))
}
#[allow(dead_code)]
unsafe fn pma_close(epoch: u64, event: u64, root: Noun) -> i32 {
raw::pma_close(epoch, event, root.as_raw())
}
/** Allocate a block of memory from the persistent memory arena.
*
* Size is in *words*, unlike the underlying pma_malloc.
*/
pub fn pma_malloc<T>(size: usize) -> *mut T {
unsafe { raw::pma_malloc(size << 3 as size_t) as *mut T }
}
#[allow(dead_code)]
unsafe fn pma_free<T>(ptr: *mut T) -> i32 {
raw::pma_free(ptr as *mut c_void)
}
unsafe fn pma_sync(epoch: u64, event: u64, root: Noun) -> i32 {
raw::pma_sync(epoch, event, root.as_raw())
}
pub fn pma_in_arena<T>(ptr: *const T) -> bool {
unsafe { raw::pma_in_arena(ptr as *const c_void) }
}
pub struct Pma {
path: PathBuf,
noun: Noun,
}
impl Pma {
pub fn new<P: AsRef<Path>>(path: P) -> Self {
let path = path.as_ref().to_path_buf();
Self { path, noun: D(0) }
}
}
impl Snapshot for Pma {
fn save(&mut self, stack: &mut NockStack, noun: &mut Noun) {
// Required so everything in the PMA has a cached mug, otherwise we would try to write
let _mug = mug_u32(stack, *noun);
unsafe { stack.copy_pma(noun) };
self.noun = *noun;
}
fn sync(&mut self, _stack: &mut NockStack, epoch: u64, event: u64) {
unsafe {
pma_sync(epoch, event, self.noun);
}
}
fn load(&mut self, _stack: &mut NockStack) -> std::io::Result<(u64, u64, Noun)> {
let path = self.path.join(".bin/page.bin");
if path.is_file() {
eprintln!("\rload: found snapshot at {:?}", path);
unsafe { Ok(pma_load(&self.path)) }
} else {
eprintln!("\rload: creating snapshot at {:?}", path);
unsafe { pma_init(&self.path) };
Ok((0, 0, D(0)))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::noun::IndirectAtom;
#[test]
fn test_pma() {
let path = "/tmp/ares_pma_test";
if let Err(err) = std::fs::remove_dir_all(path) {
if err.kind() != std::io::ErrorKind::NotFound {
panic!("failed to remove dir: {}", err);
}
}
unsafe {
pma_init(path);
let ref mut stack = NockStack::new(8 << 10 << 10, 0);
let root = IndirectAtom::new_raw(stack, 1, &0xffff_ffff_ffff_ffff).as_noun();
let eight = pma_malloc(8) as *mut u64;
*eight = 0xdeadbeef;
assert!(0 == pma_close(10, 12, root));
pma_load(path);
assert_eq!(0, pma_sync(13, 15, root));
let _ = pma_malloc(8) as *mut u64;
assert_eq!(0, pma_sync(14, 16, root));
let _ = pma_malloc(8) as *mut u64;
assert_eq!(0, pma_sync(15, 16, root));
let twenty = pma_malloc(8) as *mut u64;
pma_free(twenty as *mut c_void);
assert_eq!(0, pma_sync(16, 15, root));
let _ = pma_malloc(8) as *mut u64;
assert_eq!(0, pma_sync(17, 15, root));
let _ = pma_malloc(8) as *mut u64;
assert_eq!(0, pma_sync(18, 15, root));
let _ = pma_malloc(8) as *mut u64;
let twenty = pma_malloc(8) as *mut u64;
*twenty = 0xcafebabe;
pma_free(twenty as *mut c_void);
pma_close(123, 124, root);
}
if let Err(err) = std::fs::remove_dir_all(path) {
if err.kind() != std::io::ErrorKind::NotFound {
panic!("failed to remove dir: {}", err);
}
}
}
}