urbit/shrub
1
1
Fork 0
mirror of https://github.com/urbit/shrub.git synced 2024-12-20 17:32:11 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3235 from urbit/jb/hashcons

Browse Source 
u3: off-loom global deduplication
This commit is contained in:
Joe Bryan 2020-09-17 00:00:40 -07:00 committed by GitHub
commit 4fb3ebc145
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 7914 additions and 857 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/urbit/Makefile
View File

@ -2,12 +2,14 @@ include config.mk
jets = jets/tree.c $(wildcard jets/*/*.c)
noun = $(wildcard noun/*.c)
ur = $(wildcard ur/*.c)
vere = $(wildcard vere/*.c) $(wildcard vere/*/*.c)
daemon = $(wildcard daemon/*.c)
worker = $(wildcard worker/*.c)
tests = $(wildcard tests/*.c)
bench = $(wildcard bench/*.c)
common = $(jets) $(noun) $(vere)
common = $(jets) $(noun) $(ur) $(vere)
headers = $(shell find include -type f)
common_objs = $(shell echo $(common) | sed 's/\.c/.o/g')
@ -17,8 +19,9 @@ worker_objs = $(shell echo $(worker) | sed 's/\.c/.o/g')
all_objs = $(common_objs) $(daemon_objs) $(worker_objs)
all_srcs = $(common) $(daemon) $(worker)
test_exes = $(shell echo $(tests) | sed 's/tests\//.\/build\//g' | sed 's/\.c//g')
all_exes = $(test_exes) ./build/urbit ./build/urbit-worker
test_exes = $(shell echo $(tests) | sed 's/tests\//.\/build\//g' | sed 's/\.c//g')
bench_exes = $(shell echo $(bench) | sed 's/bench\//.\/build\//g' | sed 's/\.c//g')
all_exes = $(test_exes) $(bench_exes) ./build/urbit ./build/urbit-worker
# -Werror promotes all warnings that are enabled into errors (this is on)
# -Wall issues all types of errors. This is off (for now)
@ -40,6 +43,9 @@ test: $(test_exes)
done; \
if [ $$FAIL != 0 ]; then echo "\n" && exit 1; fi;
bench: $(bench_exes)
build/ur_bench
clean:
rm -f ./tags $(all_objs) $(all_exes)
@ -48,6 +54,11 @@ mrproper: clean
################################################################################
build/ur_bench: $(common_objs) bench/ur_bench.o
@echo CC -o $@
@mkdir -p ./build
@$(CC) $^ $(LDFLAGS) -o $@
build/%_tests: $(common_objs) tests/%_tests.o
@echo CC -o $@
@mkdir -p ./build

419
pkg/urbit/bench/ur_bench.c Normal file
View File

@ -0,0 +1,419 @@
#include "all.h"
#include "vere/vere.h"
#include "ur/ur.h"
/* _setup(): prepare for tests.
*/
static void
_setup(void)
{
u3m_init();
u3m_pave(c3y, c3n);
}
/* _ames_writ_ex(): |hi packet from fake ~zod to fake ~nec
*/
static u3_noun
_ames_writ_ex(void)
{
c3_y bod_y[63] = {
0x30, 0x90, 0x2d, 0x0, 0x0, 0x0, 0x1, 0x0, 0x9, 0xc0, 0xd0,
0x0, 0x4, 0x40, 0x30, 0xf4, 0xa, 0x3d, 0x45, 0x86, 0x66, 0x2c,
0x2, 0x38, 0xf8, 0x72, 0xa3, 0x9, 0xf6, 0x6, 0xf3, 0x0, 0xbe,
0x67, 0x61, 0x49, 0x50, 0x4, 0x3c, 0x13, 0xb2, 0x96, 0x42, 0x1b,
0x62, 0xac, 0x97, 0xff, 0x24, 0xeb, 0x69, 0x1b, 0xb2, 0x60, 0x72,
0xa, 0x53, 0xdf, 0xe8, 0x8a, 0x9c, 0x6f, 0xb3
};
u3_noun lan = u3nc(0, 1);
u3_noun cad = u3nt(c3__send, lan, u3i_bytes(sizeof(bod_y), bod_y));
u3_noun wir = u3nt(c3__newt, 0x1234, u3_nul);
u3_noun ovo = u3nc(u3nc(u3_blip, wir), cad);
u3_noun wen;
{
struct timeval tim_u;
gettimeofday(&tim_u, 0);
wen = u3_time_in_tv(&tim_u);
}
return u3nt(c3__work, 0, u3nc(wen, ovo));
}
static void
_jam_bench(void)
{
struct timeval b4, f2, d0;
c3_w mil_w, i_w, max_w = 10000;
u3_noun wit = _ames_writ_ex();
fprintf(stderr, "\r\njam microbenchmark:\r\n");
{
gettimeofday(&b4, 0);
{
c3_w* wor_w, bit_w;
for ( i_w = 0; i_w < max_w; i_w++ ) {
wor_w = u3s_jam_fib(wit, &bit_w);
u3a_wfree(wor_w);
}
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " jam og: %u ms\r\n", mil_w);
}
{
gettimeofday(&b4, 0);
{
c3_d len_d;
c3_y* byt_y;
for ( i_w = 0; i_w < max_w; i_w++ ) {
u3s_jam_xeno(wit, &len_d, &byt_y);
c3_free(byt_y);
}
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " jam xeno: %u ms\r\n", mil_w);
}
while ( 1 ) {
ur_root_t* rot_u = ur_root_init();
c3_d len_d;
c3_y* byt_y;
ur_nref ref;
u3s_jam_xeno(wit, &len_d, &byt_y);
if ( ur_cue_good != ur_cue(rot_u, len_d, byt_y, &ref) ) {
fprintf(stderr, " jam bench: cue failed wtf\r\n");
break;
}
c3_free(byt_y);
{
gettimeofday(&b4, 0);
for ( i_w = 0; i_w < max_w; i_w++ ) {
ur_jam(rot_u, ref, &len_d, &byt_y);
c3_free(byt_y);
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " jam cons: %u ms\r\n", mil_w);
}
{
gettimeofday(&b4, 0);
{
ur_dict64_t dic_u = {0};
c3_d len_d;
c3_y* byt_y;
ur_dict64_grow((ur_root_t*)0, &dic_u, ur_fib10, ur_fib11);
for ( i_w = 0; i_w < max_w; i_w++ ) {
ur_jam_unsafe(rot_u, ref, &dic_u, &len_d, &byt_y);
c3_free(byt_y);
ur_dict64_wipe(&dic_u);
}
ur_dict_free((ur_dict_t*)&dic_u);
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " jam cons unsafe: %u ms\r\n", mil_w);
}
ur_root_free(rot_u);
break;
}
u3z(wit);
}
static void
_cue_bench(void)
{
struct timeval b4, f2, d0;
c3_w mil_w, i_w, max_w = 20000;
u3_atom vat = u3ke_jam(_ames_writ_ex());
fprintf(stderr, "\r\ncue microbenchmark:\r\n");
{
gettimeofday(&b4, 0);
for ( i_w = 0; i_w < max_w; i_w++ ) {
u3z(u3s_cue(vat));
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " cue og: %u ms\r\n", mil_w);
}
{
gettimeofday(&b4, 0);
for ( i_w = 0; i_w < max_w; i_w++ ) {
u3z(u3s_cue_atom(vat));
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " cue atom: %u ms\r\n", mil_w);
}
{
gettimeofday(&b4, 0);
{
u3_noun out;
c3_w len_w = u3r_met(3, vat);
// XX assumes little-endian
//
c3_y* byt_y = ( c3y == u3a_is_cat(vat) )
? (c3_y*)&vat
: (c3_y*)((u3a_atom*)u3a_to_ptr(vat))->buf_w;
for ( i_w = 0; i_w < max_w; i_w++ ) {
u3s_cue_xeno(len_w, byt_y, &out);
u3z(out);
}
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " cue xeno: %u ms\r\n", mil_w);
}
{
gettimeofday(&b4, 0);
{
ur_dict32_t dic_u = {0};
u3_noun out;
c3_w len_w = u3r_met(3, vat);
// XX assumes little-endian
//
c3_y* byt_y = ( c3y == u3a_is_cat(vat) )
? (c3_y*)&vat
: (c3_y*)((u3a_atom*)u3a_to_ptr(vat))->buf_w;
ur_dict32_grow((ur_root_t*)0, &dic_u, ur_fib10, ur_fib11);
for ( i_w = 0; i_w < max_w; i_w++ ) {
u3s_cue_xeno_unsafe(&dic_u, len_w, byt_y, &out);
u3z(out);
ur_dict32_wipe(&dic_u);
}
ur_dict_free((ur_dict_t*)&dic_u);
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " cue xeno unsafe: %u ms\r\n", mil_w);
}
{
gettimeofday(&b4, 0);
{
c3_w len_w = u3r_met(3, vat);
// XX assumes little-endian
//
c3_y* byt_y = ( c3y == u3a_is_cat(vat) )
? (c3_y*)&vat
: (c3_y*)((u3a_atom*)u3a_to_ptr(vat))->buf_w;
for ( i_w = 0; i_w < max_w; i_w++ ) {
ur_cue_test(len_w, byt_y);
}
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " cue test: %u ms\r\n", mil_w);
}
{
gettimeofday(&b4, 0);
{
ur_dict_t dic_u = {0};
u3_noun out;
c3_w len_w = u3r_met(3, vat);
// XX assumes little-endian
//
c3_y* byt_y = ( c3y == u3a_is_cat(vat) )
? (c3_y*)&vat
: (c3_y*)((u3a_atom*)u3a_to_ptr(vat))->buf_w;
ur_dict_grow((ur_root_t*)0, &dic_u, ur_fib10, ur_fib11);
for ( i_w = 0; i_w < max_w; i_w++ ) {
ur_cue_test_unsafe(&dic_u, len_w, byt_y);
ur_dict_wipe(&dic_u);
}
ur_dict_free(&dic_u);
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " cue test unsafe: %u ms\r\n", mil_w);
}
{
gettimeofday(&b4, 0);
{
ur_root_t* rot_u = ur_root_init();
ur_nref ref;
c3_w len_w = u3r_met(3, vat);
// XX assumes little-endian
//
c3_y* byt_y = ( c3y == u3a_is_cat(vat) )
? (c3_y*)&vat
: (c3_y*)((u3a_atom*)u3a_to_ptr(vat))->buf_w;
for ( i_w = 0; i_w < max_w; i_w++ ) {
ur_cue(rot_u, len_w, byt_y, &ref);
}
ur_root_free(rot_u);
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " cue cons: %u ms\r\n", mil_w);
}
{
gettimeofday(&b4, 0);
{
ur_root_t* rot_u;
ur_nref ref;
c3_w len_w = u3r_met(3, vat);
// XX assumes little-endian
//
c3_y* byt_y = ( c3y == u3a_is_cat(vat) )
? (c3_y*)&vat
: (c3_y*)((u3a_atom*)u3a_to_ptr(vat))->buf_w;
for ( i_w = 0; i_w < max_w; i_w++ ) {
rot_u = ur_root_init();
ur_cue(rot_u, len_w, byt_y, &ref);
ur_root_free(rot_u);
}
}
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " cue re-cons: %u ms\r\n", mil_w);
}
u3z(vat);
}
static u3_noun
_cue_loop(u3_atom a)
{
c3_w i_w, max_w = 20000;
for ( i_w = 0; i_w < max_w; i_w++ ) {
u3z(u3s_cue(a));
}
return u3_blip;
}
static u3_noun
_cue_atom_loop(u3_atom a)
{
c3_w i_w, max_w = 20000;
for ( i_w = 0; i_w < max_w; i_w++ ) {
u3z(u3s_cue_atom(a));
}
return u3_blip;
}
static void
_cue_soft_bench(void)
{
struct timeval b4, f2, d0;
u3_atom vat = u3ke_jam(_ames_writ_ex());
c3_w mil_w;
fprintf(stderr, "\r\ncue virtual microbenchmark:\r\n");
{
gettimeofday(&b4, 0);
u3z(u3m_soft(0, _cue_loop, u3k(vat)));
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " cue virtual og: %u ms\r\n", mil_w);
}
{
gettimeofday(&b4, 0);
u3z(u3m_soft(0, _cue_atom_loop, u3k(vat)));
gettimeofday(&f2, 0);
timersub(&f2, &b4, &d0);
mil_w = (d0.tv_sec * 1000) + (d0.tv_usec / 1000);
fprintf(stderr, " cue virtual atom: %u ms\r\n", mil_w);
}
u3z(vat);
}
/* main(): run all benchmarks
*/
int
main(int argc, char* argv[])
{
_setup();
_jam_bench();
_cue_bench();
_cue_soft_bench();
// GC
//
u3m_grab(u3_none);
return 0;
}

1
pkg/urbit/include/all.h
View File

@ -27,6 +27,7 @@
# include "noun/serial.h" // u3s: serialization
# include "noun/trace.h" // u3t: profiling / tracing
# include "noun/xtract.h" // u3x: noun access (error crashes)
# include "noun/urth.h" // u3u: off-loom integration
# include "noun/vortex.h" // u3v: arvo kernel
# include "noun/zave.h" // u3z: memoization

20
pkg/urbit/include/noun/events.h
View File

@ -78,22 +78,12 @@
c3_w
u3e_dirty(void);
/* u3e_hold(): backup memory images
/* u3e_yolo(): disable dirty page tracking, read/write whole loom.
*/
c3_o
u3e_hold(void);
u3e_yolo(void);
/* u3e_drop(): remove backed-up memory images
/* u3e_foul(): dirty all the pages of the loom.
*/
c3_o
u3e_drop(void);
/* u3e_fall(): restore memory images
*/
c3_o
u3e_fall(void);
/* u3e_wipe(): discard memory images
*/
c3_o
u3e_wipe(void);
void
u3e_foul(void);

5
pkg/urbit/include/noun/hashtable.h
View File

@ -177,3 +177,8 @@
*/
u3p(u3h_root)
u3h_take(u3p(u3h_root) har_p);
/* u3h_wyt(): number of entries
*/
c3_w
u3h_wyt(u3p(u3h_root) har_p);

20
pkg/urbit/include/noun/manage.h
View File

@ -145,23 +145,3 @@
*/
c3_w
u3m_pack(void);
/* u3m_rock_stay(): jam state into [dir_c] at [evt_d]
*/
c3_o
u3m_rock_stay(c3_c* dir_c, c3_d evt_d);
/* u3m_rock_load(): load state from [dir_c] at [evt_d]
*/
c3_o
u3m_rock_load(c3_c* dir_c, c3_d evt_d);
/* u3m_rock_drop(): delete saved state from [dir_c] at [evt_d]
*/
c3_o
u3m_rock_drop(c3_c* dir_c, c3_d evt_d);
/* u3m_wipe(): purge and reinitialize loom, with checkpointing
*/
void
u3m_wipe(void);

46
pkg/urbit/include/noun/serial.h
View File

@ -1,6 +1,12 @@
/* i/n/serial.h
**
*/
/* forward declarations
*/
/* ur_dict32_s: off-loom 32-bit dictionary.
*/
struct ur_dict32_s;
/* Noun serialization. All noun arguments RETAINED.
*/
@ -12,25 +18,35 @@
c3_w*
u3s_jam_fib(u3_noun a, c3_w* bit_w);
/* u3s_jam_met(): measure a noun for jam, calculating backrefs
/* u3s_jam_xeno(): jam with off-loom buffer (re-)allocation.
*/
c3_d
u3s_jam_met(u3_noun a, u3p(u3h_root)* bak_p);
/* u3s_jam_buf(): jam [a] into [buf_w], without allocation
**
** using backrefs in [bak_p], as computed by u3s_jam_met
** can only encode up to c3_w bits
*/
void
u3s_jam_buf(u3_noun a, u3p(u3h_root) bak_p, c3_w* buf_w);
/* u3s_jam_file(): jam [a] into a file, overwriting
*/
c3_o
u3s_jam_file(u3_noun a, c3_c* pas_c);
u3s_jam_xeno(u3_noun a, c3_d* len_d, c3_y** byt_y);
/* u3s_cue(): cue [a]
*/
u3_noun
u3s_cue(u3_atom a);
/* u3s_cue_xeno_unsafe(): cue onto the loom, all bookkeeping off-loom.
*/
c3_o
u3s_cue_xeno_unsafe(struct ur_dict32_s* dic_u,
c3_d len_d,
const c3_y* byt_y,
u3_noun* out);
/* u3s_cue_xeno(): cue onto the loom, bookkeeping off the loom.
*/
c3_o
u3s_cue_xeno(c3_d len_d, const c3_y* byt_y, u3_noun* out);
/* u3s_cue_bytes(): cue bytes onto the loom.
*/
u3_noun
u3s_cue_bytes(c3_d len_d, const c3_y* byt_y);
/* u3s_cue_atom(): cue atom.
*/
u3_noun
u3s_cue_atom(u3_atom a);

37
pkg/urbit/include/noun/urth.h Normal file
View File

@ -0,0 +1,37 @@
/* include/noun/urth.h
*/
/** Functions.
**/
/* u3u_meld(): globally deduplicate memory.
*/
void
u3u_meld(void);
/* u3u_cram(): globably deduplicate memory, and write a rock to disk.
*/
c3_o
u3u_cram(c3_c* dir_c, c3_d eve_d);
/* u3u_uncram(): restore persistent state from a rock.
*/
c3_o
u3u_uncram(c3_c* dir_c, c3_d eve_d);
/* u3u_mmap_read(): open and mmap the file at [pat_c] for reading.
*/
c3_o
u3u_mmap_read(c3_c* cap_c, c3_c* pat_c, c3_d* out_d, c3_y** out_y);
/* u3u_mmap(): open/create file-backed mmap at [pat_c] for read/write.
*/
c3_o
u3u_mmap(c3_c* cap_c, c3_c* pat_c, c3_d len_d, c3_y** out_y);
/* u3u_mmap_save(): sync file-backed mmap.
*/
c3_o
u3u_mmap_save(c3_c* cap_c, c3_c* pat_c, c3_d len_d, c3_y* byt_y);
/* u3u_munmap(): unmap the region at [byt_y].
*/
c3_o
u3u_munmap(c3_d len_d, c3_y* byt_y);

223
pkg/urbit/include/ur/bitstream.h Normal file
View File

@ -0,0 +1,223 @@
#ifndef UR_BITSTREAM_H
#define UR_BITSTREAM_H
#include <inttypes.h>
/*
** stateful bitstream reader, backed by a byte-buffer,
** maintaing a 64-bit bit-cursor, and supporting a variety
** of read sizes and patterns.
**
** NB: ur_bsr*_any() functions behave as if the stream were infinite,
** subject to the overall limit of the bit-cursor.
**
*/
typedef struct ur_bsr_s {
uint64_t left;
uint64_t bits;
uint8_t off;
const uint8_t *bytes;
} ur_bsr_t;
/*
** generalized bitstream-reader/cue response enum
*/
typedef enum {
ur_cue_good = 0, // successful read
ur_cue_back = 1, // missing backreference
ur_cue_gone = 2, // read off the end of the stream
ur_cue_meme = 3 // exceeded memory representation
} ur_cue_res_e;
/*
** jam/cue type tag enumeration
*/
typedef enum {
ur_jam_atom = 0,
ur_jam_cell = 1,
ur_jam_back = 2
} ur_cue_tag_e;
/*
** stateful bitstream writer, backed by a byte-buffer automatically
** reallocated with fibonacc growth, maintaing a 64-bit bit-cursor,
** and supporting a variety of write sizes and patterns.
**
*/
typedef struct ur_bsw_s {
uint64_t prev;
uint64_t size;
uint64_t fill;
uint64_t bits;
uint8_t off;
uint8_t *bytes;
} ur_bsw_t;
/*
** initialize bitstream-reader and check for 64-bit bit-cursor overflow.
*/
ur_cue_res_e
ur_bsr_init(ur_bsr_t *bsr, uint64_t len, const uint8_t *bytes);
/*
** validate bitstream-reader invariants.
*/
ur_bool_t
ur_bsr_sane(ur_bsr_t *bsr);
/*
** read a bit, failing at EOS
*/
ur_cue_res_e
ur_bsr_bit(ur_bsr_t *bsr, uint8_t *out);
/*
** read a bit
*/
uint8_t
ur_bsr_bit_any(ur_bsr_t *bsr);
/*
** read N (up to 8) bits into a uint8.
*/
uint8_t
ur_bsr8_any(ur_bsr_t *bsr, uint8_t len);
/*
** read N (up to 32) bits into a uint32.
*/
uint32_t
ur_bsr32_any(ur_bsr_t *bsr, uint8_t len);
/*
** read N (up to 64) bits into a uint64.
*/
uint64_t
ur_bsr64_any(ur_bsr_t *bsr, uint8_t len);
/*
** read N bits into a zero-initialized byte array.
*/
void
ur_bsr_bytes_any(ur_bsr_t *bsr, uint64_t len, uint8_t *out);
/*
** advance the bitstream cursor as if we had read N bits.
*/
void
ur_bsr_skip_any(ur_bsr_t *bsr, uint64_t len);
/*
** read a jam/cue type tag.
*/
ur_cue_res_e
ur_bsr_tag(ur_bsr_t *bsr, ur_cue_tag_e *out);
/*
** read a binary exponent, producing the binary log.
**
** read N (up to 255) zero bits followed by a 1, produce N.
*/
ur_cue_res_e
ur_bsr_log(ur_bsr_t *bsr, uint8_t *out);
/*
** read an atomic run-length (a la +rub).
**
** read a binary log N, then read N (up to 64) bits,
** produce (N-bits ^ (1 << N))
*/
ur_cue_res_e
ur_bsr_rub_len(ur_bsr_t *bsr, uint64_t *out);
/*
** reallocate bitstream write buffer with max(fibonacci, step) growth.
*/
void
ur_bsw_grow(ur_bsw_t *bsw, uint64_t step);
/*
** validate bitstream-writer invariants.
*/
ur_bool_t
ur_bsw_sane(ur_bsw_t *bsw);
/*
** write a bit
*/
void
ur_bsw_bit(ur_bsw_t *bsw, uint8_t bit);
/*
** write N (up to 8) bits of a uint8.
*/
void
ur_bsw8(ur_bsw_t *bsw, uint8_t len, uint8_t byt);
/*
** write N (up to 32) bits of a uint32.
*/
void
ur_bsw32(ur_bsw_t *bsw, uint8_t len, uint32_t val);
/*
** write N (up to 64) bits of a uint64.
*/
void
ur_bsw64(ur_bsw_t *bsw, uint8_t len, uint64_t val);
/*
** write N bits of a byte array.
**
** NB: [byt] must contain at least N bits.
*/
void
ur_bsw_bytes(ur_bsw_t *bsw, uint64_t len, uint8_t *byt);
/*
** write a binary exponent (N zero bits, followed by a 1).
*/
void
ur_bsw_bex(ur_bsw_t *bsw, uint8_t n);
/*
** write N (up to 64) run-length prefixed bits (a la +mat).
*/
void
ur_bsw_mat64(ur_bsw_t *bsw, uint8_t len, uint64_t val);
/*
** write N run-length prefixed bits (a la +mat).
**
** NB: [byt] must contain at least N bits.
*/
void
ur_bsw_mat_bytes(ur_bsw_t *bsw, uint64_t len, uint8_t *byt);
/*
** write a backref tag (1, 1) and N (up to 64) run-length prefixed bits.
*/
void
ur_bsw_back64(ur_bsw_t *bsw, uint8_t len, uint64_t val);
/*
** write an atom tag (0) and N (up to 64) run-length prefixed bits.
*/
void
ur_bsw_atom64(ur_bsw_t *bsw, uint8_t len, uint64_t val);
/*
** write an atom tag (0) and N run-length prefixed bits.
**
** NB: [byt] must contain at least N bits.
*/
void
ur_bsw_atom_bytes(ur_bsw_t *bsw, uint64_t len, uint8_t *byt);
/*
** write a cell tag (1, 0)
*/
void
ur_bsw_cell(ur_bsw_t *bsw);
#endif

84
pkg/urbit/include/ur/defs.h Normal file
View File

@ -0,0 +1,84 @@
#ifndef UR_DEFS_H
#define UR_DEFS_H
#include <inttypes.h>
#include <assert.h>
#include <limits.h>
typedef uint8_t ur_bool_t;
#define ur_min(a, b) ( ((a) < (b)) ? (a) : (b) )
#define ur_max(a, b) ( ((a) > (b)) ? (a) : (b) )
/*
** fibonacci constants, for convenient initialization of
** objects intended to be reallocated with fibonacci growth
*/
#define ur_fib10 55
#define ur_fib11 89
#define ur_fib12 144
#define ur_fib27 196418
#define ur_fib28 317811
#define ur_fib33 3524578
#define ur_fib34 5702887
/*
** bit-masking helpers
*/
#define ur_mask_3(a) (a & 0x7)
#define ur_mask_8(a) (a & 0xff)
#define ur_mask_31(a) (a & 0x7fffffff)
#define ur_mask_62(a) (a & 0x3fffffffffffffffULL)
/*
** bloq (binary exponent) conversions
*/
#define ur_bloq_up1(a) ( (a + 0x1) >> 1 )
#define ur_bloq_up2(a) ( (a + 0x3) >> 2 )
#define ur_bloq_up3(a) ( (a + 0x7) >> 3 )
/*
** atom measurement
*/
#if (32 == (CHAR_BIT * __SIZEOF_INT__))
# define ur_lz32 __builtin_clz
# define ur_tz32 __builtin_ctz
#elif (32 == (CHAR_BIT * __SIZEOF_LONG__))
# define ur_lz32 __builtin_clzl
# define ur_tz32 __builtin_ctzl
#else
# error "port me"
#endif
#if (64 == (CHAR_BIT * __SIZEOF_LONG__))
# define ur_lz64 __builtin_clzl
# define ur_tz64 __builtin_ctzl
#elif (64 == (CHAR_BIT * __SIZEOF_LONG_LONG__))
# define ur_lz64 __builtin_clzll
# define ur_tz64 __builtin_ctzll
#else
# error "port me"
#endif
#define ur_lz8(a) ( ur_lz32(a) - 24 )
#define ur_tz8 ur_tz32
#define ur_met0_8(a) ( (a) ? 8 - ur_lz8(a) : 0 )
#define ur_met0_32(a) ( (a) ? 32 - ur_lz32(a) : 0 )
#define ur_met0_64(a) ( (a) ? 64 - ur_lz64(a) : 0 )
/*
** unsafe wrt trailing null bytes, which are invalid
*/
inline uint64_t
ur_met0_bytes_unsafe(uint64_t len, uint8_t *byt)
{
uint64_t last = len - 1;
return (last << 3) + ur_met0_8(byt[last]);
}
#define ur_met3_8(a) ur_bloq_up3(ur_met0_8(a))
#define ur_met3_32(a) ur_bloq_up3(ur_met0_32(a))
#define ur_met3_64(a) ur_bloq_up3(ur_met0_64(a))
#endif

234
pkg/urbit/include/ur/hashcons.h Normal file
View File

@ -0,0 +1,234 @@
#ifndef UR_HASHCONS_H
#define UR_HASHCONS_H
#include <inttypes.h>
#include <assert.h>
#include <limits.h>
#include <stdio.h>
#include "ur/defs.h"
/*
** 64-bit noun references, with the top 2 bits reserved for type tags.
*/
typedef uint64_t ur_nref;
typedef enum {
ur_direct = 0,
ur_iatom = 1,
ur_icell = 2,
} ur_tag;
#define ur_nref_tag(ref) ( ref >> 62 )
#define ur_nref_idx(ref) ur_mask_62(ref)
/*
** 31-bit, non-zero, murmur3-based noun hash.
*/
typedef uint32_t ur_mug;
/*
** associative structures (dictionaries) of noun references,
** distributed by mug across fixed-size buckets (pails),
** reallocated with fibonacci growth once a bucket is full.
**
** - ur_dict_t: set of noun references
** - ur_dict32_t: map from noun reference to uint32
** - ur_dict32_t: map from noun reference to uint64
*/
#define ur_pail_max 10
typedef struct ur_pail32_s {
uint8_t fill;
ur_nref refs[ur_pail_max];
uint32_t vals[ur_pail_max];
} ur_pail32_t;
typedef struct ur_dict32_s {
uint64_t prev;
uint64_t size;
ur_pail32_t *buckets;
} ur_dict32_t;
typedef struct ur_pail64_s {
uint8_t fill;
ur_nref refs[ur_pail_max];
uint64_t vals[ur_pail_max];
} ur_pail64_t;
typedef struct ur_dict64_s {
uint64_t prev;
uint64_t size;
ur_pail64_t *buckets;
} ur_dict64_t;
typedef struct ur_pail_s {
uint8_t fill;
ur_nref refs[ur_pail_max];
} ur_pail_t;
typedef struct ur_dict_s {
uint64_t prev;
uint64_t size;
ur_pail_t *buckets;
} ur_dict_t;
/*
** cells are hash-consed, atoms are deduplicated (byte-array comparison),
** mug hashes are stored, and noun references are unique within a root.
*/
typedef struct ur_cells_s {
ur_dict_t dict;
uint64_t prev;
uint64_t size;
uint64_t fill;
ur_mug *mugs;
ur_nref *heads;
ur_nref *tails;
} ur_cells_t;
typedef struct ur_atoms_s {
ur_dict_t dict;
uint64_t prev;
uint64_t size;
uint64_t fill;
ur_mug *mugs;
uint8_t **bytes;
uint64_t *lens;
} ur_atoms_t;
typedef struct ur_root_s {
ur_cells_t cells;
ur_atoms_t atoms;
} ur_root_t;
/*
** a vector of noun references.
*/
typedef struct ur_nvec_s {
uint64_t fill;
ur_nref* refs;
} ur_nvec_t;
/*
** type-specific dictionary operations.
**
** NB: [r] is only used to retrieve the stored mug of cells and
** indirect atoms. If all references are direct atoms (62-bits or less),
** [r] can be null. This option is used extensively in cue (de-serialization)
** implementations, where the dictionary keys are bit-cursors.
*/
void
ur_dict32_grow(ur_root_t *r, ur_dict32_t *dict, uint64_t prev, uint64_t size);
ur_bool_t
ur_dict32_get(ur_root_t *r, ur_dict32_t *dict, ur_nref ref, uint32_t *out);
void
ur_dict32_put(ur_root_t *r, ur_dict32_t *dict, ur_nref ref, uint32_t val);
void
ur_dict32_wipe(ur_dict32_t *dict);
void
ur_dict64_grow(ur_root_t *r, ur_dict64_t *dict, uint64_t prev, uint64_t size);
ur_bool_t
ur_dict64_get(ur_root_t *r, ur_dict64_t *dict, ur_nref ref, uint64_t *out);
void
ur_dict64_put(ur_root_t *r, ur_dict64_t *dict, ur_nref ref, uint64_t val);
void
ur_dict64_wipe(ur_dict64_t *dict);
void
ur_dict_grow(ur_root_t *r, ur_dict_t *dict, uint64_t prev, uint64_t size);
ur_bool_t
ur_dict_get(ur_root_t *r, ur_dict_t *dict, ur_nref ref);
void
ur_dict_put(ur_root_t *r, ur_dict_t *dict, ur_nref ref);
void
ur_dict_wipe(ur_dict_t *dict);
/*
** free the buckets of any dictionary (cast to ur_dict_t*).
*/
void
ur_dict_free(ur_dict_t *dict);
/*
** measure the bloq (binary-exponent) length of an atom in [r]
*/
uint64_t
ur_met(ur_root_t *r, uint8_t bloq, ur_nref ref);
/*
** find or allocate an atom in [r]
**
** unsafe variant is unsafe wrt allocation (byte arrays must be
** allocated with system malloc) and trailing null bytes (not allowed).
*/
ur_nref
ur_coin_bytes_unsafe(ur_root_t *r, uint64_t len, uint8_t *byt);
ur_nref
ur_coin_bytes(ur_root_t *r, uint64_t len, uint8_t *byt);
ur_nref
ur_coin64(ur_root_t *r, uint64_t n);
/*
** find or construct a cell in [r]
*/
ur_nref
ur_cons(ur_root_t *r, ur_nref hed, ur_nref tal);
/*
** calculate the mug of [ref], or produce the stored value in [r].
*/
ur_mug
ur_nref_mug(ur_root_t *r, ur_nref ref);
/*
** initialize a noun arena (root).
*/
ur_root_t*
ur_root_init(void);
/*
** print root details to [f]
*/
void
ur_root_info(FILE *f, ur_root_t *r);
/*
** dispose all allocations in [r]
*/
void
ur_root_free(ur_root_t *r);
/*
** initialize or dispose a vector of noun references
*/
void
ur_nvec_init(ur_nvec_t *v, uint64_t size);
void
ur_nvec_free(ur_nvec_t *v);
/*
** depth-first, pre-order noun traversal, cells can short-circuit.
*/
void
ur_walk_fore(ur_root_t *r,
ur_nref ref,
void *v,
void (*atom)(ur_root_t*, ur_nref, void*),
ur_bool_t (*cell)(ur_root_t*, ur_nref, void*));
#endif

56
pkg/urbit/include/ur/serial.h Normal file
View File

@ -0,0 +1,56 @@
#ifndef UR_SERIAL_H
#define UR_SERIAL_H
#include <inttypes.h>
#include <ur/defs.h>
#include <ur/bitstream.h>
/*
** bit-wise serialization of a noun into a byte-buffer.
** supports up to 64-bits of bit-addressed output (nearly 2 EiB).
** (as this is an impractical volume data, cursor overflow is not checked.)
**
** unsafe variant is unsafe wrt its [dict] parameter, which must be empty,
** but can be passed in order to skip reallocation inside hot loops.
**
*/
uint64_t
ur_jam_unsafe(ur_root_t *r,
ur_nref ref,
ur_dict64_t *dict,
uint64_t *len,
uint8_t **byt);
uint64_t
ur_jam(ur_root_t *r, ur_nref ref, uint64_t *len, uint8_t **byt);
/*
** bitwise deserialization of a byte-buffer into a noun.
** supports up to 62-bits of bit-addressed input (511 PiB).
** returns [ur_cue_good] on success.
**
** unsafe variant is unsafe wrt its [dict] parameter, which must be empty,
** (present in order to skip reallocation inside hot loops).
**
** test variant does not allocate nouns, but merely parses the input.
**
*/
ur_cue_res_e
ur_cue_unsafe(ur_root_t *r,
ur_dict64_t *dict,
uint64_t len,
const uint8_t *byt,
ur_nref *out);
ur_cue_res_e
ur_cue(ur_root_t *r, uint64_t len, const uint8_t *byt, ur_nref *out);
ur_cue_res_e
ur_cue_test_unsafe(ur_dict_t *dict,
uint64_t len,
const uint8_t *byt);
ur_bool_t
ur_cue_test(uint64_t len, const uint8_t *byt);
#endif

9
pkg/urbit/include/ur/ur.h Normal file
View File

@ -0,0 +1,9 @@
#ifndef UR_UR_H
#define UR_UR_H
#include "ur/defs.h"
#include "ur/bitstream.h"
#include "ur/hashcons.h"
#include "ur/serial.h"
#endif

24
pkg/urbit/include/vere/serf.h
View File

@ -6,15 +6,16 @@
/* u3_serf: worker-process state
*/
typedef struct _u3_serf {
c3_d key_d[4]; // disk key
c3_c* dir_c; // execution directory (pier)
c3_d sen_d; // last event requested
c3_d dun_d; // last event processed
c3_l mug_l; // hash of state
c3_o pac_o; // pack kernel
c3_o rec_o; // reclaim cache
c3_o mut_o; // mutated kerne
u3_noun sac; // space measurementl
c3_d key_d[4]; // disk key
c3_c* dir_c; // execution directory (pier)
c3_d sen_d; // last event requested
c3_d dun_d; // last event processed
c3_l mug_l; // hash of state
c3_o pac_o; // pack kernel
c3_o rec_o; // reclaim cache
c3_o mut_o; // mutated kerne
u3_noun sac; // space measurementl
void (*xit_f)(void); // exit callback
} u3_serf;
/** Functions.
@ -24,11 +25,6 @@
u3_noun
u3_serf_init(u3_serf* sef_u);
/* u3_serf_uncram(): initialize from rock at [eve_d].
*/
void
u3_serf_uncram(u3_serf* sef_u, c3_d eve_d);
/* u3_serf_writ(): apply writ [wit], producing plea [*pel] on c3y.
*/
c3_o

33
pkg/urbit/include/vere/vere.h
View File

@ -40,7 +40,7 @@
/* u3_moor_poke: poke callback function.
*/
typedef void (*u3_moor_poke)(void*, u3_atom);
typedef void (*u3_moor_poke)(void*, c3_d, c3_y*);
/* u3_moor_bail: bailout callback function.
*/
@ -424,15 +424,15 @@
u3_writ_play = 2,
u3_writ_save = 3,
u3_writ_cram = 4,
u3_writ_pack = 5,
u3_writ_exit = 6
u3_writ_meld = 5,
u3_writ_pack = 6,
u3_writ_exit = 7
} u3_writ_type;
/* u3_writ: ipc message from king to serf
*/
typedef struct _u3_writ {
struct timeval tim_u; // time enqueued
u3_atom mat; // serialized
struct _u3_writ* nex_u; // next in queue
u3_writ_type typ_e; // type-tagged
union { //
@ -470,6 +470,7 @@
uv_process_t cub_u; // process handle
uv_process_options_t ops_u; // process configuration
uv_stdio_container_t cod_u[3]; // process options
void* dic_u; // cue dictionary
time_t wen_t; // process creation time
u3_mojo inn_u; // client's stdin
u3_moat out_u; // client's stdout
@ -987,6 +988,16 @@
c3_o
u3_lord_cram(u3_lord* god_u);
/* u3_lord_meld(): globally deduplicate persistent state.
*/
void
u3_lord_meld(u3_lord* god_u);
/* u3_lord_pack(): defragment persistent state.
*/
void
u3_lord_pack(u3_lord* god_u);
/* u3_lord_work(): attempt work.
*/
void
@ -1183,10 +1194,10 @@
void
u3_newt_decode(u3_moat* mot_u, c3_y* buf_y, c3_d len_d);
/* u3_newt_write(): write atom to stream; free atom.
/* u3_newt_send(): write buffer to stream.
*/
void
u3_newt_write(u3_mojo* moj_u, u3_atom mat);
u3_newt_send(u3_mojo* moj_u, c3_d len_d, c3_y* byt_y);
/* u3_newt_read_sync(): start reading; multiple msgs synchronous.
*/
@ -1267,6 +1278,16 @@
c3_o
u3_pier_cram(u3_pier* pir_u);
/* u3_pier_meld(): globally deduplicate persistent state.
*/
void
u3_pier_meld(u3_pier* pir_u);
/* u3_pier_pack(): defragment persistent state.
*/
void
u3_pier_pack(u3_pier* pir_u);
/* u3_pier_info(): print status info.
*/
void

2
pkg/urbit/jets/e/cue.c
View File

@ -6,7 +6,7 @@
u3_noun
u3qe_cue(u3_atom a)
{
return u3s_cue(a);
return u3s_cue_atom(a);
}
u3_noun

165
pkg/urbit/noun/events.c
View File

@ -609,44 +609,47 @@ _ce_image_sync(u3e_image* img_u)
c3_sync(img_u->fid_i);
}
/* _ce_patch_apply(): apply patch to image.
/* _ce_image_resize(): resize image, truncating if it shrunk.
*/
static void
_ce_image_resize(u3e_image* img_u, c3_w pgs_w)
{
if ( img_u->pgs_w > pgs_w ) {
if ( ftruncate(img_u->fid_i, pgs_w << (u3a_page + 2)) ) {
fprintf(stderr, "loom: image truncate %s: %s\r\n",
img_u->nam_c,
strerror(errno));
c3_assert(0);
}
}
img_u->pgs_w = pgs_w;
}
/* _ce_patch_apply(): apply patch to images.
*/
static void
_ce_patch_apply(u3_ce_patch* pat_u)
{
c3_w i_w;
//u3l_log("image: nor_w %d, new %d\r\n", u3P.nor_u.pgs_w, pat_u->con_u->nor_w);
//u3l_log("image: sou_w %d, new %d\r\n", u3P.sou_u.pgs_w, pat_u->con_u->sou_w);
// resize images
//
_ce_image_resize(&u3P.nor_u, pat_u->con_u->nor_w);
_ce_image_resize(&u3P.sou_u, pat_u->con_u->sou_w);
if ( u3P.nor_u.pgs_w > pat_u->con_u->nor_w ) {
c3_w ret_w;
ret_w = ftruncate(u3P.nor_u.fid_i, u3P.nor_u.pgs_w << (u3a_page + 2));
if (ret_w){
fprintf(stderr, "loom: patch apply truncate north: %s\r\n", strerror(errno));
c3_assert(0);
}
}
u3P.nor_u.pgs_w = pat_u->con_u->nor_w;
if ( u3P.sou_u.pgs_w > pat_u->con_u->sou_w ) {
c3_w ret_w;
ret_w = ftruncate(u3P.sou_u.fid_i, u3P.sou_u.pgs_w << (u3a_page + 2));
if (ret_w){
fprintf(stderr, "loom: patch apply truncate south: %s\r\n", strerror(errno));
c3_assert(0);
}
}
u3P.sou_u.pgs_w = pat_u->con_u->sou_w;
if ( (-1 == lseek(pat_u->mem_i, 0, SEEK_SET)) ||
(-1 == lseek(u3P.nor_u.fid_i, 0, SEEK_SET)) ||
(-1 == lseek(u3P.sou_u.fid_i, 0, SEEK_SET)) )
// seek to begining of patch and images
//
if ( (-1 == lseek(pat_u->mem_i, 0, SEEK_SET))
|| (-1 == lseek(u3P.nor_u.fid_i, 0, SEEK_SET))
|| (-1 == lseek(u3P.sou_u.fid_i, 0, SEEK_SET)) )
{
fprintf(stderr, "loom: patch apply seek 0: %s\r\n", strerror(errno));
c3_assert(0);
}
// write patch pages into the appropriate image
//
for ( i_w = 0; i_w < pat_u->con_u->pgs_w; i_w++ ) {
c3_w pag_w = pat_u->con_u->mem_u[i_w].pag_w;
c3_w mem_w[1 << u3a_page];
@ -879,116 +882,24 @@ u3e_live(c3_o nuu_o, c3_c* dir_c)
return nuu_o;
}
static c3_o
_ce_image_move(u3e_image* img_u, c3_o bak_o)
{
c3_c old_c[8193];
c3_c new_c[8197];
snprintf(old_c, 8193, "%s/.urb/chk/%s.bin", u3P.dir_c, img_u->nam_c);
snprintf(new_c, 8197, "%s.bak", old_c);
c3_i ret_i;
if ( c3y == bak_o ) {
ret_i = rename(old_c, new_c);
}
else {
ret_i = rename(new_c, old_c);
}
if ( 0 != ret_i ) {
u3l_log("loom: %s %s failed: %s\r\n", ( c3y == bak_o ) ? "hold" : "fall",
img_u->nam_c, strerror(errno));
return c3n;
}
return c3y;
}
/* u3e_hold(): backup memory images
/* u3e_yolo(): disable dirty page tracking, read/write whole loom.
*/
c3_o
u3e_hold(void)
u3e_yolo(void)
{
if ( (c3n == _ce_image_move(&u3P.nor_u, c3y)) ||
(c3n == _ce_image_move(&u3P.sou_u, c3y)) )
{
return c3n;
}
// XX sync directory
return c3y;
}
static c3_o
_ce_image_drop(u3e_image* img_u)
{
c3_c pat_c[8193];
snprintf(pat_c, 8192, "%s/.urb/chk/%s.bin.bak", u3P.dir_c, img_u->nam_c);
if ( 0 != unlink(pat_c) ) {
u3l_log("loom: drop %s failed: %s\r\n", img_u->nam_c, strerror(errno));
// NB: u3e_save() will reinstate protection flags
//
if ( 0 != mprotect((void *)u3_Loom, u3a_bytes, (PROT_READ | PROT_WRITE)) ) {
return c3n;
}
return c3y;
}
/* u3e_drop(): remove backed-up memory images
/* u3e_foul(): dirty all the pages of the loom.
*/
c3_o
u3e_drop(void)
void
u3e_foul(void)
{
if ( (c3n == _ce_image_drop(&u3P.nor_u)) ||
(c3n == _ce_image_drop(&u3P.sou_u)) )
{
return c3n;
}
return c3y;
}
/* u3e_fall(): restore memory images
*/
c3_o
u3e_fall(void)
{
if ( (c3n == _ce_image_move(&u3P.nor_u, c3n)) ||
(c3n == _ce_image_move(&u3P.sou_u, c3n)) )
{
return c3n;
}
// XX sync directory
return c3y;
}
/* u3e_wipe(): discard memory images
*/
c3_o
u3e_wipe(void)
{
// XX ensure no patch files are present
if ( 0 != ftruncate(u3P.nor_u.fid_i, 0) ) {
u3l_log("loom: wipe %s failed: %s\r\n", u3P.nor_u.nam_c, strerror(errno));
return c3n;
}
if ( 0 != ftruncate(u3P.sou_u.fid_i, 0) ) {
u3l_log("loom: wipe %s failed: %s\r\n", u3P.sou_u.nam_c, strerror(errno));
return c3n;
}
c3_sync(u3P.nor_u.fid_i);
c3_sync(u3P.sou_u.fid_i);
close(u3P.nor_u.fid_i);
close(u3P.sou_u.fid_i);
// XX sync directory
return c3y;
memset((void*)u3P.dit_w, 0xff, u3a_pages >> 3);
}

9
pkg/urbit/noun/hashtable.c
View File

@ -1180,3 +1180,12 @@ u3h_discount(u3p(u3h_root) har_p)
return tot_w;
}
/* u3h_wyt(): number of entries
*/
c3_w
u3h_wyt(u3p(u3h_root) har_p)
{
u3h_root* har_u = u3to(u3h_root, har_p);
return har_u->use_w;
}

115
pkg/urbit/noun/manage.c
View File

@ -1729,121 +1729,6 @@ u3m_boot_lite(void)
return 0;
}
/* u3m_rock_stay(): jam state into [dir_c] at [evt_d]
*/
c3_o
u3m_rock_stay(c3_c* dir_c, c3_d evt_d)
{
c3_c nam_c[8193];
snprintf(nam_c, 8192, "%s", dir_c);
mkdir(nam_c, 0700);
snprintf(nam_c, 8192, "%s/.urb", dir_c);
mkdir(nam_c, 0700);
snprintf(nam_c, 8192, "%s/.urb/roc", dir_c);
mkdir(nam_c, 0700);
snprintf(nam_c, 8192, "%s/.urb/roc/%" PRIu64 ".jam", dir_c, evt_d);
{
u3_noun dat = u3nt(c3__fast, u3k(u3A->roc), u3j_stay());
c3_o ret_o = u3s_jam_file(dat, nam_c);
u3z(dat);
return ret_o;
}
}
/* u3m_rock_load(): load state from [dir_c] at [evt_d]
*/
c3_o
u3m_rock_load(c3_c* dir_c, c3_d evt_d)
{
c3_c nam_c[8193];
snprintf(nam_c, 8192, "%s/.urb/roc/%" PRIu64 ".jam", dir_c, evt_d);
{
u3_noun dat;
{
// XX u3m_file bails, but we'd prefer to return errors
//
u3_noun fil = u3m_file(nam_c);
u3a_print_memory(stderr, "rock: load", u3r_met(5, fil));
u3_noun pro = u3m_soft(0, u3ke_cue, fil);
if ( u3_blip != u3h(pro) ) {
fprintf(stderr, "rock: unable to cue %s\r\n", nam_c);
u3z(pro);
return c3n;
}
else {
dat = u3k(u3t(pro));
u3z(pro);
}
}
{
u3_noun roc, rel;
if ( u3r_pq(dat, c3__fast, &roc, &rel) ) {
u3z(dat);
return c3n;
}
u3A->roc = u3k(roc);
u3j_load(u3k(rel));
}
u3z(dat);
}
u3A->ent_d = evt_d;
u3j_ream();
u3n_ream();
return c3y;
}
/* u3m_rock_drop(): delete saved state from [dir_c] at [evt_d]
*/
c3_o
u3m_rock_drop(c3_c* dir_c, c3_d evt_d)
{
c3_c nam_c[8193];
snprintf(nam_c, 8192, "%s/.urb/roc/%" PRIu64 ".jam", dir_c, evt_d);
if ( 0 != unlink(nam_c) ) {
u3l_log("rock: drop %s failed: %s\r\n", nam_c, strerror(errno));
return c3n;
}
return c3y;
}
/* u3m_wipe(): purge and reinitialize loom, with checkpointing
*/
void
u3m_wipe(void)
{
// clear page flags
//
memset((void*)u3P.dit_w, 0, u3a_pages >> 3);
// reinitialize checkpoint system
//
// NB: callers must first u3e_hold() or u3e_wipe()
//
u3e_live(c3n, u3P.dir_c);
// reinitialize loom
//
u3m_pave(c3y, c3n);
// reinitialize jets
//
u3j_boot(c3y);
}
/* u3m_reclaim: clear persistent caches to reclaim memory
*/
void

784
pkg/urbit/noun/serial.c
View File

@ -6,6 +6,7 @@
#include <fcntl.h>
#include "all.h"
#include "ur/ur.h"
/* _cs_met0_w(): safe bitwidth for any c3_w
*/
@ -190,315 +191,125 @@ u3s_jam_fib(u3_noun a, c3_w* bit_w)
return fib_u.buf_w;
}
/* _cs_jam_met_mat(): the jam bitwidth of an atom of bitwidth [wid_w]
**
** equivalent to (head (rub a))
typedef struct _jam_xeno_s {
u3p(u3h_root) har_p;
ur_bsw_t rit_u;
} _jam_xeno_t;
/* _cs_coin_chub(): shortcircuit u3i_chubs().
*/
static c3_d
_cs_jam_met_mat(c3_w wid_w)
static inline u3_atom
_cs_coin_chub(c3_d a_d)
{
return ( 0 == wid_w ) ? 1ULL :
(c3_d)wid_w + (2ULL * (c3_d)_cs_met0_w(wid_w));
return ( 0x7fffffffULL >= a_d ) ? a_d : u3i_chubs(1, &a_d);
}
/* _cs_jam_met: struct for tracking the jam bitwidth of a noun
/* _cs_jam_xeno_atom(): encode in/direct atom in bitstream.
*/
struct _cs_jam_met {
u3p(u3h_root) har_p;
u3p(u3h_root) bak_p;
c3_d len_d;
};
static inline void
_cs_jam_bsw_atom(ur_bsw_t* rit_u, c3_w met_w, u3_atom a)
{
if ( c3y == u3a_is_cat(a) ) {
// XX need a ur_bsw_atom32()
//
ur_bsw_atom64(rit_u, (c3_y)met_w, (c3_d)a);
}
else {
u3a_atom* vat_u = u3a_to_ptr(a);
// XX assumes little-endian
// XX need a ur_bsw_atom_words()
//
c3_y* byt_y = (c3_y*)vat_u->buf_w;
ur_bsw_atom_bytes(rit_u, (c3_d)met_w, byt_y);
}
}
/* _cs_jam_met_atom_cb(): bitwidth of atom or backref encoding for [a]
/* _cs_jam_bsw_back(): encode in/direct backref in bitstream.
*/
static inline void
_cs_jam_bsw_back(ur_bsw_t* rit_u, c3_w met_w, u3_atom a)
{
c3_d bak_d = ( c3y == u3a_is_cat(a) )
? (c3_d)a
: u3r_chub(0, a);
// XX need a ur_bsw_back32()
//
ur_bsw_back64(rit_u, (c3_y)met_w, bak_d);
}
/* _cs_jam_xeno_atom(): encode atom or backref in bitstream.
*/
static void
_cs_jam_met_atom_cb(u3_atom a, void* ptr_v)
_cs_jam_xeno_atom(u3_atom a, void* ptr_v)
{
struct _cs_jam_met* met_u = ptr_v;
c3_w a_w = u3r_met(0, a);
u3_weak b = u3h_git(met_u->har_p, a);
_jam_xeno_t* jam_u = ptr_v;
ur_bsw_t* rit_u = &(jam_u->rit_u);
u3_weak bak = u3h_git(jam_u->har_p, a);
c3_w met_w = u3r_met(0, a);
// if we haven't haven't seen [a], put cursor into [har_p]
//
if ( u3_none == b ) {
u3h_put(met_u->har_p, a, u3i_chubs(1, &(met_u->len_d)));
met_u->len_d += 1ULL + _cs_jam_met_mat(a_w);
if ( u3_none == bak ) {
u3h_put(jam_u->har_p, a, _cs_coin_chub(rit_u->bits));
_cs_jam_bsw_atom(rit_u, met_w, a);
}
else {
c3_w b_w = u3r_met(0, b);
c3_w bak_w = u3r_met(0, bak);
// if [a] is smaller than a backref, use directly
//
if ( a_w <= b_w ) {
met_u->len_d += 1ULL + _cs_jam_met_mat(a_w);
if ( met_w <= bak_w ) {
_cs_jam_bsw_atom(rit_u, met_w, a);
}
// otherwise, save backref
//
else {
u3h_put(met_u->bak_p, a, u3k(b));
met_u->len_d += 2ULL + _cs_jam_met_mat(b_w);
_cs_jam_bsw_back(rit_u, bak_w, bak);
}
}
}
/* _cs_jam_met_cell_cb(): bitwidth of cell or backref encoding for [a]
/* _cs_jam_xeno_cell(): encode cell or backref in bitstream.
*/
static c3_o
_cs_jam_met_cell_cb(u3_noun a, void* ptr_v)
_cs_jam_xeno_cell(u3_noun a, void* ptr_v)
{
struct _cs_jam_met* met_u = ptr_v;
u3_weak b = u3h_git(met_u->har_p, a);
_jam_xeno_t* jam_u = ptr_v;
ur_bsw_t* rit_u = &(jam_u->rit_u);
u3_weak bak = u3h_git(jam_u->har_p, a);
// if we haven't haven't seen [a], put cursor into [har_p]
//
if ( u3_none == b ) {
u3h_put(met_u->har_p, a, u3i_chubs(1, &(met_u->len_d)));
met_u->len_d += 2ULL;
if ( u3_none == bak ) {
u3h_put(jam_u->har_p, a, _cs_coin_chub(rit_u->bits));
ur_bsw_cell(rit_u);
return c3y;
}
// otherwise, save backref and shortcircuit traversal
//
else {
c3_w b_w = u3r_met(0, b);
u3h_put(met_u->bak_p, a, u3k(b));
met_u->len_d += 2ULL + _cs_jam_met_mat(b_w);
_cs_jam_bsw_back(rit_u, u3r_met(0, bak), bak);
return c3n;
}
}
/* u3s_jam_met(): measure a noun for jam, calculating backrefs
/* u3s_jam_xeno(): jam with off-loom buffer (re-)allocation.
*/
c3_d
u3s_jam_met(u3_noun a, u3p(u3h_root)* bak_p)
u3s_jam_xeno(u3_noun a, c3_d* len_d, c3_y** byt_y)
{
struct _cs_jam_met met_u;
met_u.har_p = u3h_new();
met_u.bak_p = u3h_new();
met_u.len_d = 0ULL;
_jam_xeno_t jam_u = {0};
u3a_walk_fore(a, &met_u, _cs_jam_met_atom_cb,
_cs_jam_met_cell_cb);
u3h_free(met_u.har_p);
*bak_p = met_u.bak_p;
jam_u.har_p = u3h_new();
return met_u.len_d;
}
jam_u.rit_u.prev = ur_fib11;
jam_u.rit_u.size = ur_fib12;
jam_u.rit_u.bytes = c3_calloc(jam_u.rit_u.size);
/* _cs_jam_buf: struct for tracking the pre-measured jam of a noun
*/
struct _cs_jam_buf {
u3p(u3h_root) bak_p;
c3_w bit_w;
c3_w* buf_w;
};
/* _cs_jam_buf_chop(): chop [met_w] bits of [a] into [buf_u]
*/
static void
_cs_jam_buf_chop(struct _cs_jam_buf* buf_u, c3_w met_w, u3_noun a)
{
u3r_chop(0, 0, met_w, buf_u->bit_w, buf_u->buf_w, a);
buf_u->bit_w += met_w;
}
/* _cs_jam_buf_mat(): length-prefixed encode (mat) [a] into [buf_u]
*/
static void
_cs_jam_buf_mat(struct _cs_jam_buf* buf_u, u3_atom a)
{
if ( 0 == a ) {
_cs_jam_buf_chop(buf_u, 1, 1);
}
else {
c3_w a_w = u3r_met(0, a);
c3_w b_w = _cs_met0_w(a_w);
_cs_jam_buf_chop(buf_u, b_w+1, 1 << b_w);
_cs_jam_buf_chop(buf_u, b_w-1, a_w & ((1 << (b_w-1)) - 1));
_cs_jam_buf_chop(buf_u, a_w, a);
}
}
/* _cs_jam_buf_atom_cb(): encode atom or backref
*/
static void
_cs_jam_buf_atom_cb(u3_atom a, void* ptr_v)
{
struct _cs_jam_buf* buf_u = ptr_v;
u3_weak b = u3h_git(buf_u->bak_p, a);
// if [a] has no backref (or this is the referent), encode atom
// as this is a hot path, we unsafely elide overflow checks
//
if ( (u3_none == b) ||
(u3r_word(0, b) == buf_u->bit_w) )
{
_cs_jam_buf_chop(buf_u, 1, 0);
_cs_jam_buf_mat(buf_u, a);
}
else {
c3_w a_w = u3r_met(0, a);
c3_w b_w = u3r_met(0, b);
// if [a] is smaller than the backref, encode atom
//
if ( a_w <= b_w ) {
_cs_jam_buf_chop(buf_u, 1, 0);
_cs_jam_buf_mat(buf_u, a);
}
// otherwise, encode backref
//
else {
_cs_jam_buf_chop(buf_u, 2, 3);
_cs_jam_buf_mat(buf_u, b);
}
}
}
/* _cs_jam_buf_cell_cb(): encode cell or backref
*/
static c3_o
_cs_jam_buf_cell_cb(u3_noun a, void* ptr_v)
{
struct _cs_jam_buf* buf_u = ptr_v;
u3_weak b = u3h_git(buf_u->bak_p, a);
// if [a] has no backref (or this is the referent), encode cell
// a page-fault overflow detection system is urgently needed ...
//
if ( (u3_none == b) ||
(u3r_word(0, b) == buf_u->bit_w) )
{
_cs_jam_buf_chop(buf_u, 2, 1);
return c3y;
}
// otherwise, encode backref and shortcircuit traversal
//
else {
_cs_jam_buf_chop(buf_u, 2, 3);
_cs_jam_buf_mat(buf_u, b);
return c3n;
}
}
u3a_walk_fore_unsafe(a, &jam_u, _cs_jam_xeno_atom,
_cs_jam_xeno_cell);
/* u3s_jam_buf(): jam [a] into pre-allocated [buf_w], without allocation
**
** using backrefs in [bak_p], as computed by u3s_jam_met()
** NB [buf_w] must be pre-allocated with sufficient space
**
** XX can only encode up to c3_w bits, due to use of chop
*/
void
u3s_jam_buf(u3_noun a, u3p(u3h_root) bak_p, c3_w* buf_w)
{
struct _cs_jam_buf buf_u;
buf_u.bak_p = bak_p;
buf_u.buf_w = buf_w;
buf_u.bit_w = 0;
*len_d = jam_u.rit_u.fill + !!jam_u.rit_u.off;
*byt_y = jam_u.rit_u.bytes;
// this is in fact safe under normal usage, as
// the stack will have been checked in u3s_jam_met()
//
u3a_walk_fore_unsafe(a, &buf_u, _cs_jam_buf_atom_cb,
_cs_jam_buf_cell_cb);
}
u3h_free(jam_u.har_p);
/* u3s_jam_file(): jam [a] into a file, overwriting
*/
c3_o
u3s_jam_file(u3_noun a, c3_c* pas_c)
{
u3p(u3h_root) bak_p;
c3_i fid_i = open(pas_c, O_RDWR | O_CREAT | O_TRUNC, 0644);
c3_w byt_w, wor_w, len_w;
if ( fid_i < 0 ) {
fprintf(stderr, "jam: open %s: %s\r\n", pas_c, strerror(errno));
return c3n;
}
{
c3_d len_d = u3s_jam_met(a, &bak_p);
if ( len_d > 0xffffffffULL ) {
fprintf(stderr, "jam: overflow c3_w: %" PRIu64 "\r\n", len_d);
u3h_free(bak_p);
return c3n;
}
// length in bytes a la u3i_bytes
//
byt_w = (c3_w)(len_d >> 3ULL);
if ( len_d > (c3_d)(byt_w << 3) ) {
byt_w++;
}
// length in words
//
wor_w = (c3_w)(len_d >> 5ULL);
if ( len_d > (c3_d)(wor_w << 5) ) {
wor_w++;
}
// byte-length of word-length
//
len_w = 4 * wor_w;
}
// grow [fid_i] to [len_w]
//
if ( 0 != ftruncate(fid_i, len_w) ) {
fprintf(stderr, "jam: ftruncate grow %s: %s\r\n", pas_c, strerror(errno));
goto error;
}
// mmap [fid_i], jam into it, sync, and unmap
//
{
c3_w* buf_w;
void* ptr_v = mmap(0, len_w, PROT_READ|PROT_WRITE, MAP_SHARED, fid_i, 0);
if ( MAP_FAILED == ptr_v ) {
fprintf(stderr, "jam: mmap %s: %s\r\n", pas_c, strerror(errno));
goto error;
}
buf_w = ptr_v;
u3s_jam_buf(a, bak_p, buf_w);
if ( 0 != msync(ptr_v, len_w, MS_SYNC) ) {
fprintf(stderr, "jam: msync %s: %s\r\n", pas_c, strerror(errno));
// XX ignore return?
//
munmap(ptr_v, len_w);
goto error;
}
if ( 0 != munmap(ptr_v, len_w) ) {
fprintf(stderr, "jam: munmap %s: %s\r\n", pas_c, strerror(errno));
// XX fatal error?
//
goto error;
}
}
// shrink [fid_i] to [byt_w]
//
if ( 0 != ftruncate(fid_i, byt_w) ) {
fprintf(stderr, "jam: ftruncate shrink %s: %s\r\n", pas_c, strerror(errno));
goto error;
}
{
close(fid_i);
u3h_free(bak_p);
return c3y;
}
error: {
close(fid_i);
unlink(pas_c);
u3h_free(bak_p);
return c3n;
}
return jam_u.rit_u.bits;
}
#define CUE_ROOT 0
@ -726,3 +537,440 @@ u3s_cue(u3_atom a)
return pro;
}
/*
** stack frame for recording head vs tail iteration
**
** $? [u3_none bits=@]
** [hed=* bits=@]
*/
typedef struct _cue_frame_s {
u3_weak ref;
c3_d bit_d;
} _cue_frame_t;
typedef struct _cue_stack_s {
c3_w pre_w;
c3_w siz_w;
c3_w fil_w;
_cue_frame_t* fam_u;
} _cue_stack_t;
/* _cs_cue_xeno_next(): read next value from bitstream, bookkeeping off-loom.
*/
static inline ur_cue_res_e
_cs_cue_xeno_next(_cue_stack_t* tac_u,
ur_bsr_t* red_u,
ur_dict32_t* dic_u,
u3_noun* out)
{
ur_root_t* rot_u = 0;
while ( 1 ) {
c3_d len_d, bit_d = red_u->bits;
ur_cue_tag_e tag_e;
ur_cue_res_e res_e;
if ( ur_cue_good != (res_e = ur_bsr_tag(red_u, &tag_e)) ) {
return res_e;
}
switch ( tag_e ) {
default: c3_assert(0);
case ur_jam_cell: {
// reallocate the stack if full
//
if ( tac_u->fil_w == tac_u->siz_w ) {
c3_w nex_w = tac_u->pre_w + tac_u->siz_w;
tac_u->fam_u = c3_realloc(tac_u->fam_u, nex_w * sizeof(*tac_u->fam_u));
tac_u->pre_w = tac_u->siz_w;
tac_u->siz_w = nex_w;
}
// save a head-frame and read the head from the stream
//
{
_cue_frame_t* fam_u = &(tac_u->fam_u[tac_u->fil_w++]);
fam_u->ref = u3_none;
fam_u->bit_d = bit_d;
}
continue;
}
case ur_jam_back: {
if ( ur_cue_good != (res_e = ur_bsr_rub_len(red_u, &len_d)) ) {
return res_e;
}
else if ( 62 < len_d ) {
return ur_cue_meme;
}
else {
c3_d bak_d = ur_bsr64_any(red_u, len_d);
c3_w bak_w;
if ( !ur_dict32_get(rot_u, dic_u, bak_d, &bak_w) ) {
return ur_cue_back;
}
*out = u3k((u3_noun)bak_w);
return ur_cue_good;
}
}
case ur_jam_atom: {
if ( ur_cue_good != (res_e = ur_bsr_rub_len(red_u, &len_d)) ) {
return res_e;
}
if ( 31 >= len_d ) {
*out = (u3_noun)ur_bsr32_any(red_u, len_d);
}
// XX need a ur_bsr_words_any()
//
else {
c3_w* wor_w;
c3_y* byt_y;
{
c3_d byt_d = (len_d >> 3) + !!ur_mask_3(len_d);
if ( 0xffffffffULL < byt_d) {
return u3m_bail(c3__meme);
}
// XX assumes little-endian
//
wor_w = u3a_slaq(3, byt_d);
byt_y = (c3_y*)wor_w;
}
ur_bsr_bytes_any(red_u, len_d, byt_y);
*out = u3a_malt(wor_w);
}
ur_dict32_put(rot_u, dic_u, bit_d, *out);
return ur_cue_good;
}
}
}
}
/* u3s_cue_xeno_unsafe(): cue onto the loom, all bookkeeping off-loom.
**
** NB: unsafe wrt to [dic_u], which must be empty.
*/
c3_o
u3s_cue_xeno_unsafe(ur_dict32_t* dic_u,
c3_d len_d,
const c3_y* byt_y,
u3_noun* out)
{
ur_bsr_t red_u = {0};
_cue_stack_t tac_u = {0};
ur_cue_res_e res_e;
u3_noun ref;
// init bitstream-reader
//
if ( ur_cue_good != (res_e = ur_bsr_init(&red_u, len_d, byt_y)) ) {
return c3n;
}
// bit-cursor (and backreferences) must fit in 62-bit direct atoms
//
else if ( 0x7ffffffffffffffULL < len_d ) {
return c3n;
}
// setup stack
//
tac_u.pre_w = ur_fib10;
tac_u.siz_w = ur_fib11;
tac_u.fam_u = c3_malloc(tac_u.siz_w * sizeof(*tac_u.fam_u));
// advance into stream
//
res_e = _cs_cue_xeno_next(&tac_u, &red_u, dic_u, &ref);
// process result
//
while ( tac_u.fil_w && (ur_cue_good == res_e) ) {
// peek at the top of the stack
//
_cue_frame_t* fam_u = &(tac_u.fam_u[tac_u.fil_w - 1]);
// f is a head-frame; stash result and read the tail from the stream
//
if ( u3_none == fam_u->ref ) {
fam_u->ref = ref;
res_e = _cs_cue_xeno_next(&tac_u, &red_u, dic_u, &ref);
}
// f is a tail-frame; pop the stack and continue
//
else {
ur_root_t* rot_u = 0;
ref = u3nc(fam_u->ref, ref);
ur_dict32_put(rot_u, dic_u, fam_u->bit_d, ref);
tac_u.fil_w--;
}
}
if ( ur_cue_good == res_e ) {
*out = ref;
c3_free(tac_u.fam_u);
return c3y;
}
else {
// unwind the stack, disposing intermediate nouns
//
while ( tac_u.fil_w ) {
_cue_frame_t* fam_u = &(tac_u.fam_u[--tac_u.fil_w]);
if ( u3_none != fam_u->ref ) {
u3z(fam_u->ref);
}
}
c3_free(tac_u.fam_u);
return c3n;
}
}
/* u3s_cue_xeno(): cue onto the loom, bookkeeping off the loom.
*/
c3_o
u3s_cue_xeno(c3_d len_d, const c3_y* byt_y, u3_noun* out)
{
ur_dict32_t dic_u = {0};
c3_o ret_o;
c3_assert( &(u3H->rod_u) == u3R );
// XX tune the initial dictionary size for less reallocation
//
ur_dict32_grow((ur_root_t*)0, &dic_u, ur_fib10, ur_fib11);
ret_o = u3s_cue_xeno_unsafe(&dic_u, len_d, byt_y, out);
ur_dict_free((ur_dict_t*)&dic_u);
return ret_o;
}
/* _cs_cue_need(): bail on ur_cue_* read failures.
*/
static inline void
_cs_cue_need(ur_cue_res_e res_e)
{
if ( ur_cue_good == res_e ) {
return;
}
else {
c3_m mot_m = (ur_cue_meme == res_e) ? c3__meme : c3__exit;
u3m_bail(mot_m);
}
}
/* _cs_cue_get(): u3h_get wrapper handling allocation and refcounts.
*/
static inline u3_weak
_cs_cue_get(u3p(u3h_root) har_p, c3_d key_d)
{
u3_atom key = _cs_coin_chub(key_d);
u3_weak pro = u3h_get(har_p, key);
u3z(key);
return pro;
}
/* _cs_cue_put(): u3h_put wrapper handling allocation and refcounts.
*/
static inline u3_noun
_cs_cue_put(u3p(u3h_root) har_p, c3_d key_d, u3_noun val)
{
u3_atom key = _cs_coin_chub(key_d);
u3h_put(har_p, key, u3k(val));
u3z(key);
return val;
}
/* _cs_cue_full_next(): read next value from bitstream.
*/
static inline u3_noun
_cs_cue_full_next(c3_ys mov,
c3_ys off,
u3p(u3h_root) har_p,
ur_bsr_t* red_u)
{
while ( 1 ) {
c3_d len_d, bit_d = red_u->bits;
ur_cue_tag_e tag_e;
_cs_cue_need(ur_bsr_tag(red_u, &tag_e));
switch ( tag_e ) {
default: c3_assert(0);
case ur_jam_cell: {
// wind the stack
//
u3R->cap_p += mov;
// ensure we haven't overflowed (ie, run into the heap)
// (off==0 means we're on a north road)
//
if ( 0 == off ) {
if( !(u3R->cap_p > u3R->hat_p) ) {
u3m_bail(c3__meme);
}
}
else {
if( !(u3R->cap_p < u3R->hat_p) ) {
u3m_bail(c3__meme);
}
}
// save a head-frame and read the head from the stream
//
{
_cue_frame_t* fam_u = u3to(_cue_frame_t, u3R->cap_p + off);
fam_u->ref = u3_none;
fam_u->bit_d = bit_d;
}
continue;
}
case ur_jam_back: {
_cs_cue_need(ur_bsr_rub_len(red_u, &len_d));
if ( 62 < len_d ) {
return u3m_bail(c3__meme);
}
else {
c3_d bak_d = ur_bsr64_any(red_u, len_d);
u3_weak bak = _cs_cue_get(har_p, bak_d);
return u3x_good(bak);
}
}
case ur_jam_atom: {
u3_atom vat;
_cs_cue_need(ur_bsr_rub_len(red_u, &len_d));
if ( 31 >= len_d ) {
vat = (u3_noun)ur_bsr32_any(red_u, len_d);
}
// XX need a ur_bsr_words_any()
//
else {
c3_w* wor_w;
c3_y* byt_y;
{
c3_d byt_d = (len_d >> 3) + !!ur_mask_3(len_d);
if ( 0xffffffffULL < byt_d) {
return u3m_bail(c3__meme);
}
// XX assumes little-endian
//
wor_w = u3a_slaq(3, byt_d);
byt_y = (c3_y*)wor_w;
}
ur_bsr_bytes_any(red_u, len_d, byt_y);
vat = u3a_malt(wor_w);
}
return _cs_cue_put(har_p, bit_d, vat);
}
}
}
}
/* u3s_cue_bytes(): cue bytes onto the loom.
*/
u3_noun
u3s_cue_bytes(c3_d len_d, const c3_y* byt_y)
{
ur_bsr_t red_u = {0};
u3_noun ref;
// initialize a hash table for dereferencing backrefs
//
u3p(u3h_root) har_p = u3h_new();
const u3_post top_p = u3R->cap_p;
// initialize signed stack offsets (relative to north/south road)
//
c3_ys mov, off;
{
c3_o nor_o = u3a_is_north(u3R);
c3_y wis_y = c3_wiseof(_cue_frame_t);
mov = ( c3y == nor_o ? -wis_y : wis_y );
off = ( c3y == nor_o ? 0 : -wis_y );
}
// init bitstream-reader
//
_cs_cue_need(ur_bsr_init(&red_u, len_d, byt_y));
// bit-cursor (and backreferences) must fit in 62-bit direct atoms
//
if ( 0x7ffffffffffffffULL < len_d ) {
return u3m_bail(c3__meme);
}
// advance into stream
//
ref = _cs_cue_full_next(mov, off, har_p, &red_u);
// process result
//
while ( top_p != u3R->cap_p ) {
// peek at the top of the stack
//
_cue_frame_t* fam_u = u3to(_cue_frame_t, u3R->cap_p + off);
// f is a head-frame; stash result and read the tail from the stream
//
if ( u3_none == fam_u->ref ) {
fam_u->ref = ref;
ref = _cs_cue_full_next(mov, off, har_p, &red_u);
}
// f is a tail-frame; pop the stack and continue
//
else {
ref = u3nc(fam_u->ref, ref);
_cs_cue_put(har_p, fam_u->bit_d, ref);
u3R->cap_p -= mov;
}
}
u3h_free(har_p);
return ref;
}
/* u3s_cue_atom(): cue atom.
*/
u3_noun
u3s_cue_atom(u3_atom a)
{
c3_w len_w = u3r_met(3, a);
c3_y* byt_y;
// XX assumes little-endian
//
if ( c3y == u3a_is_cat(a) ) {
byt_y = (c3_y*)&a;
}
else {
u3a_atom* vat_u = u3a_to_ptr(a);
byt_y = (c3_y*)vat_u->buf_w;
}
return u3s_cue_bytes((c3_d)len_w, byt_y);
}

919
pkg/urbit/noun/urth.c Normal file
View File

@ -0,0 +1,919 @@
/* noun/urth.c
**
*/
#include <errno.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <ctype.h>
#include "all.h"
#include "ur/ur.h"
/* _cu_met_3(): atom bytewidth a la u3r_met(3, ...)
*/
static inline c3_w
_cu_met_3(u3a_atom* vat_u)
{
c3_w len_w = vat_u->len_w;
c3_w* buf_w = vat_u->buf_w;
if ( !len_w ) {
return 0;
}
else {
c3_w gal_w = len_w - 1;
c3_w daz_w = buf_w[gal_w];
return (gal_w << 2)
+ ((daz_w >> 24) ? 4 : (daz_w >> 16) ? 3 : (daz_w >> 8) ? 2 : 1);
}
}
/* _cu_atom_to_ref(): allocate indirect atom off-loom.
*/
static inline ur_nref
_cu_atom_to_ref(ur_root_t* rot_u, u3a_atom* vat_u)
{
ur_nref ref;
c3_d val_d;
switch ( vat_u->len_w ) {
case 2: {
val_d = ((c3_d)vat_u->buf_w[1]) << 32
| ((c3_d)vat_u->buf_w[0]);
ref = ur_coin64(rot_u, val_d);
} break;
case 1: {
val_d = (c3_d)vat_u->buf_w[0];
ref = ur_coin64(rot_u, val_d);
} break;
default: {
// XX assumes little-endian
//
c3_y* byt_y = (c3_y*)vat_u->buf_w;
c3_d len_d = ((c3_d)vat_u->len_w) << 2;
c3_assert( len_d );
// NB: this call will account for any trailing null bytes
// caused by an overestimate in [len_d]
//
ref = ur_coin_bytes(rot_u, len_d, byt_y);
} break;
}
return ref;
}
/* _cu_box_check(): check loom allocation box for relocation pointer.
*/
static inline c3_o
_cu_box_check(u3a_noun* som_u, ur_nref* ref)
{
u3a_box* box_u = u3a_botox(som_u);
c3_w* box_w = (void*)box_u;
if ( 0xffffffff == box_w[0] ) {
*ref = ( ((c3_d)box_w[2]) << 32
| ((c3_d)box_w[1]) );
return c3y;
}
return c3n;
}
/* _cu_box_stash(): overwrite an allocation box with relocation pointer.
*/
static inline void
_cu_box_stash(u3a_noun* som_u, ur_nref ref)
{
u3a_box* box_u = u3a_botox(som_u);
c3_w* box_w = (void*)box_u;
// overwrite u3a_atom with reallocated reference
//
box_w[0] = 0xffffffff;
box_w[1] = ref & 0xffffffff;
box_w[2] = ref >> 32;
}
/*
** stack frame for recording head vs tail iteration
**
** $? [LOM_HEAD cell=*]
** [ref=* cell=*]
*/
#define LOM_HEAD 0xffffffffffffffffULL
typedef struct _cu_frame_s
{
ur_nref ref;
u3a_cell* cel_u;
} _cu_frame;
typedef struct _cu_stack_s
{
c3_w pre_w;
c3_w siz_w;
c3_w fil_w;
_cu_frame* fam_u;
} _cu_stack;
/* _cu_from_loom_next(): advance off-loom reallocation traversal.
*/
static inline ur_nref
_cu_from_loom_next(_cu_stack* tac_u, ur_root_t* rot_u, u3_noun a)
{
while ( 1 ) {
// u3 direct == ur direct
//
if ( c3y == u3a_is_cat(a) ) {
return (ur_nref)a;
}
else {
u3a_noun* som_u = u3a_to_ptr(a);
ur_nref ref;
// check for relocation pointers
//
if ( c3y == _cu_box_check(som_u, &ref) ) {
return ref;
}
// reallocate indirect atoms, stashing relocation pointers
//
else if ( c3y == u3a_is_atom(a) ) {
ref = _cu_atom_to_ref(rot_u, (u3a_atom*)som_u);
_cu_box_stash(som_u, ref);
return ref;
}
else {
u3a_cell* cel_u = (u3a_cell*)som_u;
// reallocate the stack if full
//
if ( tac_u->fil_w == tac_u->siz_w ) {
c3_w nex_w = tac_u->pre_w + tac_u->siz_w;
tac_u->fam_u = c3_realloc(tac_u->fam_u, nex_w * sizeof(*tac_u->fam_u));
tac_u->pre_w = tac_u->siz_w;
tac_u->siz_w = nex_w;
}
// push a head-frame and continue into the head
//
{
_cu_frame* fam_u = &(tac_u->fam_u[tac_u->fil_w++]);
fam_u->ref = LOM_HEAD;
fam_u->cel_u = cel_u;
}
a = cel_u->hed;
continue;
}
}
}
}
/* _cu_from_loom(): reallocate [a] off loom, in [r].
*/
static ur_nref
_cu_from_loom(ur_root_t* rot_u, u3_noun a)
{
_cu_stack tac_u = {0};
ur_nref ref;
tac_u.pre_w = ur_fib10;
tac_u.siz_w = ur_fib11;
tac_u.fam_u = c3_malloc(tac_u.siz_w * sizeof(*tac_u.fam_u));
ref = _cu_from_loom_next(&tac_u, rot_u, a);
// incorporate reallocated ref, accounting for cells
//
while ( tac_u.fil_w ) {
// peek at the top of the stack
//
_cu_frame* fam_u = &(tac_u.fam_u[tac_u.fil_w - 1]);
// [fam_u] is a head-frame; stash ref and continue into the tail
//
if ( LOM_HEAD == fam_u->ref ) {
fam_u->ref = ref;
ref = _cu_from_loom_next(&tac_u, rot_u, fam_u->cel_u->tel);
}
// [fam_u] is a tail-frame; cons refs and pop the stack
//
else {
ref = ur_cons(rot_u, fam_u->ref, ref);
_cu_box_stash((u3a_noun*)fam_u->cel_u, ref);
tac_u.fil_w--;
}
}
c3_free(tac_u.fam_u);
return ref;
}
/* _cu_vec: parameters for cold-state hamt walk.
*/
typedef struct _cu_vec_s {
ur_nvec_t* vec_u;
ur_root_t* rot_u;
} _cu_vec;
/* _cu_hamt_walk(): reallocate key/value pair in hamt walk.
*/
static void
_cu_hamt_walk(u3_noun kev, void* ptr)
{
_cu_vec* dat_u = (_cu_vec*)ptr;
ur_nvec_t* vec_u = dat_u->vec_u;
ur_root_t* rot_u = dat_u->rot_u;
vec_u->refs[vec_u->fill++] = _cu_from_loom(rot_u, kev);
}
/* _cu_all_from_loom(): reallocate essential persistent state off-loom.
**
** NB: destroys the loom.
*/
static ur_nref
_cu_all_from_loom(ur_root_t* rot_u, ur_nvec_t* cod_u)
{
ur_nref ken = _cu_from_loom(rot_u, u3A->roc);
c3_w cod_w = u3h_wyt(u3R->jed.cod_p);
_cu_vec dat_u = { .vec_u = cod_u, .rot_u = rot_u };
ur_nvec_init(cod_u, cod_w);
u3h_walk_with(u3R->jed.cod_p, _cu_hamt_walk, &dat_u);
return ken;
}
typedef struct _cu_loom_s {
ur_dict32_t map_u; // direct->indirect mapping
u3_atom *vat; // indirect atoms
u3_noun *cel; // cells
} _cu_loom;
/* _cu_ref_to_noun(): lookup/allocate [ref] on the loom.
*/
static u3_noun
_cu_ref_to_noun(ur_root_t* rot_u, ur_nref ref, _cu_loom* lom_u)
{
switch ( ur_nref_tag(ref) ) {
default: c3_assert(0);
// all ur indirect atoms have been pre-reallocated on the loom.
//
case ur_iatom: return lom_u->vat[ur_nref_idx(ref)];
// cells were allocated off-loom in cons-order, and are traversed
// in the same order: we've already relocated any one we could need here.
//
case ur_icell: return lom_u->cel[ur_nref_idx(ref)];
// u3 direct atoms are 31-bit, while ur direct atoms are 62-bit;
// we use a hashtable to deduplicate the non-overlapping space
//
case ur_direct: {
u3_atom vat;
if ( 0x7fffffffULL >= ref ) {
return (u3_atom)ref;
}
else if ( ur_dict32_get(rot_u, &lom_u->map_u, ref, (c3_w*)&vat) ) {
return vat;
}
else {
{
c3_w wor_w[2] = { ref & 0xffffffff, ref >> 32 };
vat = (c3_w)u3i_words(2, wor_w);
}
ur_dict32_put(0, &lom_u->map_u, ref, (c3_w)vat);
return vat;
}
} break;
}
}
/* _cu_all_to_loom(): reallocate all of [rot_u] on the loom, restore roots.
** NB: requires all roots to be cells
** does *not* track refcounts, which must be
** subsequently reconstructed via tracing.
*/
static void
_cu_all_to_loom(ur_root_t* rot_u, ur_nref ken, ur_nvec_t* cod_u)
{
_cu_loom lom_u = {0};
c3_d i_d, fil_d;
ur_dict32_grow(0, &lom_u.map_u, ur_fib11, ur_fib12);
// allocate all atoms on the loom.
//
{
c3_d* len_d = rot_u->atoms.lens;
c3_y** byt_y = rot_u->atoms.bytes;
fil_d = rot_u->atoms.fill;
lom_u.vat = calloc(fil_d, sizeof(u3_atom));
for ( i_d = 0; i_d < fil_d; i_d++ ) {
lom_u.vat[i_d] = u3i_bytes(len_d[i_d], byt_y[i_d]);
}
}
// allocate all cells on the loom.
//
{
ur_nref* hed = rot_u->cells.heads;
ur_nref* tal = rot_u->cells.tails;
u3_noun cel;
fil_d = rot_u->cells.fill;
lom_u.cel = c3_calloc(fil_d * sizeof(u3_noun));
for ( i_d = 0; i_d < fil_d; i_d++ ) {
cel = u3nc(_cu_ref_to_noun(rot_u, hed[i_d], &lom_u),
_cu_ref_to_noun(rot_u, tal[i_d], &lom_u));
lom_u.cel[i_d] = cel;
u3r_mug(cel);
}
}
// restore kernel reference (always a cell)
//
u3A->roc = lom_u.cel[ur_nref_idx(ken)];
// restore cold jet state (always cells)
//
{
c3_d max_d = cod_u->fill;
c3_d i_d;
ur_nref ref;
u3_noun kev;
for ( i_d = 0; i_d < max_d; i_d++) {
ref = cod_u->refs[i_d];
kev = lom_u.cel[ur_nref_idx(ref)];
u3h_put(u3R->jed.cod_p, u3h(kev), u3k(u3t(kev)));
u3z(kev);
}
}
// dispose of relocation pointers
//
c3_free(lom_u.cel);
c3_free(lom_u.vat);
ur_dict_free((ur_dict_t*)&lom_u.map_u);
}
/* _cu_realloc(): hash-cons roots off-loom, reallocate on loom.
*/
static ur_nref
_cu_realloc(FILE* fil_u, ur_root_t** tor_u, ur_nvec_t* doc_u)
{
#ifdef U3_MEMORY_DEBUG
c3_assert(0);
#endif
// bypassing page tracking as an optimization
//
// NB: u3e_yolo() will mark all as dirty, and
// u3e_save() will reinstate protection flags
//
if ( c3n == u3e_yolo() ) {
if ( fil_u ) {
fprintf(fil_u, "uniq: unable to bypass page tracking, continuing\r\n");
}
}
// stash event number
//
c3_d eve_d = u3A->ent_d;
// reallocate kernel and cold jet state
//
ur_root_t* rot_u = ur_root_init();
ur_nvec_t cod_u;
ur_nref ken = _cu_all_from_loom(rot_u, &cod_u);
// print [rot_u] measurements
//
if ( fil_u ) {
ur_root_info(fil_u, rot_u);
fprintf(fil_u, "\r\n");
}
// reinitialize loom
//
// NB: hot jet state is not yet re-established
//
u3m_pave(c3y, c3n);
// reallocate all nouns on the loom
//
_cu_all_to_loom(rot_u, ken, &cod_u);
// establish correct refcounts via tracing
//
u3m_grab(u3_none);
// allocate new hot jet state; re-establish warm
//
u3j_boot(c3y);
u3j_ream();
// restore event number
//
u3A->ent_d = eve_d;
// mark all pages dirty
//
u3e_foul();
*tor_u = rot_u;
*doc_u = cod_u;
return ken;
}
/* u3u_meld(): globally deduplicate memory.
*/
#ifdef U3_MEMORY_DEBUG
void
u3u_meld(void)
{
fprintf(stderr, "u3: unable to meld under U3_MEMORY_DEBUG\r\n");
}
#else
void
u3u_meld(void)
{
ur_root_t* rot_u;
ur_nvec_t cod_u;
c3_assert( &(u3H->rod_u) == u3R );
_cu_realloc(stderr, &rot_u, &cod_u);
// dispose off-loom structures
//
ur_nvec_free(&cod_u);
ur_root_free(rot_u);
}
#endif
/* _cu_rock_path(): format rock path.
*/
static c3_o
_cu_rock_path(c3_c* dir_c, c3_d eve_d, c3_c** out_c)
{
c3_w nam_w = 1 + snprintf(0, 0, "%s/.urb/roc/%" PRIu64 ".jam", dir_c, eve_d);
c3_c* nam_c = c3_malloc(nam_w);
c3_i ret_i;
ret_i = snprintf(nam_c, nam_w, "%s/.urb/roc/%" PRIu64 ".jam", dir_c, eve_d);
if ( ret_i < 0 ) {
fprintf(stderr, "rock: path format failed (%s, %" PRIu64 "): %s\r\n",
dir_c, eve_d, strerror(errno));
c3_free(nam_c);
return c3n;
}
else if ( ret_i >= nam_w ) {
fprintf(stderr, "rock: path format failed (%s, %" PRIu64 "): truncated\r\n",
dir_c, eve_d);
c3_free(nam_c);
return c3n;
}
*out_c = nam_c;
return c3y;
}
/* _cu_rock_path_make(): format rock path, creating directory if necessary..
*/
static c3_o
_cu_rock_path_make(c3_c* dir_c, c3_d eve_d, c3_c** out_c)
{
c3_w nam_w = 1 + snprintf(0, 0, "%s/.urb/roc/%" PRIu64 ".jam", dir_c, eve_d);
c3_c* nam_c = c3_malloc(nam_w);
c3_i ret_i;
// create $pier/.urb/roc, if it doesn't exist
//
// NB, $pier/.urb is guaranteed to already exist
//
{
ret_i = snprintf(nam_c, nam_w, "%s/.urb/roc", dir_c);
if ( ret_i < 0 ) {
fprintf(stderr, "rock: path format failed (%s, %" PRIu64 "): %s\r\n",
dir_c, eve_d, strerror(errno));
c3_free(nam_c);
return c3n;
}
else if ( ret_i >= nam_w ) {
fprintf(stderr, "rock: path format failed (%s, %" PRIu64 "): truncated\r\n",
dir_c, eve_d);
c3_free(nam_c);
return c3n;
}
if ( mkdir(nam_c, 0700)
&& (EEXIST != errno) )
{
fprintf(stderr, "rock: directory create failed (%s, %" PRIu64 "): %s\r\n",
dir_c, eve_d, strerror(errno));
c3_free(nam_c);
return c3n;
}
}
ret_i = snprintf(nam_c, nam_w, "%s/.urb/roc/%" PRIu64 ".jam", dir_c, eve_d);
if ( ret_i < 0 ) {
fprintf(stderr, "rock: path format failed (%s, %" PRIu64 "): %s\r\n",
dir_c, eve_d, strerror(errno));
c3_free(nam_c);
return c3n;
}
else if ( ret_i >= nam_w ) {
fprintf(stderr, "rock: path format failed (%s, %" PRIu64 "): truncated\r\n",
dir_c, eve_d);
c3_free(nam_c);
return c3n;
}
*out_c = nam_c;
return c3y;
}
static c3_o
_cu_rock_save(c3_c* dir_c, c3_d eve_d, c3_d len_d, c3_y* byt_y)
{
c3_i fid_i;
// open rock file, creating the containing directory if necessary
//
{
c3_c* nam_c;
if ( c3n == _cu_rock_path_make(dir_c, eve_d, &nam_c) ) {
return c3n;
}
if ( -1 == (fid_i = open(nam_c, O_RDWR | O_CREAT | O_TRUNC, 0644)) ) {
fprintf(stderr, "rock: open failed (%s, %" PRIu64 "): %s\r\n",
dir_c, eve_d, strerror(errno));
c3_free(nam_c);
return c3n;
}
c3_free(nam_c);
}
// write jam-buffer into [fid_i]
//
// XX deduplicate with _write() wrapper in term.c
//
{
ssize_t ret_i;
while ( len_d > 0 ) {
c3_w lop_w = 0;
// retry interrupt/async errors
//
do {
// abort pathological retry loop
//
if ( 100 == ++lop_w ) {
fprintf(stderr, "rock: write loop: %s\r\n", strerror(errno));
close(fid_i);
// XX unlink file?
//
return c3n;
}
ret_i = write(fid_i, byt_y, len_d);
}
while ( (ret_i < 0)
&& ( (errno == EINTR)
|| (errno == EAGAIN)
|| (errno == EWOULDBLOCK) ));
// assert on true errors
//
// NB: can't call u3l_log here or we would re-enter _write()
//
if ( ret_i < 0 ) {
fprintf(stderr, "rock: write failed %s\r\n", strerror(errno));
close(fid_i);
// XX unlink file?
//
return c3n;
}
// continue partial writes
//
else {
len_d -= ret_i;
byt_y += ret_i;
}
}
}
close(fid_i);
return c3y;
}
/* u3u_cram(): globably deduplicate memory, and write a rock to disk.
*/
#ifdef U3_MEMORY_DEBUG
c3_o
u3u_cram(c3_c* dir_c, c3_d eve_d)
{
fprintf(stderr, "u3: unable to cram under U3_MEMORY_DEBUG\r\n");
return c3n;
}
#else
c3_o
u3u_cram(c3_c* dir_c, c3_d eve_d)
{
c3_o ret_o = c3y;
c3_d len_d;
c3_y* byt_y;
c3_assert( &(u3H->rod_u) == u3R );
{
ur_root_t* rot_u;
ur_nvec_t cod_u;
ur_nref ken = _cu_realloc(stderr, &rot_u, &cod_u);
{
ur_nref roc = u3_nul;
c3_d max_d = cod_u.fill;
c3_d i_d;
// cons vector of cold jet-state entries onto a list
//
for ( i_d = 0; i_d < max_d; i_d++) {
roc = ur_cons(rot_u, cod_u.refs[i_d], roc);
}
roc = ur_cons(rot_u, ur_coin64(rot_u, c3__fast), ur_cons(rot_u, ken, roc));
ur_jam(rot_u, roc, &len_d, &byt_y);
}
// dispose off-loom structures
//
ur_nvec_free(&cod_u);
ur_root_free(rot_u);
}
// write jam-buffer into pier
//
if ( c3n == _cu_rock_save(dir_c, eve_d, len_d, byt_y) ) {
ret_o = c3n;
}
c3_free(byt_y);
return ret_o;
}
#endif
/* u3u_mmap_read(): open and mmap the file at [pat_c] for reading.
*/
c3_o
u3u_mmap_read(c3_c* cap_c, c3_c* pat_c, c3_d* out_d, c3_y** out_y)
{
c3_i fid_i;
c3_d len_d;
// open file
//
if ( -1 == (fid_i = open(pat_c, O_RDONLY, 0644)) ) {
fprintf(stderr, "%s: open failed (%s): %s\r\n",
cap_c, pat_c, strerror(errno));
return c3n;
}
// measure file
//
{
struct stat buf_b;
if ( -1 == fstat(fid_i, &buf_b) ) {
fprintf(stderr, "%s: stat failed (%s): %s\r\n",
cap_c, pat_c, strerror(errno));
close(fid_i);
return c3n;
}
len_d = buf_b.st_size;
}
// mmap file
//
{
void* ptr_v;
if ( MAP_FAILED == (ptr_v = mmap(0, len_d, PROT_READ, MAP_SHARED, fid_i, 0)) ) {
fprintf(stderr, "%s: mmap failed (%s): %s\r\n",
cap_c, pat_c, strerror(errno));
close(fid_i);
return c3n;
}
*out_d = len_d;
*out_y = (c3_y*)ptr_v;
}
// close file
//
close(fid_i);
return c3y;
}
/* u3u_mmap(): open/create file-backed mmap at [pat_c] for read/write.
*/
c3_o
u3u_mmap(c3_c* cap_c, c3_c* pat_c, c3_d len_d, c3_y** out_y)
{
c3_i fid_i;
// open file
//
if ( -1 == (fid_i = open(pat_c, O_RDWR | O_CREAT | O_TRUNC, 0644)) ) {
fprintf(stderr, "%s: open failed (%s): %s\r\n",
cap_c, pat_c, strerror(errno));
return c3n;
}
// grow [fid_i] to [len_w]
//
// XX build with _FILE_OFFSET_BITS == 64 ?
//
if ( 0 != ftruncate(fid_i, len_d) ) {
fprintf(stderr, "%s: ftruncate grow %s: %s\r\n",
cap_c, pat_c, strerror(errno));
close(fid_i);
return c3n;
}
// mmap file
//
{
void* ptr_v;
if ( MAP_FAILED == (ptr_v = mmap(0, len_d, PROT_READ|PROT_WRITE, MAP_SHARED, fid_i, 0)) ) {
fprintf(stderr, "%s: mmap failed (%s): %s\r\n",
cap_c, pat_c, strerror(errno));
close(fid_i);
return c3n;
}
*out_y = (c3_y*)ptr_v;
}
// close file
//
close(fid_i);
return c3y;
}
/* u3u_mmap_save(): sync file-backed mmap.
*/
c3_o
u3u_mmap_save(c3_c* cap_c, c3_c* pat_c, c3_d len_d, c3_y* byt_y)
{
if ( 0 != msync(byt_y, len_d, MS_SYNC) ) {
fprintf(stderr, "%s: msync %s: %s\r\n", cap_c, pat_c, strerror(errno));
return c3n;
}
return c3y;
}
/* u3u_munmap(): unmap the region at [byt_y].
*/
c3_o
u3u_munmap(c3_d len_d, c3_y* byt_y)
{
if ( 0 != munmap(byt_y, len_d) ) {
return c3n;
}
return c3y;
}
/* u3u_uncram(): restore persistent state from a rock.
*/
c3_o
u3u_uncram(c3_c* dir_c, c3_d eve_d)
{
c3_c* nam_c;
c3_d len_d;
c3_y* byt_y;
// load rock file into buffer
//
if ( c3n == _cu_rock_path(dir_c, eve_d, &nam_c) ) {
fprintf(stderr, "uncram: failed to make rock path (%s, %" PRIu64 ")\r\n",
dir_c, eve_d);
return c3n;
}
else if ( c3n == u3u_mmap_read("rock", nam_c, &len_d, &byt_y) ) {
c3_free(nam_c);
return c3n;
}
// bypassing page tracking as an optimization
//
// NB: u3e_yolo() will mark all as dirty, and
// u3e_save() will reinstate protection flags
//
if ( c3n == u3e_yolo() ) {
fprintf(stderr, "uncram: unable to bypass page tracking, continuing\r\n");
}
// reinitialize loom
//
// NB: hot jet state is not yet re-established
//
u3m_pave(c3y, c3n);
// cue rock, restore persistent state
//
// XX errors are fatal, barring a full "u3m_reboot"-type operation.
//
{
ur_dict32_t dic_u = {0};
u3_noun roc, cod, ref;
// XX tune the initial dictionary size for less reallocation
//
ur_dict32_grow((ur_root_t*)0, &dic_u, ur_fib33, ur_fib34);
if ( c3n == u3s_cue_xeno_unsafe(&dic_u, len_d, byt_y, &ref) ) {
fprintf(stderr, "uncram: failed to cue rock\r\n");
ur_dict_free((ur_dict_t*)&dic_u);
c3_free(nam_c);
return c3n;
}
ur_dict_free((ur_dict_t*)&dic_u);
if ( c3n == u3r_pq(ref, c3__fast, &roc, &cod) ) {
fprintf(stderr, "uncram: failed: invalid rock format\r\n");
u3z(ref);
c3_free(nam_c);
return c3n;
}
u3A->roc = u3k(roc);
u3j_load(u3k(cod));
u3z(ref);
}
u3u_munmap(len_d, byt_y);
// allocate new hot jet state; re-establish warm
//
u3j_boot(c3y);
u3j_ream();
// restore event number
//
u3A->ent_d = eve_d;
// mark all pages dirty
//
u3e_foul();
// leave rocks on disk
//
// if ( 0 != unlink(nam_c) ) {
// fprintf(stderr, "uncram: failed to delete rock (%s, %" PRIu64 "): %s\r\n",
// dir_c, eve_d, strerror(errno));
// c3_free(nam_c);
// return c3n;
// }
c3_free(nam_c);
return c3y;
}

11
pkg/urbit/noun/vortex.c
View File

@ -50,30 +50,29 @@ u3v_boot(u3_noun eve)
static u3_noun
_cv_lite(u3_noun pil)
{
u3_noun arv = u3ke_cue(pil);
u3_noun eve, pro;
u3x_trel(arv, &eve, 0, 0);
u3x_trel(pil, &eve, 0, 0);
u3l_log("lite: arvo formula %x\r\n", u3r_mug(arv));
u3l_log("lite: arvo formula %x\r\n", u3r_mug(pil));
pro = u3v_life(u3k(eve));
u3l_log("lite: core %x\r\n", u3r_mug(pro));
u3z(arv);
u3z(pil);
return pro;
}
/* u3v_boot_lite(): light bootstrap sequence, just making a kernel.
*/
c3_o
u3v_boot_lite(u3_atom lit)
u3v_boot_lite(u3_noun pil)
{
// ensure zero-initialized kernel
//
u3A->roc = 0;
{
u3_noun pro = u3m_soft(0, _cv_lite, lit);
u3_noun pro = u3m_soft(0, _cv_lite, pil);
if ( u3_blip != u3h(pro) ) {
u3z(pro);

337
pkg/urbit/tests/jam_tests.c
View File

@ -1,4 +1,5 @@
#include "all.h"
#include "ur/ur.h"
/* _setup(): prepare for tests.
*/
@ -9,19 +10,21 @@ _setup(void)
u3m_pave(c3y, c3n);
}
/* _test_jam(): spot check jam/cue
/* _test_jam_spot_a(): spot check jam/cue
*/
static void
_test_jam(void)
static c3_i
_test_jam_spot_a(void)
{
c3_i ret_i = 1;
if ( 0xc != u3qe_jam(1) ) {
fprintf(stderr, "jam: fail (a)\r\n");
exit(1);
ret_i = 0;
}
if ( 1 != u3ke_cue(u3qe_jam(1)) ) {
fprintf(stderr, "jam: fail (b)\r\n");
exit(1);
ret_i = 0;
}
{
@ -29,12 +32,12 @@ _test_jam(void)
if ( 0x1231 != u3qe_jam(a) ) {
fprintf(stderr, "jam: fail (c)\r\n");
exit(1);
ret_i = 0;
}
if ( c3y != u3r_sing(a, u3ke_cue(u3qe_jam(a))) ) {
fprintf(stderr, "jam: fail (d)\r\n");
exit(1);
ret_i = 0;
}
}
@ -43,28 +46,26 @@ _test_jam(void)
if ( 0x344871 != u3qe_jam(a) ) {
fprintf(stderr, "jam: fail (e)\r\n");
exit(1);
ret_i = 0;
}
if ( c3y != u3r_sing(a, u3ke_cue(u3qe_jam(a))) ) {
fprintf(stderr, "jam: fail (f)\r\n");
exit(1);
ret_i = 0;
}
}
{
u3_noun a = u3nc(u3nc(1, 2), 3);
// fprintf(stderr, "%x\n", u3qe_jam(a));
if ( 0x3448c5 != u3qe_jam(a) ) {
fprintf(stderr, "jam: fail (g)\r\n");
exit(1);
ret_i = 0;
}
if ( c3y != u3r_sing(a, u3ke_cue(u3qe_jam(a))) ) {
fprintf(stderr, "jam: fail (h)\r\n");
exit(1);
ret_i = 0;
}
}
@ -74,7 +75,7 @@ _test_jam(void)
if ( c3y != u3r_sing(a, u3ke_cue(u3qe_jam(a))) ) {
fprintf(stderr, "jam: fail (j)\r\n");
exit(1);
ret_i = 0;
}
}
@ -84,7 +85,7 @@ _test_jam(void)
if ( c3y != u3r_sing(a, u3ke_cue(u3qe_jam(a))) ) {
fprintf(stderr, "jam: fail (k)\r\n");
exit(1);
ret_i = 0;
}
}
@ -93,16 +94,20 @@ _test_jam(void)
if ( c3y != u3r_sing(a, u3ke_cue(u3qe_jam(a))) ) {
fprintf(stderr, "jam: fail (l)\r\n");
exit(1);
ret_i = 0;
}
}
return ret_i;
}
/* _test_cue_jam(): more jam/cue spot-checking, ported from the 64-bit effort
/* _test_jam_spot_b(): more jam/cue spot-checking, ported from the 64-bit effort
*/
static void
_test_cue_jam()
static c3_i
_test_jam_spot_b()
{
c3_i ret_i = 1;
// the boot msg from the worker
{
u3_noun dat = u3_nul;
@ -114,13 +119,14 @@ _test_cue_jam()
u3_noun tail_out = u3t(out_1);
if (c3__play != head_out){
printf("*** cue_jam 0 out head \n");
fprintf(stderr, "*** cue_jam 0 out head\r\n");
ret_i = 0;
}
if (u3_nul != tail_out){
printf("*** cue_jam 0 out tail \n");
fprintf(stderr, "*** cue_jam 0 out tail\r\n");
ret_i = 0;
}
}
// the boot msg from the worker, again,
@ -144,7 +150,8 @@ _test_cue_jam()
u3_noun jam_2 = u3i_bytes(len_w, buf_y);
if ( c3n == u3r_sing(jam_1, jam_2) ) {
printf("*** error in 6 byte message\n");
fprintf(stderr, "*** error in 6 byte message\r\n");
ret_i = 0;
}
u3_noun out_1 = u3ke_cue(jam_2);
@ -153,13 +160,14 @@ _test_cue_jam()
u3_noun tail_out = u3t(out_1);
if (c3__play != head_out){
printf("*** cue_jam 0 out head \n");
fprintf(stderr, "*** cue_jam 0 out head\r\n");
ret_i = 0;
}
if (u3_nul != tail_out){
printf("*** cue_jam 0 out tail \n");
fprintf(stderr, "*** cue_jam 0 out tail\r\n");
ret_i = 0;
}
}
// 1
@ -169,13 +177,15 @@ _test_cue_jam()
u3_atom jam_1 = u3ke_jam(in_1);
if (12 != jam_1){
printf("*** cue_jam 1a \n");
fprintf(stderr, "*** cue_jam 1a\r\n");
ret_i = 0;
}
u3_noun out_1 = u3ke_cue(jam_1);
if (1 != out_1){
printf("*** cue_jam 1b \n");
fprintf(stderr, "*** cue_jam 1b\r\n");
ret_i = 0;
}
}
@ -186,7 +196,8 @@ _test_cue_jam()
u3_atom jam_1 = u3ke_jam(in_1);
if (817 != jam_1){
printf("*** cue_jam 2 in \n");
fprintf(stderr, "*** cue_jam 2 in\r\n");
ret_i = 0;
}
u3_noun out_1 = u3ke_cue(jam_1);
@ -196,11 +207,13 @@ _test_cue_jam()
u3_noun tail_out = u3t(out_1);
if (1 != head_out){
printf("*** cue_jam 2 out head \n");
fprintf(stderr, "*** cue_jam 2 out head\r\n");
ret_i = 0;
}
if (1 != tail_out){
printf("*** cue_jam 2 out tail \n");
fprintf(stderr, "*** cue_jam 2 out tail\r\n");
ret_i = 0;
}
}
@ -211,7 +224,8 @@ _test_cue_jam()
u3_atom jam_1 = u3ke_jam(in_1);
if (4657 != jam_1){
printf("*** cue_jam 2 in \n");
fprintf(stderr, "*** cue_jam 2 in\r\n");
ret_i = 0;
}
u3_noun out_1 = u3ke_cue(jam_1);
@ -220,11 +234,13 @@ _test_cue_jam()
u3_noun tail_out = u3t(out_1);
if (1 != head_out){
printf("*** cue_jam 2 out head \n");
fprintf(stderr, "*** cue_jam 2 out head\r\n");
ret_i = 0;
}
if (2 != tail_out){
printf("*** cue_jam 2 out tail \n");
fprintf(stderr, "*** cue_jam 2 out tail\r\n");
ret_i = 0;
}
}
@ -252,25 +268,261 @@ _test_cue_jam()
u3_noun a2 = u3h(out_1);
u3_noun r2 = u3t(out_1);
if (a2 != a){
printf("*** _cue_jam: complicated a\n");
fprintf(stderr, "*** _cue_jam: complicated a\r\n");
ret_i = 0;
}
u3_noun b2 = u3h(r2);
u3_noun s2 = u3t(r2);
if (b2 != b){
printf("*** _cue_jam: complicated b\n");
fprintf(stderr, "*** _cue_jam: complicated b\r\n");
ret_i = 0;
}
u3_noun c2 = u3h(s2);
u3_noun d2 = u3t(s2);
if (c2 != c){
printf("*** _cue_jam: complicated c\n");
fprintf(stderr, "*** _cue_jam: complicated c\r\n");
ret_i = 0;
}
if (d2 != d){
printf("*** _cue_jam: complicated d\n");
fprintf(stderr, "*** _cue_jam: complicated d\r\n");
ret_i = 0;
}
}
return 1;
}
static void
_byte_print(c3_d out_d,
c3_y* out_y,
c3_w len_w,
const c3_y* byt_y)
{
c3_d i_d;
fprintf(stderr, " actual: { ");
for ( i_d = 0; i_d < out_d; i_d++ ) {
fprintf(stderr, "0x%x, ", out_y[i_d]);
}
fprintf(stderr, "}\r\n");
fprintf(stderr, " expect: { ");
for ( i_d = 0; i_d < len_w; i_d++ ) {
fprintf(stderr, "0x%x, ", byt_y[i_d]);
}
fprintf(stderr, "}\r\n");
}
static c3_i
_test_jam_spec(const c3_c* cap_c,
u3_noun ref,
c3_w len_w,
const c3_y* byt_y)
{
c3_i ret_i = 1;
c3_d out_d;
c3_y* out_y;
{
u3s_jam_xeno(ref, &out_d, &out_y);
if ( 0 != memcmp(out_y, byt_y, len_w) ) {
fprintf(stderr, "\033[31mjam xeno %s fail\033[0m\r\n", cap_c);
_byte_print(out_d, out_y, len_w, byt_y);
ret_i = 0;
}
free(out_y);
}
{
c3_w bit_w;
c3_w* wor_w = u3s_jam_fib(ref, &bit_w);
out_d = (bit_w >> 3) + !!ur_mask_3(bit_w);
// XX assumes little-endian
//
out_y = (c3_y*)wor_w;
if ( 0 != memcmp(out_y, byt_y, len_w) ) {
fprintf(stderr, "\033[31mjam fib %s fail\033[0m\r\n", cap_c);
_byte_print(out_d, out_y, len_w, byt_y);
ret_i = 0;
}
u3a_wfree(wor_w);
}
return ret_i;
}
static c3_i
_test_cue_spec(const c3_c* cap_c,
u3_noun ref,
c3_w len_w,
const c3_y* byt_y)
{
c3_i ret_i = 1;
{
ur_dict32_t dic_u = {0};
u3_noun out;
ur_dict32_grow((ur_root_t*)0, &dic_u, ur_fib10, ur_fib11);
if ( c3n == u3s_cue_xeno_unsafe(&dic_u, len_w, byt_y, &out) ) {
fprintf(stderr, "\033[31mcue %s fail 1\033[0m\r\n", cap_c);
ret_i = 0;
}
else if ( c3n == u3r_sing(ref, out) ) {
fprintf(stderr, "\033[31mcue %s fail 2\033[0m\r\n", cap_c);
u3m_p("ref", ref);
u3m_p("out", out);
ret_i = 0;
}
u3z(out);
ur_dict_free((ur_dict_t*)&dic_u);
}
{
u3_noun pro = u3m_soft(0, u3s_cue_atom, u3i_bytes(len_w, byt_y));
u3_noun tag, out;
u3x_cell(pro, &tag, &out);
if ( u3_blip != tag ) {
fprintf(stderr, "\033[31mcue %s fail 3\033[0m\r\n", cap_c);
ret_i = 0;
}
else if ( c3n == u3r_sing(ref, out) ) {
fprintf(stderr, "\033[31mcue %s fail 4\033[0m\r\n", cap_c);
u3m_p("ref", ref);
u3m_p("out", out);
ret_i = 0;
}
u3z(pro);
}
return ret_i;
}
static c3_i
_test_jam_roundtrip(void)
{
c3_i ret_i = 1;
# define TEST_CASE(a, b) \
const c3_c* cap_c = a; \
u3_noun ref = b; \
ret_i &= _test_jam_spec(cap_c, ref, sizeof(res_y), res_y); \
ret_i &= _test_cue_spec(cap_c, ref, sizeof(res_y), res_y); \
u3z(ref);
{
c3_y res_y[1] = { 0x2 };
TEST_CASE("0", 0);
}
{
c3_y res_y[1] = { 0xc };
TEST_CASE("1", 1);
}
{
c3_y res_y[1] = { 0x48 };
TEST_CASE("2", 2);
}
{
c3_y res_y[6] = { 0xc0, 0x37, 0xb, 0x9b, 0xa3, 0x3 };
TEST_CASE("%fast", c3__fast);
}
{
c3_y res_y[6] = { 0xc0, 0x37, 0xab, 0x63, 0x63, 0x3 };
TEST_CASE("%full", c3__full);
}
{
c3_y res_y[1] = { 0x29 };
TEST_CASE("[0 0]", u3nc(0, 0));
}
{
c3_y res_y[2] = { 0x31, 0x3 };
TEST_CASE("[1 1]", u3nc(1, 1));
}
{
c3_y res_y[2] = { 0x21, 0xd1 };
TEST_CASE("[2 3]", u3nc(2, 3));
}
{
c3_y res_y[11] = { 0x1, 0xdf, 0x2c, 0x6c, 0x8e, 0xe, 0x7c, 0xb3, 0x3a, 0x36, 0x36 };
TEST_CASE("[%fast %full]", u3nc(c3__fast, c3__full));
}
{
c3_y res_y[2] = { 0x71, 0xcc };
TEST_CASE("[1 1 1]", u3nc(1, u3nc(1, 1)));
}
{
c3_y res_y[12] = { 0x1, 0xdf, 0x2c, 0x6c, 0x8e, 0x1e, 0xf0, 0xcd, 0xea, 0xd8, 0xd8, 0x93 };
TEST_CASE("[%fast %full %fast]", u3nc(c3__fast, u3nc(c3__full, c3__fast)));
}
{
c3_y res_y[6] = { 0xa5, 0x35, 0x19, 0xf3, 0x18, 0x5 };
TEST_CASE("[[0 0] [[0 0] 1 1] 1 1]", u3nc(u3nc(0, 0), u3nc(u3nc(u3nc(0, 0), u3nc(1, 1)), u3nc(1, 1))));
}
{
c3_y res_y[14] = { 0x15, 0x17, 0xb2, 0xd0, 0x85, 0x59, 0xb8, 0x61, 0x87, 0x5f, 0x10, 0x54, 0x55, 0x5 };
TEST_CASE("deep", u3nc(u3nc(u3nc(1, u3nc(u3nc(2, u3nc(u3nc(3, u3nc(u3nc(4, u3nc(u3nt(5, 6, u3nc(7, u3nc(u3nc(8, 0), 0))), 0)), 0)), 0)), 0)), 0), 0));
}
{
c3_y inp_y[33] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
c3_y res_y[35] = { 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8 };
TEST_CASE("wide", u3i_bytes(sizeof(inp_y), inp_y));
}
{
c3_y inp_y[16] = { 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0xa8, 0xab, 0x60, 0xef, 0x2d, 0xd, 0x0, 0x0, 0x80 };
c3_y res_y[19] = { 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x18, 0x50, 0x57, 0xc1, 0xde, 0x5b, 0x1a, 0x0, 0x0, 0x0, 0x1 };
TEST_CASE("date", u3i_bytes(sizeof(inp_y), inp_y));
}
return ret_i;
}
static c3_i
_test_jam(void)
{
c3_i ret_i = 1;
if ( !_test_jam_spot_a() ) {
fprintf(stderr, "test jam: spot a: failed\r\n");
ret_i = 0;
}
if ( !_test_jam_spot_b() ) {
fprintf(stderr, "test jam: spot b: failed\r\n");
ret_i = 0;
}
if ( !_test_jam_roundtrip() ) {
fprintf(stderr, "test jam: roundtrip: failed\r\n");
ret_i = 0;
}
return ret_i;
}
/* main(): run all test cases.
@ -280,10 +532,11 @@ main(int argc, char* argv[])
{
_setup();
_test_jam();
_test_cue_jam();
fprintf(stderr, "test_jam: ok\n");
if ( !_test_jam() ) {
fprintf(stderr, "test jam: failed\r\n");
exit(1);
}
fprintf(stderr, "test jam: ok\r\n");
return 0;
}

1860
pkg/urbit/tests/ur_tests.c Normal file
View File

File diff suppressed because it is too large Load Diff

1267
pkg/urbit/ur/bitstream.c Normal file
View File

File diff suppressed because it is too large Load Diff

946
pkg/urbit/ur/hashcons.c Normal file
View File

@ -0,0 +1,946 @@
#include <inttypes.h>
#include <stdlib.h>
#include <stddef.h>
#include <string.h>
#include <assert.h>
#include <stdio.h>
#include <limits.h>
#include <murmur3.h>
#include "ur/defs.h"
#include "ur/hashcons.h"
// declarations of inline functions
//
uint64_t
ur_met0_bytes_unsafe(uint64_t len, uint8_t *byt);
static void*
_oom(const char* cap, void* v)
{
if ( !v ) {
fprintf(stderr,
"ur: hashcons: %s: allocation failed, out of memory\r\n", cap);
abort();
}
return v;
}
void
ur_dict32_grow(ur_root_t *r, ur_dict32_t *dict, uint64_t prev, uint64_t size)
{
ur_pail32_t *buckets, *old_buckets = dict->buckets;
uint64_t old_size = dict->size;
uint64_t i, next = prev + size;
buckets = _oom("dict32_grow", calloc(next, sizeof(*buckets)));
if ( old_buckets ) {
for ( i = 0; i < old_size; i++ ) {
ur_pail32_t *old_bucket = &(old_buckets[i]);
uint8_t j, old_fill = old_bucket->fill;
for ( j = 0; j < old_fill; j++ ) {
uint32_t val = old_bucket->vals[j];
ur_nref ref = old_bucket->refs[j];
ur_mug mug = ur_nref_mug(r, ref);
uint64_t idx = ( mug % next );
ur_pail32_t *bucket = &(buckets[idx]);
uint8_t new_fill = bucket->fill;
if ( ur_pail_max == new_fill ) {
free(buckets);
return ur_dict32_grow(r, dict, size, next);
}
bucket->refs[new_fill] = ref;
bucket->vals[new_fill] = val;
bucket->fill = 1 + new_fill;
}
}
free(old_buckets);
}
dict->prev = size;
dict->size = next;
dict->buckets = buckets;
}
ur_bool_t
ur_dict32_get(ur_root_t *r, ur_dict32_t *dict, ur_nref ref, uint32_t *out)
{
ur_mug mug = ur_nref_mug(r, ref);
uint64_t idx = ( mug % dict->size );
ur_pail32_t *bucket = &(dict->buckets[idx]);
uint8_t i, fill = bucket->fill;
for ( i = 0; i < fill; i++ ) {
if ( ref == bucket->refs[i] ) {
*out = bucket->vals[i];
return 1;
}
}
return 0;
}
void
ur_dict32_put(ur_root_t *r, ur_dict32_t *dict, ur_nref ref, uint32_t val)
{
ur_mug mug = ur_nref_mug(r, ref);
while ( 1 ) {
uint64_t idx = ( mug % dict->size );
ur_pail32_t *bucket = &(dict->buckets[idx]);
uint8_t i, fill = bucket->fill;
for ( i = 0; i < fill; i++ ) {
if ( ref == bucket->refs[i] ) {
bucket->vals[i] = val;
return;
}
}
if ( ur_pail_max == fill ) {
ur_dict32_grow(r, dict, dict->prev, dict->size);
continue;
}
bucket->refs[fill] = ref;
bucket->vals[fill] = val;
bucket->fill = 1 + fill;
break;
}
}
void
ur_dict32_wipe(ur_dict32_t *dict)
{
ur_pail32_t *buckets = dict->buckets;
uint64_t i, size = dict->size;
for ( i = 0; i < size; i++ ) {
buckets[i].fill = 0;
}
}
void
ur_dict64_grow(ur_root_t *r, ur_dict64_t *dict, uint64_t prev, uint64_t size)
{
ur_pail64_t *buckets, *old_buckets = dict->buckets;
uint64_t old_size = dict->size;
uint64_t i, next = prev + size;
buckets = _oom("dict64_grow", calloc(next, sizeof(*buckets)));
if ( old_buckets ) {
for ( i = 0; i < old_size; i++ ) {
ur_pail64_t *old_bucket = &(old_buckets[i]);
uint8_t j, old_fill = old_bucket->fill;
for ( j = 0; j < old_fill; j++ ) {
uint64_t val = old_bucket->vals[j];
ur_nref ref = old_bucket->refs[j];
ur_mug mug = ur_nref_mug(r, ref);
uint64_t idx = ( mug % next );
ur_pail64_t *bucket = &(buckets[idx]);
uint8_t new_fill = bucket->fill;
if ( ur_pail_max == new_fill ) {
free(buckets);
return ur_dict64_grow(r, dict, size, next);
}
bucket->refs[new_fill] = ref;
bucket->vals[new_fill] = val;
bucket->fill = 1 + new_fill;
}
}
free(old_buckets);
}
dict->prev = size;
dict->size = next;
dict->buckets = buckets;
}
ur_bool_t
ur_dict64_get(ur_root_t *r, ur_dict64_t *dict, ur_nref ref, uint64_t *out)
{
ur_mug mug = ur_nref_mug(r, ref);
uint64_t idx = ( mug % dict->size );
ur_pail64_t *bucket = &(dict->buckets[idx]);
uint8_t i, fill = bucket->fill;
for ( i = 0; i < fill; i++ ) {
if ( ref == bucket->refs[i] ) {
*out = bucket->vals[i];
return 1;
}
}
return 0;
}
void
ur_dict64_put(ur_root_t *r, ur_dict64_t *dict, ur_nref ref, uint64_t val)
{
ur_mug mug = ur_nref_mug(r, ref);
while ( 1 ) {
uint64_t idx = ( mug % dict->size );
ur_pail64_t *bucket = &(dict->buckets[idx]);
uint8_t i, fill = bucket->fill;
for ( i = 0; i < fill; i++ ) {
if ( ref == bucket->refs[i] ) {
bucket->vals[i] = val;
return;
}
}
if ( ur_pail_max == fill ) {
ur_dict64_grow(r, dict, dict->prev, dict->size);
continue;
}
bucket->refs[fill] = ref;
bucket->vals[fill] = val;
bucket->fill = 1 + fill;
break;
}
}
void
ur_dict64_wipe(ur_dict64_t *dict)
{
ur_pail64_t *buckets = dict->buckets;
uint64_t i, size = dict->size;
for ( i = 0; i < size; i++ ) {
buckets[i].fill = 0;
}
}
void
ur_dict_grow(ur_root_t *r, ur_dict_t *dict, uint64_t prev, uint64_t size)
{
ur_pail_t *buckets, *old_buckets = dict->buckets;
uint64_t old_size = dict->size;
uint64_t i, next = prev + size;
buckets = _oom("dict_grow", calloc(next, sizeof(*buckets)));
if ( old_buckets ) {
for ( i = 0; i < old_size; i++ ) {
ur_pail_t *old_bucket = &(old_buckets[i]);
uint8_t j, old_fill = old_bucket->fill;
for ( j = 0; j < old_fill; j++ ) {
ur_nref ref = old_bucket->refs[j];
ur_mug mug = ur_nref_mug(r, ref);
uint64_t idx = ( mug % next );
ur_pail_t *bucket = &(buckets[idx]);
uint8_t new_fill = bucket->fill;
if ( ur_pail_max == new_fill ) {
free(buckets);
return ur_dict_grow(r, dict, size, next);
}
bucket->refs[new_fill] = ref;
bucket->fill = 1 + new_fill;
}
}
free(old_buckets);
}
dict->prev = size;
dict->size = next;
dict->buckets = buckets;
}
ur_bool_t
ur_dict_get(ur_root_t *r, ur_dict_t *dict, ur_nref ref)
{
ur_mug mug = ur_nref_mug(r, ref);
uint64_t idx = ( mug % dict->size );
ur_pail_t *bucket = &(dict->buckets[idx]);
uint8_t i, fill = bucket->fill;
for ( i = 0; i < fill; i++ ) {
if ( ref == bucket->refs[i] ) {
return 1;
}
}
return 0;
}
void
ur_dict_put(ur_root_t *r, ur_dict_t *dict, ur_nref ref)
{
ur_mug mug = ur_nref_mug(r, ref);
while ( 1 ) {
uint64_t idx = ( mug % dict->size );
ur_pail_t *bucket = &(dict->buckets[idx]);
uint8_t i, fill = bucket->fill;
for ( i = 0; i < fill; i++ ) {
if ( ref == bucket->refs[i] ) {
return;
}
}
if ( ur_pail_max == fill ) {
ur_dict_grow(r, dict, dict->prev, dict->size);
continue;
}
bucket->refs[fill] = ref;
bucket->fill = 1 + fill;
break;
}
}
void
ur_dict_wipe(ur_dict_t *dict)
{
ur_pail_t *buckets = dict->buckets;
uint64_t i, size = dict->size;
for ( i = 0; i < size; i++ ) {
buckets[i].fill = 0;
}
}
void
ur_dict_free(ur_dict_t *dict)
{
free(dict->buckets);
dict->buckets = 0;
}
ur_mug
ur_mug_bytes(const uint8_t *byt, uint64_t len)
{
uint32_t seed = 0xcafebabe;
ur_mug mug;
while ( 1 ) {
uint32_t raw;
MurmurHash3_x86_32(byt, len, seed, &raw);
mug = (raw >> 31) ^ ( ur_mask_31(raw) );
if ( 0 == mug ) {
seed++;
}
else {
return mug;
}
}
}
ur_mug
ur_mug32(uint32_t x)
{
uint8_t byt[4] = {
ur_mask_8(x >> 0),
ur_mask_8(x >> 8),
ur_mask_8(x >> 16),
ur_mask_8(x >> 24)
};
return ur_mug_bytes(byt, ur_met3_32(x));
}
ur_mug
ur_mug64(uint64_t x)
{
uint8_t byt[8] = {
ur_mask_8(x >> 0),
ur_mask_8(x >> 8),
ur_mask_8(x >> 16),
ur_mask_8(x >> 24),
ur_mask_8(x >> 32),
ur_mask_8(x >> 40),
ur_mask_8(x >> 48),
ur_mask_8(x >> 56)
};
return ur_mug_bytes(byt, ur_met3_64(x));
}
ur_mug
ur_mug_both(ur_mug hed, ur_mug tal)
{
// XX not correct per u3r_mug, but necessary to avoid collisions
//
return ur_mug32(hed ^ (0x7fffffff ^ ur_mug32(tal)));
}
ur_mug
ur_nref_mug(ur_root_t *r, ur_nref ref)
{
switch ( ur_nref_tag(ref) ) {
default: assert(0);
case ur_direct: return ur_mug64(ref);
case ur_iatom: return r->atoms.mugs[ur_nref_idx(ref)];
case ur_icell: return r->cells.mugs[ur_nref_idx(ref)];
}
}
ur_bool_t
ur_deep(ur_nref ref)
{
return ur_icell == ur_nref_tag(ref);
}
ur_nref
ur_head(ur_root_t *r, ur_nref ref)
{
assert( ur_deep(ref) );
return r->cells.heads[ur_nref_idx(ref)];
}
ur_nref
ur_tail(ur_root_t *r, ur_nref ref)
{
assert( ur_deep(ref) );
return r->cells.tails[ur_nref_idx(ref)];
}
void
ur_atoms_grow(ur_atoms_t *atoms)
{
uint64_t prev = atoms->prev;
uint64_t size = atoms->size;
uint64_t next = prev + size;
uint8_t **bytes = atoms->bytes;
uint64_t *lens = atoms->lens;
ur_mug *mugs = atoms->mugs;
atoms->bytes = _oom("atoms_grow", malloc(next * ( sizeof(*atoms->bytes)
+ sizeof(*atoms->lens)
+ sizeof(*atoms->mugs) )));
atoms->lens = (void*)((char*)atoms->bytes + (next * sizeof(*atoms->bytes)));
atoms->mugs = (void*)((char*)atoms->lens + (next * sizeof(*atoms->lens)));
if ( bytes ) {
memcpy(atoms->bytes, bytes, size * (sizeof(*bytes)));
memcpy(atoms->lens, lens, size * (sizeof(*lens)));
memcpy(atoms->mugs, mugs, size * (sizeof(*mugs)));
free(bytes);
}
atoms->prev = size;
atoms->size = next;
}
void
ur_cells_grow(ur_cells_t *cells)
{
uint64_t prev = cells->prev;
uint64_t size = cells->size;
uint64_t next = prev + size;
ur_nref *heads = cells->heads;
ur_nref *tails = cells->tails;
ur_mug *mugs = cells->mugs;
cells->heads = _oom("cells_grow", malloc(next * ( sizeof(*cells->heads)
+ sizeof(*cells->heads)
+ sizeof(*cells->mugs) )));
cells->tails = (void*)((char*)cells->heads + (next * sizeof(*cells->heads)));
cells->mugs = (void*)((char*)cells->tails + (next * sizeof(*cells->tails)));
if ( heads ) {
memcpy(cells->heads, heads, size * (sizeof(*heads)));
memcpy(cells->tails, tails, size * (sizeof(*tails)));
memcpy(cells->mugs, mugs, size * (sizeof(*mugs)));
free(heads);
}
cells->prev = size;
cells->size = next;
}
void
ur_bytes(ur_root_t *r, ur_nref ref, uint8_t **byt, uint64_t *len)
{
assert( !ur_deep(ref) );
switch ( ur_nref_tag(ref) ) {
default: assert(0);
case ur_direct: {
*len = ur_met3_64(ref);
// XX little-endian
//
*byt = (uint8_t*)&ref;
} break;
case ur_iatom: {
uint64_t idx = ur_nref_idx(ref);
*len = r->atoms.lens[idx];
*byt = r->atoms.bytes[idx];
} break;
}
}
uint64_t
ur_met(ur_root_t *r, uint8_t bloq, ur_nref ref)
{
uint64_t m_bit;
// XX return bool for cells, length in out parameter
//
assert( !ur_deep(ref) );
if ( ur_direct == ur_nref_tag(ref) ) {
m_bit = ur_met0_64(ref);
}
else {
uint64_t idx = ur_nref_idx(ref);
uint64_t len = r->atoms.lens[idx];
uint8_t *byt = r->atoms.bytes[idx];
m_bit = ur_met0_bytes_unsafe(len, byt);
}
switch ( bloq ) {
case 0: return m_bit;
case 1: return ur_bloq_up1(m_bit);
case 2: return ur_bloq_up2(m_bit);
{
uint64_t m_byt = ur_bloq_up3(m_bit);
case 3: return m_byt;
default: {
uint8_t off = (bloq - 3);
return (m_byt + ((1ULL << off) - 1)) >> off;
}
}
}
}
static ur_nref
_coin_unsafe(ur_atoms_t *atoms, ur_mug mug, uint64_t len, uint8_t *byt)
{
uint64_t fill = atoms->fill;
ur_tag tag = ur_iatom;
ur_nref tom = ( fill | ((uint64_t)tag << 62) );
// XX necessary?
//
assert( 62 >= ur_met0_64(fill) );
atoms->bytes[fill] = byt;
atoms->lens[fill] = len;
atoms->mugs[fill] = mug;
atoms->fill = 1 + fill;
return tom;
}
static ur_nref
_cons_unsafe(ur_cells_t *cells, ur_mug mug, ur_nref hed, ur_nref tal)
{
uint64_t fill = cells->fill;
ur_tag tag = ur_icell;
ur_nref cel = ( fill | ((uint64_t)tag << 62) );
// XX necessary?
//
assert( 62 >= ur_met0_64(fill) );
cells->mugs[fill] = mug;
cells->heads[fill] = hed;
cells->tails[fill] = tal;
cells->fill = 1 + fill;
return cel;
}
ur_nref
ur_coin_bytes_unsafe(ur_root_t *r, uint64_t len, uint8_t *byt)
{
ur_atoms_t *atoms = &(r->atoms);
ur_dict_t *dict = &(atoms->dict);
ur_mug mug = ur_mug_bytes(byt, len);
while ( 1 ) {
uint64_t idx = ( mug % dict->size );
ur_pail_t *bucket = &(dict->buckets[idx]);
uint8_t i, b_fill = bucket->fill;
ur_nref tom;
for ( i = 0; i < b_fill; i++ ) {
uint8_t *t_byt;
uint64_t t_len;
tom = bucket->refs[i];
ur_bytes(r, tom, &t_byt, &t_len);
if ( (t_len == len)
&& (0 == memcmp(t_byt, byt, len)) )
{
return tom;
}
}
if ( ur_pail_max == b_fill ) {
ur_dict_grow(r, dict, dict->prev, dict->size);
continue;
}
if ( atoms->fill == atoms->size ) {
ur_atoms_grow(atoms);
}
tom = _coin_unsafe(atoms, mug, len, byt);
bucket->refs[b_fill] = tom;
bucket->fill = 1 + b_fill;
return tom;
}
}
ur_nref
ur_coin_bytes(ur_root_t *r, uint64_t len, uint8_t *byt)
{
// strip trailing zeroes
//
while ( len && !byt[len - 1] ) {
len--;
}
// produce a direct atom if possible
//
if ( 62 >= ur_met0_bytes_unsafe(len, byt) ) {
uint64_t i, direct = 0;
for ( i = 0; i < len; i++ ) {
direct |= byt[i] << (8 * i);
}
return (ur_nref)direct;
}
else {
uint8_t *copy = _oom("coin_bytes", malloc(len));
memcpy(copy, byt, len);
return ur_coin_bytes_unsafe(r, len, copy);
}
}
ur_nref
ur_coin64(ur_root_t *r, uint64_t n)
{
if ( ur_direct == ur_nref_tag(n) ) {
return n;
}
else {
uint8_t *byt;
assert( 8 == ur_met3_64(n) );
byt = _oom("coin64", malloc(8));
byt[0] = ur_mask_8(n);
byt[1] = ur_mask_8(n >> 8);
byt[2] = ur_mask_8(n >> 16);
byt[3] = ur_mask_8(n >> 24);
byt[4] = ur_mask_8(n >> 32);
byt[5] = ur_mask_8(n >> 40);
byt[6] = ur_mask_8(n >> 48);
byt[7] = ur_mask_8(n >> 56);
return ur_coin_bytes_unsafe(r, 8, byt);
}
}
ur_nref
ur_cons(ur_root_t *r, ur_nref hed, ur_nref tal)
{
ur_cells_t *cells = &(r->cells);
ur_dict_t *dict = &(cells->dict);
ur_mug mug = ur_mug_both(ur_nref_mug(r, hed),
ur_nref_mug(r, tal));
while ( 1 ) {
uint64_t idx = ( mug % dict->size );
ur_pail_t *bucket = &(dict->buckets[idx]);
uint8_t i, b_fill = bucket->fill;
ur_nref cel;
for ( i = 0; i < b_fill; i++ ) {
cel = bucket->refs[i];
if ( (hed == ur_head(r, cel))
&& (tal == ur_tail(r, cel)) )
{
return cel;
}
}
if ( ur_pail_max == b_fill ) {
ur_dict_grow(r, dict, dict->prev, dict->size);
continue;
}
if ( cells->fill == cells->size ) {
ur_cells_grow(cells);
}
cel = _cons_unsafe(cells, mug, hed, tal);
bucket->refs[b_fill] = cel;
bucket->fill = 1 + b_fill;
return cel;
}
}
static void
_print_memory(FILE *f, const char *c, uint64_t bytes)
{
if ( !bytes ) {
fprintf(f, "%s: B/0\r\n", c);
}
else {
uint32_t g = (bytes / 1000000000);
uint32_t m = (bytes % 1000000000) / 1000000;
uint32_t k = (bytes % 1000000) / 1000;
uint32_t b = (bytes % 1000);
if ( g ) {
fprintf(f, "%s: GB/%d.%03d.%03d.%03d\r\n", c, g, m, k, b);
}
else if ( m ) {
fprintf(f, "%s: MB/%d.%03d.%03d\r\n", c, m, k, b);
}
else if ( k ) {
fprintf(f, "%s: KB/%d.%03d\r\n", c, k, b);
}
else if ( b ) {
fprintf(f, "%s: B/%d\r\n", c, b);
}
}
}
static uint64_t
_dict_info(FILE *f, ur_dict_t *dict)
{
uint64_t data = dict->size * sizeof(*dict->buckets);
_print_memory(f, " dict", data);
return data;
}
static uint64_t
_atoms_info(FILE *f, ur_atoms_t *atoms)
{
uint64_t total = 0;
uint64_t size = atoms->size;
uint64_t fill = atoms->fill;
uint64_t refs = size * ( sizeof(*atoms->bytes)
+ sizeof(*atoms->lens)
+ sizeof(*atoms->mugs) );
uint64_t i, data = 0;
fprintf(f, " atoms (%" PRIu64 "):\r\n", fill);
_print_memory(f, " refs", refs);
total += refs;
for ( i = 0; i < fill; i++ ) {
data += atoms->lens[i];
}
_print_memory(f, " data", data);
total += data;
total += _dict_info(f, &(atoms->dict));
_print_memory(f, " total", total);
return total;
}
static uint64_t
_cells_info(FILE *f, ur_cells_t *cells)
{
uint64_t total = 0;
uint64_t size = cells->size;
uint64_t fill = cells->fill;
uint64_t refs = size * ( sizeof(*cells->heads)
+ sizeof(*cells->heads)
+ sizeof(*cells->mugs) );
fprintf(f, " cells (%" PRIu64 "):\r\n", fill);
_print_memory(f, " refs", refs);
total += refs;
total += _dict_info(f, &(cells->dict));
_print_memory(f, " total", total);
return total;
}
void
ur_root_info(FILE *f, ur_root_t *r)
{
uint64_t total = 0;
fprintf(stderr, "hash-cons arena:\r\n");
{
uint64_t root = sizeof(*r);
_print_memory(f, " root", root);
total += root;
}
total += _atoms_info(f, &(r->atoms));
total += _cells_info(f, &(r->cells));
_print_memory(f, "total", total);
}
static void
_atoms_free(ur_atoms_t *atoms)
{
uint8_t **bytes = atoms->bytes;
uint64_t i, fill = atoms->fill;
for ( i = 0; i < fill; i++ ) {
free(bytes[i]);
}
ur_dict_free(&(atoms->dict));
free(bytes);
}
static void
_cells_free(ur_cells_t *cells)
{
ur_dict_free(&(cells->dict));
free(cells->heads);
}
void
ur_root_free(ur_root_t *r)
{
_atoms_free(&(r->atoms));
_cells_free(&(r->cells));
free(r);
}
ur_root_t*
ur_root_init(void)
{
ur_root_t *r = _oom("root_init", calloc(1, sizeof(*r)));
{
ur_dict_t *dict;
// allocate atom storage
//
r->atoms.prev = ur_fib11;
r->atoms.size = ur_fib12;
ur_atoms_grow(&(r->atoms));
// allocate atom hashtable
//
dict = &(r->atoms.dict);
ur_dict_grow(r, dict, ur_fib11, ur_fib12);
// allocate cell storage
//
r->cells.prev = ur_fib11;
r->cells.size = ur_fib12;
ur_cells_grow(&(r->cells));
// allocate cell hashtable
//
dict = &(r->cells.dict);
ur_dict_grow(r, dict, ur_fib11, ur_fib12);
}
return r;
}
void
ur_nvec_free(ur_nvec_t *v)
{
free(v->refs);
}
void
ur_nvec_init(ur_nvec_t *v, uint64_t size)
{
v->fill = 0;
v->refs = _oom("nvec_init", calloc(size, sizeof(ur_nref)));
}
void
ur_walk_fore(ur_root_t *r,
ur_nref ref,
void *v,
void (*atom)(ur_root_t*, ur_nref, void*),
ur_bool_t (*cell)(ur_root_t*, ur_nref, void*))
{
uint64_t prev = ur_fib11, size = ur_fib12, fill = 0;
ur_nref *top, *don;
don = _oom("walk_fore", malloc(size * sizeof(*don)));
top = don + ++fill;
*top = ref;
while ( top != don ) {
// visit atom, pop stack
//
if ( !ur_deep(ref) ) {
atom(r, ref, v);
top--; fill--;
}
// visit cell, pop stack if false
//
else if ( !cell(r, ref, v) ) {
top--; fill--;
}
// push the tail, continue into the head
//
else {
*top = ur_tail(r, ref);
// reallocate "stack" if full
//
if ( size == fill ) {
uint64_t next = prev + size;
don = _oom("walk_fore", realloc(don, next * sizeof(*don)));
top = don + fill;
prev = size;
size = next;
}
top++; fill++;
*top = ur_head(r, ref);
}
ref = *top;
}
free(don);
}

455
pkg/urbit/ur/serial.c Normal file
View File

@ -0,0 +1,455 @@
#include <stdlib.h>
#include <assert.h>
#include "ur/ur.h"
static void*
_oom(const char* cap, void* v)
{
if ( !v ) {
fprintf(stderr,
"ur: hashcons: %s: allocation failed, out of memory\r\n", cap);
abort();
}
return v;
}
static inline void
_bsw_atom(ur_root_t *r, ur_nref ref, ur_bsw_t *bsw, uint64_t len)
{
switch ( ur_nref_tag(ref) ) {
default: assert(0);
case ur_direct: return ur_bsw_atom64(bsw, len, ref);
case ur_iatom: {
uint8_t *byt = r->atoms.bytes[ur_nref_idx(ref)];
return ur_bsw_atom_bytes(bsw, len, byt);
}
}
}
typedef struct _jam_s {
ur_dict64_t *dict;
ur_bsw_t bsw;
} _jam_t;
static void
_jam_atom(ur_root_t *r, ur_nref ref, void *ptr)
{
_jam_t *j = ptr;
ur_dict64_t *dict = j->dict;
ur_bsw_t *bsw = &j->bsw;
uint64_t bak, len = ur_met(r, 0, ref);
if ( !ur_dict64_get(r, dict, ref, &bak) ) {
ur_dict64_put(r, dict, ref, bsw->bits);
_bsw_atom(r, ref, bsw, len);
}
else {
uint64_t len_bak = ur_met0_64(bak);
if ( len <= len_bak ) {
_bsw_atom(r, ref, bsw, len);
}
else {
ur_bsw_back64(bsw, len_bak, bak);
}
}
}
static ur_bool_t
_jam_cell(ur_root_t *r, ur_nref ref, void *ptr)
{
_jam_t *j = ptr;
ur_dict64_t *dict = j->dict;
ur_bsw_t *bsw = &j->bsw;
uint64_t bak;
if ( !ur_dict64_get(r, dict, ref, &bak) ) {
ur_dict64_put(r, dict, ref, bsw->bits);
ur_bsw_cell(bsw);
return 1;
}
else {
ur_bsw_back64(bsw, ur_met0_64(bak), bak);
return 0;
}
}
uint64_t
ur_jam_unsafe(ur_root_t *r,
ur_nref ref,
ur_dict64_t *dict,
uint64_t *len,
uint8_t **byt)
{
_jam_t j = {0};
j.dict = dict;
j.bsw.prev = ur_fib11;
j.bsw.size = ur_fib12;
j.bsw.bytes = _oom("jam", calloc(j.bsw.size, 1));
ur_walk_fore(r, ref, &j, _jam_atom, _jam_cell);
*len = j.bsw.fill + !!j.bsw.off;
*byt = j.bsw.bytes;
return j.bsw.bits;
}
uint64_t
ur_jam(ur_root_t *r, ur_nref ref, uint64_t *len, uint8_t **byt)
{
ur_dict64_t dict = {0};
ur_dict64_grow(r, &dict, ur_fib11, ur_fib12);
{
uint64_t bits = ur_jam_unsafe(r, ref, &dict, len, byt);
ur_dict_free((ur_dict_t*)&dict);
return bits;
}
}
/*
** stack frame for recording head vs tail iteration
**
** $? [CUE_HEAD bits=@]
** [hed=* bits=@]
*/
#define CUE_HEAD 0xffffffffffffffffULL
typedef struct _cue_frame_s {
uint64_t ref;
uint64_t bits;
} _cue_frame_t;
typedef struct _cue_stack_s {
uint32_t prev;
uint32_t size;
uint32_t fill;
_cue_frame_t* f;
} _cue_stack_t;
static inline ur_cue_res_e
_cue_next(ur_root_t *r,
_cue_stack_t *s,
ur_bsr_t *bsr,
ur_dict64_t *dict,
ur_nref *out)
{
while ( 1 ) {
uint64_t len, bits = bsr->bits;
ur_cue_tag_e tag;
ur_cue_res_e res;
if ( ur_cue_good != (res = ur_bsr_tag(bsr, &tag)) ) {
return res;
}
switch ( tag ) {
default: assert(0);
case ur_jam_cell: {
// reallocate the stack if full
//
if ( s->fill == s->size ) {
uint32_t next = s->prev + s->size;
s->f = _oom("cue_next stack", realloc(s->f, next * sizeof(*s->f)));
s->prev = s->size;
s->size = next;
}
// save a head-frame and read the head from the stream
//
{
_cue_frame_t* f = &(s->f[s->fill++]);
f->ref = CUE_HEAD;
f->bits = bits;
}
continue;
}
case ur_jam_back: {
if ( ur_cue_good != (res = ur_bsr_rub_len(bsr, &len)) ) {
return res;
}
else if ( 62 < len ) {
return ur_cue_meme;
}
else {
uint64_t val, bak = ur_bsr64_any(bsr, len);
if ( !ur_dict64_get(r, dict, bak, &val) ) {
return ur_cue_back;
}
*out = (ur_nref)val;
return ur_cue_good;
}
}
case ur_jam_atom: {
if ( ur_cue_good != (res = ur_bsr_rub_len(bsr, &len)) ) {
return res;
}
else if ( 62 >= len ) {
*out = (ur_nref)ur_bsr64_any(bsr, len);
}
else {
uint64_t len_byt = (len >> 3) + !!ur_mask_3(len);
uint8_t *byt = _oom("cue_next bytes", calloc(len_byt, 1));
ur_bsr_bytes_any(bsr, len, byt);
// strip trailing zeroes
//
while ( len_byt && !byt[len_byt - 1] ) {
len_byt--;
}
*out = ur_coin_bytes_unsafe(r, len_byt, byt);
}
ur_dict64_put(r, dict, bits, (uint64_t)*out);
return ur_cue_good;
}
}
}
}
ur_cue_res_e
ur_cue_unsafe(ur_root_t *r,
ur_dict64_t *dict,
uint64_t len,
const uint8_t *byt,
ur_nref *out)
{
ur_bsr_t bsr = {0};
_cue_stack_t s = {0};
ur_cue_res_e res;
ur_nref ref;
// init bitstream-reader
//
if ( ur_cue_good != (res = ur_bsr_init(&bsr, len, byt)) ) {
return res;
}
// bit-cursor (and backreferences) must fit in 62-bit direct atoms
//
else if ( 0x7ffffffffffffffULL < len ) {
return ur_cue_meme;
}
// setup stack
//
s.prev = ur_fib10;
s.size = ur_fib11;
s.f = _oom("cue stack", malloc(s.size * sizeof(*s.f)));
// advance into stream
//
res = _cue_next(r, &s, &bsr, dict, &ref);
// process result
//
while ( s.fill && (ur_cue_good == res) ) {
// peek at the top of the stack
//
_cue_frame_t *f = &(s.f[s.fill - 1]);
// f is a head-frame; stash result and read the tail from the stream
//
if ( CUE_HEAD == f->ref ) {
f->ref = ref;
res = _cue_next(r, &s, &bsr, dict, &ref);
}
// f is a tail-frame; pop the stack and continue
//
else {
ref = ur_cons(r, f->ref, ref);
ur_dict64_put(r, dict, f->bits, (uint64_t)ref);
s.fill--;
}
}
free(s.f);
if ( ur_cue_good == res ) {
*out = ref;
}
return res;
}
ur_cue_res_e
ur_cue(ur_root_t *r,
uint64_t len,
const uint8_t *byt,
ur_nref *out)
{
ur_dict64_t dict = {0};
ur_dict64_grow(r, &dict, ur_fib11, ur_fib12);
ur_cue_res_e res = ur_cue_unsafe(r, &dict, len, byt, out);
ur_dict_free((ur_dict_t*)&dict);
return res;
}
/*
** stack frame for recording head vs tail iteration
**
** [hed=? bits=@]
**
*/
typedef struct _cue_test_frame_s {
ur_bool_t tal;
uint64_t bits;
} _cue_test_frame_t;
typedef struct _cue_test_stack_s {
uint32_t prev;
uint32_t size;
uint32_t fill;
_cue_test_frame_t* f;
} _cue_test_stack_t;
static inline ur_cue_res_e
_cue_test_next(_cue_test_stack_t *s,
ur_bsr_t *bsr,
ur_dict_t *dict)
{
while ( 1 ) {
uint64_t len, bits = bsr->bits;
ur_cue_tag_e tag;
ur_cue_res_e res;
if ( ur_cue_good != (res = ur_bsr_tag(bsr, &tag)) ) {
return res;
}
switch ( tag ) {
default: assert(0);
case ur_jam_cell: {
// reallocate the stack if full
//
if ( s->fill == s->size ) {
uint32_t next = s->prev + s->size;
s->f = _oom("cue_test", realloc(s->f, next * sizeof(*s->f)));
s->prev = s->size;
s->size = next;
}
// save a head-frame and read the head from the stream
//
{
_cue_test_frame_t* f = &(s->f[s->fill++]);
f->tal = 0;
f->bits = bits;
}
continue;
}
case ur_jam_back: {
if ( ur_cue_good != (res = ur_bsr_rub_len(bsr, &len)) ) {
return res;
}
else if ( 62 < len ) {
return ur_cue_meme;
}
else {
uint64_t bak = ur_bsr64_any(bsr, len);
return ur_dict_get((ur_root_t*)0, dict, bak)
? ur_cue_good
: ur_cue_back;
}
}
case ur_jam_atom: {
if ( ur_cue_good != (res = ur_bsr_rub_len(bsr, &len)) ) {
return res;
}
ur_bsr_skip_any(bsr, len);
ur_dict_put((ur_root_t*)0, dict, bits);
return ur_cue_good;
}
}
}
}
ur_cue_res_e
ur_cue_test_unsafe(ur_dict_t *dict,
uint64_t len,
const uint8_t *byt)
{
ur_bsr_t bsr = {0};
_cue_test_stack_t s = {0};
ur_cue_res_e res;
// init bitstream-reader
//
if ( ur_cue_good != (res = ur_bsr_init(&bsr, len, byt)) ) {
return res;
}
// bit-cursor (and backreferences) must fit in 62-bit direct atoms
//
else if ( 0x7ffffffffffffffULL < len ) {
return ur_cue_meme;
}
// setup stack
//
s.prev = ur_fib10;
s.size = ur_fib11;
s.f = _oom("cue_test", malloc(s.size * sizeof(*s.f)));
// advance into stream
//
res = _cue_test_next(&s, &bsr, dict);
// process result
//
while ( s.fill && (ur_cue_good == res) ) {
// peek at the top of the stack
//
_cue_test_frame_t *f = &(s.f[s.fill - 1]);
// f is a head-frame; stash result and read the tail from the stream
//
if ( !f->tal ) {
f->tal = 1;
res = _cue_test_next(&s, &bsr, dict);
}
// f is a tail-frame; pop the stack and continue
//
else {
ur_dict_put((ur_root_t*)0, dict, f->bits);
s.fill--;
}
}
free(s.f);
return res;
}
ur_bool_t
ur_cue_test(uint64_t len, const uint8_t *byt)
{
ur_dict_t dict = {0};
ur_dict_grow((ur_root_t*)0, &dict, ur_fib11, ur_fib12);
ur_bool_t ret = ur_cue_good == ur_cue_test_unsafe(&dict, len, byt);
ur_dict_free(&dict);
return ret;
}

48
pkg/urbit/vere/disk.c
View File

@ -39,6 +39,8 @@ struct _cd_save {
};
#undef VERBOSE_DISK
#undef DISK_TRACE_JAM
#undef DISK_TRACE_CUE
static void
_disk_commit(u3_disk* log_u);
@ -168,11 +170,23 @@ _disk_commit_start(struct _cd_save* req_u)
static c3_w
_disk_serialize_v0(u3_fact* tac_u, c3_y** dat_y)
{
u3_atom mat = u3ke_jam(u3nc(tac_u->bug_l, u3k(tac_u->job)));
c3_w len_w = u3r_met(3, mat);
u3_noun val = u3nc(tac_u->bug_l, u3k(tac_u->job));
u3_atom mat;
c3_w len_w;
#ifdef DISK_TRACE_JAM
u3t_event_trace("king disk jam", 'B');
#endif
mat = u3ke_jam(val);
len_w = u3r_met(3, mat);
*dat_y = c3_malloc(len_w);
u3r_bytes(0, len_w, *dat_y, mat);
#ifdef DISK_TRACE_JAM
u3t_event_trace("king disk jam", 'E');
#endif
u3z(mat);
return len_w;
@ -368,19 +382,27 @@ _disk_read_one_cb(void* ptr_v, c3_d eve_d, size_t val_i, void* val_p)
u3_fact* tac_u;
{
// XX u3m_soft?
//
u3_noun dat = u3ke_cue(u3i_bytes(val_i, val_p));
u3_noun mug, job;
u3_noun val, mug, job;
c3_l bug_l;
#ifdef DISK_TRACE_CUE
u3t_event_trace("king disk cue", 'B');
#endif
if ( (c3n == u3r_cell(dat, &mug, &job))
// XX u3m_soft?
//
val = u3ke_cue(u3i_bytes(val_i, val_p));
#ifdef DISK_TRACE_CUE
u3t_event_trace("king disk cue", 'E');
#endif
if ( (c3n == u3r_cell(val, &mug, &job))
|| (c3n == u3r_safe_word(mug, &bug_l)) ) // XX
{
// failure here triggers cleanup in _disk_read_start_cb()
//
u3z(dat);
u3z(val);
return c3n;
}
@ -389,7 +411,7 @@ _disk_read_one_cb(void* ptr_v, c3_d eve_d, size_t val_i, void* val_p)
tac_u = u3_fact_init(eve_d, 0, u3k(job));
tac_u->bug_l = bug_l;
u3z(dat);
u3z(val);
}
if ( !red_u->ent_u ) {
@ -643,6 +665,10 @@ u3_disk_exit(u3_disk* log_u)
u3_dire_free(log_u->com_u);
c3_free(log_u);
#if defined(DISK_TRACE_JAM) || defined(DISK_TRACE_CUE)
u3t_trace_close();
#endif
}
/* u3_disk_info(): print status info.
@ -786,5 +812,9 @@ u3_disk_init(c3_c* pax_c, u3_disk_cb cb_u)
log_u->liv_o = c3y;
#if defined(DISK_TRACE_JAM) || defined(DISK_TRACE_CUE)
u3t_trace_open(pax_c);
#endif
return log_u;
}

11
pkg/urbit/vere/io/term.c
View File

@ -1679,13 +1679,14 @@ _term_io_kick(u3_auto* car_u, u3_noun wir, u3_noun cad)
// uv_timer_start(&u3K.tim_u, (uv_timer_cb)u3_king_grab, 0, 0);
} break;
// ignore pack (processed in worker)
//
case c3__meld: {
ret_o = c3y;
u3_pier_meld(car_u->pir_u);
} break;
case c3__pack: {
ret_o = c3y;
// XX would be
//
// u3_assure(u3_pier_pack(car_u->pir_u));
u3_pier_pack(car_u->pir_u);
} break;
}
}

64
pkg/urbit/vere/king.c
View File

@ -5,8 +5,10 @@
#include <curl/curl.h>
#include <unistd.h>
#include <uv.h>
#include "all.h"
#include "vere/vere.h"
#include "ur/ur.h"
#include "ivory.h"
@ -692,6 +694,52 @@ _king_loop_exit()
unlink(u3K.certs_c);
}
static void
_king_boot_ivory(void)
{
c3_d len_d;
c3_y* byt_y;
if ( u3_Host.ops_u.lit_c ) {
if ( c3n == u3u_mmap_read("lite", u3_Host.ops_u.lit_c, &len_d, &byt_y) ) {
u3l_log("lite: unable to load ivory pill at %s\n",
u3_Host.ops_u.lit_c);
exit(1);
}
}
else {
len_d = u3_Ivory_pill_len;
byt_y = u3_Ivory_pill;
}
{
ur_dict32_t dic_u = {0};
u3_noun pil;
ur_dict32_grow((ur_root_t*)0, &dic_u, ur_fib27, ur_fib28);
if ( c3n == u3s_cue_xeno_unsafe(&dic_u, len_d, byt_y, &pil) ) {
u3l_log("lite: unable to cue ivory pill\r\n");
exit(1);
}
ur_dict_free((ur_dict_t*)&dic_u);
if ( c3n == u3v_boot_lite(pil)) {
u3l_log("lite: boot failed\r\n");
exit(1);
}
}
if ( u3_Host.ops_u.lit_c ) {
if ( c3n == u3u_munmap(len_d, byt_y) ) {
u3l_log("lite: unable to unmap ivory pill at %s\n",
u3_Host.ops_u.lit_c);
exit(1);
}
}
}
/* u3_king_commence(): start the daemon
*/
void
@ -727,21 +775,7 @@ u3_king_commence()
// boot the ivory pill
//
{
u3_noun lit;
if ( 0 != u3_Host.ops_u.lit_c ) {
lit = u3m_file(u3_Host.ops_u.lit_c);
}
else {
lit = u3i_bytes(u3_Ivory_pill_len, u3_Ivory_pill);
}
if ( c3n == u3v_boot_lite(lit)) {
u3l_log("lite: boot failed\r\n");
exit(1);
}
}
_king_boot_ivory();
// disable core dumps (due to lmdb size)
//

184
pkg/urbit/vere/lord.c
View File

@ -18,6 +18,10 @@
#include "all.h"
#include "vere/vere.h"
#include "ur/hashcons.h"
#undef LORD_TRACE_JAM
#undef LORD_TRACE_CUE
/*
|%
@ -28,6 +32,7 @@
$% [%cram eve=@]
[%exit cod=@]
[%save eve=@]
[%meld ~]
[%pack ~]
== ==
[%peek mil=@ now=@da lyc=gang pat=path]
@ -69,6 +74,7 @@ _lord_stop_cb(void* ptr_v,
void (*exit_f)(void*) = god_u->cb_u.exit_f;
void* exit_v = god_u->cb_u.ptr_v;
ur_dict_free((ur_dict_t*)god_u->dic_u);
c3_free(god_u);
if ( exit_f ) {
@ -111,6 +117,7 @@ _lord_writ_free(u3_writ* wit_u)
case u3_writ_save:
case u3_writ_cram:
case u3_writ_meld:
case u3_writ_pack:
case u3_writ_exit: {
} break;
@ -152,6 +159,10 @@ _lord_stop(u3_lord* god_u)
u3_newt_mojo_stop(&god_u->inn_u, _lord_bail_noop);
uv_close((uv_handle_t*)&god_u->cub_u, 0);
#if defined(LORD_TRACE_JAM) || defined(LORD_TRACE_CUE)
u3t_trace_close();
#endif
}
/* _lord_bail(): serf/lord error.
@ -201,6 +212,7 @@ _lord_writ_str(u3_writ_type typ_e)
case u3_writ_play: return "play";
case u3_writ_save: return "save";
case u3_writ_cram: return "cram";
case u3_writ_meld: return "meld";
case u3_writ_pack: return "pack";
case u3_writ_exit: return "exit";
}
@ -267,6 +279,12 @@ _lord_plea_live(u3_lord* god_u, u3_noun dat)
god_u->cb_u.cram_f(god_u->cb_u.ptr_v);
} break;
case u3_writ_meld: {
// XX wire into cb
//
u3l_log("pier: meld complete\n");
} break;
case u3_writ_pack: {
// XX wire into cb
//
@ -654,14 +672,29 @@ _lord_plea_work(u3_lord* god_u, u3_noun dat)
/* _lord_on_plea(): handle plea from serf.
*/
static void
_lord_on_plea(void* ptr_v, u3_noun mat)
_lord_on_plea(void* ptr_v, c3_d len_d, c3_y* byt_y)
{
u3_lord* god_u = ptr_v;
u3_noun jar = u3ke_cue(mat);
u3_noun tag, dat;
u3_lord* god_u = ptr_v;
ur_dict32_t* dic_u = god_u->dic_u;
u3_noun tag, dat, jar = u3_blip;
c3_o ret_o;
if ( c3n == u3r_cell(jar, &tag, &dat) ) {
u3m_p("jar", jar);
#ifdef LORD_TRACE_CUE
u3t_event_trace("king ipc cue", 'B');
#endif
ret_o = u3s_cue_xeno_unsafe(dic_u, len_d, byt_y, &jar);
// XX check if the dictionary grew too much and shrink?
//
ur_dict32_wipe(dic_u);
#ifdef LORD_TRACE_CUE
u3t_event_trace("king ipc cue", 'E');
#endif
if ( (c3n == ret_o)
|| (c3n == u3r_cell(jar, &tag, &dat)) )
{
return _lord_plea_foul(god_u, 0, jar);
}
@ -708,64 +741,66 @@ _lord_writ_new(u3_lord* god_u)
return wit_u;
}
/* _lord_writ_jam(): serialize writ.
/* _lord_writ_make(): cons writ.
*/
static void
_lord_writ_jam(u3_lord* god_u, u3_writ* wit_u)
static u3_noun
_lord_writ_make(u3_lord* god_u, u3_writ* wit_u)
{
if ( 0 == wit_u->mat ) {
u3_noun msg;
u3_noun msg;
switch ( wit_u->typ_e ) {
default: c3_assert(0);
switch ( wit_u->typ_e ) {
default: c3_assert(0);
case u3_writ_work: {
u3_noun mil = u3i_words(1, &wit_u->wok_u.egg_u->mil_w);
msg = u3nt(c3__work, mil, u3k(wit_u->wok_u.job));
} break;
case u3_writ_work: {
u3_noun mil = u3i_words(1, &wit_u->wok_u.egg_u->mil_w);
msg = u3nt(c3__work, mil, u3k(wit_u->wok_u.job));
} break;
case u3_writ_peek: {
msg = u3nc(c3__peek, u3nq(0, // XX support timeouts
u3k(wit_u->pek_u->now),
u3k(wit_u->pek_u->gan),
u3k(wit_u->pek_u->ful)));
} break;
case u3_writ_peek: {
msg = u3nc(c3__peek, u3nq(0, // XX support timeouts
u3k(wit_u->pek_u->now),
u3k(wit_u->pek_u->gan),
u3k(wit_u->pek_u->ful)));
} break;
case u3_writ_play: {
u3_fact* tac_u = wit_u->fon_u.ext_u;
c3_d eve_d = tac_u->eve_d;
u3_noun lit = u3_nul;
case u3_writ_play: {
u3_fact* tac_u = wit_u->fon_u.ext_u;
c3_d eve_d = tac_u->eve_d;
u3_noun lit = u3_nul;
while ( tac_u ) {
lit = u3nc(u3k(tac_u->job), lit);
tac_u = tac_u->nex_u;
}
while ( tac_u ) {
lit = u3nc(u3k(tac_u->job), lit);
tac_u = tac_u->nex_u;
}
msg = u3nt(c3__play, u3i_chubs(1, &eve_d), u3kb_flop(lit));
msg = u3nt(c3__play, u3i_chubs(1, &eve_d), u3kb_flop(lit));
} break;
} break;
case u3_writ_save: {
msg = u3nt(c3__live, c3__save, u3i_chubs(1, &god_u->eve_d));
} break;
case u3_writ_save: {
msg = u3nt(c3__live, c3__save, u3i_chubs(1, &god_u->eve_d));
} break;
case u3_writ_cram: {
msg = u3nt(c3__live, c3__cram, u3i_chubs(1, &god_u->eve_d));
} break;
case u3_writ_cram: {
msg = u3nt(c3__live, c3__cram, u3i_chubs(1, &god_u->eve_d));
} break;
case u3_writ_pack: {
msg = u3nt(c3__live, c3__pack, u3_nul);
} break;
case u3_writ_meld: {
msg = u3nt(c3__live, c3__meld, u3_nul);
} break;
case u3_writ_exit: {
// requested exit code is always 0
//
msg = u3nt(c3__live, c3__exit, 0);
} break;
}
case u3_writ_pack: {
msg = u3nt(c3__live, c3__pack, u3_nul);
} break;
wit_u->mat = u3ke_jam(msg);
case u3_writ_exit: {
// requested exit code is always 0
//
msg = u3nt(c3__live, c3__exit, 0);
} break;
}
return msg;
}
/* _lord_writ_send(): send writ to serf.
@ -780,9 +815,24 @@ _lord_writ_send(u3_lord* god_u, u3_writ* wit_u)
god_u->inn_u.bal_f = _lord_bail_noop;
}
_lord_writ_jam(god_u, wit_u);
u3_newt_write(&god_u->inn_u, wit_u->mat);
wit_u->mat = 0;
{
u3_noun jar = _lord_writ_make(god_u, wit_u);
c3_d len_d;
c3_y* byt_y;
#ifdef LORD_TRACE_JAM
u3t_event_trace("king ipc jam", 'B');
#endif
u3s_jam_xeno(jar, &len_d, &byt_y);
#ifdef LORD_TRACE_JAM
u3t_event_trace("king ipc jam", 'E');
#endif
u3_newt_send(&god_u->inn_u, len_d, byt_y);
u3z(jar);
}
}
/* _lord_writ_plan(): enqueue a writ and send.
@ -929,6 +979,26 @@ u3_lord_cram(u3_lord* god_u)
}
}
/* u3_lord_meld(): globally deduplicate persistent state.
*/
void
u3_lord_meld(u3_lord* god_u)
{
u3_writ* wit_u = _lord_writ_new(god_u);
wit_u->typ_e = u3_writ_meld;
_lord_writ_plan(god_u, wit_u);
}
/* u3_lord_pack(): defragment persistent state.
*/
void
u3_lord_pack(u3_lord* god_u)
{
u3_writ* wit_u = _lord_writ_new(god_u);
wit_u->typ_e = u3_writ_pack;
_lord_writ_plan(god_u, wit_u);
}
/* u3_lord_exit(): shutdown gracefully.
*/
void
@ -1097,6 +1167,16 @@ u3_lord_init(c3_c* pax_c, c3_w wag_w, c3_d key_d[4], u3_lord_cb cb_u)
}
}
#if defined(LORD_TRACE_JAM) || defined(LORD_TRACE_CUE)
u3t_trace_open(god_u->pax_c);
#endif
{
ur_dict32_t* dic_u = c3_calloc(sizeof(*dic_u));
ur_dict32_grow((ur_root_t*)0, dic_u, ur_fib10, ur_fib11);
god_u->dic_u = dic_u;
}
// start reading from proc
//
{

43
pkg/urbit/vere/newt.c
View File

@ -70,8 +70,7 @@ _newt_meat_plan(u3_moat* mot_u, u3_meat* met_u)
static void
_newt_meat_poke(u3_moat* mot_u, u3_meat* met_u)
{
u3_noun mat = u3i_bytes((c3_w)met_u->len_d, met_u->hun_y);
mot_u->pok_f(mot_u->ptr_v, mat);
mot_u->pok_f(mot_u->ptr_v, met_u->len_d, met_u->hun_y);
c3_free(met_u);
}
@ -388,7 +387,8 @@ u3_newt_moat_info(u3_moat* mot_u)
typedef struct _n_req {
uv_write_t wri_u;
u3_mojo* moj_u;
c3_y buf_y[0];
c3_y* buf_y;
c3_y len_y[8];
} n_req;
/* _newt_write_cb(): generic write callback.
@ -399,6 +399,7 @@ _newt_write_cb(uv_write_t* wri_u, c3_i sas_i)
n_req* req_u = (n_req*)wri_u;
u3_mojo* moj_u = req_u->moj_u;
c3_free(req_u->buf_y);
c3_free(req_u);
if ( 0 != sas_i ) {
@ -435,35 +436,37 @@ u3_newt_mojo_stop(u3_mojo* moj_u, u3_moor_bail bal_f)
uv_close((uv_handle_t*)&moj_u->pyp_u, _mojo_stop_cb);
}
/* u3_newt_write(): write atom to stream; free atom.
/* u3_newt_send(): write buffer to stream.
*/
void
u3_newt_write(u3_mojo* moj_u, u3_atom mat)
u3_newt_send(u3_mojo* moj_u, c3_d len_d, c3_y* byt_y)
{
c3_w len_w = u3r_met(3, mat);
n_req* req_u = c3_malloc(8 + len_w + sizeof(*req_u));
n_req* req_u = c3_malloc(sizeof(*req_u));
req_u->moj_u = moj_u;
req_u->buf_y = byt_y;
// write header; c3_d is futureproofing
// write header
//
req_u->buf_y[0] = ((len_w >> 0) & 0xff);
req_u->buf_y[1] = ((len_w >> 8) & 0xff);
req_u->buf_y[2] = ((len_w >> 16) & 0xff);
req_u->buf_y[3] = ((len_w >> 24) & 0xff);
req_u->buf_y[4] = req_u->buf_y[5] = req_u->buf_y[6] = req_u->buf_y[7] = 0;
// write payload
//
u3r_bytes(0, len_w, req_u->buf_y + 8, mat);
u3z(mat);
req_u->len_y[0] = ( len_d & 0xff);
req_u->len_y[1] = ((len_d >> 8) & 0xff);
req_u->len_y[2] = ((len_d >> 16) & 0xff);
req_u->len_y[3] = ((len_d >> 24) & 0xff);
req_u->len_y[4] = ((len_d >> 32) & 0xff);
req_u->len_y[5] = ((len_d >> 40) & 0xff);
req_u->len_y[6] = ((len_d >> 48) & 0xff);
req_u->len_y[7] = ((len_d >> 56) & 0xff);
{
uv_buf_t buf_u = uv_buf_init((c3_c*)req_u->buf_y, 8 + len_w);
uv_buf_t buf_u[2] = {
uv_buf_init((c3_c*)req_u->len_y, 8),
uv_buf_init((c3_c*)req_u->buf_y, len_d)
};
c3_i sas_i;
if ( 0 != (sas_i = uv_write(&req_u->wri_u,
(uv_stream_t*)&moj_u->pyp_u,
&buf_u, 1,
buf_u, 2,
_newt_write_cb)) )
{
c3_free(req_u);

32
pkg/urbit/vere/pier.c
View File

@ -1571,6 +1571,8 @@ u3_pier_boot(c3_w wag_w, // config flags
return pir_u;
}
/* _pier_save_cb(): save snapshot upon serf/disk synchronization.
*/
static void
_pier_save_cb(void* ptr_v, c3_d eve_d)
{
@ -1604,6 +1606,8 @@ u3_pier_save(u3_pier* pir_u)
return c3n;
}
/* _pier_cram_cb(): save snapshot upon serf/disk synchronization.
*/
static void
_pier_cram_cb(void* ptr_v, c3_d eve_d)
{
@ -1638,6 +1642,32 @@ u3_pier_cram(u3_pier* pir_u)
return c3n;
}
/* u3_pier_meld(): globally deduplicate persistent state.
*/
void
u3_pier_meld(u3_pier* pir_u)
{
#ifdef VERBOSE_PIER
fprintf(stderr, "pier: (%" PRIu64 "): meld: plan\r\n", pir_u->god_u->eve_d);
#endif
u3_lord_meld(pir_u->god_u);
}
/* u3_pier_pack(): defragment persistent state.
*/
void
u3_pier_pack(u3_pier* pir_u)
{
#ifdef VERBOSE_PIER
fprintf(stderr, "pier: (%" PRIu64 "): meld: plan\r\n", pir_u->god_u->eve_d);
#endif
u3_lord_pack(pir_u->god_u);
}
/* _pier_work_close_cb(): dispose u3_work after closing handles.
*/
static void
_pier_work_close_cb(uv_handle_t* idl_u)
{
@ -1645,6 +1675,8 @@ _pier_work_close_cb(uv_handle_t* idl_u)
c3_free(wok_u);
}
/* _pier_work_close(): close drivers/handles in the u3_psat_work state.
*/
static void
_pier_work_close(u3_work* wok_u)
{

151
pkg/urbit/worker/main.c
View File

@ -22,9 +22,15 @@
#include <vere/vere.h>
#include <vere/serf.h>
static u3_serf u3V; // one serf per process
static u3_moat inn_u; // input stream
static u3_mojo out_u; // output stream
#include "ur/hashcons.h"
static u3_serf u3V; // one serf per process
static u3_moat inn_u; // input stream
static u3_mojo out_u; // output stream
static ur_dict32_t dic_u; // cue dictionary
#undef SERF_TRACE_JAM
#undef SERF_TRACE_CUE
/* _cw_serf_fail(): failure stub.
*/
@ -46,7 +52,21 @@ _cw_serf_fail(void* ptr_v, ssize_t err_i, const c3_c* err_c)
static void
_cw_serf_send(u3_noun pel)
{
u3_newt_write(&out_u, u3ke_jam(pel));
c3_d len_d;
c3_y* byt_y;
#ifdef SERF_TRACE_JAM
u3t_event_trace("serf ipc jam", 'B');
#endif
u3s_jam_xeno(pel, &len_d, &byt_y);
#ifdef SERF_TRACE_JAM
u3t_event_trace("serf ipc jam", 'E');
#endif
u3_newt_send(&out_u, len_d, byt_y);
u3z(pel);
}
/* _cw_serf_send_slog(): send hint output (hod is [priority tank]).
@ -65,14 +85,49 @@ _cw_serf_send_stdr(c3_c* str_c)
_cw_serf_send_slog(u3nc(0, u3i_string(str_c)));
}
/* _cw_serf_writ():
/* _cw_serf_step_trace(): initialize or rotate trace file.
*/
static void
_cw_serf_writ(void* vod_p, u3_noun mat)
_cw_serf_step_trace(void)
{
u3_noun ret;
if ( u3C.wag_w & u3o_trace ) {
if ( u3_Host.tra_u.con_w == 0 && u3_Host.tra_u.fun_w == 0 ) {
u3t_trace_open(u3V.dir_c);
}
else if ( u3_Host.tra_u.con_w >= 100000 ) {
u3t_trace_close();
u3t_trace_open(u3V.dir_c);
}
}
}
if ( c3n == u3_serf_writ(&u3V, u3ke_cue(mat), &ret) ) {
/* _cw_serf_writ(): process a command from the king.
*/
static void
_cw_serf_writ(void* vod_p, c3_d len_d, c3_y* byt_y)
{
u3_noun ret, jar;
c3_o ret_o;
_cw_serf_step_trace();
#ifdef SERF_TRACE_CUE
u3t_event_trace("serf ipc cue", 'B');
#endif
ret_o = u3s_cue_xeno_unsafe(&dic_u, len_d, byt_y, &jar);
// XX check if the dictionary grew too much and shrink?
//
ur_dict32_wipe(&dic_u);
#ifdef SERF_TRACE_CUE
u3t_event_trace("serf ipc cue", 'E');
#endif
if ( (c3n == ret_o)
|| (c3n == u3_serf_writ(&u3V, jar, &ret)) )
{
_cw_serf_fail(0, -1, "bad jar");
}
else {
@ -105,6 +160,15 @@ _cw_serf_stdio(c3_i* inn_i, c3_i* out_i)
close(nul_i);
}
/* _cw_serf_stdio(): cleanup on serf exit.
*/
static void
_cw_serf_exit(void)
{
ur_dict_free((ur_dict_t*)&dic_u);
u3t_trace_close();
}
/* _cw_serf_commence(); initialize and run serf
*/
static void
@ -175,6 +239,8 @@ _cw_serf_commence(c3_i argc, c3_c* argv[])
uv_stream_set_blocking((uv_stream_t*)&out_u.pyp_u, 1);
}
ur_dict32_grow((ur_root_t*)0, &dic_u, ur_fib10, ur_fib11);
// set up writing
//
out_u.ptr_v = &u3V;
@ -193,7 +259,15 @@ _cw_serf_commence(c3_i argc, c3_c* argv[])
u3V.sen_d = u3V.dun_d = u3m_boot(dir_c);
if ( eve_d ) {
u3_serf_uncram(&u3V, eve_d);
// XX need not be fatal, need a u3m_reboot equivalent
// XX can spuriously fail do to corrupt memory-image checkpoint,
// need a u3m_half_boot equivalent
// workaround is to delete/move the checkpoint in case of corruption
//
if ( c3n == u3u_uncram(u3V.dir_c, eve_d) ) {
fprintf(stderr, "serf (%" PRIu64 "): rock load failed\r\n", eve_d);
exit(1);
}
}
}
@ -206,6 +280,12 @@ _cw_serf_commence(c3_i argc, c3_c* argv[])
u3C.slog_f = _cw_serf_send_slog;
}
u3V.xit_f = _cw_serf_exit;
#if defined(SERF_TRACE_JAM) || defined(SERF_TRACE_CUE)
u3t_trace_open(u3V.dir_c);
#endif
// start serf
//
{
@ -255,15 +335,24 @@ _cw_cram(c3_i argc, c3_c* argv[])
c3_c* dir_c = argv[2];
c3_d eve_d = u3m_boot(dir_c);
c3_o ret_o;
fprintf(stderr, "urbit-worker: cram: preparing\r\n");
if ( c3n == u3m_rock_stay(dir_c, eve_d) ) {
if ( c3n == (ret_o = u3u_cram(dir_c, eve_d)) ) {
fprintf(stderr, "urbit-worker: cram: unable to jam state\r\n");
exit(1);
}
else {
fprintf(stderr, "urbit-worker: cram: rock saved at event %" PRIu64 "\r\n", eve_d);
}
fprintf(stderr, "urbit-worker: cram: rock saved at event %" PRIu64 "\r\n", eve_d);
// save even on failure, as we just did all the work of deduplication
//
u3e_save();
if ( c3n == ret_o ) {
exit(1);
}
}
/* _cw_queu(); cue rock, save, and exit.
@ -287,13 +376,42 @@ _cw_queu(c3_i argc, c3_c* argv[])
memset(&u3V, 0, sizeof(u3V));
u3V.dir_c = strdup(dir_c);
u3V.sen_d = u3V.dun_d = u3m_boot(dir_c);
u3_serf_uncram(&u3V, eve_d);
// XX can spuriously fail do to corrupt memory-image checkpoint,
// need a u3m_half_boot equivalent
// workaround is to delete/move the checkpoint in case of corruption
//
if ( c3n == u3u_uncram(dir_c, eve_d) ) {
fprintf(stderr, "urbit-worker: queu: failed\r\n");
exit(1);
}
u3e_save();
fprintf(stderr, "urbit-worker: queu: rock loaded at event %" PRIu64 "\r\n", eve_d);
}
}
/* _cw_uniq(); deduplicate persistent nouns
*/
static void
_cw_meld(c3_i argc, c3_c* argv[])
{
c3_assert( 3 <= argc );
c3_c* dir_c = argv[2];
u3m_boot(dir_c);
u3_serf_grab();
u3u_meld();
u3_serf_grab();
u3e_save();
}
/* _cw_pack(); compact memory, save, and exit.
*/
static void
@ -322,13 +440,15 @@ _cw_usage(c3_i argc, c3_c* argv[])
" %s grab <pier>\n\n"
" compact persistent state:\n"
" %s pack <pier>\n\n"
" deduplicate persistent state:\n"
" %s meld <pier>\n\n"
" jam persistent state:\n"
" %s cram <pier>\n\n"
" cue persistent state:\n"
" %s queu <pier> <at-event>\n\n"
" run as a 'serf':\n"
" %s serf <pier> <key> <flags> <cache-size> <at-event>\n",
argv[0], argv[0], argv[0], argv[0], argv[0], argv[0]);
argv[0], argv[0], argv[0], argv[0], argv[0], argv[0], argv[0]);
}
/* main(): main() when run as urbit-worker
@ -368,6 +488,9 @@ main(c3_i argc, c3_c* argv[])
else if ( 0 == strcmp("queu", argv[1]) ) {
_cw_queu(argc, argv);
}
else if ( 0 == strcmp("meld", argv[1]) ) {
_cw_meld(argc, argv);
}
else if ( 0 == strcmp("pack", argv[1]) ) {
_cw_pack(argc, argv);
}

140
pkg/urbit/worker/serf.c
View File

@ -34,6 +34,7 @@
$% [%cram eve=@]
[%exit cod=@]
[%save eve=@]
[%meld ~]
[%pack ~]
== ==
[%peek mil=@ now=@da lyc=gang pat=path]
@ -277,27 +278,6 @@ u3_serf_grab(void)
fflush(stderr);
}
/* _serf_cram(): deduplicate and compact memory. ORPHANED
*/
static void
_serf_cram(u3_serf* sef_u)
{
u3_serf_grab();
u3l_log("serf (%" PRIu64 "): compacting loom\r\n", sef_u->dun_d);
if ( c3n == u3m_rock_stay(sef_u->dir_c, sef_u->dun_d) ) {
u3l_log("serf: unable to jam state\r\n");
return;
}
u3_serf_uncram(sef_u, sef_u->dun_d);
u3l_log("serf (%" PRIu64 "): compacted loom\r\n", sef_u->dun_d);
u3_serf_grab();
}
/* u3_serf_post(): update serf state post-writ.
*/
void
@ -363,12 +343,6 @@ _serf_sure_feck(u3_serf* sef_u, c3_w pre_w, u3_noun vir)
rec_o = c3y;
}
// pack memory on |pack
//
if ( c3__pack == u3h(fec) ) {
pac_o = c3y;
}
riv = u3t(riv);
i_w++;
}
@ -387,6 +361,9 @@ _serf_sure_feck(u3_serf* sef_u, c3_w pre_w, u3_noun vir)
// low-priority: 2^27 contiguous words remaining (~536 MB)
// XX maybe use 2^23 (~16 MB) and 2^26 (~268 MB?
//
// XX these thresholds should trigger notifications sent to the king
// instead of directly triggering these remedial actions.
//
{
u3_noun pri = u3_none;
c3_w pos_w = u3a_open(u3R);
@ -608,7 +585,7 @@ _serf_work(u3_serf* sef_u, c3_w mil_w, u3_noun job)
u3_noun
u3_serf_work(u3_serf* sef_u, c3_w mil_w, u3_noun job)
{
c3_t tac_t = ( 0 != u3_Host.tra_u.fil_u );
c3_t tac_t = ( u3C.wag_w & u3o_trace );
c3_c lab_c[2056];
u3_noun pro;
@ -859,7 +836,7 @@ u3_serf_peek(u3_serf* sef_u, c3_w mil_w, u3_noun sam)
/* _serf_writ_live_exit(): exit on command.
*/
static void
_serf_writ_live_exit(c3_w cod_w)
_serf_writ_live_exit(u3_serf* sef_u, c3_w cod_w)
{
if ( u3C.wag_w & u3o_debug_cpu ) {
FILE* fil_u;
@ -896,6 +873,8 @@ _serf_writ_live_exit(c3_w cod_w)
//
c3_free(u3D.ray_u);
sef_u->xit_f();
exit(cod_w);
}
@ -945,7 +924,7 @@ u3_serf_live(u3_serf* sef_u, u3_noun com, u3_noun* ret)
u3z(com);
// NB, doesn't return
//
_serf_writ_live_exit(cod_y);
_serf_writ_live_exit(sef_u, cod_y);
*ret = u3nc(c3__live, u3_nul);
return c3y;
}
@ -971,11 +950,18 @@ u3_serf_live(u3_serf* sef_u, u3_noun com, u3_noun* ret)
u3l_log("serf (%" PRIu64 "): saving rock\r\n", sef_u->dun_d);
if ( c3n == u3m_rock_stay(sef_u->dir_c, eve_d) ) {
if ( c3n == u3u_cram(sef_u->dir_c, eve_d) ) {
fprintf(stderr, "serf (%" PRIu64 "): unable to jam state\r\n", eve_d);
return c3n;
}
if ( u3r_mug(u3A->roc) != sef_u->mug_l ) {
fprintf(stderr, "serf (%" PRIu64 "): mug mismatch 0x%08x 0x%08x\r\n",
eve_d, sef_u->mug_l, u3r_mug(u3A->roc));
return c3n;
}
u3e_save();
u3_serf_grab();
*ret = u3nc(c3__live, u3_nul);
@ -995,6 +981,19 @@ u3_serf_live(u3_serf* sef_u, u3_noun com, u3_noun* ret)
}
}
case c3__meld: {
if ( u3_nul != dat ) {
u3z(com);
return c3n;
}
else {
u3z(com);
u3u_meld();
*ret = u3nc(c3__live, u3_nul);
return c3y;
}
}
case c3__save: {
c3_d eve_d;
@ -1011,22 +1010,6 @@ u3_serf_live(u3_serf* sef_u, u3_noun com, u3_noun* ret)
}
}
/* _serf_step_trace(): initialize or rotate trace file.
*/
static void
_serf_step_trace(u3_serf* sef_u)
{
if ( u3C.wag_w & u3o_trace ) {
if ( u3_Host.tra_u.con_w == 0 && u3_Host.tra_u.fun_w == 0 ) {
u3t_trace_open(sef_u->dir_c);
}
else if ( u3_Host.tra_u.con_w >= 100000 ) {
u3t_trace_close();
u3t_trace_open(sef_u->dir_c);
}
}
}
/* u3_serf_writ(): apply writ [wit], producing plea [*pel] on c3y.
*/
c3_o
@ -1039,8 +1022,6 @@ u3_serf_writ(u3_serf* sef_u, u3_noun wit, u3_noun* pel)
ret_o = c3n;
}
else {
_serf_step_trace(sef_u);
switch ( tag ) {
default: {
ret_o = c3n;
@ -1120,67 +1101,6 @@ _serf_ripe(u3_serf* sef_u)
return u3nc(u3i_chubs(1, &sef_u->dun_d), sef_u->mug_l);
}
/* u3_serf_uncram(): initialize from rock at [eve_d].
*/
void
u3_serf_uncram(u3_serf* sef_u, c3_d eve_d)
{
c3_o roc_o;
c3_c nam_c[8193];
snprintf(nam_c, 8192, "%s/.urb/roc/%" PRIu64 ".jam", sef_u->dir_c, eve_d);
struct stat buf_b;
c3_i fid_i = open(nam_c, O_RDONLY, 0644);
if ( (fid_i < 0) || (fstat(fid_i, &buf_b) < 0) ) {
fprintf(stderr, "serf: rock: %s not found\r\n", nam_c);
roc_o = c3n;
}
else {
fprintf(stderr, "serf: rock: %s found\r\n", nam_c);
roc_o = c3y;
}
close(fid_i);
if ( c3y == roc_o ) {
if ( c3n == u3e_hold() ) {
fprintf(stderr, "serf: unable to backup checkpoint\r\n");
}
else {
u3m_wipe();
if ( c3n == u3m_rock_load(sef_u->dir_c, eve_d) ) {
fprintf(stderr, "serf: compaction failed, restoring checkpoint\r\n");
if ( c3n == u3e_fall() ) {
fprintf(stderr, "serf: unable to restore checkpoint\r\n");
c3_assert(0);
}
}
if ( c3n == u3e_drop() ) {
fprintf(stderr, "serf: warning: orphaned backup checkpoint file\r\n");
}
// leave rocks on disk
//
// if ( c3n == u3m_rock_drop(sef_u->dir_c, sef_u->dun_d) ) {
// u3l_log("serf: warning: orphaned state file\r\n");
// }
fprintf(stderr, "serf (%" PRIu64 "): compacted loom\r\n", eve_d);
sef_u->sen_d = sef_u->dun_d = eve_d;
// save now for flexibility
//
u3e_save();
}
}
}
/* u3_serf_init(): init or restore, producing status.
*/
u3_noun