diff --git a/.travis.yml b/.travis.yml index f04e45d33..12caa62e5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,15 +2,15 @@ language: nix nix: 2.1.3 install: - # nix-env -iA cachix -f https://cachix.org/api/v1/install + - nix-env -iA cachix -f https://cachix.org/api/v1/install before_install: - git lfs pull script: - # cachix authtoken "$CACHIX_AUTH_TOKEN" >/dev/null - # cachix use urbit + - cachix authtoken "$CACHIX_AUTH_TOKEN" >/dev/null + - cachix use urbit2 - make - make test - sh/update-brass-pill # To verify that this works. - # ./sh/cachix + - ./sh/cachix diff --git a/nix/dep-repos.nix b/nix/dep-repos.nix index bd3463a89..3022c02dd 100644 --- a/nix/dep-repos.nix +++ b/nix/dep-repos.nix @@ -7,10 +7,8 @@ with (import ./deps {}); { argon2-src = argon2.src; ed25519-src = ed25519.src; - ent-src = ent.src; h2o-src = h2o.src; murmur3-src = murmur3.src; - nodehello-src = nodehello.src; scrypt-src = scrypt.src; secp256k1-src = secp256k1.src; sni-src = sni.src; diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index c7b3b2368..293f9c6c9 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -16,6 +16,8 @@ #include #include #include +#include +#include #define U3_GLOBAL #define C3_GLOBAL @@ -441,6 +443,7 @@ u3_ve_sysopt() static void report(void) { + printf("urbit %s\n", URBIT_VERSION); printf("---------\nLibraries\n---------\n"); printf("gmp: %s\n", gmp_version); printf("sigsegv: %d.%d\n", @@ -453,6 +456,15 @@ report(void) H2O_LIBRARY_VERSION_MAJOR, H2O_LIBRARY_VERSION_MINOR, H2O_LIBRARY_VERSION_PATCH); + printf("lmdb: %d.%d.%d\n", + MDB_VERSION_MAJOR, + MDB_VERSION_MINOR, + MDB_VERSION_PATCH); + printf("curl: %d.%d.%d\n", + LIBCURL_VERSION_MAJOR, + LIBCURL_VERSION_MINOR, + LIBCURL_VERSION_PATCH); + printf("argon2: 0x%x\n", ARGON2_VERSION_NUMBER); } static void @@ -603,7 +615,7 @@ main(c3_i argc, sigemptyset(&set); sigaddset(&set, SIGPROF); if ( 0 != pthread_sigmask(SIG_BLOCK, &set, NULL) ) { - perror("pthread_sigmask"); + u3l_log("boot: thread mask SIGPROF: %s\r\n", strerror(errno)); exit(1); } } @@ -717,21 +729,6 @@ main(c3_i argc, SSL_library_init(); SSL_load_error_strings(); - { - c3_i rad; - c3_y buf[4096]; - - // RAND_status, at least on OS X, never returns true. - // 4096 bytes should be enough entropy for anyone, right? - rad = open("/dev/urandom", O_RDONLY); - if ( 4096 != read(rad, &buf, 4096) ) { - perror("rand-seed"); - exit(1); - } - RAND_seed(buf, 4096); - close(rad); - } - u3_daemon_commence(); } return 0; diff --git a/pkg/urbit/include/noun/vortex.h b/pkg/urbit/include/noun/vortex.h index 866fe740c..b809550e7 100644 --- a/pkg/urbit/include/noun/vortex.h +++ b/pkg/urbit/include/noun/vortex.h @@ -4,18 +4,6 @@ */ /** Data structures. **/ - /* u3_cart: ovum carton. - */ - struct _u3v_arvo; - - typedef struct _u3v_cart { - u3_noun vir; // effects of ovum - c3_o did; // cart considered for commit? - c3_o cit; // cart committed? - c3_d ent_d; // event number - u3p(struct _u3v_cart) nex_p; - } u3v_cart; - /* u3v_arvo: modern arvo structure. */ typedef struct _u3v_arvo { @@ -28,18 +16,9 @@ u3_noun our; // identity u3_noun fak; // c3y is fake - u3_noun sac; // space profiling - - u3_noun roe; // temporary unsaved events - u3_noun key; // log key, or 0 u3_noun sys; // system pill u3_noun roc; // kernel core - - struct { // ova waiting to process - u3p(u3v_cart) egg_p; // exit of ovum queue - u3p(u3v_cart) geg_p; // entry of ovum queue - } ova; } u3v_arvo; /* u3v_home: all internal (within image) state. @@ -145,11 +124,6 @@ void u3v_plow(u3_noun ova); - /* u3v_hose(): clear initial ovum queue. - */ - void - u3v_hose(void); - /* u3v_mark(): mark arvo kernel. */ c3_w diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index 352c7bb82..4f5ee22f6 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -691,6 +691,7 @@ u3_writ* ent_u; // entry of queue u3_writ* ext_u; // exit of queue uv_prepare_t pep_u; // preloop registration + uv_idle_t idl_u; // postloop registration } u3_pier; /* u3_king: all executing piers. diff --git a/pkg/urbit/noun/allocate.c b/pkg/urbit/noun/allocate.c index e6057bc5f..3e4f230e1 100644 --- a/pkg/urbit/noun/allocate.c +++ b/pkg/urbit/noun/allocate.c @@ -1851,7 +1851,7 @@ u3a_sweep(void) weq_w += box_u->siz_w; } else { - _ca_print_leak("weak", box_u, box_u->eus_w, box_u->use_w); + _ca_print_leak("leak", box_u, box_u->eus_w, box_u->use_w); leq_w += box_u->siz_w; } diff --git a/pkg/urbit/noun/events.c b/pkg/urbit/noun/events.c index 39f9b0271..929ea73f0 100644 --- a/pkg/urbit/noun/events.c +++ b/pkg/urbit/noun/events.c @@ -155,7 +155,7 @@ u3e_fault(void* adr_v, c3_i ser_i) (1 << (u3a_page + 2)), (PROT_READ | PROT_WRITE)) ) { - perror("mprotect"); + u3l_log("loom: fault mprotect: %s\r\n", strerror(errno)); c3_assert(0); return 0; } @@ -182,14 +182,14 @@ _ce_image_open(u3e_image* img_u) snprintf(ful_c, 8192, "%s/.urb/chk/%s.bin", u3P.dir_c, img_u->nam_c); if ( -1 == (img_u->fid_i = open(ful_c, mod_i, 0666)) ) { - perror(ful_c); + u3l_log("loom: open %s: %s\r\n", ful_c, strerror(errno)); return c3n; } else { struct stat buf_u; if ( -1 == fstat(img_u->fid_i, &buf_u) ) { - perror(ful_c); + u3l_log("loom: stat %s: %s\r\n", ful_c, strerror(errno)); c3_assert(0); return c3n; } @@ -273,13 +273,13 @@ _ce_patch_create(u3_ce_patch* pat_u) snprintf(ful_c, 8192, "%s/.urb/chk/control.bin", u3P.dir_c); if ( -1 == (pat_u->ctl_i = open(ful_c, O_RDWR | O_CREAT | O_EXCL, 0600)) ) { - perror(ful_c); + u3l_log("loom: patch open control.bin: %s\r\n", strerror(errno)); c3_assert(0); } snprintf(ful_c, 8192, "%s/.urb/chk/memory.bin", u3P.dir_c); if ( -1 == (pat_u->mem_i = open(ful_c, O_RDWR | O_CREAT | O_EXCL, 0600)) ) { - perror(ful_c); + u3l_log("loom: patch open memory.bin: %s\r\n", strerror(errno)); c3_assert(0); } } @@ -311,12 +311,12 @@ _ce_patch_verify(u3_ce_patch* pat_u) c3_w mem_w[1 << u3a_page]; if ( -1 == lseek(pat_u->mem_i, (i_w << (u3a_page + 2)), SEEK_SET) ) { - perror("seek"); + u3l_log("loom: patch seek: %s\r\n", strerror(errno)); c3_assert(0); return c3n; } if ( -1 == read(pat_u->mem_i, mem_w, (1 << (u3a_page + 2))) ) { - perror("read"); + u3l_log("loom: patch read: %s\r\n", strerror(errno)); c3_assert(0); return c3n; } @@ -636,7 +636,7 @@ _ce_patch_apply(u3_ce_patch* pat_u) c3_w ret_w; ret_w = ftruncate(u3P.nor_u.fid_i, u3P.nor_u.pgs_w << (u3a_page + 2)); if (ret_w){ - perror("_ce_patch_apply"); + u3l_log("loom: patch apply truncate north: %s\r\n", strerror(errno)); c3_assert(0); } } @@ -646,7 +646,7 @@ _ce_patch_apply(u3_ce_patch* pat_u) c3_w ret_w; ret_w = ftruncate(u3P.sou_u.fid_i, u3P.sou_u.pgs_w << (u3a_page + 2)); if (ret_w){ - perror("_ce_patch_apply"); + u3l_log("loom: patch apply truncate south: %s\r\n", strerror(errno)); c3_assert(0); } } @@ -656,7 +656,7 @@ _ce_patch_apply(u3_ce_patch* pat_u) (-1 == lseek(u3P.nor_u.fid_i, 0, SEEK_SET)) || (-1 == lseek(u3P.sou_u.fid_i, 0, SEEK_SET)) ) { - perror("apply: seek"); + u3l_log("loom: patch apply seek 0: %s\r\n", strerror(errno)); c3_assert(0); } @@ -676,16 +676,16 @@ _ce_patch_apply(u3_ce_patch* pat_u) } if ( -1 == read(pat_u->mem_i, mem_w, (1 << (u3a_page + 2))) ) { - perror("apply: read"); + u3l_log("loom: patch apply read: %s\r\n", strerror(errno)); c3_assert(0); } else { if ( -1 == lseek(fid_i, (off_w << (u3a_page + 2)), SEEK_SET) ) { - perror("apply: lseek"); + u3l_log("loom: patch apply seek: %s\r\n", strerror(errno)); c3_assert(0); } if ( -1 == write(fid_i, mem_w, (1 << (u3a_page + 2))) ) { - perror("apply: write"); + u3l_log("loom: patch apply write: %s\r\n", strerror(errno)); c3_assert(0); } } @@ -707,7 +707,7 @@ _ce_image_blit(u3e_image* img_u, lseek(img_u->fid_i, 0, SEEK_SET); for ( i_w=0; i_w < img_u->pgs_w; i_w++ ) { if ( -1 == read(img_u->fid_i, ptr_w, (1 << (u3a_page + 2))) ) { - perror("read"); + u3l_log("loom: image blit read: %s\r\n", strerror(errno)); c3_assert(0); } #if 0 @@ -739,7 +739,7 @@ _ce_image_fine(u3e_image* img_u, c3_w mem_w, fil_w; if ( -1 == read(img_u->fid_i, buf_w, (1 << (u3a_page + 2))) ) { - perror("read"); + u3l_log("loom: image fine read: %s\r\n", strerror(errno)); c3_assert(0); } mem_w = u3r_mug_words(ptr_w, (1 << u3a_page)); @@ -882,7 +882,7 @@ u3e_live(c3_o nuu_o, c3_c* dir_c) -(1 << u3a_page)); if ( 0 != mprotect((void *)u3_Loom, u3a_bytes, PROT_READ) ) { - perror("protect"); + u3l_log("loom: live mprotect: %s\r\n", strerror(errno)); c3_assert(0); } diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index f329e9d14..b53b0c101 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -1501,7 +1501,7 @@ _cm_limits(void) ? (65536 << 10) : rlm.rlim_max; if ( 0 != setrlimit(RLIMIT_STACK, &rlm) ) { - perror("stack"); + u3l_log("boot: stack size: %s\r\n", strerror(errno)); exit(1); } } @@ -1513,7 +1513,7 @@ _cm_limits(void) c3_assert(0 == ret_i); rlm.rlim_cur = 10240; // default OSX max, not in rlim_max irritatingly if ( 0 != setrlimit(RLIMIT_NOFILE, &rlm) ) { - // perror("file limit"); + u3l_log("boot: open file limit: %s\r\n", strerror(errno)); // no exit, not a critical limit } } @@ -1524,7 +1524,7 @@ _cm_limits(void) getrlimit(RLIMIT_CORE, &rlm); rlm.rlim_cur = RLIM_INFINITY; if ( 0 != setrlimit(RLIMIT_CORE, &rlm) ) { - perror("core limit"); + u3l_log("boot: core limit: %s\r\n", strerror(errno)); // no exit, not a critical limit } } @@ -1536,7 +1536,7 @@ static void _cm_signals(void) { if ( 0 != sigsegv_install_handler(u3e_fault) ) { - u3l_log("sigsegv install failed\n"); + u3l_log("boot: sigsegv install failed\n"); exit(1); } // signal(SIGINT, _loom_stop); @@ -1552,7 +1552,7 @@ _cm_signals(void) sigaddset(&set, SIGPROF); if ( 0 != pthread_sigmask(SIG_BLOCK, &set, NULL) ) { - perror("pthread_sigmask"); + u3l_log("boot: thread mask SIGPROF: %s\r\n", strerror(errno)); exit(1); } } @@ -1911,7 +1911,6 @@ u3m_boot_new(c3_c* dir_c) /* Reactivate jets on old kernel. */ if ( !_(nuu_o) ) { - u3v_hose(); u3j_ream(); u3n_ream(); diff --git a/pkg/urbit/noun/trace.c b/pkg/urbit/noun/trace.c index bc4454d6c..c6c1f3bf5 100644 --- a/pkg/urbit/noun/trace.c +++ b/pkg/urbit/noun/trace.c @@ -167,7 +167,7 @@ _t_samp_process(u3_road* rod_u) pef = t_pef; } - // fprintf(stderr, "sample: stack length %d\r\n", u3kb_lent(u3k(pal))); + // u3l_log("sample: stack length %d\r\n", u3kb_lent(u3k(pal))); return pal; } } @@ -536,7 +536,7 @@ u3t_boot(void) sigemptyset(&set); sigaddset(&set, SIGPROF); if ( 0 != pthread_sigmask(SIG_UNBLOCK, &set, NULL) ) { - perror("pthread_sigmask"); + u3l_log("trace: thread mask SIGPROF: %s\r\n", strerror(errno)); } } @@ -569,7 +569,7 @@ u3t_boff(void) sigemptyset(&set); sigaddset(&set, SIGPROF); if ( 0 != pthread_sigmask(SIG_BLOCK, &set, NULL) ) { - perror("pthread_sigmask"); + u3l_log("trace: thread mask SIGPROF: %s\r\n", strerror(errno)); } } diff --git a/pkg/urbit/noun/vortex.c b/pkg/urbit/noun/vortex.c index 44fd0205b..ec0624fc7 100644 --- a/pkg/urbit/noun/vortex.c +++ b/pkg/urbit/noun/vortex.c @@ -159,50 +159,6 @@ u3v_boot_lite(u3_atom lit) u3z(pru); } -/* u3v_hose(): clear initial ovum queue. -*/ -void -u3v_hose(void) -{ - u3p(u3v_cart) egg_p = u3A->ova.egg_p; - - while ( egg_p ) { - u3v_cart* egg_u = u3to(u3v_cart, egg_p); - u3p(u3v_cart) nex_p = egg_u->nex_p; - - u3a_lose(egg_u->vir); - u3a_free(egg_u); - - egg_p = nex_p; - } - u3A->ova.egg_p = u3A->ova.geg_p = 0; - u3z(u3A->roe); - u3A->roe = u3_nul; -} - -// XX deprecated, remove -#if 0 -/* u3v_start(): start time. -*/ -void -u3v_start(u3_noun now) -{ - u3v_time(now); - u3v_numb(); - u3A->sac = u3_nul; - - { - c3_c* wen_c = u3r_string(u3A->wen); - - u3l_log("arvo: time: %s\n", wen_c); - free(wen_c); - } - - if ( u3C.wag_w & u3o_trace ) - u3t_trace_open(); -} -#endif - /* u3v_wish(): text expression with cache. */ u3_noun @@ -485,56 +441,6 @@ u3v_sway(u3_noun blu, c3_l tab_l, u3_noun tax) u3z(mok); } -// XX deprecated, remove -#if 0 -/* u3v_plan(): queue ovum (external). -*/ -void -u3v_plan(u3_noun pax, u3_noun fav) -{ - u3_noun egg = u3nc(pax, fav); - u3A->roe = u3nc(u3nc(u3_nul, egg), u3A->roe); -} -#endif - -// XX deprecated, remove -#if 0 -/* u3v_plow(): queue multiple ova (external). -*/ -void -u3v_plow(u3_noun ova) -{ - u3_noun ovi = ova; - - while ( u3_nul != ovi ) { - u3_noun ovo=u3h(ovi); - - u3v_plan(u3k(u3h(ovo)), u3k(u3t(ovo))); - ovi = u3t(ovi); - } - u3z(ova); -} -#endif - -/* _cv_mark_ova(): mark ova queue. -*/ -c3_w -_cv_mark_ova(u3p(u3v_cart) egg_p) -{ - c3_w tot_w = 0; - - while ( egg_p ) { - u3v_cart* egg_u = u3to(u3v_cart, egg_p); - - tot_w += u3a_mark_mptr(egg_u); - tot_w += u3a_mark_noun(egg_u->vir); - - egg_p = egg_u->nex_p; - } - - return tot_w; -} - /* u3v_mark(): mark arvo kernel. */ c3_w @@ -549,9 +455,6 @@ u3v_mark(FILE* fil_u) tot_w += u3a_maid(fil_u, " instance string", u3a_mark_noun(arv_u->sen)); tot_w += u3a_maid(fil_u, " identity", u3a_mark_noun(arv_u->our)); tot_w += u3a_maid(fil_u, " fake", u3a_mark_noun(arv_u->fak)); - tot_w += u3a_maid(fil_u, " pending events", u3a_mark_noun(arv_u->roe)); - tot_w += u3a_maid(fil_u, " event-log key", u3a_mark_noun(arv_u->key)); tot_w += u3a_maid(fil_u, " kernel", u3a_mark_noun(arv_u->roc)); - tot_w += u3a_maid(fil_u, " egg basket", _cv_mark_ova(arv_u->ova.egg_p)); return u3a_maid(fil_u, "total arvo stuff", tot_w); } diff --git a/pkg/urbit/vere/dawn.c b/pkg/urbit/vere/dawn.c index 1e341699a..80a3856aa 100644 --- a/pkg/urbit/vere/dawn.c +++ b/pkg/urbit/vere/dawn.c @@ -197,7 +197,7 @@ _dawn_fail(u3_noun who, u3_noun rac, u3_noun sas) } } - u3l_log("dawn: invalid keys for %s '%s'\r\n", rac_c, how_c); + u3l_log("boot: invalid keys for %s '%s'\r\n", rac_c, how_c); // XX deconstruct sas, print helpful error messages u3m_p("pre-boot error", u3t(sas)); @@ -306,20 +306,22 @@ u3_dawn_vent(u3_noun seed) // load snapshot from file // if ( 0 != u3_Host.ops_u.ets_c ) { - u3l_log("boot: loading ethereum snapshot\r\n"); + u3l_log("boot: loading azimuth snapshot\r\n"); u3_noun raw_snap = u3ke_cue(u3m_file(u3_Host.ops_u.ets_c)); sap = u3nc(u3_nul, raw_snap); } // load snapshot from HTTP URL // else if ( 0 != u3_Host.ops_u.sap_c ) { + u3l_log("boot: downloading azimuth snapshot from %s\r\n", + u3_Host.ops_u.sap_c); u3_noun raw_snap = _dawn_get_jam(u3_Host.ops_u.sap_c); sap = u3nc(u3_nul, raw_snap); } // no snapshot // else { - u3l_log("dawn: no ethereum snapshot specified\n"); + u3l_log("boot: no azimuth snapshot specified\n"); sap = u3_nul; } diff --git a/pkg/urbit/vere/pier.c b/pkg/urbit/vere/pier.c index 82ac6e717..541f812ae 100644 --- a/pkg/urbit/vere/pier.c +++ b/pkg/urbit/vere/pier.c @@ -69,6 +69,7 @@ static void _pier_boot_complete(u3_pier* pir_u); static void _pier_boot_ready(u3_pier* pir_u); static void _pier_boot_set_ship(u3_pier* pir_u, u3_noun who, u3_noun fak); static void _pier_exit_done(u3_pier* pir_u); +static void _pier_loop_resume(u3_pier* pir_u); /* _pier_db_bail(): bail from disk i/o. */ @@ -118,7 +119,7 @@ _pier_db_commit_complete(c3_o success, log_u->com_d += 1ULL; } - _pier_apply(pir_u); + _pier_loop_resume(pir_u); } /* _pier_db_commit_request(): start commit. @@ -667,20 +668,6 @@ _pier_work_complete(u3_writ* wit_u, fprintf(stderr, "pier: (%" PRIu64 "): compute: complete\r\n", wit_u->evt_d); #endif - if ( u3_psat_pace == pir_u->sat_e && - wit_u->nex_u && - mug_l != wit_u->nex_u->mug_l ) { - // While we are replaying the event log, we also perform checks that the - // resulting mug_l for this urbit's state is equivalent to the expected - // input state of the next event. If it isn't, we have either corruption or - // non-determinism during replay and either should cause a bail. - u3l_log("Invalid recomputed state. For event %" PRIu64 ", the computed mug " - "was %x but event %" PRIu64 " expected %x.\r\n", - wit_u->evt_d, mug_l, wit_u->nex_u->evt_d, wit_u->nex_u->mug_l); - - u3_pier_bail(); - } - god_u->dun_d += 1; c3_assert(god_u->dun_d == wit_u->evt_d); @@ -780,6 +767,7 @@ _pier_work_play(u3_pier* pir_u, // all events in the worker are complete // god_u->rel_d = god_u->dun_d = god_u->sen_d = (lav_d - 1ULL); + god_u->mug_l = mug_l; _pier_boot_ready(pir_u); } @@ -1001,16 +989,14 @@ _pier_work_poke(void* vod_p, goto error; } else { - // XXX: The wit_u pointer will almost always be 0 because of how the - // worker process manages the difference between u3V.evt_d vs - // u3A->ent_d. Either stop communicating the evt_d in the wire protocol - // or fix the worker to keep track of and communicate the correct event - // number. c3_d evt_d = u3r_chub(0, p_jar); c3_w pri_w = u3r_word(0, q_jar); u3_writ* wit_u = _pier_writ_find(pir_u, evt_d); - // Only print this slog if the event is uncommitted. + // skip slog during replay + // + // XX also update the worker to skip *sending* the slog during replay + // if ( u3_psat_pace != pir_u->sat_e ) { _pier_work_slog(wit_u, pri_w, u3k(r_jar)); } @@ -1020,7 +1006,7 @@ _pier_work_poke(void* vod_p, } u3z(jar); u3z(mat); - _pier_apply(pir_u); + _pier_loop_resume(pir_u); return; error: { @@ -1118,7 +1104,7 @@ _pier_loop_time(void) u3v_time(u3_time_in_tv(&tim_tv)); } -/* _pier_loop_prepare(): +/* _pier_loop_prepare(): run on every loop iteration before i/o polling. */ static void _pier_loop_prepare(uv_prepare_t* pep_u) @@ -1126,6 +1112,27 @@ _pier_loop_prepare(uv_prepare_t* pep_u) _pier_loop_time(); } +/* _pier_loop_idle_cb(): run on every loop iteration after i/o polling. +*/ +static void +_pier_loop_idle_cb(uv_idle_t* idl_u) +{ + u3_pier* pir_u = idl_u->data; + _pier_apply(pir_u); + + uv_idle_stop(idl_u); +} + +/* _pier_loop_resume(): (re-)activate idle handler +*/ +static void +_pier_loop_resume(u3_pier* pir_u) +{ + if ( !uv_is_active((uv_handle_t*)&pir_u->idl_u) ) { + uv_idle_start(&pir_u->idl_u, _pier_loop_idle_cb); + } +} + /* _pier_loop_init_pier(): initialize loop handlers. */ static void @@ -1188,7 +1195,7 @@ _pier_loop_wake(u3_pier* pir_u) u3_noun wir = u3nt(u3_blip, c3__arvo, u3_nul); u3_noun car = u3nc(c3__wack, u3i_words(16, eny_w)); - _pier_writ_insert_ovum(pir_u, 0, u3nc(wir, car)); + u3_pier_work(pir_u, wir, car); } cod_l = u3a_lush(c3__unix); @@ -1690,8 +1697,7 @@ start: // remove from queue // - // XX must be done before releasing effects - // which is currently reentrant + // Must be done before releasing effects // _pier_writ_unlink(wit_u); @@ -1842,7 +1848,7 @@ u3_pier_discover(u3_pier* pir_u, u3_noun job) { _pier_writ_insert(pir_u, msc_l, job); - _pier_apply(pir_u); + _pier_loop_resume(pir_u); } /* u3_pier_send(): modern send with target and path. @@ -2024,6 +2030,29 @@ u3_pier_stub(void) } } +/* _pier_init(): initialize pier i/o handles +*/ +static void +_pier_init(u3_pier* pir_u) +{ + // initialize i/o handlers + // + _pier_loop_init(pir_u); + + // initialize pre i/o polling handle + // + uv_prepare_init(u3_Host.lup_u, &pir_u->pep_u); + pir_u->pep_u.data = pir_u; + uv_prepare_start(&pir_u->pep_u, _pier_loop_prepare); + + // initialize post i/o polling handle + // + uv_idle_init(u3_Host.lup_u, &pir_u->idl_u); + pir_u->idl_u.data = pir_u; + + _pier_loop_resume(pir_u); +} + /* u3_pier_boot(): start the new pier system. */ void @@ -2037,6 +2066,12 @@ u3_pier_boot(c3_w wag_w, // config flags // u3_pier* pir_u = _pier_create(wag_w, u3r_string(pax)); + if ( 0 == pir_u ) { + u3l_log("pier: failed to create\r\n"); + u3_daemon_bail(); + exit(1); + } + // set boot params // { @@ -2045,14 +2080,7 @@ u3_pier_boot(c3_w wag_w, // config flags _pier_boot_set_ship(pir_u, u3k(who), ( c3__fake == u3h(ven) ) ? c3y : c3n); } - // initialize i/o handlers - // - _pier_loop_init(pir_u); - - // initialize polling handle - // - uv_prepare_init(u3_Host.lup_u, &pir_u->pep_u); - uv_prepare_start(&pir_u->pep_u, _pier_loop_prepare); + _pier_init(pir_u); u3z(who); u3z(ven); u3z(pil); u3z(pax); } @@ -2066,14 +2094,13 @@ u3_pier_stay(c3_w wag_w, u3_noun pax) // u3_pier* pir_u = _pier_create(wag_w, u3r_string(pax)); - // initialize i/o handlers - // - _pier_loop_init(pir_u); + if ( 0 == pir_u ) { + u3l_log("pier: failed to create\r\n"); + u3_daemon_bail(); + exit(1); + } - // initialize polling handle - // - uv_prepare_init(u3_Host.lup_u, &pir_u->pep_u); - uv_prepare_start(&pir_u->pep_u, _pier_loop_prepare); + _pier_init(pir_u); u3z(pax); } diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index e219c3282..924c2d0a3 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -26,7 +26,9 @@ typedef struct _u3_worker { c3_w len_w; // boot sequence length - c3_d evt_d; // last event processed + u3_noun roe; // lifecycle formulas + c3_d sen_d; // last event requested + c3_d dun_d; // last event processed c3_l mug_l; // hash of state c3_d key_d[4]; // disk key u3_moat inn_u; // message input @@ -59,7 +61,7 @@ :: $: %done :: p: event number - :: q: mug of state (or 0) + :: q: mug of kernel :: r: effects :: [p=@ q=@ r=(list ovum)] @@ -68,7 +70,7 @@ :: $: %work :: p: event number - :: q: mug of state (or 0) + :: q: mug of kernel :: r: replacement event (at date) :: [p=@ q=@ r=(pair date ovum)] @@ -200,9 +202,9 @@ _worker_prof(FILE* fil_u, c3_w den, u3_noun mas) _worker_print_memory(fil_u, tot_w); #if 1 - /* The basic issue here is that tt_mas is included in - * u3A->sac, so they can't both be roots in the normal - * sense. When we mark u3A->sac later on, we want tt_mas + /* The basic issue here is that tt_mas is included in .sac + * (the whole profile), so they can't both be roots in the + * normal sense. When we mark .sac later on, we want tt_mas * to appear unmarked, but its children should be already * marked. */ @@ -262,7 +264,7 @@ _worker_grab(u3_noun sac, u3_noun ovo, u3_noun vir) } } else { - c3_w usr_w = 0, man_w = 0, sac_w = 0, ova_w = 0, vir_w = 0; + c3_w usr_w = 0, man_w = 0, sac_w = 0, ova_w = 0, roe_w = 0, vir_w = 0; FILE* fil_u; @@ -308,6 +310,9 @@ _worker_grab(u3_noun sac, u3_noun ovo, u3_noun vir) ova_w = u3a_mark_noun(ovo); u3a_print_memory(fil_u, "event", ova_w); + roe_w = u3a_mark_noun(u3V.roe); + u3a_print_memory(fil_u, "lifecycle events", roe_w); + vir_w = u3a_mark_noun(vir); u3a_print_memory(fil_u, "effects", vir_w); @@ -343,17 +348,15 @@ _worker_send(u3_noun job) /* _worker_send_replace(): send replacement job back to daemon. */ static void -_worker_send_replace(c3_d evt_d, u3_noun ovo) +_worker_send_replace(c3_d evt_d, u3_noun job) { u3l_log("worker_send_replace %" PRIu64 " %s\r\n", evt_d, - u3r_string(u3h(u3t(ovo)))); + u3r_string(u3h(u3t(u3t(job))))); _worker_send(u3nt(c3__work, u3i_chubs(1, &evt_d), - u3ke_jam(u3nt(u3V.mug_l, - u3k(u3A->now), - ovo)))); + u3ke_jam(u3nc(u3V.mug_l, job)))); } /* _worker_send_complete(): report completion. @@ -362,7 +365,7 @@ static void _worker_send_complete(u3_noun vir) { _worker_send(u3nq(c3__done, - u3i_chubs(1, &u3V.evt_d), + u3i_chubs(1, &u3V.dun_d), u3V.mug_l, vir)); } @@ -372,7 +375,7 @@ _worker_send_complete(u3_noun vir) static void _worker_send_stdr(c3_c* str_c) { - _worker_send(u3nt(c3__stdr, u3i_chubs(1, &u3V.evt_d), u3i_string(str_c))); + _worker_send(u3nt(c3__stdr, u3i_chubs(1, &u3V.sen_d), u3i_string(str_c))); } /* _worker_send_slog(): send hint output (hod is [priority tank]). @@ -380,13 +383,13 @@ _worker_send_stdr(c3_c* str_c) static void _worker_send_slog(u3_noun hod) { - _worker_send(u3nt(c3__slog, u3i_chubs(1, &u3V.evt_d), hod)); + _worker_send(u3nt(c3__slog, u3i_chubs(1, &u3V.sen_d), hod)); } /* _worker_lame(): event failed, replace with error event. */ static void -_worker_lame(c3_d evt_d, u3_noun ovo, u3_noun why, u3_noun tan) +_worker_lame(c3_d evt_d, u3_noun now, u3_noun ovo, u3_noun why, u3_noun tan) { u3_noun rep; u3_noun wir, tag, cad; @@ -435,7 +438,7 @@ _worker_lame(c3_d evt_d, u3_noun ovo, u3_noun why, u3_noun tan) rep = u3nc(u3k(wir), u3nt(c3__crud, u3k(tag), nat)); } - _worker_send_replace(evt_d, rep); + _worker_send_replace(evt_d, u3nc(now, rep)); u3z(ovo); u3z(why); u3z(tan); } @@ -446,8 +449,9 @@ static void _worker_sure(u3_noun ovo, u3_noun vir, u3_noun cor) { u3z(u3A->roc); - u3A->roc = cor; - u3V.mug_l = u3r_mug(u3A->roc); + u3A->roc = cor; + u3A->ent_d = u3V.dun_d; + u3V.mug_l = u3r_mug(u3A->roc); u3_noun sac = u3_nul; @@ -500,25 +504,18 @@ _worker_sure(u3_noun ovo, u3_noun vir, u3_noun cor) /* _worker_work_live(): apply event. */ static void -_worker_work_live(c3_d evt_d, // event number - c3_l mug_l, // mug of state - u3_noun job) // event date +_worker_work_live(c3_d evt_d, u3_noun job) { u3_noun now, ovo, gon; + u3_noun last_date; - c3_assert(evt_d == u3V.evt_d + 1ULL); - if ( 0 != mug_l ) { - c3_assert(u3V.mug_l == mug_l); - } + c3_assert(evt_d == u3V.dun_d + 1ULL); + u3V.sen_d = evt_d; u3x_cell(job, &now, &ovo); - u3z(u3A->now); - u3A->now = u3k(now); - - // XX why is this set before u3v_poke? - // - u3A->ent_d = evt_d; + last_date = u3A->now; + u3A->now = u3k(now); #ifdef U3_EVENT_TIME_DEBUG { @@ -557,20 +554,26 @@ _worker_work_live(c3_d evt_d, // event number if ( u3_blip != u3h(gon) ) { // event rejected // + u3V.sen_d = u3V.dun_d; + // restore previous time + // + u3_noun nex = u3A->now; + u3A->now = last_date; + u3_noun why, tan; u3x_cell(gon, &why, &tan); u3k(ovo); u3k(why); u3k(tan); u3z(gon); u3z(job); - _worker_lame(evt_d, ovo, why, tan); + _worker_lame(evt_d, nex, ovo, why, tan); } else { // event accepted // - // XX reconcile/dedupe with u3A->ent_d - // - u3V.evt_d = evt_d; + u3V.dun_d = u3V.sen_d; + u3z(last_date); + // vir/(list ovum) list of effects // cor/arvo arvo core // @@ -582,9 +585,13 @@ _worker_work_live(c3_d evt_d, // event number _worker_sure(ovo, vir, cor); - // reclaim memory from persistent caches on |reset + // reclaim memory from persistent caches periodically // - if ( 0 == (u3A->ent_d % 1000ULL) ) { + // XX this is a hack to work around the fact that + // the bytecode caches grow rapidly and are not + // able to be simply capped (due to internal posts). + // + if ( 0 == (evt_d % 1000ULL) ) { u3m_reclaim(); } } @@ -609,26 +616,23 @@ _worker_boot_fire(u3_noun eve) /* _worker_work_boot(): apply initial-stage event. */ static void -_worker_work_boot(c3_d evt_d, - c3_l mug_l, - u3_noun job) +_worker_work_boot(c3_d evt_d, u3_noun job) { - c3_assert(evt_d == u3V.evt_d + 1ULL); - if ( 0 != mug_l ) { - c3_assert(u3V.mug_l == mug_l); - } + // here we asset on u3V.sen_d, because u3V.dun_d isn't set until + // after u3V.sen_d == u3V.len_w (ie, after the lifecycle evaluation) + // + c3_assert(evt_d == u3V.sen_d + 1ULL); + u3V.sen_d = evt_d; - u3V.evt_d = evt_d; - - u3A->roe = u3nc(job, u3A->roe); + u3V.roe = u3nc(job, u3V.roe); u3l_log("work: (%" PRIu64 ")| boot\r\n", evt_d); if ( u3V.len_w == evt_d ) { u3_noun eve, pru; - eve = u3kb_flop(u3A->roe); - u3A->roe = 0; + eve = u3kb_flop(u3V.roe); + u3V.roe = u3_nul; u3l_log("work: (%" PRIu64 ")| pill: %x\r\n", evt_d, u3r_mug(eve)); @@ -639,16 +643,19 @@ _worker_work_boot(c3_d evt_d, exit(1); } - u3A->roc = u3k(u3t(pru)); - u3V.mug_l = u3r_mug(u3A->roc); + u3V.dun_d = evt_d; + u3A->ent_d = u3V.dun_d; + u3A->roc = u3k(u3t(pru)); + u3V.mug_l = u3r_mug(u3A->roc); u3l_log("work: (%" PRIu64 ")| core: %x\r\n", evt_d, u3V.mug_l); - // XX set u3A->evt_d ? - // u3z(pru); } else { + // prior to the evaluation of the entire lifecycle sequence, + // we simply use the mug of the formula as the kernel mug + // u3V.mug_l = u3r_mug(job); } @@ -675,12 +682,29 @@ _worker_poke_work(c3_d evt_d, // event number } } + // Require mugs to match + // + // We use mugs to enforce that %work is always performed against + // the exact kernel we expect it to be. If it isn't, we have either + // event-log corruption or non-determism on replay, or programmer error + // in normal operation. In either case, we immediately exit. + // + if ( u3V.mug_l != mug_l ) { + u3l_log("work: invalid %%work for event %" PRIu64 ".\r\n", evt_d); + u3l_log("work: computed mug is %x but event %" PRIu64 " expected %x.\r\n", + u3V.mug_l, + evt_d, + mug_l); + _worker_fail(0, "bad jar"); + return; + } + if ( evt_d <= u3V.len_w ) { c3_c lab_c[8]; snprintf(lab_c, 8, "boot: %" PRIu64 "", evt_d); u3t_event_trace(lab_c, 'B'); - _worker_work_boot(evt_d, mug_l, job); + _worker_work_boot(evt_d, job); u3t_event_trace(lab_c, 'E'); } else { @@ -692,7 +716,7 @@ _worker_poke_work(c3_d evt_d, // event number u3m_pretty_path(wir), u3m_pretty(cad)); u3t_event_trace(lab_c, 'B'); - _worker_work_live(evt_d, mug_l, job); + _worker_work_live(evt_d, job); u3t_event_trace(lab_c, 'E'); } } @@ -773,7 +797,7 @@ _worker_poke(void* vod_p, u3_noun mat) goto error; } - u3_noun entry = u3ke_cue(u3k(jammed_entry)); + u3_noun entry = u3qe_cue(jammed_entry); if ( (c3y != u3du(entry)) || (c3n == u3r_cell(entry, &mug, &job)) || (c3n == u3ud(mug)) || @@ -818,10 +842,9 @@ _worker_poke(void* vod_p, u3_noun mat) } evt_d = u3r_chub(0, evt); - u3z(evt); u3z(jar); - c3_assert( evt_d == u3V.evt_d ); + c3_assert( evt_d == u3V.dun_d ); return u3e_save(); } @@ -844,7 +867,7 @@ u3_worker_boot(void) if ( u3_none != u3A->our ) { u3V.mug_l = u3r_mug(u3A->roc); - nex_d = u3A->ent_d + 1ULL; + nex_d = u3V.dun_d + 1ULL; dat = u3nc(u3_nul, u3nt(u3i_chubs(1, &nex_d), u3V.mug_l, u3nc(u3k(u3A->our), u3k(u3A->fak)))); @@ -877,6 +900,9 @@ main(c3_i argc, c3_c* argv[]) c3_assert(4 == argc); + memset(&u3V, 0, sizeof(u3V)); + memset(&u3_Host.tra_u, 0, sizeof(u3_Host.tra_u)); + /* load passkey */ { @@ -899,19 +925,10 @@ main(c3_i argc, c3_c* argv[]) u3V.dir_c = strdup(dir_c); } - /* clear tracing struct - */ - { - u3_Host.tra_u.nid_w = 0; - u3_Host.tra_u.fil_u = NULL; - u3_Host.tra_u.con_w = 0; - u3_Host.tra_u.fun_w = 0; - } - /* boot image */ { - u3V.evt_d = u3m_boot_new(dir_c); + u3V.sen_d = u3V.dun_d = u3m_boot_new(dir_c); u3C.stderr_log_f = _worker_send_stdr; u3C.slog_f = _worker_send_slog; } diff --git a/sh/cachix b/sh/cachix index 102d7c89f..c29af7be9 100755 --- a/sh/cachix +++ b/sh/cachix @@ -5,6 +5,10 @@ fail () { exit 1 } +if [ -z "$CACHIX_SIGNING_KEY" ] +then fail "The CACHIX_AUTH_TOKEN environment variable needs to be set." +fi + if [ -z "$CACHIX_AUTH_TOKEN" ] then fail "The CACHIX_AUTH_TOKEN environment variable needs to be set." fi @@ -16,7 +20,7 @@ cleanup () { trap cleanup EXIT cachix authtoken "$CACHIX_AUTH_TOKEN" >/dev/null -cachix use urbit -nix-build nix/dep-repos.nix > .cache.list -nix-build default.nix >> .cache.list -cachix push urbit < .cache.list +cachix use urbit2 +nix-build --no-out-link nix/dep-repos.nix > .cache.list +nix-build --no-out-link default.nix >> .cache.list +cachix push urbit2 < .cache.list