From 4a48e16f414ed809efb6313c5846d1d5875a9491 Mon Sep 17 00:00:00 2001
From: Joe Bryan <joemfb@gmail.com>
Date: Fri, 19 Jun 2020 14:32:07 -0700
Subject: [PATCH] vere: refactors king/pier/lord shutdown

---
 pkg/urbit/daemon/main.c       |   4 +-
 pkg/urbit/include/vere/vere.h |  56 +++++--
 pkg/urbit/vere/io/ames.c      |   2 +-
 pkg/urbit/vere/io/term.c      |   5 +-
 pkg/urbit/vere/king.c         | 109 ++++++++++++-
 pkg/urbit/vere/lord.c         | 259 ++++++++++++++++++++++--------
 pkg/urbit/vere/pier.c         | 294 ++++++++++++++++++----------------
 7 files changed, 495 insertions(+), 234 deletions(-)

diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c
index 1c1c66fa1..dec85ab8c 100644
--- a/pkg/urbit/daemon/main.c
+++ b/pkg/urbit/daemon/main.c
@@ -483,7 +483,7 @@ _stop_signal(c3_i int_i)
 {
   //  if we have a pier, unmap the event log before dumping core
   //
-  u3_pier_halt();
+  u3_king_halt();
 }
 
 /*
@@ -581,7 +581,7 @@ _fork_into_background_process()
 static void
 _stop_on_boot_completed_cb()
 {
-  u3_pier_exit(u3_pier_stub());
+  u3_king_exit();
 }
 
 c3_i
diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h
index 85b97a29e..aceb09a64 100644
--- a/pkg/urbit/include/vere/vere.h
+++ b/pkg/urbit/include/vere/vere.h
@@ -413,7 +413,6 @@
             u3_peek*       pek_u;               //  peek
             u3_info        fon_u;               //  recompute
             c3_d           eve_d;               //  save/pack at
-            c3_w           xit_w;               //  exit code
           };
         } u3_writ;
 
@@ -431,7 +430,8 @@
           void (*work_bail_f)(void*, u3_ovum*, u3_noun lud);
           void (*save_f)(void*);
           void (*pack_f)(void*);
-          void (*exit_f)(void*, c3_o);
+          void (*bail_f)(void*);
+          void (*exit_f)(void*);
         } u3_lord_cb;
 
       /* u3_lord: serf controller.
@@ -600,15 +600,14 @@
           // XX remove
           c3_s             por_s;               //  UDP port
           u3_save*         sav_u;               //  autosave
+          struct _u3_pier* nex_u;               //  next in list
         } u3_pier;
 
       /* u3_king: all executing piers.
       */
         typedef struct _u3_king {
           c3_c*          certs_c;               //  ssl certificate dump
-          c3_w             len_w;               //  number used
-          c3_w             all_w;               //  number allocated
-          u3_pier**        tab_u;               //  pier table
+          u3_pier*         pir_u;               //  pier list
           uv_timer_t       tim_u;               //  gc timer
         } u3_king;
 
@@ -885,7 +884,17 @@
       /* u3_lord_exit(): shutdown gracefully.
       */
         void
-        u3_lord_exit(u3_lord* god_u, c3_w cod_w);
+        u3_lord_exit(u3_lord* god_u);
+
+      /* u3_lord_stall(): send SIGINT
+      */
+        void
+        u3_lord_stall(u3_lord* god_u);
+
+      /* u3_lord_halt(): shutdown immediately
+      */
+        void
+        u3_lord_halt(u3_lord* god_u);
 
       /* u3_lord_save(): save a snapshot.
       */
@@ -1204,12 +1213,12 @@
       /* u3_pier_bail(): immediately shutdown..
       */
         void
-        u3_pier_bail(void);
+        u3_pier_bail(u3_pier* pir_u);
 
-      /* u3_pier_halt(): emergency release.
+      /* u3_pier_halt(): emergency resource release (ie, on SIGABRT).
       */
         void
-        u3_pier_halt(void);
+        u3_pier_halt(u3_pier* pir_u);
 
       /* u3_pier_save(): request checkpoint.
       */
@@ -1221,14 +1230,9 @@
         c3_o
         u3_pier_pack(u3_pier* pir_u);
 
-      /* u3_pier_stub(): get the One Pier for unreconstructed code.
-      */
-        u3_pier*
-        u3_pier_stub(void);
-
       /* u3_pier_boot(): start the new pier system.
       */
-        void
+        u3_pier*
         u3_pier_boot(c3_w    wag_w,                 //  config flags
                      u3_noun who,                   //  identity
                      u3_noun ven,                   //  boot event
@@ -1237,7 +1241,7 @@
 
       /* u3_pier_stay(): restart the new pier system.
       */
-        void
+        u3_pier*
         u3_pier_stay(c3_w wag_w, u3_noun pax);
 
       /* u3_pier_tank(): dump single tank.
@@ -1285,6 +1289,26 @@
         void
         u3_king_commence();
 
+      /* u3_king_stub(): get the One Pier for unreconstructed code.
+      */
+        u3_pier*
+        u3_king_stub(void);
+
+      /* u3_king_done(): all piers closed
+      */
+        void
+        u3_king_done(void);
+
+      /* u3_king_exit(): shutdown gracefully
+      */
+        void
+        u3_king_exit(void);
+
+      /* u3_king_halt(): emergency release.
+      */
+        void
+        u3_king_halt(void);
+
       /* u3_king_bail(): immediately shutdown.
       */
         void
diff --git a/pkg/urbit/vere/io/ames.c b/pkg/urbit/vere/io/ames.c
index edd2ac01b..67de82bf2 100644
--- a/pkg/urbit/vere/io/ames.c
+++ b/pkg/urbit/vere/io/ames.c
@@ -463,7 +463,7 @@ _ames_io_start(u3_ames* sam_u)
 
       //  XX revise
       //
-      u3_pier_exit(u3_pier_stub());
+      u3_pier_bail(u3_king_stub());
     }
 
     uv_udp_getsockname(&sam_u->wax_u, (struct sockaddr *)&add_u, &add_i);
diff --git a/pkg/urbit/vere/io/term.c b/pkg/urbit/vere/io/term.c
index f92c3748d..890bd407c 100644
--- a/pkg/urbit/vere/io/term.c
+++ b/pkg/urbit/vere/io/term.c
@@ -737,7 +737,10 @@ _term_suck(u3_utty* uty_u, const c3_y* buf, ssize_t siz_i)
       //  then corrupts the event log), so we force shutdown.
       //
       u3l_log("term: hangup (EOF)\r\n");
-      u3_pier_exit(u3_pier_stub());
+
+      //  XX revise
+      //
+      u3_pier_bail(u3_king_stub());
     }
     else if ( siz_i < 0 ) {
       u3l_log("term %d: read: %s\n", uty_u->tid_l, uv_strerror(siz_i));
diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c
index 021e61573..a859cc06d 100644
--- a/pkg/urbit/vere/king.c
+++ b/pkg/urbit/vere/king.c
@@ -168,7 +168,10 @@ _daemon_boot(u3_noun bul)
 void
 _daemon_fake(u3_noun ship, u3_noun pill, u3_noun path)
 {
-  u3_pier_boot(sag_w, ship, u3nc(c3__fake, u3k(ship)), pill, path);
+  //  XX link properly
+  //
+  u3_noun vent = u3nc(c3__fake, u3k(ship));
+  u3K.pir_u    = u3_pier_boot(sag_w, ship, vent, pill, path);
 }
 
 /* _daemon_come(): mine a comet under star (unit)
@@ -197,7 +200,10 @@ _daemon_dawn(u3_noun seed, u3_noun pill, u3_noun path)
   //
   u3C.slog_f = _daemon_slog;
 
-  u3_pier_boot(sag_w, u3k(u3h(seed)), u3_dawn_vent(seed), pill, path);
+  //  XX link properly
+  //
+  u3_noun vent = u3_dawn_vent(seed);
+  u3K.pir_u = u3_pier_boot(sag_w, u3k(u3h(seed)), vent, pill, path);
 
   // disable ivory slog printfs
   //
@@ -215,7 +221,7 @@ _daemon_pier(u3_noun pier)
     exit(1);
   }
 
-  u3_pier_stay(sag_w, u3k(u3t(pier)));
+  u3K.pir_u = u3_pier_stay(sag_w, u3k(u3t(pier)));
   u3z(pier);
 }
 
@@ -574,7 +580,7 @@ _daemon_sign_cb(uv_signal_t* sil_u, c3_i num_i)
     }
 
     case SIGTERM: {
-      u3_pier_exit(u3_pier_stub());
+      u3_king_exit();
       break;
     }
 
@@ -722,13 +728,104 @@ u3_king_commence()
   _daemon_loop_exit();
 }
 
+/* u3_king_stub(): get the One Pier for unreconstructed code.
+*/
+u3_pier*
+u3_king_stub(void)
+{
+  if ( !u3K.pir_u ) {
+    c3_assert(!"king: no pier");
+  }
+  else {
+    return u3K.pir_u;
+  }
+}
+
+/* _king_forall(): run on all piers
+*/
+static void
+_king_forall(void (*pir_f)(u3_pier*))
+{
+  u3_pier* pir_u = u3K.pir_u;
+
+  while ( pir_u ) {
+    pir_f(pir_u);
+    pir_u = pir_u->nex_u;
+  }
+}
+
+/* _king_forall_unlink(): run on all piers, unlinking from king.
+*/
+static void
+_king_forall_unlink(void (*pir_f)(u3_pier*))
+{
+  u3_pier* pir_u = u3K.pir_u;
+
+  while ( u3K.pir_u ) {
+    u3_pier* pir_u = u3K.pir_u;
+    u3K.pir_u = pir_u->nex_u;
+    pir_f(pir_u);
+  }
+}
+
+/* _king_done_cb():
+*/
+static void
+_king_done_cb(uv_handle_t* han_u)
+{
+  if( UV_EBUSY == uv_loop_close(u3L) ) {
+    //  XX uncomment to debug
+    //
+    // fprintf(stderr, "\r\nking: open libuv handles\r\n");
+    // uv_print_all_handles(u3L, stderr);
+    // fprintf(stderr, "\r\nking: force shutdown\r\n");
+
+    uv_stop(u3L);
+  }
+}
+
+/* u3_king_done(): all piers closed. s/b callback
+*/
+void
+u3_king_done(void)
+{
+  uv_handle_t* han_u = (uv_handle_t*)&u3K.tim_u;
+
+  //  XX hack, if pier's are still linked, we're not actually done
+  //
+  if ( !u3K.pir_u && !uv_is_closing(han_u) ) {
+    uv_close((uv_handle_t*)&u3K.tim_u, _king_done_cb);
+    _daemon_sign_close();
+
+    u3_term_log_exit();
+    fflush(stdout);
+  }
+}
+
+/* u3_king_exit(): shutdown gracefully
+*/
+void
+u3_king_exit(void)
+{
+  _king_forall(u3_pier_exit);
+}
+
+/* u3_king_halt(): emergency release
+*/
+void
+u3_king_halt(void)
+{
+  _king_forall_unlink(u3_pier_halt);
+}
+
 /* u3_king_bail(): immediately shutdown.
 */
 void
 u3_king_bail(void)
 {
+  _king_forall_unlink(u3_pier_bail);
   _daemon_loop_exit();
-  u3_pier_bail();
+  u3_king_done();
   exit(1);
 }
 
@@ -750,7 +847,7 @@ u3_king_grab(void* vod_p)
     c3_c* wen_c = u3r_string(wen);
 
     c3_c nam_c[2048];
-    snprintf(nam_c, 2048, "%s/.urb/put/mass", u3_pier_stub()->pax_c);
+    snprintf(nam_c, 2048, "%s/.urb/put/mass", u3_king_stub()->pax_c);
 
     struct stat st;
     if ( -1 == stat(nam_c, &st) ) {
diff --git a/pkg/urbit/vere/lord.c b/pkg/urbit/vere/lord.c
index 3f918535f..640ae0306 100644
--- a/pkg/urbit/vere/lord.c
+++ b/pkg/urbit/vere/lord.c
@@ -53,6 +53,112 @@
 --
 */
 
+/* _lord_stop_cb(): finally all done.
+*/
+static void
+_lord_stop_cb(void*       ptr_v,
+              const c3_c* err_c)
+{
+  u3_lord* god_u = ptr_v;
+
+  void (*exit_f)(void*) = god_u->cb_u.exit_f;
+  void* exit_v = god_u->cb_u.vod_p;
+
+  c3_free(god_u);
+
+  if ( exit_f ) {
+    exit_f(exit_v);
+  }
+}
+
+/* _lord_writ_free(): dispose of pending writ.
+*/
+static void
+_lord_writ_free(u3_writ* wit_u)
+{
+  switch ( wit_u->typ_e ) {
+    default: c3_assert(0);
+
+    case u3_writ_work: {
+      //  XX confirm
+      //
+      u3_ovum* egg_u = wit_u->wok_u.egg_u;
+      u3_auto_drop(egg_u->car_u, egg_u);
+      u3z(wit_u->wok_u.job);
+    } break;
+
+    case u3_writ_peek: {
+      u3z(wit_u->pek_u->now);
+      u3z(wit_u->pek_u->gan);
+      u3z(wit_u->pek_u->ful);
+    } break;
+
+    case u3_writ_play: {
+      u3_fact* tac_u = wit_u->fon_u.ext_u;
+      u3_fact* nex_u;
+
+      while ( tac_u ) {
+        nex_u = tac_u->nex_u;
+        u3_fact_free(tac_u);
+        tac_u = nex_u;
+      }
+    } break;
+
+    case u3_writ_save:
+    case u3_writ_pack:
+    case u3_writ_exit: {
+    } break;
+  }
+
+  c3_free(wit_u);
+}
+
+/* _lord_bail_noop(): ignore subprocess error on shutdown
+*/
+static void
+_lord_bail_noop(void*       ptr_v,
+                const c3_c* err_c)
+{
+}
+
+/* _lord_stop(): close and dispose all resources.
+*/
+static void
+_lord_stop(u3_lord* god_u)
+{
+  //  dispose outstanding writs
+  //
+  {
+    u3_writ* wit_u = god_u->ext_u;
+    u3_writ* nex_u;
+
+    while ( wit_u ) {
+      nex_u = wit_u->nex_u;
+      _lord_writ_free(wit_u);
+      wit_u = nex_u;
+    }
+
+    god_u->ent_u = god_u->ext_u = 0;
+  }
+
+  u3_newt_moat_stop(&god_u->out_u, _lord_stop_cb);
+  u3_newt_mojo_stop(&god_u->inn_u, _lord_bail_noop);
+
+  uv_close((uv_handle_t*)&god_u->cub_u, 0);
+}
+
+/* _lord_bail(): serf/lord error.
+*/
+static void
+_lord_bail(u3_lord* god_u)
+{
+  void (*bail_f)(void*) = god_u->cb_u.bail_f;
+  void* bail_v = god_u->cb_u.vod_p;
+
+  u3_lord_halt(god_u);
+  bail_f(bail_v);
+}
+
 /* _lord_writ_pop(): pop the writ stack.
 */
 static u3_writ*
@@ -103,58 +209,13 @@ _lord_writ_need(u3_lord* god_u, u3_writ_type typ_e)
     fprintf(stderr, "lord: unexpected %%%s, expected %%%s\r\n",
                     _lord_writ_str(typ_e),
                     _lord_writ_str(wit_u->typ_e));
-    u3_pier_bail();
-    exit(1);
+    _lord_bail(god_u);
+    return 0;
   }
 
   return wit_u;
 }
 
-/* _lord_on_k(): handle subprocess exit.
-*/
-static void
-_lord_on_exit(uv_process_t* req_u,
-              c3_ds         sas_i,
-              c3_i          sig_i)
-{
-  u3_lord* god_u = (void*)req_u;
-
-  {
-    void (*exit_f)(void*, c3_o) = god_u->cb_u.exit_f;
-    void* vod_p = god_u->cb_u.vod_p;
-
-    //  XX correct comparison?
-    //
-    c3_o  ret_o = ( u3_writ_exit != god_u->ext_u->typ_e )
-                  ? c3n
-                  : ( god_u->ext_u->xit_w == sas_i )
-                    ? c3y : c3n;
-
-    //  XX dispose god_u
-    //
-    exit_f(vod_p, c3y);
-  }
-}
-
-/* _lord_bail_noop(): ignore subprocess error on shutdown
-*/
-static void
-_lord_bail_noop(void*       vod_p,
-                const c3_c* err_c)
-{
-}
-
-/* _lord_bail(): handle subprocess error.
-*/
-static void
-_lord_bail(void*       vod_p,
-           const c3_c* err_c)
-{
-  // XX exit?
-  //
-  fprintf(stderr, "\rpier: work error: %s\r\n", err_c);
-}
-
 /* _lord_plea_foul():
 */
 static void
@@ -167,9 +228,11 @@ _lord_plea_foul(u3_lord* god_u, c3_m mot_m, u3_noun dat)
     fprintf(stderr, "lord: received invalid %%%.4s $plea\r\n", (c3_c*)&mot_m);
   }
 
-  u3m_p("plea", dat);
-  u3_pier_bail();
-  exit(1);
+  //  XX can't unconditionally print
+  //
+  // u3m_p("plea", dat);
+
+  _lord_bail(god_u);
 }
 
 /* _lord_plea_live(): hear serf %live ack
@@ -207,8 +270,8 @@ _lord_plea_ripe(u3_lord* god_u, u3_noun dat)
 {
   if ( c3y == god_u->liv_o ) {
     fprintf(stderr, "lord: received unexpected %%ripe\n");
-    u3_pier_bail();
-    exit(1);
+    _lord_bail(god_u);
+    return;
   }
 
   {
@@ -230,8 +293,8 @@ _lord_plea_ripe(u3_lord* god_u, u3_noun dat)
 
     if ( 1 != pro_y ) {
       fprintf(stderr, "pier: unsupported ipc protocol version %u\r\n", pro_y);
-      u3_pier_bail();
-      exit(1);
+      _lord_bail(god_u);
+      return;
     }
 
     god_u->eve_d = eve_d;
@@ -276,6 +339,7 @@ _lord_plea_peek(u3_lord* god_u, u3_noun dat)
   {
     u3_writ* wit_u = _lord_writ_need(god_u, u3_writ_peek);
     pek_u = wit_u->pek_u;
+    c3_free(wit_u);
   }
 
   //  XX cache [dat] (unless last)
@@ -533,12 +597,12 @@ _lord_plea_work(u3_lord* god_u, u3_noun dat)
   u3z(dat);
 }
 
-/* _lord_plea(): handle plea from serf.
+/* _lord_on_plea(): handle plea from serf.
 */
 static void
-_lord_plea(void* vod_p, u3_noun mat)
+_lord_on_plea(void* ptr_v, u3_noun mat)
 {
-  u3_lord* god_u = vod_p;
+  u3_lord* god_u = ptr_v;
   u3_noun    jar = u3ke_cue(mat);
   u3_noun    tag, dat;
 
@@ -634,9 +698,9 @@ _lord_writ_jam(u3_lord* god_u, u3_writ* wit_u)
       } break;
 
       case u3_writ_exit: {
-        //  XX u3_newt_close on send
+        //  requested exit code is always 0
         //
-        msg = u3nt(c3__live, c3__exit, u3i_words(1, &wit_u->xit_w));
+        msg = u3nt(c3__live, c3__exit, 0);
       } break;
     }
 
@@ -649,16 +713,16 @@ _lord_writ_jam(u3_lord* god_u, u3_writ* wit_u)
 static void
 _lord_writ_send(u3_lord* god_u, u3_writ* wit_u)
 {
-  _lord_writ_jam(god_u, wit_u);
-  u3_newt_write(&god_u->inn_u, wit_u->mat);
-  wit_u->mat = 0;
-
-  //  ignore subprocess error on shutdown
+  //  exit expected
   //
   if ( u3_writ_exit == wit_u->typ_e ) {
     god_u->out_u.bal_f = _lord_bail_noop;
     god_u->inn_u.bal_f = _lord_bail_noop;
   }
+
+  _lord_writ_jam(god_u, wit_u);
+  u3_newt_write(&god_u->inn_u, wit_u->mat);
+  wit_u->mat = 0;
 }
 
 /* _lord_writ_plan(): enqueue a writ and send.
@@ -848,12 +912,67 @@ u3_lord_pack(u3_lord* god_u)
 /* u3_lord_exit(): shutdown gracefully.
 */
 void
-u3_lord_exit(u3_lord* god_u, c3_w cod_w)
+u3_lord_exit(u3_lord* god_u)
 {
   u3_writ* wit_u = _lord_writ_new(god_u);
   wit_u->typ_e = u3_writ_exit;
-  wit_u->xit_w = cod_w;
   _lord_writ_plan(god_u, wit_u);
+
+  //  XX set timer, then halt
+}
+
+/* u3_lord_stall(): send SIGINT
+*/
+void
+u3_lord_stall(u3_lord* god_u)
+{
+  uv_process_kill(&god_u->cub_u, SIGINT);
+}
+
+/* u3_lord_halt(): shutdown immediately
+*/
+void
+u3_lord_halt(u3_lord* god_u)
+{
+  //  no exit callback on halt
+  //
+  god_u->cb_u.exit_f = 0;
+
+  uv_process_kill(&god_u->cub_u, SIGKILL);
+  _lord_stop(god_u);
+}
+
+/* _lord_on_serf_exit(): handle subprocess exit.
+*/
+static void
+_lord_on_serf_exit(uv_process_t* req_u,
+                   c3_ds         sas_i,
+                   c3_i          sig_i)
+{
+
+  u3_lord* god_u = (void*)req_u;
+
+  if (  !god_u->ext_u
+     || !(u3_writ_exit == god_u->ext_u->typ_e) )
+  {
+    fprintf(stderr, "pier: work exit: status %" PRId64 ", signal %d\r\n",
+                  sas_i, sig_i);
+    _lord_bail(god_u);
+  }
+  else {
+    _lord_stop(god_u);
+  }
+}
+
+/* _lord_on_serf_bail(): handle subprocess error.
+*/
+static void
+_lord_on_serf_bail(void*       ptr_v,
+                   const c3_c* err_c)
+{
+  u3_lord* god_u = ptr_v;
+  u3l_log("pier: serf error: %s\r\n", err_c);
+  _lord_bail(god_u);
 }
 
 /* u3_lord_init(): instantiate child process.
@@ -921,7 +1040,7 @@ u3_lord_init(c3_c* pax_c, c3_w wag_w, c3_d key_d[4], u3_lord_cb cb_u)
     god_u->ops_u.stdio = god_u->cod_u;
     god_u->ops_u.stdio_count = 3;
 
-    god_u->ops_u.exit_cb = _lord_on_exit;
+    god_u->ops_u.exit_cb = _lord_on_serf_exit;
     god_u->ops_u.file = arg_c[0];
     god_u->ops_u.args = arg_c;
 
@@ -936,13 +1055,13 @@ u3_lord_init(c3_c* pax_c, c3_w wag_w, c3_d key_d[4], u3_lord_cb cb_u)
   //
   {
     god_u->out_u.ptr_v = god_u;
-    god_u->out_u.pok_f = _lord_plea;
-    god_u->out_u.bal_f = _lord_bail;
+    god_u->out_u.pok_f = _lord_on_plea;
+    god_u->out_u.bal_f = _lord_on_serf_bail;
 
     //  XX distinguish from out_u.bal_f ?
     //
     god_u->inn_u.ptr_v = god_u;
-    god_u->inn_u.bal_f = _lord_bail;
+    god_u->inn_u.bal_f = _lord_on_serf_bail;
 
     u3_newt_read(&god_u->out_u);
   }
diff --git a/pkg/urbit/vere/pier.c b/pkg/urbit/vere/pier.c
index 7e6ead75b..d12dd843d 100644
--- a/pkg/urbit/vere/pier.c
+++ b/pkg/urbit/vere/pier.c
@@ -592,7 +592,7 @@ _pier_play(u3_play* pay_u)
       // //  XX graceful shutdown
       // //
       // u3_lord_save(pir_u->god_u);
-      // u3_pier_bail();
+      // u3_pier_bail(pir_u);
       // exit(0);
 
       //  XX temporary hack
@@ -631,7 +631,7 @@ _pier_on_lord_play_done(void* vod_p, u3_info fon_u, c3_l mug_l)
             tac_u->eve_d,
             tac_u->mug_l,
             mug_l);
-    // u3_pier_bail();
+    // u3_pier_bail(pir_u);
   }
 
   //  dispose successful
@@ -680,7 +680,7 @@ _pier_on_lord_play_bail(void* vod_p, u3_info fon_u,
              (c3_d)(eve_d - 1ULL),
              las_l,
              mug_l);
-      // u3_pier_bail();
+      // u3_pier_bail(pir_u);
     }
 
     //  XX enable to retry
@@ -716,7 +716,7 @@ _pier_on_lord_play_bail(void* vod_p, u3_info fon_u,
         u3_pier_punt_ovum("play", u3k(wir), u3k(tag));
       }
 
-      u3_pier_bail();
+      u3_pier_bail(pir_u);
       exit(1);
     }
 #endif
@@ -785,7 +785,7 @@ _pier_on_disk_read_bail(void* vod_p, c3_d eve_d)
   //  
   fprintf(stderr, "pier: disk read bail\r\n");
   u3_term_stop_spinner();
-  u3_pier_bail();
+  u3_pier_bail(pir_u);
 }
 
 /* _pier_on_disk_write_done(): event log write success.
@@ -833,7 +833,7 @@ _pier_on_disk_write_bail(void* vod_p, c3_d eve_d)
   // XX
   //
   fprintf(stderr, "pier: disk write bail\r\n");
-  u3_pier_bail();
+  u3_pier_bail(pir_u);
 }
 
 /* _pier_on_lord_slog(): debug printf from worker.
@@ -883,35 +883,56 @@ _pier_on_lord_pack(void* vod_p)
   //
   if ( u3_psat_play == pir_u->sat_e ) {
     u3l_log("pier: pack complete, shutting down\r\n");
-    u3_pier_bail();
+    u3_pier_bail(pir_u);
     exit(0);
   }
 
   // if ( u3_psat_done == pir_u->sat_e ) {
   //   fprintf(stderr, "snap cb exit\r\n");
-  //   u3_lord_exit(pir_u->god_u, 0);
+  //   u3_lord_exit(pir_u->god_u);
   // }
   // else {
     // _pier_next(pir_u);
   // }
 }
 
+static void
+_pier_done(u3_pier* pir_u);
+
 /* _pier_on_lord_exit(): worker shutdown.
 */
 static void
-_pier_on_lord_exit(void* vod_p, c3_o ret_o)
+_pier_on_lord_exit(void* vod_p)
 {
   u3_pier* pir_u = vod_p;
 
-  if ( u3_psat_done == pir_u->sat_e ) {
-    if ( c3n == ret_o ) {
-      u3l_log("pier: serf shutdown dirty\r\n");
-    }
-  }
-  else {
+  //  the lord has already gone
+  //
+  pir_u->god_u = 0;
+
+  if ( u3_psat_done != pir_u->sat_e ) {
     u3l_log("pier: serf shutdown unexpected\r\n");
-    u3_pier_bail();
+    u3_pier_bail(pir_u);
   }
+  //  if we made it all the way here, it's our jab to wrap up
+  //
+  else {
+    _pier_done(pir_u);
+  }
+}
+
+/* _pier_on_lord_bail(): worker error.
+*/
+static void
+_pier_on_lord_bail(void* vod_p)
+{
+  u3_pier* pir_u = vod_p;
+
+  //  the lord has already gone
+  //
+  pir_u->god_u = 0;
+
+  u3_pier_bail(pir_u);
 }
 
 /* _pier_on_lord_live(): worker is ready.
@@ -1031,6 +1052,7 @@ _pier_init(c3_w wag_w, c3_c* pax_c)
       .work_bail_f = _pier_on_lord_work_bail,
       .save_f = _pier_on_lord_save,
       .pack_f = _pier_on_lord_pack,
+      .bail_f = _pier_on_lord_bail,
       .exit_f = _pier_on_lord_exit
     };
 
@@ -1042,26 +1064,12 @@ _pier_init(c3_w wag_w, c3_c* pax_c)
     }
   }
 
-  //  install in the pier table
-  //
-  //    XX  u3_king_plan
-  //
-  if ( 0 == u3K.all_w ) {
-    u3K.all_w = 16;
-    u3K.tab_u = c3_malloc(16 * sizeof(u3_pier*));
-  }
-  if ( u3K.len_w == u3K.all_w ) {
-    u3K.all_w = 2 * u3K.all_w;
-    u3K.tab_u = c3_realloc(u3K.tab_u, u3K.all_w * sizeof(u3_pier*));
-  }
-  u3K.tab_u[u3K.len_w++] = pir_u;
-
   return pir_u;
 }
 
 /* u3_pier_stay(): restart an existing pier.
 */
-void
+u3_pier*
 u3_pier_stay(c3_w wag_w, u3_noun pax)
 {
   u3_pier* pir_u = _pier_init(wag_w, u3r_string(pax));
@@ -1072,11 +1080,13 @@ u3_pier_stay(c3_w wag_w, u3_noun pax)
     fprintf(stderr, "pier: disk read meta fail\r\n");
     //  XX dispose
     //
-    u3_pier_bail();
+    u3_pier_bail(pir_u);
     exit(1);
   }
 
   u3z(pax);
+
+  return pir_u;
 }
 
 /* _pier_pill_parse(): extract boot formulas and module/userspace ova from pill
@@ -1265,7 +1275,7 @@ _pier_boot_plan(u3_pier* pir_u, u3_noun who, u3_noun ven, u3_noun pil)
 
 /* u3_pier_boot(): start a new pier.
 */
-void
+u3_pier*
 u3_pier_boot(c3_w  wag_w,                   //  config flags
              u3_noun who,                   //  identity
              u3_noun ven,                   //  boot event
@@ -1278,11 +1288,13 @@ u3_pier_boot(c3_w  wag_w,                   //  config flags
     fprintf(stderr, "pier: boot plan fail\r\n");
     //  XX dispose
     //
-    u3_pier_bail();
+    u3_pier_bail(pir_u);
     exit(1);
   }
 
   u3z(pax);
+
+  return pir_u;
 }
 
 static void
@@ -1353,29 +1365,51 @@ u3_pier_pack(u3_pier* pir_u)
 }
 
 static void
-_pier_exit_work_cb(uv_handle_t* idl_u)
+_pier_work_close_cb(uv_handle_t* idl_u)
 {
   u3_work* wok_u = idl_u->data;
   c3_free(wok_u);
 }
 
-/* _pier_exit_cb(): synchronous shutdown.
+static void
+_pier_work_close(u3_work* wok_u)
+{
+  u3_auto_exit(wok_u->car_u);
+
+  //  free pending effects
+  //
+  {
+    u3_gift* gif_u = wok_u->fec_u.ext_u;
+    u3_gift* nex_u;
+
+    while ( gif_u ) {
+      nex_u = gif_u->nex_u;
+      u3_gift_free(gif_u);
+      gif_u = nex_u;
+    }
+  }
+
+  uv_close((uv_handle_t*)&wok_u->pep_u, _pier_work_close_cb);
+  uv_close((uv_handle_t*)&wok_u->cek_u, 0);
+  uv_close((uv_handle_t*)&wok_u->idl_u, 0);
+  wok_u->pep_u.data = wok_u;
+}
+
+static void
+_pier_done(u3_pier* pir_u)
+{
+  //  XX unlink properly
+  //
+  u3K.pir_u = 0;
+  u3_king_done();
+}
+
+/* _pier_exit(): synchronous shutdown.
 */
 static void
-_pier_exit_cb(void* vod_p, c3_d eve_d)
+_pier_exit(u3_pier* pir_u)
 {
-  u3_pier* pir_u = vod_p;
-
-  if ( pir_u->wok_u ) {
-    u3_work* wok_u = pir_u->wok_u;
-    u3_auto_exit(wok_u->car_u);
-    //  XX confirm, libuv close callback are fired with a stack discipline
-    //
-    uv_close((uv_handle_t*)&wok_u->pep_u, _pier_exit_work_cb);
-    uv_close((uv_handle_t*)&wok_u->cek_u, 0);
-    uv_close((uv_handle_t*)&wok_u->idl_u, 0);
-    pir_u->wok_u = 0;
-  }
+  c3_assert( u3_psat_done == pir_u->sat_e );
   
   if ( pir_u->log_u ) {
     u3_disk_exit(pir_u->log_u);
@@ -1383,17 +1417,46 @@ _pier_exit_cb(void* vod_p, c3_d eve_d)
   }
 
   if ( pir_u->god_u ) {
-    u3_lord_exit(pir_u->god_u, 0);
+    u3_lord_exit(pir_u->god_u);
     pir_u->god_u = 0;
   }
+  else {
+    //  otherwise called in _pier_on_lord_exit()
+    //
+    _pier_done(pir_u);
+  }
+}
 
-  u3_term_log_exit();
+/* _pier_work_exit(): commence graceful shutdown.
+*/
+static void
+_pier_work_exit_cb(void* vod_p, c3_d eve_d)
+{
+  u3_pier* pir_u = vod_p;
 
-  //  XX uninstall pier from u3K.tab_u, dispose
+  _pier_work_close(pir_u->wok_u);
+  pir_u->wok_u = 0;
 
-  //  XX no can do
+  _pier_exit(pir_u);
+}
+
+/* _pier_work_exit(): setup graceful shutdown callbacks.
+*/
+static void
+_pier_work_exit(u3_pier* pir_u)
+{
+  _pier_wall_plan(pir_u, 0, pir_u, _pier_save_cb);
+  _pier_wall_plan(pir_u, 0, pir_u, _pier_work_exit_cb);
+
+  //  XX moveme, XX bails if not started
   //
-  uv_stop(u3L);
+  {
+    c3_l cod_l = u3a_lush(c3__save);
+    u3_save_io_exit(pir_u);
+    u3a_lop(cod_l);
+  }
+
+  pir_u->sat_e = u3_psat_done;
 }
 
 /* u3_pier_exit(): graceful shutdown.
@@ -1401,60 +1464,21 @@ _pier_exit_cb(void* vod_p, c3_d eve_d)
 void
 u3_pier_exit(u3_pier* pir_u)
 {
-  // fprintf(stderr, "pier: exit\r\n");
-
   switch ( pir_u->sat_e ) {
-    //  XX specifically handle init/done?
-    //
     default: {
-      fprintf(stderr, "pier: unexpected exit: %u\r\n", pir_u->sat_e);
+      fprintf(stderr, "pier: unknown exit: %u\r\n", pir_u->sat_e);
       c3_assert(0);
     }
 
-    case u3_psat_boot: {
-      //  XX properly dispose boot
-      //
-      c3_free(pir_u->bot_u);
-      pir_u->bot_u = 0;
-      _pier_exit_cb(pir_u, 0);
-    } break;
+    case u3_psat_done: return;
 
-    case u3_psat_play: {
-      //  XX dispose play q
-      //
-      c3_free(pir_u->pay_u);
-      pir_u->pay_u = 0;
-      _pier_exit_cb(pir_u, 0);
-    } break;
+    case u3_psat_work: return _pier_work_exit(pir_u);
 
-    case u3_psat_work: {
-      _pier_wall_plan(pir_u, 0, pir_u, _pier_save_cb);
-      _pier_wall_plan(pir_u, 0, pir_u, _pier_exit_cb);
-
-      //  XX moveme
-      //
-      {
-        c3_l cod_l = u3a_lush(c3__save);
-        u3_save_io_exit(pir_u);
-        u3a_lop(cod_l);
-      }
-    } break;
-  }
-
-  u3K.len_w = 0;
-  pir_u->sat_e = u3_psat_done;
-}
-
-/* _pier_exit_done(): force synchronous shut down.
-*/
-static void
-_pier_exit_done(u3_pier* pir_u)
-{
-  switch ( pir_u->sat_e ) {
-    default: break;
+    case u3_psat_init: break;
 
     case u3_psat_boot: {
       //  XX properly dispose boot
+      //  XX also on actual boot
       //
       c3_free(pir_u->bot_u);
       pir_u->bot_u = 0;
@@ -1466,47 +1490,54 @@ _pier_exit_done(u3_pier* pir_u)
       c3_free(pir_u->pay_u);
       pir_u->pay_u = 0;
     } break;
-
-    case u3_psat_work: {
-      //  XX moveme
-      //
-      {
-        c3_l cod_l = u3a_lush(c3__save);
-        u3_save_io_exit(pir_u);
-        u3a_lop(cod_l);
-      }
-    } break;
   }
 
-  u3K.len_w = 0;
   pir_u->sat_e = u3_psat_done;
-  _pier_exit_cb(pir_u, 0);
+  _pier_exit(pir_u);
 }
 
-/* u3_pier_bail(): immediately shutdown.
+/* u3_pier_bail(): immediately shutdown due to error.
 */
 void
-u3_pier_bail(void)
+u3_pier_bail(u3_pier* pir_u)
 {
-  if ( 0 != u3K.len_w ) {
-    _pier_exit_done(u3_pier_stub());
+  pir_u->sat_e = u3_psat_done;
+
+  //
+  if ( pir_u->god_u ) {
+    u3_lord_halt(pir_u->god_u);
+    pir_u->god_u = 0;
   }
 
-  fflush(stdout);
-  exit(1);
+  //  exig i/o drivers
+  //
+  if ( pir_u->wok_u ) {
+    _pier_work_close(pir_u->wok_u);
+    pir_u->wok_u = 0;
+  }
+
+  //  close db
+  //
+  if ( pir_u->log_u ) {
+    u3_disk_exit(pir_u->log_u);
+    pir_u->log_u = 0;
+  }
+
+  _pier_done(pir_u);
 }
 
-/* u3_pier_halt(): emergency release.
+/* u3_pier_halt(): emergency resource release (ie, on SIGABRT).
 */
 void
-u3_pier_halt(void)
+u3_pier_halt(u3_pier* pir_u)
 {
-  if ( 0 != u3K.len_w ) {
-    u3_disk_exit(u3_pier_stub()->log_u);
-
-    //  we should only ever try this trick once
-    //
-    u3K.len_w = 0;
+  //  unmap disk if present
+  //
+  //    XX maybe skip close/cancel/free. and just unmap
+  //
+  if ( pir_u->log_u ) {
+    u3_disk_exit(pir_u->log_u);
+    pir_u->log_u = 0;
   }
 }
 
@@ -1519,7 +1550,7 @@ c3_rand(c3_w* rad_w)
     fprintf(stderr, "c3_rand getentropy: %s\n", strerror(errno));
     //  XX review
     //
-    u3_pier_bail();
+    u3_king_bail();
   }
 }
 
@@ -1691,19 +1722,6 @@ u3_pier_sway(c3_l tab_l, u3_noun tax)
   u3z(mok);
 }
 
-/* u3_pier_stub(): get the One Pier for unreconstructed code.
-*/
-u3_pier*
-u3_pier_stub(void)
-{
-  if ( 0 == u3K.len_w ) {
-    c3_assert(!"plan: no pier");
-  }
-  else {
-    return u3K.tab_u[0];
-  }
-}
-
 /* u3_pier_mark(): mark all Loom allocations in all u3_pier structs.
 */
 c3_w