Merge pull request #3942 from urbit/ixv/vere-imp-exp

vere: add flags for import and export

pkg/arvo/app/dojo.hoon

@@ -1144,6 +1144,8 @@
             $listen-api  !!
             $export      !!
             $import      !!
+            $export-all  !!
+            $import-all  !!
             $as
           :*  %as  mar.source.com
               $(num +(num), source.com next.source.com)

pkg/arvo/app/lens.hoon

@@ -14,6 +14,28 @@
           job=(unit [eyre-id=@ta com=command:lens])
       ==
   ==
+::
+++  export-app
+  |=  [app=@tas our=@p now=@da]
+  .^(@ %gx /(scot %p our)/[app]/(scot %da now)/export/noun)
+++  export-all
+  |=  [our=@p now=@da]
+  ^-  (list [@tas @])
+  %+  turn
+    ^-  (list @tas)
+    :~  %group-store
+        %metadata-store
+        %metadata-hook
+        %contact-store
+        %contact-hook
+        %invite-store
+        %chat-store
+        %chat-hook
+        %publish
+        %graph-store
+    ==
+  |=  app=@tas
+  [app (export-app app our now)]
 --
 ::
 =|  =state
@@ -43,12 +65,15 @@
   =/  com=command:lens
     (json:grab:lens-mark jon)
   ::
-  ?:  ?=(%export -.source.com)
+  ?+  -.source.com
-    ~&  [%export app.source.com]
+    :_  this(job.state (some [eyre-id com]))
+    [%pass /sole %agent [our.bowl %dojo] %watch /sole/[eyre-id]]~
+  ::
+      %export
     :_  this(job.state (some [eyre-id com]))
     [%pass /export %agent [our.bowl app.source.com] %watch /export]~
   ::
-  ?:  ?=(%import -.source.com)
+      %import
     ?~  enc=(de:base64 base64-jam.source.com)
       !!
     ::
@@ -57,8 +82,28 @@
     :_  this(job.state (some [eyre-id com]))
     [%pass /import %agent [our.bowl app.source.com] %poke %import !>(c)]~
   ::
-  :_  this(job.state (some [eyre-id com]))
+      %export-all
-  [%pass /sole %agent [our.bowl %dojo] %watch /sole/[eyre-id]]~
+    =/  output  (crip "{<our.bowl>}-export/atom")
+    =/  jon
+      =/  =atom  (jam (export-all our.bowl now.bowl))
+      =/  =octs  [(met 3 atom) atom]
+      =/  enc    (en:base64 octs)
+      (pairs:enjs:format file+s+output data+s+enc ~)
+    :_  this
+    %+  give-simple-payload:app  eyre-id
+    (json-response:gen jon)
+  ::
+      %import-all
+    =/  enc  (de:base64 base64-jam.source.com)
+    ?~  enc  !!
+    =/  by-app  ;;((list [@tas @]) (cue q.u.enc))
+    :_  this
+    %+  weld  (give-simple-payload:app eyre-id not-found:gen)
+    %+  turn  by-app
+    |=  [app=@tas data=@]
+    ^-  card:agent:gall
+    [%pass /import-all %agent [our.bowl app] %poke %import !>(data)]
+  ==
 ::
 ++  on-watch
   |=  =path
@@ -68,7 +113,13 @@
   (on-watch:def path)
 ::
 ++  on-leave  on-leave:def
-++  on-peek   on-peek:def
+++  on-peek
+  |=  =path
+  ^-  (unit (unit cage))
+  ?+  path  (on-peek:def path)
+    [%x %export-all ~]
+    ``noun+!>((jam (export-all our.bowl now.bowl)))
+  ==
 ++  on-agent
   |=  [=wire =sign:agent:gall]
   ^-  (quip card:agent:gall _this)

pkg/arvo/mar/lens/command.hoon

@@ -42,6 +42,8 @@
         listen-api+(su ;~(plug sym ;~(pfix col sym)))
         export+so
         import+(ot app+so base64-jam+so ~)
+        export-all+none
+        import-all+(ot base64-jam+so ~)
         as+(ot mark+(su sym) next+source ~)
         hoon+(ot code+so next+source ~)
       ==

pkg/arvo/sur/lens.hoon

@@ -17,6 +17,8 @@
       {$listen-api api/term event/term}
       {$export app/@t}
       {$import app/@t base64-jam/@t}
+      {$export-all ~}
+      {$import-all base64-jam/@t}
   ==
 ++  sink
   $%  {$stdout ~}

pkg/herb/herb

@@ -145,6 +145,34 @@ class importFileAction(argparse.Action):
 
         res.source = {"import": {"app": new_value, "base64-jam": base_data}}
 
+class importAllAction(argparse.Action):
+    """Handles the import-all statement.
+
+    The --import-all statement reads in a jammed noun file from the path passed
+    in and stuffs it the base64 encoded version which gets passed into your
+    Urbit.
+
+    """
+    def __call__(self, parser, res, new_value, option_string):
+        logging.debug('%r %r' % (new_value, option_string))
+        logging.debug('source %s' % res.source)
+        logging.debug('level %s' % res.level)
+
+        # We check to see if there's a "{new_value}" file in the current
+        # working directory. If there isn't, we error
+        data = ""
+        filename = new_value
+        with open(filename, 'rb') as f:
+            data = f.read()
+
+        if data == "":
+            raise ValueError('Failed to read jamfile')
+
+        base_data = base64.b64encode(data)
+
+        res.source = {"import-all": {"base64-jam": base_data}}
+
+
 class transformerAction(argparse.Action):
     """Handle transformer flag.
 
@@ -350,6 +378,13 @@ parser.add_argument('-i', '--import',
                     metavar='app-name',
                     help='imports the application state',
                     action=importFileAction)
+parser.add_argument('-E', '--export-all', const={'export-all': None},
+                    help='exports data from all landscape apps',
+                    action='store_const', dest='source')
+parser.add_argument('-I', '--import-all',
+                    metavar='jam-file',
+                    help='imports data for all landscape apps',
+                    action=importAllAction)
 parser.add_argument('-m', '--mark', which='as',
                     metavar='mark',
                     help='transform a source to another mark',
@@ -398,10 +433,8 @@ sinks.add_argument('-p', '--app', which='app',
                    metavar='app',
                    action=sinkAction)
 
-
 args = parser.parse_args(args)
 
-
 if args.source is None:
     args.source = {"data": ''.join(sys.stdin)}
 

pkg/urbit/daemon/main.c

@@ -74,6 +74,7 @@ _main_getopt(c3_i argc, c3_c** argv)
   u3_Host.ops_u.abo = c3n;
   u3_Host.ops_u.dem = c3n;
   u3_Host.ops_u.dry = c3n;
+  u3_Host.ops_u.exp = c3n;
   u3_Host.ops_u.gab = c3n;
   u3_Host.ops_u.git = c3n;
 
@@ -96,7 +97,7 @@ _main_getopt(c3_i argc, c3_c** argv)
   u3_Host.ops_u.kno_w = DefaultKernel;
 
   while ( -1 != (ch_i=getopt(argc, argv,
-                 "X:Y:G:J:B:K:A:H:I:C:w:u:e:F:k:n:p:r:LljacdgqstvxPDRS")) )
+                 "X:Y:G:J:B:K:A:H:I:C:w:u:e:F:k:n:p:r:i:LljacdgoqstvxPDRS")) )
   {
     switch ( ch_i ) {
       case 'X': {
@@ -187,6 +188,10 @@ _main_getopt(c3_i argc, c3_c** argv)
         u3_Host.ops_u.roc_c = strdup(optarg);
         break;
       }
+      case 'i': {
+        u3_Host.ops_u.imp_c = strdup(optarg);
+        break;
+      }
       case 'L': { u3_Host.ops_u.net = c3n; break; }
       case 'l': { u3_Host.ops_u.lit = c3y; break; }
       case 'j': { u3_Host.ops_u.tra = c3y; break; }
@@ -194,6 +199,7 @@ _main_getopt(c3_i argc, c3_c** argv)
       case 'c': { u3_Host.ops_u.nuu = c3y; break; }
       case 'd': { u3_Host.ops_u.dem = c3y; break; }
       case 'g': { u3_Host.ops_u.gab = c3y; break; }
+      case 'o': { u3_Host.ops_u.exp = c3y; break; }
       case 'P': { u3_Host.ops_u.pro = c3y; break; }
       case 'D': { u3_Host.ops_u.dry = c3y; break; }
       case 'q': { u3_Host.ops_u.qui = c3y; break; }
@@ -398,10 +404,12 @@ u3_ve_usage(c3_i argc, c3_c** argv)
     "-e url        Ethereum gateway\n",
     "-F ship       Fake keys; also disables networking\n",
     "-g            Set GC flag\n",
+    "-i jam_file   import pier state\n",
     "-j            Create json trace file in .urb/put/trace\n",
     "-K stage      Start at Hoon kernel version stage\n",
     "-k keys       Private key file\n",
     "-L            local networking only\n",
+    "-o            export pier state\n",
     "-P            Profiling\n",
     "-p ames_port  Set the ames port to bind to\n",
     "-q            Quiet\n",
@@ -415,7 +423,7 @@ u3_ve_usage(c3_i argc, c3_c** argv)
     "-w name       Boot as ~name\n",
     "-X path       Scry, jam to file, then exit\n"
     "-x            Exit immediately\n",
-    "-Y file       Optional name of jamfile (for -X)\n"
+    "-Y file       Optional name of jamfile (for -X and -o)\n"
     "\n",
     "Development Usage:\n",
     "   To create a development ship, use a fakezod:\n",

pkg/urbit/include/vere/vere.h

@@ -265,6 +265,8 @@
         c3_o    gab;                        //  -g, test garbage collection
         c3_c*   dns_c;                      //  -H, ames bootstrap domain
         c3_c*   jin_c;                      //  -I, inject raw event
+        c3_c*   imp_c;                      //  -i, import pier state
+        c3_o    exp;                        //  -o, export pier state
         c3_w    hap_w;                      //  -C, cap memo cache
         c3_c*   lit_c;                      //  -J, ivory (fastboot) kernel
         c3_o    tra;                        //  -j, json trace

pkg/urbit/vere/io/fore.c

@@ -22,6 +22,17 @@ _fore_inject_bail(u3_ovum* egg_u, u3_noun lud)
   u3_ovum_free(egg_u);
 }
 
+/* _fore_import_bail(): handle failure on arbitrary injection.
+*/
+static void
+_fore_import_bail(u3_ovum* egg_u, u3_noun lud)
+{
+  u3_auto_bail_slog(egg_u, lud);
+  u3l_log("pier: import failed\n");
+
+  u3_ovum_free(egg_u);
+}
+
 /* _fore_inject(): inject an arbitrary ovum from a jammed file at [pax_c].
 */
 static void
@@ -69,6 +80,37 @@ _fore_inject(u3_auto* car_u, c3_c* pax_c)
   u3z(ovo);
 }
 
+/* _fore_import(): form an ovum from jammed archive at [pax_c] and inject it.
+*/
+static void
+_fore_import(u3_auto* car_u, c3_c* pax_c)
+{
+  // With apologies
+  u3_noun arc  = u3ke_cue(u3m_file(pax_c));
+  u3_noun b64  = u3do("crip", u3do("en-base64:mimes:html", arc));
+  c3_c * b64_c = u3r_string(b64);
+
+  c3_w siz_w   = strlen(b64_c) + 120;
+  c3_c * bod_c = (c3_c *) c3_malloc(siz_w);
+  snprintf(bod_c, siz_w,
+    "{\"source\": {\"import-all\": {\"base64-jam\": \"%s\"}}, \
+    \"sink\": {\"stdout\": null}}", b64_c);
+
+  u3_noun dat = u3nt(u3_nul, u3i_word(strlen(bod_c)), u3i_string(bod_c));
+  u3_noun req = u3nt(c3n,
+    u3nc(u3i_string("ipv4"), u3i_word(0x7f000001)),
+    u3nq(u3i_string("POST"), u3i_string("/"), u3_nul, dat));
+  u3_noun wir = u3nc(u3i_string("http-server"), u3_nul);
+  u3_noun cad = u3nc(u3i_string("request-local"), req);
+  u3_auto_peer(
+    u3_auto_plan(car_u, u3_ovum_init(0, c3__e, wir, cad)),
+    0, 0, _fore_import_bail);
+
+  u3z(b64);
+  c3_free(b64_c);
+  c3_free(bod_c);
+}
+
 /* _fore_io_talk():
 */
 static void
@@ -106,6 +148,10 @@ _fore_io_talk(u3_auto* car_u)
   if ( u3_Host.ops_u.jin_c ) {
     _fore_inject(car_u, u3_Host.ops_u.jin_c);
   }
+
+  if ( u3_Host.ops_u.imp_c ) {
+    _fore_import(car_u, u3_Host.ops_u.imp_c);
+  }
 }
 
 /* _fore_io_kick(): handle no effects.

pkg/urbit/vere/pier.c

@@ -477,6 +477,51 @@ u3_pier_peek_last(u3_pier*   pir_u,
   _pier_peek_plan(pir_u, pic_u);
 }
 
+/* _pier_on_scry_done(): scry callback.
+*/
+static void
+_pier_on_scry_done(void* ptr_v, u3_noun nun)
+{
+  u3_pier* pir_u = ptr_v;
+  u3_weak res = u3r_at(7, nun);
+
+  if (u3_none == res) {
+    u3l_log("pier: scry failed\n");
+  }
+  else {
+    u3l_log("pier: scry succeeded\n");
+
+    c3_c* pac_c = u3_Host.ops_u.puk_c;
+    if (!pac_c) {
+      pac_c = u3_Host.ops_u.pek_c;
+    }
+
+    u3_noun pad;
+    {
+      //  XX crashes if [pac_c] is not a valid path
+      //  XX virtualize or fix
+      //
+      u3_noun pax = u3do("stab", u3i_string(pac_c));
+      c3_w len_w = u3kb_lent(u3k(pax));
+      pad = u3nt(c3_s4('.','u','r','b'),
+                 c3_s3('p','u','t'),
+                 u3qb_scag(len_w - 1, pax));
+      u3z(pax);
+    }
+
+    c3_c fil_c[2048];
+    snprintf(fil_c, 2048, "%s/.urb/put/%s.jam", pir_u->pax_c, pac_c+1);
+
+    u3_walk_save(fil_c, 0, u3qe_jam(res), pir_u->pax_c, pad);
+    u3l_log("pier: scry in %s\n", fil_c);
+  }
+
+  u3l_log("pier: exit\n");
+  u3_pier_exit(pir_u);
+
+  u3z(nun);
+}
+
 /* _pier_work_init(): begin processing new events
 */
 static void
@@ -529,11 +574,6 @@ _pier_work_init(u3_pier* pir_u)
   uv_idle_init(u3L, &wok_u->idl_u);
   wok_u->idl_u.data = wok_u;
 
-  //  initialize i/o drivers
-  //
-  wok_u->car_u = u3_auto_init(pir_u);
-  u3_auto_talk(wok_u->car_u);
-
   // //  setup u3_lord work callbacks
   // //
   // u3_lord_work_cb cb_u = {
@@ -545,6 +585,51 @@ _pier_work_init(u3_pier* pir_u)
   // };
   // u3_lord_work_init(pir_u->god_u, cb_u);
 
+  //  XX this is messy, revise
+  //
+  if ( u3_Host.ops_u.pek_c ) {
+    u3_noun pex = u3do("stab", u3i_string(u3_Host.ops_u.pek_c));
+    u3_noun car;
+    u3_noun dek;
+    u3_noun pax;
+    if ( c3n == u3r_trel(pex, &car, &dek, &pax)
+      || c3n == u3a_is_cat(car) )
+    {
+      u3m_p("pier: invalid scry", pex);
+      _pier_on_scry_done(pir_u, u3_nul);
+    } else {
+      //  run the requested scry, jam to disk, then exit
+      //
+      u3l_log("pier: scry\n");
+      u3_pier_peek_last(pir_u, u3_nul, u3k(car), u3k(dek), u3k(pax),
+                        pir_u, _pier_on_scry_done);
+    }
+    u3z(pex);
+  }
+  else if ( _(u3_Host.ops_u.exp) ) {
+    u3_noun pex = u3do("stab", u3i_string("/gx/lens/export-all/noun"));
+    u3_noun car;
+    u3_noun dek;
+    u3_noun pax;
+    u3r_trel(pex, &car, &dek, &pax);
+    if (!u3_Host.ops_u.puk_c) {
+      u3_Host.ops_u.puk_c = strdup("/archive");
+    }
+    //  run the requested scry, jam to disk, then exit
+    //
+    u3l_log("pier: scry\n");
+    u3_pier_peek_last(pir_u, u3_nul, u3k(car), u3k(dek), u3k(pax),
+                      pir_u, _pier_on_scry_done);
+    u3z(pex);
+
+  }
+  else {
+    //  initialize i/o drivers
+    //
+    wok_u->car_u = u3_auto_init(pir_u);
+    u3_auto_talk(wok_u->car_u);
+  }
+
   _pier_work(wok_u);
 }
 
@@ -1068,51 +1153,6 @@ _pier_on_lord_bail(void* ptr_v)
   u3_pier_bail(pir_u);
 }
 
-/* _pier_on_scry_done(): scry callback.
-*/
-static void
-_pier_on_scry_done(void* ptr_v, u3_noun nun)
-{
-  u3_pier* pir_u = ptr_v;
-  u3_weak res = u3r_at(7, nun);
-
-  if (u3_none == res) {
-    u3l_log("pier: scry failed\n");
-  }
-  else {
-    u3l_log("pier: scry succeeded\n");
-
-    c3_c* pac_c = u3_Host.ops_u.puk_c;
-    if (!pac_c) {
-      pac_c = u3_Host.ops_u.pek_c;
-    }
-
-    u3_noun pad;
-    {
-      //  XX crashes if [pac_c] is not a valid path
-      //  XX virtualize or fix
-      //
-      u3_noun pax = u3do("stab", u3i_string(pac_c));
-      c3_w len_w = u3kb_lent(u3k(pax));
-      pad = u3nt(c3_s4('.','u','r','b'),
-                 c3_s3('p','u','t'),
-                 u3qb_scag(len_w - 1, pax));
-      u3z(pax);
-    }
-
-    c3_c fil_c[2048];
-    snprintf(fil_c, 2048, "%s/.urb/put/%s.jam", pir_u->pax_c, pac_c+1);
-
-    u3_walk_save(fil_c, 0, u3qe_jam(res), pir_u->pax_c, pad);
-    u3l_log("pier: scry in %s\n", fil_c);
-  }
-
-  u3l_log("pier: exit");
-  u3_pier_exit(pir_u);
-
-  u3z(nun);
-}
-
 /* _pier_on_lord_live(): worker is ready.
 */
 static void
@@ -1143,26 +1183,7 @@ _pier_on_lord_live(void* ptr_v)
     c3_assert( u3_psat_init == pir_u->sat_e );
     c3_assert( log_u->sen_d == log_u->dun_d );
 
-    if (u3_Host.ops_u.pek_c) {
+    if ( god_u->eve_d < log_u->dun_d ) {
-      u3_noun pex = u3do("stab", u3i_string(u3_Host.ops_u.pek_c));
-      u3_noun car;
-      u3_noun dek;
-      u3_noun pax;
-      if ( c3n == u3r_trel(pex, &car, &dek, &pax)
-        || c3n == u3a_is_cat(car) )
-      {
-        u3m_p("pier: invalid scry", pex);
-        _pier_on_scry_done(pir_u, u3_nul);
-      } else {
-        //  run the requested scry, jam to disk, then exit
-        //
-        u3l_log("pier: scry\n");
-        u3_pier_peek_last(pir_u, u3_nul, u3k(car), u3k(dek), u3k(pax),
-                          pir_u, _pier_on_scry_done);
-      }
-      u3z(pex);
-    }
-    else if ( god_u->eve_d < log_u->dun_d ) {
       c3_d eve_d;
 
       //  XX revisit