From bd4edb7fa4b57bc4ff28cc5b9ca50a34f8bcdc9c Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Fri, 13 Nov 2020 14:31:24 -0800 Subject: [PATCH 1/4] vere: refactors pier initialization to fix -X --- pkg/urbit/vere/pier.c | 144 ++++++++++++++++++++++-------------------- 1 file changed, 74 insertions(+), 70 deletions(-) diff --git a/pkg/urbit/vere/pier.c b/pkg/urbit/vere/pier.c index 0da9f6bb19..fd0828a40e 100644 --- a/pkg/urbit/vere/pier.c +++ b/pkg/urbit/vere/pier.c @@ -477,6 +477,51 @@ u3_pier_peek_last(u3_pier* pir_u, _pier_peek_plan(pir_u, pic_u); } +/* _pier_on_scry_done(): scry callback. +*/ +static void +_pier_on_scry_done(void* ptr_v, u3_noun nun) +{ + u3_pier* pir_u = ptr_v; + u3_weak res = u3r_at(7, nun); + + if (u3_none == res) { + u3l_log("pier: scry failed\n"); + } + else { + u3l_log("pier: scry succeeded\n"); + + c3_c* pac_c = u3_Host.ops_u.puk_c; + if (!pac_c) { + pac_c = u3_Host.ops_u.pek_c; + } + + u3_noun pad; + { + // XX crashes if [pac_c] is not a valid path + // XX virtualize or fix + // + u3_noun pax = u3do("stab", u3i_string(pac_c)); + c3_w len_w = u3kb_lent(u3k(pax)); + pad = u3nt(c3_s4('.','u','r','b'), + c3_s3('p','u','t'), + u3qb_scag(len_w - 1, pax)); + u3z(pax); + } + + c3_c fil_c[2048]; + snprintf(fil_c, 2048, "%s/.urb/put/%s.jam", pir_u->pax_c, pac_c+1); + + u3_walk_save(fil_c, 0, u3qe_jam(res), pir_u->pax_c, pad); + u3l_log("pier: scry in %s\n", fil_c); + } + + u3l_log("pier: exit\n"); + u3_pier_exit(pir_u); + + u3z(nun); +} + /* _pier_work_init(): begin processing new events */ static void @@ -529,11 +574,6 @@ _pier_work_init(u3_pier* pir_u) uv_idle_init(u3L, &wok_u->idl_u); wok_u->idl_u.data = wok_u; - // initialize i/o drivers - // - wok_u->car_u = u3_auto_init(pir_u); - u3_auto_talk(wok_u->car_u); - // // setup u3_lord work callbacks // // // u3_lord_work_cb cb_u = { @@ -545,6 +585,34 @@ _pier_work_init(u3_pier* pir_u) // }; // u3_lord_work_init(pir_u->god_u, cb_u); + // XX this is messy, revise + // + if ( u3_Host.ops_u.pek_c ) { + u3_noun pex = u3do("stab", u3i_string(u3_Host.ops_u.pek_c)); + u3_noun car; + u3_noun dek; + u3_noun pax; + if ( c3n == u3r_trel(pex, &car, &dek, &pax) + || c3n == u3a_is_cat(car) ) + { + u3m_p("pier: invalid scry", pex); + _pier_on_scry_done(pir_u, u3_nul); + } else { + // run the requested scry, jam to disk, then exit + // + u3l_log("pier: scry\n"); + u3_pier_peek_last(pir_u, u3_nul, u3k(car), u3k(dek), u3k(pax), + pir_u, _pier_on_scry_done); + } + u3z(pex); + } + else { + // initialize i/o drivers + // + wok_u->car_u = u3_auto_init(pir_u); + u3_auto_talk(wok_u->car_u); + } + _pier_work(wok_u); } @@ -1068,51 +1136,6 @@ _pier_on_lord_bail(void* ptr_v) u3_pier_bail(pir_u); } -/* _pier_on_scry_done(): scry callback. -*/ -static void -_pier_on_scry_done(void* ptr_v, u3_noun nun) -{ - u3_pier* pir_u = ptr_v; - u3_weak res = u3r_at(7, nun); - - if (u3_none == res) { - u3l_log("pier: scry failed\n"); - } - else { - u3l_log("pier: scry succeeded\n"); - - c3_c* pac_c = u3_Host.ops_u.puk_c; - if (!pac_c) { - pac_c = u3_Host.ops_u.pek_c; - } - - u3_noun pad; - { - // XX crashes if [pac_c] is not a valid path - // XX virtualize or fix - // - u3_noun pax = u3do("stab", u3i_string(pac_c)); - c3_w len_w = u3kb_lent(u3k(pax)); - pad = u3nt(c3_s4('.','u','r','b'), - c3_s3('p','u','t'), - u3qb_scag(len_w - 1, pax)); - u3z(pax); - } - - c3_c fil_c[2048]; - snprintf(fil_c, 2048, "%s/.urb/put/%s.jam", pir_u->pax_c, pac_c+1); - - u3_walk_save(fil_c, 0, u3qe_jam(res), pir_u->pax_c, pad); - u3l_log("pier: scry in %s\n", fil_c); - } - - u3l_log("pier: exit"); - u3_pier_exit(pir_u); - - u3z(nun); -} - /* _pier_on_lord_live(): worker is ready. */ static void @@ -1143,26 +1166,7 @@ _pier_on_lord_live(void* ptr_v) c3_assert( u3_psat_init == pir_u->sat_e ); c3_assert( log_u->sen_d == log_u->dun_d ); - if (u3_Host.ops_u.pek_c) { - u3_noun pex = u3do("stab", u3i_string(u3_Host.ops_u.pek_c)); - u3_noun car; - u3_noun dek; - u3_noun pax; - if ( c3n == u3r_trel(pex, &car, &dek, &pax) - || c3n == u3a_is_cat(car) ) - { - u3m_p("pier: invalid scry", pex); - _pier_on_scry_done(pir_u, u3_nul); - } else { - // run the requested scry, jam to disk, then exit - // - u3l_log("pier: scry\n"); - u3_pier_peek_last(pir_u, u3_nul, u3k(car), u3k(dek), u3k(pax), - pir_u, _pier_on_scry_done); - } - u3z(pex); - } - else if ( god_u->eve_d < log_u->dun_d ) { + if ( god_u->eve_d < log_u->dun_d ) { c3_d eve_d; // XX revisit From 41ce22d6ae18678f9979f7ab5bef54f5c65fb6e7 Mon Sep 17 00:00:00 2001 From: Isaac Visintainer Date: Mon, 16 Nov 2020 15:57:35 -0800 Subject: [PATCH 2/4] herb/lens: add utilites for import/export --- pkg/arvo/app/dojo.hoon | 2 ++ pkg/arvo/app/lens.hoon | 63 ++++++++++++++++++++++++++++++---- pkg/arvo/mar/lens/command.hoon | 2 ++ pkg/arvo/sur/lens.hoon | 2 ++ pkg/herb/herb | 37 ++++++++++++++++++-- 5 files changed, 98 insertions(+), 8 deletions(-) diff --git a/pkg/arvo/app/dojo.hoon b/pkg/arvo/app/dojo.hoon index 2716c009f7..d225ef9e0b 100644 --- a/pkg/arvo/app/dojo.hoon +++ b/pkg/arvo/app/dojo.hoon @@ -1144,6 +1144,8 @@ $listen-api !! $export !! $import !! + $export-all !! + $import-all !! $as :* %as mar.source.com $(num +(num), source.com next.source.com) diff --git a/pkg/arvo/app/lens.hoon b/pkg/arvo/app/lens.hoon index 8c86f8f8d6..e08d3e395d 100644 --- a/pkg/arvo/app/lens.hoon +++ b/pkg/arvo/app/lens.hoon @@ -14,6 +14,28 @@ job=(unit [eyre-id=@ta com=command:lens]) == == +:: +++ export-app + |= [app=@tas our=@p now=@da] + .^(@ %gx /(scot %p our)/[app]/(scot %da now)/export/noun) +++ export-all + |= [our=@p now=@da] + ^- (list [@tas @]) + %+ turn + ^- (list @tas) + :~ %group-store + %metadata-store + %metadata-hook + %contact-store + %contact-hook + %invite-store + %chat-store + %chat-hook + %publish + %graph-store + == + |= app=@tas + [app (export-app app our now)] -- :: =| =state @@ -43,12 +65,15 @@ =/ com=command:lens (json:grab:lens-mark jon) :: - ?: ?=(%export -.source.com) - ~& [%export app.source.com] + ?+ -.source.com + :_ this(job.state (some [eyre-id com])) + [%pass /sole %agent [our.bowl %dojo] %watch /sole/[eyre-id]]~ + :: + %export :_ this(job.state (some [eyre-id com])) [%pass /export %agent [our.bowl app.source.com] %watch /export]~ :: - ?: ?=(%import -.source.com) + %import ?~ enc=(de:base64 base64-jam.source.com) !! :: @@ -57,8 +82,28 @@ :_ this(job.state (some [eyre-id com])) [%pass /import %agent [our.bowl app.source.com] %poke %import !>(c)]~ :: - :_ this(job.state (some [eyre-id com])) - [%pass /sole %agent [our.bowl %dojo] %watch /sole/[eyre-id]]~ + %export-all + =/ output (crip "{}-export/atom") + =/ jon + =/ =atom (jam (export-all our.bowl now.bowl)) + =/ =octs [(met 3 atom) atom] + =/ enc (en:base64 octs) + (pairs:enjs:format file+s+output data+s+enc ~) + :_ this + %+ give-simple-payload:app eyre-id + (json-response:gen jon) + :: + %import-all + =/ enc (de:base64 base64-jam.source.com) + ?~ enc !! + =/ by-app ;;((list [@tas @]) (cue q.u.enc)) + :_ this + %+ weld (give-simple-payload:app eyre-id not-found:gen) + %+ turn by-app + |= [app=@tas data=@] + ^- card:agent:gall + [%pass /import-all %agent [our.bowl app] %poke %import !>(data)] + == :: ++ on-watch |= =path @@ -68,7 +113,13 @@ (on-watch:def path) :: ++ on-leave on-leave:def -++ on-peek on-peek:def +++ on-peek + |= =path + ^- (unit (unit cage)) + ?+ path (on-peek:def path) + [%x %export-all ~] + ``noun+!>((jam (export-all our.bowl now.bowl))) + == ++ on-agent |= [=wire =sign:agent:gall] ^- (quip card:agent:gall _this) diff --git a/pkg/arvo/mar/lens/command.hoon b/pkg/arvo/mar/lens/command.hoon index b8f0532f2e..bd8a88cc9f 100644 --- a/pkg/arvo/mar/lens/command.hoon +++ b/pkg/arvo/mar/lens/command.hoon @@ -42,6 +42,8 @@ listen-api+(su ;~(plug sym ;~(pfix col sym))) export+so import+(ot app+so base64-jam+so ~) + export-all+none + import-all+(ot base64-jam+so ~) as+(ot mark+(su sym) next+source ~) hoon+(ot code+so next+source ~) == diff --git a/pkg/arvo/sur/lens.hoon b/pkg/arvo/sur/lens.hoon index 5503aa7581..0c75afa8c7 100644 --- a/pkg/arvo/sur/lens.hoon +++ b/pkg/arvo/sur/lens.hoon @@ -17,6 +17,8 @@ {$listen-api api/term event/term} {$export app/@t} {$import app/@t base64-jam/@t} + {$export-all ~} + {$import-all base64-jam/@t} == ++ sink $% {$stdout ~} diff --git a/pkg/herb/herb b/pkg/herb/herb index 94c02942a8..7f5bd69f88 100755 --- a/pkg/herb/herb +++ b/pkg/herb/herb @@ -145,6 +145,34 @@ class importFileAction(argparse.Action): res.source = {"import": {"app": new_value, "base64-jam": base_data}} +class importAllAction(argparse.Action): + """Handles the import-all statement. + + The --import-all statement reads in a jammed noun file from the path passed + in and stuffs it the base64 encoded version which gets passed into your + Urbit. + + """ + def __call__(self, parser, res, new_value, option_string): + logging.debug('%r %r' % (new_value, option_string)) + logging.debug('source %s' % res.source) + logging.debug('level %s' % res.level) + + # We check to see if there's a "{new_value}" file in the current + # working directory. If there isn't, we error + data = "" + filename = new_value + with open(filename, 'rb') as f: + data = f.read() + + if data == "": + raise ValueError('Failed to read jamfile') + + base_data = base64.b64encode(data) + + res.source = {"import-all": {"base64-jam": base_data}} + + class transformerAction(argparse.Action): """Handle transformer flag. @@ -350,6 +378,13 @@ parser.add_argument('-i', '--import', metavar='app-name', help='imports the application state', action=importFileAction) +parser.add_argument('-E', '--export-all', const={'export-all': None}, + help='exports data from all landscape apps', + action='store_const', dest='source') +parser.add_argument('-I', '--import-all', + metavar='jam-file', + help='imports data for all landscape apps', + action=importAllAction) parser.add_argument('-m', '--mark', which='as', metavar='mark', help='transform a source to another mark', @@ -398,10 +433,8 @@ sinks.add_argument('-p', '--app', which='app', metavar='app', action=sinkAction) - args = parser.parse_args(args) - if args.source is None: args.source = {"data": ''.join(sys.stdin)} From 050bcf0ab9908e5dcc1707db3a7bbea7f7cc4d38 Mon Sep 17 00:00:00 2001 From: Isaac Visintainer Date: Mon, 16 Nov 2020 15:59:40 -0800 Subject: [PATCH 3/4] vere: add -i and -o options for import and export --- pkg/urbit/daemon/main.c | 12 ++++++++++-- pkg/urbit/include/vere/vere.h | 2 ++ pkg/urbit/vere/io/fore.c | 30 ++++++++++++++++++++++++++++++ pkg/urbit/vere/pier.c | 17 +++++++++++++++++ 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 9918726406..4ef173240c 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -74,6 +74,7 @@ _main_getopt(c3_i argc, c3_c** argv) u3_Host.ops_u.abo = c3n; u3_Host.ops_u.dem = c3n; u3_Host.ops_u.dry = c3n; + u3_Host.ops_u.exp = c3n; u3_Host.ops_u.gab = c3n; u3_Host.ops_u.git = c3n; @@ -96,7 +97,7 @@ _main_getopt(c3_i argc, c3_c** argv) u3_Host.ops_u.kno_w = DefaultKernel; while ( -1 != (ch_i=getopt(argc, argv, - "X:Y:G:J:B:K:A:H:I:C:w:u:e:F:k:n:p:r:LljacdgqstvxPDRS")) ) + "X:Y:G:J:B:K:A:H:I:C:w:u:e:F:k:n:p:r:i:LljacdgoqstvxPDRS")) ) { switch ( ch_i ) { case 'X': { @@ -187,6 +188,10 @@ _main_getopt(c3_i argc, c3_c** argv) u3_Host.ops_u.roc_c = strdup(optarg); break; } + case 'i': { + u3_Host.ops_u.imp_c = strdup(optarg); + break; + } case 'L': { u3_Host.ops_u.net = c3n; break; } case 'l': { u3_Host.ops_u.lit = c3y; break; } case 'j': { u3_Host.ops_u.tra = c3y; break; } @@ -194,6 +199,7 @@ _main_getopt(c3_i argc, c3_c** argv) case 'c': { u3_Host.ops_u.nuu = c3y; break; } case 'd': { u3_Host.ops_u.dem = c3y; break; } case 'g': { u3_Host.ops_u.gab = c3y; break; } + case 'o': { u3_Host.ops_u.exp = c3y; break; } case 'P': { u3_Host.ops_u.pro = c3y; break; } case 'D': { u3_Host.ops_u.dry = c3y; break; } case 'q': { u3_Host.ops_u.qui = c3y; break; } @@ -398,10 +404,12 @@ u3_ve_usage(c3_i argc, c3_c** argv) "-e url Ethereum gateway\n", "-F ship Fake keys; also disables networking\n", "-g Set GC flag\n", + "-i jam_file import pier state\n", "-j Create json trace file in .urb/put/trace\n", "-K stage Start at Hoon kernel version stage\n", "-k keys Private key file\n", "-L local networking only\n", + "-o export pier state\n", "-P Profiling\n", "-p ames_port Set the ames port to bind to\n", "-q Quiet\n", @@ -415,7 +423,7 @@ u3_ve_usage(c3_i argc, c3_c** argv) "-w name Boot as ~name\n", "-X path Scry, jam to file, then exit\n" "-x Exit immediately\n", - "-Y file Optional name of jamfile (for -X)\n" + "-Y file Optional name of jamfile (for -X and -o)\n" "\n", "Development Usage:\n", " To create a development ship, use a fakezod:\n", diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index de62a3fdd1..ba38d0c666 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -265,6 +265,8 @@ c3_o gab; // -g, test garbage collection c3_c* dns_c; // -H, ames bootstrap domain c3_c* jin_c; // -I, inject raw event + c3_c* imp_c; // -i, import pier state + c3_o exp; // -o, export pier state c3_w hap_w; // -C, cap memo cache c3_c* lit_c; // -J, ivory (fastboot) kernel c3_o tra; // -j, json trace diff --git a/pkg/urbit/vere/io/fore.c b/pkg/urbit/vere/io/fore.c index 96f6dd53b7..8eea61f226 100644 --- a/pkg/urbit/vere/io/fore.c +++ b/pkg/urbit/vere/io/fore.c @@ -69,6 +69,32 @@ _fore_inject(u3_auto* car_u, c3_c* pax_c) u3z(ovo); } +/* _fore_import(): form an ovum from jammed archive at [pax_c] and inject it. +*/ +static void +_fore_import(u3_auto* car_u, c3_c* pax_c) +{ + // With apologies + u3_noun arc = u3ke_cue(u3m_file(pax_c)); + c3_c * b64_c = u3r_string(u3do("crip", u3do("en-base64:mimes:html", arc))); + c3_w siz = strlen(b64_c) + 120; + + c3_c bod_c[siz]; + snprintf(bod_c, siz, + "{\"source\": {\"import-all\": {\"base64-jam\": \"%s\"}}, \ + \"sink\": {\"stdout\": null}}", b64_c); + u3_noun dat = u3nt(u3_nul, u3i_word(strlen(bod_c)), u3i_string(bod_c)); + + u3_noun req = u3nt(c3n, + u3nc(u3i_string("ipv4"), u3i_word(0x7f000001)), + u3nq(u3i_string("POST"), u3i_string("/"), u3_nul, dat)); + u3_noun wir = u3nc(u3i_string("http-server"), u3_nul); + u3_noun cad = u3nc(u3i_string("request-local"), req); + u3_auto_peer( + u3_auto_plan(car_u, u3_ovum_init(0, c3__e, wir, cad)), + 0, 0, _fore_inject_bail); +} + /* _fore_io_talk(): */ static void @@ -106,6 +132,10 @@ _fore_io_talk(u3_auto* car_u) if ( u3_Host.ops_u.jin_c ) { _fore_inject(car_u, u3_Host.ops_u.jin_c); } + + if ( u3_Host.ops_u.imp_c ) { + _fore_import(car_u, u3_Host.ops_u.imp_c); + } } /* _fore_io_kick(): handle no effects. diff --git a/pkg/urbit/vere/pier.c b/pkg/urbit/vere/pier.c index fd0828a40e..289f1768a9 100644 --- a/pkg/urbit/vere/pier.c +++ b/pkg/urbit/vere/pier.c @@ -606,6 +606,23 @@ _pier_work_init(u3_pier* pir_u) } u3z(pex); } + else if ( _(u3_Host.ops_u.exp) ) { + u3_noun pex = u3do("stab", u3i_string("/gx/lens/export-all/noun")); + u3_noun car; + u3_noun dek; + u3_noun pax; + u3r_trel(pex, &car, &dek, &pax); + if (!u3_Host.ops_u.puk_c) { + u3_Host.ops_u.puk_c = strdup("/archive"); + } + // run the requested scry, jam to disk, then exit + // + u3l_log("pier: scry\n"); + u3_pier_peek_last(pir_u, u3_nul, u3k(car), u3k(dek), u3k(pax), + pir_u, _pier_on_scry_done); + u3z(pex); + + } else { // initialize i/o drivers // From 16b9d810f980f7ff5957cb3c8f2ca6aa04ce1816 Mon Sep 17 00:00:00 2001 From: Isaac Visintainer Date: Wed, 18 Nov 2020 12:56:31 -0800 Subject: [PATCH 4/4] vere: manage memory properly in _fore_import --- pkg/urbit/vere/io/fore.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/urbit/vere/io/fore.c b/pkg/urbit/vere/io/fore.c index 8eea61f226..1da96bcc6d 100644 --- a/pkg/urbit/vere/io/fore.c +++ b/pkg/urbit/vere/io/fore.c @@ -22,6 +22,17 @@ _fore_inject_bail(u3_ovum* egg_u, u3_noun lud) u3_ovum_free(egg_u); } +/* _fore_import_bail(): handle failure on arbitrary injection. +*/ +static void +_fore_import_bail(u3_ovum* egg_u, u3_noun lud) +{ + u3_auto_bail_slog(egg_u, lud); + u3l_log("pier: import failed\n"); + + u3_ovum_free(egg_u); +} + /* _fore_inject(): inject an arbitrary ovum from a jammed file at [pax_c]. */ static void @@ -75,16 +86,17 @@ static void _fore_import(u3_auto* car_u, c3_c* pax_c) { // With apologies - u3_noun arc = u3ke_cue(u3m_file(pax_c)); - c3_c * b64_c = u3r_string(u3do("crip", u3do("en-base64:mimes:html", arc))); - c3_w siz = strlen(b64_c) + 120; + u3_noun arc = u3ke_cue(u3m_file(pax_c)); + u3_noun b64 = u3do("crip", u3do("en-base64:mimes:html", arc)); + c3_c * b64_c = u3r_string(b64); - c3_c bod_c[siz]; - snprintf(bod_c, siz, + c3_w siz_w = strlen(b64_c) + 120; + c3_c * bod_c = (c3_c *) c3_malloc(siz_w); + snprintf(bod_c, siz_w, "{\"source\": {\"import-all\": {\"base64-jam\": \"%s\"}}, \ \"sink\": {\"stdout\": null}}", b64_c); - u3_noun dat = u3nt(u3_nul, u3i_word(strlen(bod_c)), u3i_string(bod_c)); + u3_noun dat = u3nt(u3_nul, u3i_word(strlen(bod_c)), u3i_string(bod_c)); u3_noun req = u3nt(c3n, u3nc(u3i_string("ipv4"), u3i_word(0x7f000001)), u3nq(u3i_string("POST"), u3i_string("/"), u3_nul, dat)); @@ -92,7 +104,11 @@ _fore_import(u3_auto* car_u, c3_c* pax_c) u3_noun cad = u3nc(u3i_string("request-local"), req); u3_auto_peer( u3_auto_plan(car_u, u3_ovum_init(0, c3__e, wir, cad)), - 0, 0, _fore_inject_bail); + 0, 0, _fore_import_bail); + + u3z(b64); + c3_free(b64_c); + c3_free(bod_c); } /* _fore_io_talk():