From e8c5dd9806417cb819f745e7cbb99070c29b3351 Mon Sep 17 00:00:00 2001 From: Anton Dyudin Date: Mon, 19 Oct 2020 11:05:31 -0700 Subject: [PATCH 001/136] [hoon co] Fix float printer overuse of scientific notation I.e, print `.10` as `.10` not `.1e1`. Only use `e+-N` when it would be shorter. Before: ``` > (turn (gulf 1 10) |=(a=@u (turn (gulf 0 9) |=(b=@u (grd:rs d/[& -5 (mul (pow 10 a) +((pow 10 b)))]))))) ~[ ~[.2e-3 .1.1e-3 .1.01e-2 .1.001e-1 .1.0001 .1.00001e1 .1.000001e2 .1.0000001e3 .1e4 .1e5] ~[.2e-2 .1.1e-2 .1.01e-1 .1.001 .1.0001e1 .1.00001e2 .1.000001e3 .1.0000001e4 .1e5 .1e6] ~[.2e-1 .1.1e-1 .1.01 .1.001e1 .1.0001e2 .1.00001e3 .1.000001e4 .1.0000001e5 .1e6 .1e7] ~[.2 .1.1 .1.01e1 .1.001e2 .1.0001e3 .1.00001e4 .1.000001e5 .1.0000001e6 .1e7 .1e8] ~[.2e1 .1.1e1 .1.01e2 .1.001e3 .1.0001e4 .1.00001e5 .1.000001e6 .1.0000001e7 .1e8 .1e9] ~[.2e2 .1.1e2 .1.01e3 .1.001e4 .1.0001e5 .1.00001e6 .1.000001e7 .1.0000001e8 .1e9 .1e10] ~[.2e3 .1.1e3 .1.01e4 .1.001e5 .1.0001e6 .1.00001e7 .1.00000096e8 .1.0000001e9 .1e10 .1e11] ~[.2e4 .1.1e4 .1.01e5 .1.001e6 .1.0001e7 .1.00001e8 .1.000001e9 .1.0000001e10 .1e11 .1e12] ~[.2e5 .1.1e5 .1.01e6 .1.001e7 .1.0001e8 .1.00001e9 .1.000001e10 .1.0000001e11 .1e12 .1e13] ~[.2e6 .1.1e6 .1.01e7 .1.001e8 .1.00009997e9 .1.00001e10 .1.000001e11 .1.0000001e12 .1e13 .1e14] ] ``` After: ``` > (turn (gulf 1 10) |=(a=@u (turn (gulf 0 9) |=(b=@u (grd:rs d/[& -5 (mul (pow 10 a) +((pow 10 b)))]))))) ~[ ~[.2e-4 .1.1e-3 .0.0101 .0.1001 .1.0001 .10.0001 .100.0001 .1000.0001 .1e4 .1e5] ~[.2e-3 .0.011 .0.101 .1.001 .10.001 .100.001 .1000.001 .10000.001 .1e5 .1e6] ~[.0.02 .0.11 .1.01 .10.01 .100.01 .1000.01 .10000.01 .100000.01 .1e6 .1e7] ~[.0.2 .1.1 .10.1 .100.1 .1000.1 .10000.1 .100000.1 .1000000.1 .1e7 .1e8] ~[.2 .11 .101 .1001 .10001 .100001 .1000001 .10000001 .1e8 .1e9] ~[.20 .110 .1010 .10010 .100010 .1000010 .10000010 .100000010 .1e9 .1e10] ~[.200 .1100 .10100 .100100 .1000100 .10000100 .100000096 .1000000100 .1e10 .1e11] ~[.2e3 .1.1e4 .1.01e5 .1.001e6 .1.0001e7 .1.00001e8 .1.000001e9 .1.0000001e10 .1e11 .1e12] ~[.2e4 .1.1e5 .1.01e6 .1.001e7 .1.0001e8 .1.00001e9 .1.000001e10 .1.0000001e11 .1e12 .1e13] ~[.2e5 .1.1e6 .1.01e7 .1.001e8 .1000099970 .1.00001e10 .1.000001e11 .1.0000001e12 .1e13 .1e14] ] ``` --- pkg/arvo/sys/hoon.hoon | 43 ++++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/pkg/arvo/sys/hoon.hoon b/pkg/arvo/sys/hoon.hoon index b3602bfd43..43386d4890 100644 --- a/pkg/arvo/sys/hoon.hoon +++ b/pkg/arvo/sys/hoon.hoon @@ -6024,17 +6024,17 @@ |= a/dn ?: ?=({$i *} a) (weld ?:(s.a "inf" "-inf") rep) ?: ?=({$n *} a) (weld "nan" rep) - =+ ^= e %+ ed-co [10 1] - |= {a/? b/@ c/tape} - ?: a [~(d ne b) '.' c] - [~(d ne b) c] - =+ ^= f - =>(.(rep ~) (e a.a)) - =. e.a (sum:si e.a (sun:si (dec +.f))) - =+ b=?:((syn:si e.a) "e" "e-") - => .(rep ?~(e.a rep (weld b ((d-co 1) (abs:si e.a))))) - => .(rep (weld -.f rep)) - ?:(s.a rep ['-' rep]) + =; rep ?:(s.a rep ['-' rep]) + =/ f ((d-co 1) a.a) + =^ e e.a + =/ e=@s (sun:si (lent f)) + =/ sci :(sum:si e.a e -1) + ?: (syn:si (dif:si e.a --3)) [--1 sci] :: 12000 -> 12e3 e>+2 + ?: !(syn:si (dif:si sci -2)) [--1 sci] :: 0.001 -> 1e-3 e<-2 + [(sum:si sci --1) --0] :: 1.234e2 -> '.'@3 -> 123 .4 + =? rep !=(--0 e.a) + :(weld ?:((syn:si e.a) "e" "e-") ((d-co 1) (abs:si e.a))) + (weld (ed-co e f) rep) :: ++ s-co |= esc/(list @) ^- tape @@ -6064,20 +6064,13 @@ == :: ++ ed-co - |= {{bas/@ min/@} par/$-({? @ tape} tape)} - =+ [fir=& cou=0] - |= hol/@ - ^- {tape @} - ?: &(=(0 hol) =(0 min)) - [rep cou] - =+ [rad=(mod hol bas) dar=(div hol bas)] - %= $ - min ?:(=(0 min) 0 (dec min)) - hol dar - rep (par &(=(0 dar) !fir) rad rep) - fir | - cou +(cou) - == + |= [exp=@s int=tape] ^- tape + =/ [pos=? dig=@u] [=(--1 (cmp:si exp --0)) (abs:si exp)] + ?. pos + (into (weld (reap +(dig) '0') int) 1 '.') + =/ len (lent int) + ?: (lth dig len) (into int dig '.') + (weld int (reap (sub dig len) '0')) :: ++ ox-co |= {{bas/@ gop/@} dug/$-(@ @)} From b55417a1c325eaf8eea5727e08ae268b40816306 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Fri, 5 Mar 2021 16:53:35 -0500 Subject: [PATCH 002/136] clay: remove +bunt from marks --- pkg/arvo/app/dojo.hoon | 2 +- pkg/arvo/lib/strandio.hoon | 2 +- pkg/arvo/sys/lull.hoon | 2 -- pkg/arvo/sys/vane/clay.hoon | 11 ++++------- pkg/arvo/tests/sys/vane/clay.hoon | 6 +++--- pkg/arvo/tests/sys/vane/eyre.hoon | 1 - 6 files changed, 9 insertions(+), 15 deletions(-) diff --git a/pkg/arvo/app/dojo.hoon b/pkg/arvo/app/dojo.hoon index 70e07d618d..2252a3d01f 100644 --- a/pkg/arvo/app/dojo.hoon +++ b/pkg/arvo/app/dojo.hoon @@ -875,7 +875,7 @@ %ge (dy-run-generator (dy-cage p.p.p.bil) q.p.bil) %sa =+ .^(=dais:clay cb+(en-beam he-beak /[p.bil])) - (dy-hand p.bil bunt:dais) + (dy-hand p.bil *vale:dais) :: %as =/ cag=cage (dy-cage p.q.bil) diff --git a/pkg/arvo/lib/strandio.hoon b/pkg/arvo/lib/strandio.hoon index a8b52ede1f..9d8161d104 100644 --- a/pkg/arvo/lib/strandio.hoon +++ b/pkg/arvo/lib/strandio.hoon @@ -490,7 +490,7 @@ =/ m (strand ,vase) ^- form:m ;< =riot:clay bind:m - (warp ship desk ~ %sing %b case /[mak]) + (warp ship desk ~ %sing %e case /[mak]) ?~ riot (strand-fail %build-nave >arg< ~) ?> =(%nave p.r.u.riot) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 23d08ae5bf..3a0e752972 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -955,7 +955,6 @@ $_ ^? |% - ++ bunt *typ ++ diff |~([old=typ new=typ] *dif) ++ form *mark ++ join |~([a=dif b=dif] *(unit (unit dif))) @@ -970,7 +969,6 @@ +$ dais $_ ^| |_ sam=vase - ++ bunt sam ++ diff |~(new=_sam *vase) ++ form *mark ++ join |~([a=vase b=vase] *(unit (unit vase))) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index af2f653037..a46282f527 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -559,7 +559,6 @@ =/ dif diff:deg ^- (nave typ dif) |% - ++ bunt +<.cor ++ diff |= [old=typ new=typ] ^- dif @@ -581,7 +580,6 @@ =/ dif _*diff:grad:cor ^- (nave:clay typ dif) |% - ++ bunt +<.cor ++ diff |=([old=typ new=typ] (diff:~(grad cor old) new)) ++ form form:grad:cor ++ join @@ -622,7 +620,6 @@ :_ nub ^- dais |_ sam=vase - ++ bunt (slap nav limb/%bunt) ++ diff |= new=vase (slam (slap nav limb/%diff) (slop sam new)) @@ -649,7 +646,7 @@ |= diff=vase (slam (slap nav limb/%pact) (slop sam diff)) ++ vale - |= =noun + |: noun=q:(slap nav !,(*hoon *vale)) (slam (slap nav limb/%vale) noun/noun) -- :: +build-cast: produce gate to convert mark .a to, statically typed @@ -2313,7 +2310,7 @@ =+ (slag (dec (lent path)) path) ?~(- %$ i.-) =/ =dais (get-dais mark) - =/ res=(unit (unit vase)) (~(join dais bunt:dais) q.cal q.cob) + =/ res=(unit (unit vase)) (~(join dais *vale:dais) q.cal q.cob) ?~ res `[form:dais q.cob] ?~ u.res @@ -3528,11 +3525,11 @@ [[~ ~] fod.dom] =/ cached=(unit [=vase *]) (~(get by naves.fod.dom) i.path) ?^ cached - :_(fod.dom [~ ~ %& %nave !>(vase.u.cached)]) + :_(fod.dom [~ ~ %& %nave vase.u.cached]) =^ =vase fod.dom %- wrap:fusion (build-nave:(ford:fusion static-ford-args) i.path) - :_(fod.dom [~ ~ %& %nave !>(vase)]) + :_(fod.dom [~ ~ %& %nave vase]) :: ++ read-f !. diff --git a/pkg/arvo/tests/sys/vane/clay.hoon b/pkg/arvo/tests/sys/vane/clay.hoon index 636efd3a35..5b201e6273 100644 --- a/pkg/arvo/tests/sys/vane/clay.hoon +++ b/pkg/arvo/tests/sys/vane/clay.hoon @@ -112,7 +112,7 @@ ;: weld %+ expect-eq !>(*mime) - (slap res limb/%bunt) + (slap res !,(*hoon *vale)) :: %+ expect-eq !> (~(gas in *(set path)) /mar/mime/hoon ~) @@ -139,7 +139,7 @@ ;: weld %+ expect-eq !>(*@t) - (slap res limb/%bunt) + (slap res !,(*hoon *vale)) :: %+ expect-eq !> (~(gas in *(set path)) /mar/udon/hoon /lib/cram/hoon ~) @@ -170,7 +170,7 @@ =/ changes %- my :~ [/mar/mime/hoon &+hoon+mar-mime] - [/lib/foo/hoon &+hoon+'/% moo %mime\0abunt:moo'] + [/lib/foo/hoon &+hoon+'/% moo %mime\0a*vale:moo'] == =/ ford %: ford:fusion diff --git a/pkg/arvo/tests/sys/vane/eyre.hoon b/pkg/arvo/tests/sys/vane/eyre.hoon index 0b83d3a843..6710fa2aee 100644 --- a/pkg/arvo/tests/sys/vane/eyre.hoon +++ b/pkg/arvo/tests/sys/vane/eyre.hoon @@ -2353,7 +2353,6 @@ :^ ~ ~ %dais !> ^- dais:clay |_ sam=vase - ++ bunt !! ++ diff !! ++ form !! ++ join !! From f76a8b79111e7e627319582de7d5f2c28293deba Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Fri, 5 Mar 2021 17:44:40 -0500 Subject: [PATCH 003/136] pill: update solid --- bin/solid.pill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 65f89637de..82ae034f4c 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:ec80a42446a1d80974f32bf87283435547441cb7ea3fcd340711df2ce6cbec81 -size 9146390 +oid sha256:4c9d91a355b034f50de24c959b5f11751b1d75c92c6a9e26e7e22c865c773144 +size 9726392 From ef1b125c3ebe7b69cd2358920bea550dba4fb9a5 Mon Sep 17 00:00:00 2001 From: J Date: Fri, 12 Mar 2021 19:54:04 +0000 Subject: [PATCH 004/136] zuse: fix time fix regression Fixes #4598. #4474 made the JSON time conversion no longer invertible, which caused problems for chat, which uses message timestamp in milliseconds as a key -- so chat would send a message with ms timestamp x, it would get encoded as @da x, but then when it went back through the conversion to milliseconds, it would often (not always) get encoded as x-1. I still do not fully understand why this is -- and why it doesn't seem to be a problem with seconds based on cursory testing -- but integer multiplication and division generally do not invert. And adding a half a millisecond to the input date before converting it resolves the issue and makes the functions invertible. I added a regression test, so hopefully the next courageous adventurer who winds up here after wondering why +unm looks funny will have a safeguard against some of the mistakes I made. --- pkg/arvo/sys/zuse.hoon | 3 ++- pkg/arvo/tests/sys/zuse/format.hoon | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index eaf098937b..14c34390d6 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5535,7 +5535,8 @@ :: :: ++unm:chrono:userlib ++ unm :: Urbit to Unix ms |= a=@da - (div (mul (sub a ~1970.1.1) 1.000) ~s1) + =- (div (mul - 1.000) ~s1) + (sub (add a (div ~s1 2.000)) ~1970.1.1) :: :: ++unt:chrono:userlib ++ unt :: Urbit to Unix time |= a=@da diff --git a/pkg/arvo/tests/sys/zuse/format.hoon b/pkg/arvo/tests/sys/zuse/format.hoon index 8719ab241e..a8ee826a4a 100644 --- a/pkg/arvo/tests/sys/zuse/format.hoon +++ b/pkg/arvo/tests/sys/zuse/format.hoon @@ -179,6 +179,11 @@ %+ expect-eq !> [%n '1000'] !> (time ~1970.1.1..0.0.1) + :: timestamps should invert + :: + %+ expect-eq + !> [%n '1001'] + !> (time (from-unix-ms:chrono:userlib 1.001)) :: ship - store ship identity as a string :: %+ expect-eq From 549c5e5db9f999604d1d09667b458274a13494da Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Wed, 17 Mar 2021 14:21:58 -0700 Subject: [PATCH 005/136] hoon: update seminoun bunt to be fully blocked --- pkg/arvo/sys/hoon.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/hoon.hoon b/pkg/arvo/sys/hoon.hoon index 834cbcf390..ed2e92fad5 100644 --- a/pkg/arvo/sys/hoon.hoon +++ b/pkg/arvo/sys/hoon.hoon @@ -6627,7 +6627,7 @@ +$ seminoun :: partial noun; blocked subtrees are ~ :: - $~ [[%full ~] ~] + $~ [[%full / ~ ~] ~] [mask=stencil data=noun] :: :: +stencil: noun knowledge map From 19d5bd0de4228198b867d41c8d3b534bf6b49801 Mon Sep 17 00:00:00 2001 From: Joe Bryan Date: Thu, 18 Mar 2021 17:04:08 -0700 Subject: [PATCH 006/136] ames: gate "fragment crashed" printf on msg.veb --- pkg/arvo/sys/vane/ames.hoon | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index bd267477e4..ccf31d062f 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1944,11 +1944,11 @@ =/ =bone bone.shut-packet :: ?: ?=(%& -.meat.shut-packet) - =+ ?~ dud ~ + =+ ?. &(?=(^ dud) msg.veb) ~ %. ~ - %+ slog - leaf+"ames: {} fragment crashed {}" - ?.(msg.veb ~ tang.u.dud) + %- slog + :_ tang.u.dud + leaf+"ames: {} fragment crashed {}" (run-message-sink bone %hear lane shut-packet ?=(~ dud)) :: Just try again on error, printing trace :: From 186fe8d5777bf95fc3fb11d834dda477bfa8459d Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 19:54:01 +0000 Subject: [PATCH 007/136] jets: rename jets/c/con.c due to clash with legacy CP/M device names --- pkg/urbit/jets/c/{con.c => c0n.c} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/urbit/jets/c/{con.c => c0n.c} (100%) diff --git a/pkg/urbit/jets/c/con.c b/pkg/urbit/jets/c/c0n.c similarity index 100% rename from pkg/urbit/jets/c/con.c rename to pkg/urbit/jets/c/c0n.c From d554c18dc35253e0158353932944f8563c34b788 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 19:58:18 +0000 Subject: [PATCH 008/136] vere: request binary mode in fopen/open calls The C runtime MingW relies on, msvcrt.dll, has the notion of accessing files in text and binary mode. In text mode, it does CRLF conversion and handles some control characters, which breaks binary files. This change adds O_BINARY in every fopen/open call and #defines it to 0 on other platforms. --- pkg/urbit/include/c/portable.h | 4 ++++ pkg/urbit/noun/events.c | 10 +++++----- pkg/urbit/noun/manage.c | 2 +- pkg/urbit/noun/trace.c | 2 +- pkg/urbit/noun/urth.c | 6 +++--- pkg/urbit/vere/dawn.c | 2 +- pkg/urbit/vere/io/http.c | 2 +- pkg/urbit/vere/io/unix.c | 14 +++++++------- pkg/urbit/vere/king.c | 2 +- pkg/urbit/vere/walk.c | 6 +++--- pkg/urbit/worker/main.c | 2 +- pkg/urbit/worker/serf.c | 4 ++-- 12 files changed, 30 insertions(+), 26 deletions(-) diff --git a/pkg/urbit/include/c/portable.h b/pkg/urbit/include/c/portable.h index 62c410b851..fbe52b7632 100644 --- a/pkg/urbit/include/c/portable.h +++ b/pkg/urbit/include/c/portable.h @@ -76,6 +76,10 @@ #error "port: headers" # endif +# ifndef O_BINARY +# define O_BINARY 0 +# endif + # ifndef __has_feature # define __has_feature(x) 0 # endif diff --git a/pkg/urbit/noun/events.c b/pkg/urbit/noun/events.c index df82bb9cfd..2d6286c9e4 100644 --- a/pkg/urbit/noun/events.c +++ b/pkg/urbit/noun/events.c @@ -171,7 +171,7 @@ u3e_fault(void* adr_v, c3_i ser_i) static c3_o _ce_image_open(u3e_image* img_u) { - c3_i mod_i = O_RDWR | O_CREAT; + c3_i mod_i = O_RDWR | O_BINARY | O_CREAT; c3_c ful_c[8193]; snprintf(ful_c, 8192, "%s", u3P.dir_c); @@ -275,13 +275,13 @@ _ce_patch_create(u3_ce_patch* pat_u) mkdir(ful_c, 0700); snprintf(ful_c, 8192, "%s/.urb/chk/control.bin", u3P.dir_c); - if ( -1 == (pat_u->ctl_i = open(ful_c, O_RDWR | O_CREAT | O_EXCL, 0600)) ) { + if ( -1 == (pat_u->ctl_i = open(ful_c, O_RDWR | O_BINARY | O_CREAT | O_EXCL, 0600)) ) { fprintf(stderr, "loom: patch open control.bin: %s\r\n", strerror(errno)); c3_assert(0); } snprintf(ful_c, 8192, "%s/.urb/chk/memory.bin", u3P.dir_c); - if ( -1 == (pat_u->mem_i = open(ful_c, O_RDWR | O_CREAT | O_EXCL, 0600)) ) { + if ( -1 == (pat_u->mem_i = open(ful_c, O_RDWR | O_BINARY | O_CREAT | O_EXCL, 0600)) ) { fprintf(stderr, "loom: patch open memory.bin: %s\r\n", strerror(errno)); c3_assert(0); } @@ -377,12 +377,12 @@ _ce_patch_open(void) mkdir(ful_c, 0700); snprintf(ful_c, 8192, "%s/.urb/chk/control.bin", u3P.dir_c); - if ( -1 == (ctl_i = open(ful_c, O_RDWR)) ) { + if ( -1 == (ctl_i = open(ful_c, O_RDWR | O_BINARY)) ) { return 0; } snprintf(ful_c, 8192, "%s/.urb/chk/memory.bin", u3P.dir_c); - if ( -1 == (mem_i = open(ful_c, O_RDWR)) ) { + if ( -1 == (mem_i = open(ful_c, O_RDWR | O_BINARY)) ) { close(ctl_i); _ce_patch_delete(); diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 23ad01e537..669fd1f3fe 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -412,7 +412,7 @@ u3_noun u3m_file(c3_c* pas_c) { struct stat buf_b; - c3_i fid_i = open(pas_c, O_RDONLY, 0644); + c3_i fid_i = open(pas_c, O_RDONLY | O_BINARY, 0644); c3_w fln_w, red_w; c3_y* pad_y; diff --git a/pkg/urbit/noun/trace.c b/pkg/urbit/noun/trace.c index f8fe81320c..53a1ff92cc 100644 --- a/pkg/urbit/noun/trace.c +++ b/pkg/urbit/noun/trace.c @@ -288,7 +288,7 @@ u3t_trace_open(c3_c* dir_c) c3_c lif_c[2056]; snprintf(lif_c, 2056, "%s/%d.json", fil_c, u3_Host.tra_u.fun_w); - u3_Host.tra_u.fil_u = fopen(lif_c, "w"); + u3_Host.tra_u.fil_u = fopen(lif_c, "wb"); u3_Host.tra_u.nid_w = (int)getpid(); fprintf(u3_Host.tra_u.fil_u, "[ "); diff --git a/pkg/urbit/noun/urth.c b/pkg/urbit/noun/urth.c index 6e0117a4d6..31b38d95b8 100644 --- a/pkg/urbit/noun/urth.c +++ b/pkg/urbit/noun/urth.c @@ -550,7 +550,7 @@ _cu_rock_save(c3_c* dir_c, c3_d eve_d, c3_d len_d, c3_y* byt_y) return c3n; } - if ( -1 == (fid_i = open(nam_c, O_RDWR | O_CREAT | O_TRUNC, 0644)) ) { + if ( -1 == (fid_i = open(nam_c, O_RDWR | O_BINARY | O_CREAT | O_TRUNC, 0644)) ) { fprintf(stderr, "rock: open failed (%s, %" PRIu64 "): %s\r\n", dir_c, eve_d, strerror(errno)); c3_free(nam_c); @@ -689,7 +689,7 @@ u3u_mmap_read(c3_c* cap_c, c3_c* pat_c, c3_d* out_d, c3_y** out_y) // open file // - if ( -1 == (fid_i = open(pat_c, O_RDONLY, 0644)) ) { + if ( -1 == (fid_i = open(pat_c, O_RDONLY | O_BINARY, 0644)) ) { fprintf(stderr, "%s: open failed (%s): %s\r\n", cap_c, pat_c, strerror(errno)); return c3n; @@ -742,7 +742,7 @@ u3u_mmap(c3_c* cap_c, c3_c* pat_c, c3_d len_d, c3_y** out_y) // open file // - if ( -1 == (fid_i = open(pat_c, O_RDWR | O_CREAT | O_TRUNC, 0644)) ) { + if ( -1 == (fid_i = open(pat_c, O_RDWR | O_BINARY | O_CREAT | O_TRUNC, 0644)) ) { fprintf(stderr, "%s: open failed (%s): %s\r\n", cap_c, pat_c, strerror(errno)); return c3n; diff --git a/pkg/urbit/vere/dawn.c b/pkg/urbit/vere/dawn.c index a2e09ab1cb..b278adefdd 100644 --- a/pkg/urbit/vere/dawn.c +++ b/pkg/urbit/vere/dawn.c @@ -521,7 +521,7 @@ _dawn_come(u3_noun stars) c3_c pat_c[64]; snprintf(pat_c, 64, "%s.key", who_c + 1); - FILE* fil_u = fopen(pat_c, "w"); + FILE* fil_u = fopen(pat_c, "wb"); fprintf(fil_u, "%s\n", key_c); fclose(fil_u); } diff --git a/pkg/urbit/vere/io/http.c b/pkg/urbit/vere/io/http.c index 229584cd94..7705a3bc48 100644 --- a/pkg/urbit/vere/io/http.c +++ b/pkg/urbit/vere/io/http.c @@ -1609,7 +1609,7 @@ _http_write_ports_file(u3_httd* htd_u, c3_c *pax_c) c3_c* paf_c = c3_malloc(len_w); snprintf(paf_c, len_w, "%s/%s", pax_c, nam_c); - c3_i por_i = open(paf_c, O_WRONLY | O_CREAT | O_TRUNC, 0666); + c3_i por_i = open(paf_c, O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, 0666); c3_free(paf_c); u3_http* htp_u = htd_u->htp_u; diff --git a/pkg/urbit/vere/io/unix.c b/pkg/urbit/vere/io/unix.c index b27921936f..1207b6548c 100644 --- a/pkg/urbit/vere/io/unix.c +++ b/pkg/urbit/vere/io/unix.c @@ -233,7 +233,7 @@ _unix_mkdir(c3_c* pax_c) static c3_w _unix_write_file_hard(c3_c* pax_c, u3_noun mim) { - c3_i fid_i = open(pax_c, O_WRONLY | O_CREAT | O_TRUNC, 0666); + c3_i fid_i = open(pax_c, O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, 0666); c3_w len_w, rit_w, siz_w, mug_w = 0; c3_y* dat_y; @@ -276,7 +276,7 @@ static void _unix_write_file_soft(u3_ufil* fil_u, u3_noun mim) { struct stat buf_u; - c3_i fid_i = open(fil_u->pax_c, O_RDONLY, 0644); + c3_i fid_i = open(fil_u->pax_c, O_RDONLY | O_BINARY, 0644); c3_ws len_ws, red_ws; c3_w old_w; c3_y* old_y; @@ -700,7 +700,7 @@ _unix_update_file(u3_unix* unx_u, u3_ufil* fil_u) fil_u->dry = c3n; struct stat buf_u; - c3_i fid_i = open(fil_u->pax_c, O_RDONLY, 0644); + c3_i fid_i = open(fil_u->pax_c, O_RDONLY | O_BINARY, 0644); c3_ws len_ws, red_ws; c3_y* dat_y; @@ -803,7 +803,7 @@ _unix_update_dir(u3_unix* unx_u, u3_udir* dir_u) } else { struct stat buf_u; - c3_i fid_i = open(nod_u->pax_c, O_RDONLY, 0644); + c3_i fid_i = open(nod_u->pax_c, O_RDONLY | O_BINARY, 0644); if ( (fid_i < 0) || (fstat(fid_i, &buf_u) < 0) ) { if ( ENOENT != errno ) { @@ -973,7 +973,7 @@ static u3_noun _unix_initial_update_file(c3_c* pax_c, c3_c* bas_c) { struct stat buf_u; - c3_i fid_i = open(pax_c, O_RDONLY, 0644); + c3_i fid_i = open(pax_c, O_RDONLY | O_BINARY, 0644); c3_ws len_ws, red_ws; c3_y* dat_y; @@ -1289,7 +1289,7 @@ u3_unix_acquire(c3_c* pax_c) c3_w pid_w; FILE* loq_u; - if ( NULL != (loq_u = fopen(paf_c, "r")) ) { + if ( NULL != (loq_u = fopen(paf_c, "rb")) ) { if ( 1 != fscanf(loq_u, "%" SCNu32, &pid_w) ) { u3l_log("lockfile %s is corrupt!\n", paf_c); kill(getpid(), SIGTERM); @@ -1327,7 +1327,7 @@ u3_unix_acquire(c3_c* pax_c) unlink(paf_c); } - if ( NULL == (loq_u = fopen(paf_c, "w")) ) { + if ( NULL == (loq_u = fopen(paf_c, "wb")) ) { u3l_log("unix: unable to open %s\n", paf_c); c3_assert(0); } diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c index fd8ba8c693..7899cfd691 100644 --- a/pkg/urbit/vere/king.c +++ b/pkg/urbit/vere/king.c @@ -913,7 +913,7 @@ u3_king_grab(void* vod_p) c3_c man_c[2048]; snprintf(man_c, 2048, "%s/%s-daemon.txt", nam_c, wen_c); - fil_u = fopen(man_c, "w"); + fil_u = fopen(man_c, "wb"); fprintf(fil_u, "%s\r\n", wen_c); c3_free(wen_c); diff --git a/pkg/urbit/vere/walk.c b/pkg/urbit/vere/walk.c index 1f7095a188..2cf70591c9 100644 --- a/pkg/urbit/vere/walk.c +++ b/pkg/urbit/vere/walk.c @@ -39,7 +39,7 @@ u3_noun u3_walk_safe(c3_c* pas_c) { struct stat buf_b; - c3_i fid_i = open(pas_c, O_RDONLY, 0644); + c3_i fid_i = open(pas_c, O_RDONLY | O_BINARY, 0644); c3_w fln_w, red_w; c3_y* pad_y; @@ -71,7 +71,7 @@ u3_noun u3_walk_load(c3_c* pas_c) { struct stat buf_b; - c3_i fid_i = open(pas_c, O_RDONLY, 0644); + c3_i fid_i = open(pas_c, O_RDONLY | O_BINARY, 0644); c3_w fln_w, red_w; c3_y* pad_y; @@ -137,7 +137,7 @@ _walk_mkdirp(c3_c* bas_c, u3_noun pax) void u3_walk_save(c3_c* pas_c, u3_noun tim, u3_atom pad, c3_c* bas_c, u3_noun pax) { - c3_i fid_i = open(pas_c, O_WRONLY | O_CREAT | O_TRUNC, 0666); + c3_i fid_i = open(pas_c, O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, 0666); c3_w fln_w, rit_w; c3_y* pad_y; diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index 46895f093b..183cea70e7 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -146,7 +146,7 @@ _cw_serf_stdio(c3_i* inn_i, c3_i* out_i) // we replace [FD 0] (stdin) with a fd pointing to /dev/null // we replace [FD 1] (stdout) with a dup of [FD 2] (stderr) // - c3_i nul_i = open("/dev/null", O_RDWR, 0); + c3_i nul_i = open("/dev/null", O_RDWR | O_BINARY, 0); *inn_i = dup(0); *out_i = dup(1); diff --git a/pkg/urbit/worker/serf.c b/pkg/urbit/worker/serf.c index 1173ae344f..3774cbfc01 100644 --- a/pkg/urbit/worker/serf.c +++ b/pkg/urbit/worker/serf.c @@ -226,7 +226,7 @@ _serf_grab(u3_serf* sef_u) c3_c man_c[2054]; snprintf(man_c, 2053, "%s/%s-serf.txt", nam_c, wen_c); - fil_u = fopen(man_c, "w"); + fil_u = fopen(man_c, "wb"); fprintf(fil_u, "%s\r\n", wen_c); c3_free(wen_c); @@ -825,7 +825,7 @@ _serf_writ_live_exit(u3_serf* sef_u, c3_w cod_w) c3_c man_c[2054]; snprintf(man_c, 2053, "%s/%s.txt", nam_c, wen_c); - fil_u = fopen(man_c, "w"); + fil_u = fopen(man_c, "wb"); c3_free(wen_c); u3z(wen); From 577a267ed3c255fb01fca5ad057aabee67330287 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 20:03:09 +0000 Subject: [PATCH 009/136] vere: cleanup and reorder #includes Most C files have multiple #includes that are duplicated in include/c/portable.h. Removing them helps keep include-related #ifdefs in include/c/portable.h. #including library files after all.h avoids clashes on MingW and allows portable.h to add compatibility #defines where necessary. --- pkg/urbit/daemon/main.c | 17 ++++------------- pkg/urbit/noun/events.c | 3 +-- pkg/urbit/noun/manage.c | 5 ++--- pkg/urbit/noun/serial.c | 5 ++--- pkg/urbit/noun/urth.c | 5 ++--- pkg/urbit/noun/vortex.c | 2 +- pkg/urbit/vere/auto.c | 16 ---------------- pkg/urbit/vere/dawn.c | 5 ++--- pkg/urbit/vere/db/lmdb.c | 4 +--- pkg/urbit/vere/disk.c | 16 ---------------- pkg/urbit/vere/foil.c | 16 ---------------- pkg/urbit/vere/io/ames.c | 9 --------- pkg/urbit/vere/io/behn.c | 7 ------- pkg/urbit/vere/io/cttp.c | 12 ++---------- pkg/urbit/vere/io/fore.c | 7 ------- pkg/urbit/vere/io/hind.c | 7 ------- pkg/urbit/vere/io/http.c | 13 ++----------- pkg/urbit/vere/io/term.c | 5 ++--- pkg/urbit/vere/io/unix.c | 8 -------- pkg/urbit/vere/king.c | 6 ++---- pkg/urbit/vere/lord.c | 16 ---------------- pkg/urbit/vere/newt.c | 16 ---------------- pkg/urbit/vere/pier.c | 18 +----------------- pkg/urbit/vere/save.c | 7 ------- pkg/urbit/vere/time.c | 6 ------ pkg/urbit/vere/walk.c | 7 ------- pkg/urbit/vere/ward.c | 16 ---------------- pkg/urbit/worker/main.c | 16 ---------------- pkg/urbit/worker/serf.c | 16 ---------------- 29 files changed, 24 insertions(+), 262 deletions(-) diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 9364dbc221..9526df5987 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -1,16 +1,12 @@ /* vere/main.c ** */ -#include -#include -#include -#include -#include -#include +#define U3_GLOBAL +#define C3_GLOBAL +#include "all.h" +#include "vere/vere.h" #include -#include #include -#include #include #include #include @@ -20,11 +16,6 @@ #include #include -#define U3_GLOBAL -#define C3_GLOBAL -#include "all.h" -#include "vere/vere.h" - #include "ca-bundle.h" /* Require unsigned char diff --git a/pkg/urbit/noun/events.c b/pkg/urbit/noun/events.c index 2d6286c9e4..49818618e8 100644 --- a/pkg/urbit/noun/events.c +++ b/pkg/urbit/noun/events.c @@ -1,12 +1,11 @@ /* g/e.c ** */ +#include "all.h" #include #include #include -#include "all.h" - #ifdef U3_SNAPSHOT_VALIDATION /* Image check. */ diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 669fd1f3fe..ee9a5a1673 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -1,6 +1,8 @@ /* n/m.c ** */ +#include "all.h" +#include "vere/vere.h" #include #include #include @@ -9,9 +11,6 @@ #include #include -#include "all.h" -#include "vere/vere.h" - // XX stack-overflow recovery should be gated by -a // #undef NO_OVERFLOW diff --git a/pkg/urbit/noun/serial.c b/pkg/urbit/noun/serial.c index dac9970a5f..cc92ff9c3f 100644 --- a/pkg/urbit/noun/serial.c +++ b/pkg/urbit/noun/serial.c @@ -2,11 +2,10 @@ ** */ -#include -#include - #include "all.h" #include "ur/ur.h" +#include +#include /* _cs_jam_buf: struct for tracking the fibonacci-allocated jam of a noun */ diff --git a/pkg/urbit/noun/urth.c b/pkg/urbit/noun/urth.c index 31b38d95b8..8fc0b33145 100644 --- a/pkg/urbit/noun/urth.c +++ b/pkg/urbit/noun/urth.c @@ -1,14 +1,13 @@ /* noun/urth.c ** */ +#include "all.h" +#include "ur/ur.h" #include #include #include #include -#include "all.h" -#include "ur/ur.h" - /* _cu_atom_to_ref(): allocate indirect atom off-loom. */ static inline ur_nref diff --git a/pkg/urbit/noun/vortex.c b/pkg/urbit/noun/vortex.c index 4bd17746af..acf7eea9d2 100644 --- a/pkg/urbit/noun/vortex.c +++ b/pkg/urbit/noun/vortex.c @@ -1,8 +1,8 @@ /* g/v.c ** */ -#include #include "all.h" +#include #define _CVX_LOAD 4 #define _CVX_PEEK 22 diff --git a/pkg/urbit/vere/auto.c b/pkg/urbit/vere/auto.c index 759f9a6e47..884cc75f45 100644 --- a/pkg/urbit/vere/auto.c +++ b/pkg/urbit/vere/auto.c @@ -1,21 +1,5 @@ /* vere/auto.c */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" diff --git a/pkg/urbit/vere/dawn.c b/pkg/urbit/vere/dawn.c index b278adefdd..6cb6e71420 100644 --- a/pkg/urbit/vere/dawn.c +++ b/pkg/urbit/vere/dawn.c @@ -2,11 +2,10 @@ ** ** ethereum-integrated pre-boot validation */ -#include -#include - #include "all.h" #include "vere/vere.h" +#include +#include /* _dawn_oct_to_buf(): +octs to uv_buf_t */ diff --git a/pkg/urbit/vere/db/lmdb.c b/pkg/urbit/vere/db/lmdb.c index 6160c2b4c9..e720eeed24 100644 --- a/pkg/urbit/vere/db/lmdb.c +++ b/pkg/urbit/vere/db/lmdb.c @@ -1,11 +1,9 @@ /* vere/db/lmdb.c */ - -#include - #include "c/portable.h" #include "c/types.h" #include "c/defs.h" +#include #include diff --git a/pkg/urbit/vere/disk.c b/pkg/urbit/vere/disk.c index 691f2ae683..6f8240daae 100644 --- a/pkg/urbit/vere/disk.c +++ b/pkg/urbit/vere/disk.c @@ -1,21 +1,5 @@ /* vere/disk.c */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" #include diff --git a/pkg/urbit/vere/foil.c b/pkg/urbit/vere/foil.c index 9c78cd2969..1d844f5d41 100644 --- a/pkg/urbit/vere/foil.c +++ b/pkg/urbit/vere/foil.c @@ -4,22 +4,6 @@ */ #include "all.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "vere/vere.h" /* assumptions: diff --git a/pkg/urbit/vere/io/ames.c b/pkg/urbit/vere/io/ames.c index 64cee5fcea..ac828ab6fa 100644 --- a/pkg/urbit/vere/io/ames.c +++ b/pkg/urbit/vere/io/ames.c @@ -1,15 +1,6 @@ /* vere/ames.c ** */ -#include -#include -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" #include "ur/serial.h" diff --git a/pkg/urbit/vere/io/behn.c b/pkg/urbit/vere/io/behn.c index a6a7b0736f..c9394564e1 100644 --- a/pkg/urbit/vere/io/behn.c +++ b/pkg/urbit/vere/io/behn.c @@ -1,13 +1,6 @@ /* vere/behn.c ** */ -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" diff --git a/pkg/urbit/vere/io/cttp.c b/pkg/urbit/vere/io/cttp.c index 0d0fbb1bae..56bbb8604d 100644 --- a/pkg/urbit/vere/io/cttp.c +++ b/pkg/urbit/vere/io/cttp.c @@ -1,18 +1,10 @@ /* vere/cttp.c ** */ -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" +#include +#include /* u3_csat: client connection state. */ diff --git a/pkg/urbit/vere/io/fore.c b/pkg/urbit/vere/io/fore.c index 1834cc7e11..81476d92a1 100644 --- a/pkg/urbit/vere/io/fore.c +++ b/pkg/urbit/vere/io/fore.c @@ -1,13 +1,6 @@ /* vere/root.c ** */ -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" diff --git a/pkg/urbit/vere/io/hind.c b/pkg/urbit/vere/io/hind.c index b8b1577ee6..b0d207e838 100644 --- a/pkg/urbit/vere/io/hind.c +++ b/pkg/urbit/vere/io/hind.c @@ -1,13 +1,6 @@ /* vere/root.c ** */ -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" diff --git a/pkg/urbit/vere/io/http.c b/pkg/urbit/vere/io/http.c index 7705a3bc48..451a4e26f4 100644 --- a/pkg/urbit/vere/io/http.c +++ b/pkg/urbit/vere/io/http.c @@ -1,21 +1,12 @@ /* vere/http.c ** */ -#include -#include -#include -#include -#include -#include -#include -#include +#include "all.h" +#include "vere/vere.h" #include #include #include -#include "all.h" -#include "vere/vere.h" - typedef struct _u3_h2o_serv { h2o_globalconf_t fig_u; // h2o global config h2o_context_t ctx_u; // h2o ctx diff --git a/pkg/urbit/vere/io/term.c b/pkg/urbit/vere/io/term.c index fe58abc32c..3f48b49908 100644 --- a/pkg/urbit/vere/io/term.c +++ b/pkg/urbit/vere/io/term.c @@ -1,6 +1,8 @@ /* vere/term.c ** */ +#include "all.h" +#include "vere/vere.h" #include #include #include @@ -10,9 +12,6 @@ #include #include -#include "all.h" -#include "vere/vere.h" - // macros for string literal args/buffers // // since (sizeof(s) - 1) is used for vector length, parameters diff --git a/pkg/urbit/vere/io/unix.c b/pkg/urbit/vere/io/unix.c index 1207b6548c..2885686c4a 100644 --- a/pkg/urbit/vere/io/unix.c +++ b/pkg/urbit/vere/io/unix.c @@ -2,15 +2,7 @@ ** */ #include "all.h" -#include -#include -#include -#include -#include -#include -#include #include - #include "vere/vere.h" struct _u3_umon; diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c index 7899cfd691..ff6a997903 100644 --- a/pkg/urbit/vere/king.c +++ b/pkg/urbit/vere/king.c @@ -2,13 +2,11 @@ ** ** the main loop of the daemon process */ -#include -#include -#include - #include "all.h" #include "vere/vere.h" #include "ur/ur.h" +#include +#include #include "ivory.h" diff --git a/pkg/urbit/vere/lord.c b/pkg/urbit/vere/lord.c index b910da1fd3..501427d3d7 100644 --- a/pkg/urbit/vere/lord.c +++ b/pkg/urbit/vere/lord.c @@ -1,21 +1,5 @@ /* vere/lord.c */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" #include "ur/hashcons.h" diff --git a/pkg/urbit/vere/newt.c b/pkg/urbit/vere/newt.c index d96c7197d2..6325f0ef90 100644 --- a/pkg/urbit/vere/newt.c +++ b/pkg/urbit/vere/newt.c @@ -9,22 +9,6 @@ ** the implementation is relatively inefficient and could ** lose a few copies, mallocs, etc. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" diff --git a/pkg/urbit/vere/pier.c b/pkg/urbit/vere/pier.c index bc0a1ce466..edee2db242 100644 --- a/pkg/urbit/vere/pier.c +++ b/pkg/urbit/vere/pier.c @@ -1,24 +1,8 @@ /* vere/pier.c */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" +#include #define PIER_READ_BATCH 1000ULL #define PIER_PLAY_BATCH 500ULL diff --git a/pkg/urbit/vere/save.c b/pkg/urbit/vere/save.c index e2c042b243..2f093e69be 100644 --- a/pkg/urbit/vere/save.c +++ b/pkg/urbit/vere/save.c @@ -1,13 +1,6 @@ /* vere/save.c ** */ -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" diff --git a/pkg/urbit/vere/time.c b/pkg/urbit/vere/time.c index e67046ba22..993a04a9f3 100644 --- a/pkg/urbit/vere/time.c +++ b/pkg/urbit/vere/time.c @@ -1,12 +1,6 @@ /* vere/time.c ** */ -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" diff --git a/pkg/urbit/vere/walk.c b/pkg/urbit/vere/walk.c index 2cf70591c9..4d03f12bdd 100644 --- a/pkg/urbit/vere/walk.c +++ b/pkg/urbit/vere/walk.c @@ -1,13 +1,6 @@ /* vere/walk.c ** */ -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" diff --git a/pkg/urbit/vere/ward.c b/pkg/urbit/vere/ward.c index f4db6badf7..f0c4260200 100644 --- a/pkg/urbit/vere/ward.c +++ b/pkg/urbit/vere/ward.c @@ -1,21 +1,5 @@ /* vere/ward.c */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "all.h" #include "vere/vere.h" diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index 183cea70e7..5377ce90d9 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -2,22 +2,6 @@ ** ** the main loop of a serf process. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "all.h" #include #include diff --git a/pkg/urbit/worker/serf.c b/pkg/urbit/worker/serf.c index 3774cbfc01..2376d8e48a 100644 --- a/pkg/urbit/worker/serf.c +++ b/pkg/urbit/worker/serf.c @@ -1,21 +1,5 @@ /* worker/serf.c */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include "all.h" #include #include From 98c002bc267bdc08e5595c59d4753682f6169249 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 20:04:39 +0000 Subject: [PATCH 010/136] u3: fix typo: show suggested loom base if fixed address is unavailable --- pkg/urbit/noun/manage.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index ee9a5a1673..2387c24396 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -1668,7 +1668,7 @@ u3m_init(void) u3l_log("boot: mapping %dMB failed\r\n", (len_w / (1024 * 1024))); u3l_log("see urbit.org/using/install/#about-swap-space" " for adding swap space\r\n"); - if ( -1 != (c3_ps)map_v ) { + if ( -1 != (c3_ps)dyn_v ) { u3l_log("if porting to a new platform, try U3_OS_LoomBase %p\r\n", dyn_v); } From 0e47c37f9cfdf9f2d02718db1e4d07cced72b3e0 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 20:08:22 +0000 Subject: [PATCH 011/136] vere: use embedded CA store directly rather than save it to a tempfile Vere writes the embedded CA store to a tempfile and directs libcurl and libh2o to pick it up from there, but on MingW this is inconvenient because temp paths are different and mkstemp(3) is not available. Loading CA store certificates directly from memory is tidier. --- pkg/urbit/daemon/main.c | 69 +++++++++++++++++++++++++++-------- pkg/urbit/include/vere/vere.h | 3 +- pkg/urbit/vere/dawn.c | 4 +- pkg/urbit/vere/io/cttp.c | 2 +- pkg/urbit/vere/king.c | 3 +- 5 files changed, 59 insertions(+), 22 deletions(-) diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 9526df5987..e2810e16b6 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -352,26 +352,62 @@ _main_getopt(c3_i argc, c3_c** argv) return c3y; } -/* _setup_cert_store: writes our embedded certificate database to a temp file +/* _cert_store: decoded CA certificates + */ +static STACK_OF(X509_INFO)* _cert_store; + +/* _setup_cert_store(): decodes embedded CA certificates */ static void -_setup_cert_store(char* tmp_cert_file_name) +_setup_cert_store() { - errno = 0; - int fd = mkstemp(tmp_cert_file_name); - if (fd < 1) { - printf("boot: failed to write local ssl temporary certificate store: %s\n", - strerror(errno)); + BIO* cbio = BIO_new_mem_buf(include_ca_bundle_crt, include_ca_bundle_crt_len); + if ( !cbio || !(_cert_store = PEM_X509_INFO_read_bio(cbio, NULL, NULL, NULL)) ) { + u3l_log("boot: failed to decode embedded CA certificates\r\n"); exit(1); } - if (-1 == write(fd, include_ca_bundle_crt, include_ca_bundle_crt_len)) { - printf("boot: failed to write local ssl temporary certificate store: %s\n", - strerror(errno)); - exit(1); - } + BIO_free(cbio); +} - setenv("SSL_CERT_FILE", tmp_cert_file_name, 1); +/* _setup_ssl_x509(): adds embedded CA certificates to a X509_STORE + */ +static void +_setup_ssl_x509(X509_STORE* cts) +{ + int i; + for ( i = 0; i < sk_X509_INFO_num(_cert_store); i++ ) { + X509_INFO *itmp = sk_X509_INFO_value(_cert_store, i); + if(itmp->x509) { + X509_STORE_add_cert(cts, itmp->x509); + } + if(itmp->crl) { + X509_STORE_add_crl(cts, itmp->crl); + } + } +} + +/* _curl_ssl_ctx_cb(): curl SSL context callback + */ +static CURLcode +_curl_ssl_ctx_cb(CURL* curl, SSL_CTX* sslctx, void* param) +{ + X509_STORE* cts = SSL_CTX_get_cert_store(sslctx); + if (!cts || !_cert_store) + return CURLE_ABORTED_BY_CALLBACK; + + _setup_ssl_x509(cts); + return CURLE_OK; +} + +/* _setup_ssl_curl(): adds embedded CA certificates to a curl context + */ +static void +_setup_ssl_curl(CURL* curl) +{ + curl_easy_setopt(curl, CURLOPT_CAINFO, NULL); + curl_easy_setopt(curl, CURLOPT_CAPATH, NULL); + curl_easy_setopt(curl, CURLOPT_SSL_CTX_FUNCTION, _curl_ssl_ctx_cb); } @@ -681,9 +717,6 @@ main(c3_i argc, } // printf("vere: hostname is %s\n", u3_Host.ops_u.nam_c); - u3K.certs_c = strdup("/tmp/urbit-ca-cert-XXXXXX"); - _setup_cert_store(u3K.certs_c); - if ( c3y == u3_Host.ops_u.dem ) { printf("boot: running as daemon\n"); } @@ -764,6 +797,10 @@ main(c3_i argc, exit(1); } + _setup_cert_store(); + u3K.ssl_curl_f = _setup_ssl_curl; + u3K.ssl_x509_f = _setup_ssl_x509; + u3_king_commence(); // uninitialize curl diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index 657d1860d7..67f1ad543e 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -640,7 +640,8 @@ /* u3_king: all executing piers. */ typedef struct _u3_king { - c3_c* certs_c; // ssl certificate dump + void (*ssl_curl_f)(void*); // setup ssl CAs in CURL* + void (*ssl_x509_f)(void*); // setup ssl CAs in X509_STORE* u3_pier* pir_u; // pier list uv_timer_t tim_u; // gc timer } u3_king; diff --git a/pkg/urbit/vere/dawn.c b/pkg/urbit/vere/dawn.c index 6cb6e71420..a7663235b3 100644 --- a/pkg/urbit/vere/dawn.c +++ b/pkg/urbit/vere/dawn.c @@ -79,7 +79,7 @@ _dawn_post_json(c3_c* url_c, uv_buf_t lod_u) // XX require TLS, pin default cert? // - curl_easy_setopt(curl, CURLOPT_CAINFO, u3K.certs_c); + u3K.ssl_curl_f(curl); curl_easy_setopt(curl, CURLOPT_URL, url_c); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, _dawn_curl_alloc); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf_u); @@ -126,7 +126,7 @@ _dawn_get_jam(c3_c* url_c) // XX require TLS, pin default cert? // - curl_easy_setopt(curl, CURLOPT_CAINFO, u3K.certs_c); + u3K.ssl_curl_f(curl); curl_easy_setopt(curl, CURLOPT_URL, url_c); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, _dawn_curl_alloc); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf_u); diff --git a/pkg/urbit/vere/io/cttp.c b/pkg/urbit/vere/io/cttp.c index 56bbb8604d..4db12432b3 100644 --- a/pkg/urbit/vere/io/cttp.c +++ b/pkg/urbit/vere/io/cttp.c @@ -939,8 +939,8 @@ _cttp_init_tls(void) // SSL_OP_NO_TLSv1 | // XX test SSL_OP_NO_COMPRESSION); + u3K.ssl_x509_f(SSL_CTX_get_cert_store(tls_u)); SSL_CTX_set_verify(tls_u, SSL_VERIFY_PEER, 0); - SSL_CTX_set_default_verify_paths(tls_u); SSL_CTX_set_session_cache_mode(tls_u, SSL_SESS_CACHE_OFF); SSL_CTX_set_cipher_list(tls_u, "ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:" diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c index ff6a997903..ad1dc14304 100644 --- a/pkg/urbit/vere/king.c +++ b/pkg/urbit/vere/king.c @@ -244,7 +244,7 @@ _king_get_atom(c3_c* url_c) exit(1); } - curl_easy_setopt(curl, CURLOPT_CAINFO, u3K.certs_c); + u3K.ssl_curl_f(curl); curl_easy_setopt(curl, CURLOPT_URL, url_c); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, _king_curl_alloc); curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void*)&buf_u); @@ -677,7 +677,6 @@ _king_loop_init() void _king_loop_exit() { - unlink(u3K.certs_c); } static void From 0d9de4f30406617c8369c8a9df346870ced985ee Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 20:11:20 +0000 Subject: [PATCH 012/136] u3: fix patch cleanup In some places, _ce_patch_delete is invoked before _ce_patch_free. This works on Unix, where an open file can be unlinked, but on Windows one must open files with an extra flag FILE_SHARE_DELETE for this to work, and there is no way to pass this flag to open(). --- pkg/urbit/noun/events.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/noun/events.c b/pkg/urbit/noun/events.c index 49818618e8..0085df8ed7 100644 --- a/pkg/urbit/noun/events.c +++ b/pkg/urbit/noun/events.c @@ -814,8 +814,8 @@ u3e_save(void) _ce_image_sync(&u3P.nor_u); _ce_image_sync(&u3P.sou_u); - _ce_patch_delete(); _ce_patch_free(pat_u); + _ce_patch_delete(); } /* u3e_live(): start the checkpointing system. @@ -852,8 +852,8 @@ u3e_live(c3_o nuu_o, c3_c* dir_c) _ce_patch_apply(pat_u); _ce_image_sync(&u3P.nor_u); _ce_image_sync(&u3P.sou_u); - _ce_patch_delete(); _ce_patch_free(pat_u); + _ce_patch_delete(); } /* Write image files to memory; reinstate protection. From bf38e79520e3cf27e76e59267fffca689f954740 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 20:12:27 +0000 Subject: [PATCH 013/136] u3: centralize use of u3_Signal --- pkg/urbit/noun/manage.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 2387c24396..89f3d67f51 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -126,7 +126,7 @@ static void _cm_overflow(void *arg1, void *arg2, void *arg3) (void)(arg1); (void)(arg2); (void)(arg3); - siglongjmp(u3_Signal, c3__over); + u3m_signal(c3__over); } /* _cm_signal_handle(): handle a signal in general. @@ -138,7 +138,7 @@ _cm_signal_handle(c3_l sig_l) sigsegv_leave_handler(_cm_overflow, NULL, NULL, NULL); } else { - siglongjmp(u3_Signal, sig_l); + u3m_signal(sig_l); } } From d31d6f656f2ba3dc2d91535406c649510c2f7c8a Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 20:13:19 +0000 Subject: [PATCH 014/136] u3: rename function to avoid name clash on MingW --- pkg/urbit/noun/manage.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 89f3d67f51..11f865a0c8 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -97,7 +97,7 @@ _cm_punt(u3_noun tax) } #endif -static void _write(int fd, const void *buf, size_t count) +static void _write_or_fail(int fd, const void *buf, size_t count) { if (count != write(fd, buf, count)){ u3l_log("write failed\r\n"); @@ -111,14 +111,14 @@ static void _write(int fd, const void *buf, size_t count) static void _cm_emergency(c3_c* cap_c, c3_l sig_l) { - _write(2, "\r\n", 2); - _write(2, cap_c, strlen(cap_c)); + _write_or_fail(2, "\r\n", 2); + _write_or_fail(2, cap_c, strlen(cap_c)); if ( sig_l ) { - _write(2, ": ", 2); - _write(2, &sig_l, 4); + _write_or_fail(2, ": ", 2); + _write_or_fail(2, &sig_l, 4); } - _write(2, "\r\n", 2); + _write_or_fail(2, "\r\n", 2); } static void _cm_overflow(void *arg1, void *arg2, void *arg3) From 6ed4b7077b07e11701206882a58b7c0a4fae8b98 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 20:15:17 +0000 Subject: [PATCH 015/136] vere: replace dprintf (missing on MingW) with snprintf+write --- pkg/urbit/vere/io/http.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/vere/io/http.c b/pkg/urbit/vere/io/http.c index 451a4e26f4..8a26a35ae9 100644 --- a/pkg/urbit/vere/io/http.c +++ b/pkg/urbit/vere/io/http.c @@ -1605,11 +1605,12 @@ _http_write_ports_file(u3_httd* htd_u, c3_c *pax_c) u3_http* htp_u = htd_u->htp_u; + c3_c temp[32]; while ( 0 != htp_u ) { if ( 0 < htp_u->por_s ) { - dprintf(por_i, "%u %s %s\n", htp_u->por_s, + write(por_i, temp, snprintf(temp, 32, "%u %s %s\n", htp_u->por_s, (c3y == htp_u->sec) ? "secure" : "insecure", - (c3y == htp_u->lop) ? "loopback" : "public"); + (c3y == htp_u->lop) ? "loopback" : "public")); } htp_u = htp_u->nex_u; From ad81a017c522eca0edb4ac2c33ee88fe4fae2c40 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 20:15:56 +0000 Subject: [PATCH 016/136] libent: use BCryptGenRandom on Windows --- pkg/ent/configure | 1 + pkg/ent/ent.c | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/ent/configure b/pkg/ent/configure index 3e98a5108d..8171293f3f 100755 --- a/pkg/ent/configure +++ b/pkg/ent/configure @@ -7,6 +7,7 @@ log () { for impl in ENT_GETENTROPY_UNISTD \ ENT_GETENTROPY_SYSRANDOM \ ENT_GETRANDOM_SYSCALL \ + ENT_GETENTROPY_BCRYPTGENRANDOM \ ENT_DEV_URANDOM do export IMPL=$impl diff --git a/pkg/ent/ent.c b/pkg/ent/ent.c index 7aee7f48d9..e7309ff339 100644 --- a/pkg/ent/ent.c +++ b/pkg/ent/ent.c @@ -57,6 +57,16 @@ int ent_getentropy(void* buf, size_t len) { return 0; } + +// Use `BCryptGenRandom` on Windows //////////////////////////////////////////// + +#elif defined(ENT_GETENTROPY_BCRYPTGENRANDOM) +#include +#include +int ent_getentropy(void* buf, size_t len) { + return BCryptGenRandom(NULL, (PUCHAR)buf, len, BCRYPT_USE_SYSTEM_PREFERRED_RNG); +} + #else -#error "One of these must be set: ENT_DEV_URANDOM, ENT_GETENTROPY_UNISTD, ENT_GETENTROPY_SYSRANDOM, ENT_GETRANDOM_SYSCALL" +#error "One of these must be set: ENT_GETENTROPY_BCRYPTGENRANDOM, ENT_DEV_URANDOM, ENT_GETENTROPY_UNISTD, ENT_GETENTROPY_SYSRANDOM, ENT_GETRANDOM_SYSCALL" #endif From b105ce946a6900099df7ee33b53d4cb91d77ef3d Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 20:17:09 +0000 Subject: [PATCH 017/136] u3: add U3_OS_PROF macro to gate profiling-related code --- pkg/urbit/configure | 3 +++ pkg/urbit/daemon/main.c | 6 ++++-- pkg/urbit/noun/manage.c | 2 ++ pkg/urbit/noun/trace.c | 13 ++----------- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index cfc42f83e3..fa70c1997e 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -60,12 +60,15 @@ esac case $(tr A-Z a-z <<< $os) in *linux*) defmacro U3_OS_linux 1 + defmacro U3_OS_PROF 1 ;; *darwin*) defmacro U3_OS_osx 1 + defmacro U3_OS_PROF 1 ;; *apple*) defmacro U3_OS_osx 1 + defmacro U3_OS_PROF 1 ;; *freebsd*) defmacro U3_OS_bsd 1 diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index e2810e16b6..09e1a3af70 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -207,9 +207,9 @@ _main_getopt(c3_i argc, c3_c** argv) } } -#if defined(U3_OS_bsd) +#if !defined(U3_OS_PROF) if (u3_Host.ops_u.pro == c3y) { - fprintf(stderr, "profiling isn't yet supported on BSD\r\n"); + fprintf(stderr, "profiling isn't yet supported on your OS\r\n"); return c3n; } #endif @@ -670,6 +670,7 @@ main(c3_i argc, // // XX review, may be unnecessary due to similar in u3m_init() // +#if defined(U3_OS_PROF) if ( _(u3_Host.ops_u.pro) ) { sigset_t set; @@ -680,6 +681,7 @@ main(c3_i argc, exit(1); } } +#endif // Handle SIGTSTP as if it was SIGTERM. // diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 11f865a0c8..922620f286 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -1618,6 +1618,7 @@ _cm_signals(void) // signal(SIGINT, _loom_stop); +# if defined(U3_OS_PROF) // Block SIGPROF, so that if/when we reactivate it on the // main thread for profiling, we won't get hits in parallel // on other threads. @@ -1632,6 +1633,7 @@ _cm_signals(void) exit(1); } } +# endif } /* u3m_init(): start the environment. diff --git a/pkg/urbit/noun/trace.c b/pkg/urbit/noun/trace.c index 53a1ff92cc..2b53ae9388 100644 --- a/pkg/urbit/noun/trace.c +++ b/pkg/urbit/noun/trace.c @@ -519,7 +519,7 @@ u3t_boot(void) { if ( u3C.wag_w & u3o_debug_cpu ) { _ct_lop_o = c3n; -#if defined(U3_OS_osx) || defined(U3_OS_linux) +#if defined(U3_OS_PROF) // skip profiling if we don't yet have an arvo kernel // if ( 0 == u3A->roc ) { @@ -551,10 +551,6 @@ u3t_boot(void) itm_v.it_value = itm_v.it_interval; setitimer(ITIMER_PROF, &itm_v, 0); } -#elif defined(U3_OS_bsd) - // XX "Profiling isn't yet supported on BSD" -#else -# error "port: profiling" #endif } } @@ -565,7 +561,7 @@ void u3t_boff(void) { if ( u3C.wag_w & u3o_debug_cpu ) { -#if defined(U3_OS_osx) || defined(U3_OS_linux) +#if defined(U3_OS_PROF) // Mask SIGPROF signals in this thread (and this is the only // thread that unblocked them). { @@ -590,11 +586,6 @@ u3t_boff(void) sig_s.sa_handler = SIG_IGN; sigaction(SIGPROF, &sig_s, 0); } - -#elif defined(U3_OS_bsd) - // XX "Profiling isn't yet supported on BSD" -#else -# error "port: profiling" #endif } } From a9d166ced1f95cddc5df13786dea09bf199520a8 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 27 Mar 2021 20:18:00 +0000 Subject: [PATCH 018/136] vere: comment out unused code that doesn't compile on MingW --- pkg/urbit/vere/save.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/urbit/vere/save.c b/pkg/urbit/vere/save.c index 2f093e69be..b2fc5b3904 100644 --- a/pkg/urbit/vere/save.c +++ b/pkg/urbit/vere/save.c @@ -13,6 +13,7 @@ _save_time_cb(uv_timer_t* tim_u) u3_pier_save(pir_u); } +#if 0 /* u3_save_ef_chld(): report save termination. */ void @@ -34,6 +35,7 @@ u3_save_ef_chld(u3_pier *pir_u) } sav_u->pid_w = 0; } +#endif /* u3_save_io_init(): initialize autosave. */ From 7b55097ef4c4703d960029ebc3b88c1959e000bb Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Sun, 28 Mar 2021 00:21:30 -0700 Subject: [PATCH 019/136] hoon: memoize rend:co We commonly print many names in a row, often the same ones. For example, on landscape's initial load, we send all the members of all the groups we're in, and there's substantial overlap. At least half the cost is in +fein, which is not currently jetted, but I believe there's an old jet in the git history. --- pkg/arvo/sys/hoon.hoon | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/sys/hoon.hoon b/pkg/arvo/sys/hoon.hoon index 834cbcf390..e5298ea746 100644 --- a/pkg/arvo/sys/hoon.hoon +++ b/pkg/arvo/sys/hoon.hoon @@ -5455,9 +5455,10 @@ ~% %co ..co ~ =< |_ lot=coin ++ rear |=(rom=tape rend(rep rom)) - ++ rent `@ta`(rap 3 rend) + ++ rent ~+ `@ta`(rap 3 rend) ++ rend ^- tape + ~+ ?: ?=(%blob -.lot) ['~' '0' ((v-co 1) (jam p.lot))] ?: ?=(%many -.lot) From 56348ed61e5e2be26ae7d79000c140f342a7dae4 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sun, 28 Mar 2021 07:37:10 +0000 Subject: [PATCH 020/136] vere: add -fcommon to be compatible with gcc 10 gcc 9 and earlier default to -fcommon, i.e. linker merges identical symbols. gcc 10 defaults to -fno-common, and the build breaks because c3_global is not defined anywhere. --- pkg/urbit/configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index fa70c1997e..3ea527fdd0 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -93,7 +93,7 @@ do LDFLAGS="${LDFLAGS-} -I$header" done cat >config.mk < Date: Sun, 28 Mar 2021 00:47:37 -0700 Subject: [PATCH 021/136] clay, eyre: add profiling hints --- pkg/arvo/sys/vane/clay.hoon | 4 ++++ pkg/arvo/sys/vane/eyre.hoon | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index af2f653037..55e3ea7afc 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -3962,8 +3962,10 @@ ruf=raft :: revision tree == :: |= [now=@da eny=@uvJ rof=roof] :: current invocation +~% %clay-top ..part ~ |% :: ++ call :: handle request + ~/ %clay-call |= $: hen=duct dud=(unit goof) wrapped-task=(hobo task) @@ -4273,6 +4275,7 @@ -- :: ++ scry :: inspect + ~/ %clay-scry ^- roon |= [lyc=gang car=term bem=beam] ^- (unit (unit cage)) @@ -4334,6 +4337,7 @@ == :: ++ take :: accept response + ~/ %clay-take |= [tea=wire hen=duct dud=(unit goof) hin=sign] ^+ [*(list move) ..^$] ?^ dud diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index b509e71314..e878c64334 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -1597,6 +1597,7 @@ :: +channel-event-to-sign: attempt to recover a sign from a channel-event :: ++ channel-event-to-sign + ~% %eyre-channel-event-to-sign ..part ~ |= event=channel-event ^- (unit sign:agent:gall) ?. ?=(%fact -.event) `event @@ -1677,6 +1678,7 @@ == :: ++ event-json-to-wall + ~% %eyre-json-to-wall ..part ~ |= [event-id=@ud =json] ^- wall :~ (weld "id: " (format-ud-as-integer event-id)) @@ -2094,6 +2096,7 @@ ~% %http-server ..part ~ |% ++ call + ~/ %eyre-call |= [=duct dud=(unit goof) wrapped-task=(hobo task)] ^- [(list move) _http-server-gate] :: @@ -2296,6 +2299,7 @@ == :: ++ take + ~/ %eyre-take |= [=wire =duct dud=(unit goof) =sign] ^- [(list move) _http-server-gate] ?^ dud @@ -2483,6 +2487,7 @@ :: +scry: request a path in the urbit namespace :: ++ scry + ~/ %eyre-scry ^- roon |= [lyc=gang car=term bem=beam] ^- (unit (unit cage)) From a777567bb65b2ead63a6b8b08820dabc0e5ef792 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Sun, 28 Mar 2021 00:57:33 -0700 Subject: [PATCH 022/136] eyre: faster wain-to-octs conversion Avoid allocating hundreds of thousands of cells when giving large requests. This took the footprint of this function on initial landscape load from 1 second to 100 ms. --- pkg/arvo/sys/vane/eyre.hoon | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index e878c64334..bc09d4dd03 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -425,10 +425,12 @@ :- ~ %- as-octs:mimes:html %- crip - %- zing + %- zing ^- ^wall + %- zing ^- (list ^wall) %+ turn wall |= t=tape - "{t}\0a" + ^- ^wall + ~[t "\0a"] :: +internal-server-error: 500 page, with a tang :: ++ internal-server-error From fda5e4b36139e1c2a7e8e17314c3cec39d838306 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Sun, 28 Mar 2021 00:59:08 -0700 Subject: [PATCH 023/136] zuse: hack enjs to speed up ship printing In +en-json, the vast majority of our time is in +jesc (json string escaping). Since ships will always be string-safe, we pretend they're numbers to bypass the escaper. This saves about a second on initial landscape load. --- pkg/arvo/sys/zuse.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index eaf098937b..7ca6e825ef 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -3286,7 +3286,7 @@ ++ ship :: string from ship |= a=^ship ^- json - (tape (slag 1 (scow %p a))) + [%n (rap 3 '"' (scot %p a) '"' ~)] :: :: ++numb:enjs:format ++ numb :: number from unsigned |= a=@u From 820aee04781b3f389efc262bbeae134a0abca2a2 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Sun, 28 Mar 2021 01:58:21 -0700 Subject: [PATCH 024/136] zuse: strip ~ from ships in enjs --- pkg/arvo/sys/zuse.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 7ca6e825ef..8f11ba3595 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -3286,7 +3286,7 @@ ++ ship :: string from ship |= a=^ship ^- json - [%n (rap 3 '"' (scot %p a) '"' ~)] + [%n (rap 3 '"' (rsh [3 1] (scot %p a)) '"' ~)] :: :: ++numb:enjs:format ++ numb :: number from unsigned |= a=@u From a2ad6365b705719dd5cdc626e2baefc6488c5395 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sun, 28 Mar 2021 09:41:12 +0000 Subject: [PATCH 025/136] vere: lmdb: wrap mdb_strerror in mdb_logerror To avoid allocating a buffer on MingW, mdb_strerror uses a horrible stack hack which explodes in vere. Source patch replaces it with mdb_logerror on MingW. This change adds the default implementation for other platforms. --- pkg/urbit/vere/db/lmdb.c | 77 ++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 38 deletions(-) diff --git a/pkg/urbit/vere/db/lmdb.c b/pkg/urbit/vere/db/lmdb.c index e720eeed24..ab9b92dbdf 100644 --- a/pkg/urbit/vere/db/lmdb.c +++ b/pkg/urbit/vere/db/lmdb.c @@ -5,6 +5,10 @@ #include "c/defs.h" #include +/* mdb_logerror(): writes an error message and lmdb error code to f. +*/ +void mdb_logerror(FILE* f, int err, const char* fmt, ...); + #include // lmdb api wrapper @@ -36,31 +40,28 @@ u3_lmdb_init(const c3_c* pax_c, size_t siz_i) c3_w ret_w; if ( (ret_w = mdb_env_create(&env_u)) ) { - fprintf(stderr, "lmdb: init fail: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: init fail"); return 0; } // Our databases have two tables: META and EVENTS // if ( (ret_w = mdb_env_set_maxdbs(env_u, 2)) ) { - fprintf(stderr, "lmdb: failed to set number of databases: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: failed to set number of databases"); // XX dispose env_u // return 0; } if ( (ret_w = mdb_env_set_mapsize(env_u, siz_i)) ) { - fprintf(stderr, "lmdb: failed to set database size: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: failed to set database size"); // XX dispose env_u // return 0; } if ( (ret_w = mdb_env_open(env_u, pax_c, 0, 0664)) ) { - fprintf(stderr, "lmdb: failed to open event log: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: failed to open event log"); // XX dispose env_u // return 0; @@ -91,7 +92,7 @@ u3_lmdb_gulf(MDB_env* env_u, c3_d* low_d, c3_d* hig_d) // XX why no MDB_RDONLY? // if ( (ret_w = mdb_txn_begin(env_u, 0, 0, &txn_u)) ) { - fprintf(stderr, "lmdb: gulf: txn_begin fail: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: gulf: txn_begin fail"); return c3n; } @@ -101,7 +102,7 @@ u3_lmdb_gulf(MDB_env* env_u, c3_d* low_d, c3_d* hig_d) c3_w ops_w = MDB_CREATE | MDB_INTEGERKEY; if ( (ret_w = mdb_dbi_open(txn_u, "EVENTS", ops_w, &mdb_u)) ) { - fprintf(stderr, "lmdb: gulf: dbi_open fail: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: gulf: dbi_open fail"); // XX confirm // mdb_txn_abort(txn_u); @@ -118,8 +119,7 @@ u3_lmdb_gulf(MDB_env* env_u, c3_d* low_d, c3_d* hig_d) // creates a cursor to point to the last event // if ( (ret_w = mdb_cursor_open(txn_u, mdb_u, &cur_u)) ) { - fprintf(stderr, "lmdb: gulf: cursor_open fail: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: gulf: cursor_open fail"); // XX confirm // mdb_txn_abort(txn_u); @@ -138,8 +138,7 @@ u3_lmdb_gulf(MDB_env* env_u, c3_d* low_d, c3_d* hig_d) return c3y; } else if ( ret_w ) { - fprintf(stderr, "lmdb: gulf: head fail: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: gulf: head fail"); mdb_cursor_close(cur_u); mdb_txn_abort(txn_u); return c3n; @@ -162,7 +161,7 @@ u3_lmdb_gulf(MDB_env* env_u, c3_d* low_d, c3_d* hig_d) mdb_txn_abort(txn_u); if ( ret_w ) { - fprintf(stderr, "lmdb: gulf: last fail: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: gulf: last fail"); return c3n; } else { @@ -189,7 +188,7 @@ u3_lmdb_read(MDB_env* env_u, // create a read-only transaction. // if ( (ret_w = mdb_txn_begin(env_u, 0, MDB_RDONLY, &txn_u)) ) { - fprintf(stderr, "lmdb: read txn_begin fail: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: read txn_begin fail"); return c3n; } @@ -199,7 +198,7 @@ u3_lmdb_read(MDB_env* env_u, c3_w ops_w = MDB_CREATE | MDB_INTEGERKEY; if ( (ret_w = mdb_dbi_open(txn_u, "EVENTS", ops_w, &mdb_u)) ) { - fprintf(stderr, "lmdb: read: dbi_open fail: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: read: dbi_open fail"); // XX confirm // mdb_txn_abort(txn_u); @@ -218,8 +217,7 @@ u3_lmdb_read(MDB_env* env_u, // creates a cursor to iterate over keys starting at [eve_d] // if ( (ret_w = mdb_cursor_open(txn_u, mdb_u, &cur_u)) ) { - fprintf(stderr, "lmdb: read: cursor_open fail: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: read: cursor_open fail"); // XX confirm // mdb_txn_abort(txn_u); @@ -229,9 +227,7 @@ u3_lmdb_read(MDB_env* env_u, // set the cursor to the position of [eve_d] // if ( (ret_w = mdb_cursor_get(cur_u, &key_u, &val_u, MDB_SET_KEY)) ) { - fprintf(stderr, "lmdb: read: initial cursor_get failed at %" PRIu64 ": %s\r\n", - eve_d, - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: read: initial cursor_get failed at %" PRIu64, eve_d); mdb_cursor_close(cur_u); // XX confirm // @@ -276,8 +272,7 @@ u3_lmdb_read(MDB_env* env_u, if ( (ret_w = mdb_cursor_get(cur_u, &key_u, &val_u, MDB_NEXT)) && (MDB_NOTFOUND != ret_w) ) { - fprintf(stderr, "lmdb: read: error: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: read: error"); ret_o = c3n; break; } @@ -310,7 +305,7 @@ u3_lmdb_save(MDB_env* env_u, // create a write transaction // if ( (ret_w = mdb_txn_begin(env_u, 0, 0, &txn_u)) ) { - fprintf(stderr, "lmdb: write: txn_begin fail: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: write: txn_begin fail"); return c3n; } @@ -320,7 +315,7 @@ u3_lmdb_save(MDB_env* env_u, c3_w ops_w = MDB_CREATE | MDB_INTEGERKEY; if ( (ret_w = mdb_dbi_open(txn_u, "EVENTS", ops_w, &mdb_u)) ) { - fprintf(stderr, "lmdb: write: dbi_open fail: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: write: dbi_open fail"); mdb_txn_abort(txn_u); return c3n; } @@ -352,7 +347,7 @@ u3_lmdb_save(MDB_env* env_u, // commit transaction // if ( (ret_w = mdb_txn_commit(txn_u)) ) { - fprintf(stderr, "lmdb: write failed: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: write failed"); return c3n; } @@ -374,16 +369,14 @@ u3_lmdb_read_meta(MDB_env* env_u, // create a read transaction // if ( (ret_w = mdb_txn_begin(env_u, 0, MDB_RDONLY, &txn_u)) ) { - fprintf(stderr, "lmdb: meta read: txn_begin fail: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: meta read: txn_begin fail"); return read_f(ptr_v, 0, 0); } // open the database in the transaction // if ( (ret_w = mdb_dbi_open(txn_u, "META", 0, &mdb_u)) ) { - fprintf(stderr, "lmdb: meta read: dbi_open fail: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: meta read: dbi_open fail"); mdb_txn_abort(txn_u); return read_f(ptr_v, 0, 0); } @@ -394,7 +387,7 @@ u3_lmdb_read_meta(MDB_env* env_u, MDB_val val_u; if ( (ret_w = mdb_get(txn_u, mdb_u, &key_u, &val_u)) ) { - fprintf(stderr, "lmdb: read failed: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: read failed"); mdb_txn_abort(txn_u); return read_f(ptr_v, 0, 0); } @@ -423,16 +416,14 @@ u3_lmdb_save_meta(MDB_env* env_u, // create a write transaction // if ( (ret_w = mdb_txn_begin(env_u, 0, 0, &txn_u)) ) { - fprintf(stderr, "lmdb: meta write: txn_begin fail: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: meta write: txn_begin fail"); return c3n; } // opens the database in the transaction // if ( (ret_w = mdb_dbi_open(txn_u, "META", MDB_CREATE, &mdb_u)) ) { - fprintf(stderr, "lmdb: meta write: dbi_open fail: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: meta write: dbi_open fail"); mdb_txn_abort(txn_u); return c3n; } @@ -444,7 +435,7 @@ u3_lmdb_save_meta(MDB_env* env_u, MDB_val val_u = { .mv_size = val_i, .mv_data = val_p }; if ( (ret_w = mdb_put(txn_u, mdb_u, &key_u, &val_u, 0)) ) { - fprintf(stderr, "lmdb: write failed: %s\r\n", mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: write failed"); mdb_txn_abort(txn_u); return c3n; } @@ -453,10 +444,20 @@ u3_lmdb_save_meta(MDB_env* env_u, // commit txn // if ( (ret_w = mdb_txn_commit(txn_u)) ) { - fprintf(stderr, "lmdb: meta write: commit failed: %s\r\n", - mdb_strerror(ret_w)); + mdb_logerror(stderr, ret_w, "lmdb: meta write: commit failed"); return c3n; } return c3y; } + +/* mdb_logerror(): writes an error message and lmdb error code to f. +*/ +void mdb_logerror(FILE* f, int err, const char* fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vfprintf(f, fmt, ap); + va_end(ap); + fprintf(f, ": %s\r\n", mdb_strerror(err)); +} From ef0b87a1cb7181ea9631d083289826c99a4548ac Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sun, 28 Mar 2021 17:15:07 +0000 Subject: [PATCH 026/136] vere: #include termios.h from vere.h where it is actually used --- pkg/urbit/daemon/main.c | 1 - pkg/urbit/include/vere/vere.h | 1 + pkg/urbit/vere/io/term.c | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 09e1a3af70..da3d1b29f7 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -6,7 +6,6 @@ #include "all.h" #include "vere/vere.h" #include -#include #include #include #include diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index 67f1ad543e..23679c9b52 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -2,6 +2,7 @@ */ #include +#include /** Quasi-tunable parameters. **/ diff --git a/pkg/urbit/vere/io/term.c b/pkg/urbit/vere/io/term.c index 3f48b49908..8c769e43cc 100644 --- a/pkg/urbit/vere/io/term.c +++ b/pkg/urbit/vere/io/term.c @@ -10,7 +10,6 @@ #include #include #include -#include // macros for string literal args/buffers // From 4d14b410d5bfe65d3834177babd0d00950ce37bd Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sun, 28 Mar 2021 19:27:26 +0000 Subject: [PATCH 027/136] vere: basic MingW compatibility changes This commit adds code changes, compatibility functions, stubs and a build script to build urbit binaries on MingW64. Some functionality is limited or missing: terminal input and daemon mode is not available, graceful exit does not work, and the binaries are not completely static and use (portable) MingW dlls. To build the binaries, install the MSYS2 environment, check out or copy the urbit repo and pill binaries, open a MingW64 shell and `cd pkg/urbit && ./build-mingw`. --- nix/sources-mingw.json | 47 + nix/sources.json | 30 +- pkg/urbit/Makefile | 12 +- pkg/urbit/build-mingw | 61 + pkg/urbit/compat/mingw/argon2u.patch | 22 + pkg/urbit/compat/mingw/compat.c | 436 ++++++ pkg/urbit/compat/mingw/compat.h | 41 + pkg/urbit/compat/mingw/ed25519.patch | 13 + pkg/urbit/compat/mingw/h2o.patch | 1834 +++++++++++++++++++++++ pkg/urbit/compat/mingw/libscrypt.patch | 17 + pkg/urbit/compat/mingw/lmdb.patch | 74 + pkg/urbit/compat/mingw/mman.h | 26 + pkg/urbit/compat/mingw/murmur3.patch | 13 + pkg/urbit/compat/mingw/softfloat3.patch | 36 + pkg/urbit/compat/mingw/termios.h | 8 + pkg/urbit/configure | 15 +- pkg/urbit/daemon/main.c | 10 + pkg/urbit/include/c/portable.h | 35 +- pkg/urbit/include/vere/vere.h | 2 +- pkg/urbit/noun/manage.c | 5 + pkg/urbit/vere/db/lmdb.c | 2 + pkg/urbit/vere/io/dumb.c | 171 +++ pkg/urbit/vere/king.c | 6 + pkg/urbit/vere/time.c | 4 +- pkg/urbit/worker/main.c | 4 + 25 files changed, 2904 insertions(+), 20 deletions(-) create mode 100644 nix/sources-mingw.json create mode 100755 pkg/urbit/build-mingw create mode 100755 pkg/urbit/compat/mingw/argon2u.patch create mode 100644 pkg/urbit/compat/mingw/compat.c create mode 100644 pkg/urbit/compat/mingw/compat.h create mode 100644 pkg/urbit/compat/mingw/ed25519.patch create mode 100755 pkg/urbit/compat/mingw/h2o.patch create mode 100755 pkg/urbit/compat/mingw/libscrypt.patch create mode 100644 pkg/urbit/compat/mingw/lmdb.patch create mode 100644 pkg/urbit/compat/mingw/mman.h create mode 100644 pkg/urbit/compat/mingw/murmur3.patch create mode 100644 pkg/urbit/compat/mingw/softfloat3.patch create mode 100644 pkg/urbit/compat/mingw/termios.h create mode 100644 pkg/urbit/vere/io/dumb.c diff --git a/nix/sources-mingw.json b/nix/sources-mingw.json new file mode 100644 index 0000000000..9fbb3f5f07 --- /dev/null +++ b/nix/sources-mingw.json @@ -0,0 +1,47 @@ +{ + "lmdb": { + "branch": "mdb.master", + "description": "LMDB library", + "homepage": "http://www.lmdb.tech/", + "mingw": { + "strip": 2, + "make": "liblmdb.a" + }, + "owner": "LMDB", + "repo": "lmdb", + "rev": "48a7fed59a8aae623deff415dda27097198ca0c1", + "type": "tarball", + "url": "https://github.com/LMDB/lmdb/archive/48a7fed59a8aae623deff415dda27097198ca0c1.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "secp256k1": { + "branch": "master", + "description": "Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1.", + "homepage": null, + "mingw": { + "include": "include", + "lib": ".libs", + "prepare": "./autogen.sh && ./configure --enable-module-recovery" + }, + "owner": "bitcoin-core", + "repo": "secp256k1", + "rev": "26de4dfeb1f1436dae1fcf17f57bdaa43540f940", + "type": "tarball", + "url": "https://github.com/bitcoin-core/secp256k1/archive/26de4dfeb1f1436dae1fcf17f57bdaa43540f940.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "ent": { + "mingw": { + "prepare": "./configure" + } + }, + "ge-additions": { + "mingw": { + "make": "CFLAGS=-I../ed25519" + } + }, + "libaes_siv": { + "mingw": { + } + } +} \ No newline at end of file diff --git a/nix/sources.json b/nix/sources.json index abe295cd04..6bb8aa172b 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -3,6 +3,10 @@ "branch": "master", "description": "With argon2u. Based off https://github.com/P-H-C/phc-winner-argon2", "homepage": "", + "mingw": { + "include": ["include", "src/blake2"], + "make": "libargon2.a" + }, "owner": "urbit", "repo": "argon2", "rev": "4da94a611ee62bad87ab2b131ffda3bcc0723d9c", @@ -15,6 +19,10 @@ "branch": "master", "description": "Submodule included by Urbit", "homepage": null, + "mingw": { + "strip": 1, + "make": "all" + }, "owner": "urbit", "repo": "ed25519", "rev": "76385f2ebbbc9580a9c236952d68d11d73a6135c", @@ -27,12 +35,17 @@ "branch": "master", "description": "H2O - the optimized HTTP/1, HTTP/2, HTTP/3 server", "homepage": "https://h2o.examp1e.net", + "mingw": { + "include": "include", + "prepare": "cmake -G\"MSYS Makefiles\" .", + "make": "libh2o" + }, "owner": "h2o", "repo": "h2o", - "rev": "v2.2.4", - "sha256": "0176x0bzjry19zs074a9i5vhncc842xikmx43wj61jky318nq4w4", + "rev": "v2.2.6", + "sha256": "0qni676wqvxx0sl0pw9j0ph7zf2krrzqc1zwj73mgpdnsr8rsib7", "type": "tarball", - "url": "https://github.com/h2o/h2o/archive/v2.2.4.tar.gz", + "url": "https://github.com/h2o/h2o/archive/v2.2.6.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "hackage.nix": { @@ -63,6 +76,9 @@ "branch": "master", "description": null, "homepage": null, + "mingw": { + "make": "libscrypt.a CFLAGS_EXTRA=-ffast-math" + }, "owner": "urbit", "repo": "libscrypt", "rev": "029693ff1cbe4f69d3a2da87d0f4f034f92cc0c2", @@ -75,6 +91,9 @@ "branch": "master", "description": null, "homepage": null, + "mingw": { + "make": "static" + }, "owner": "urbit", "repo": "murmur3", "rev": "71a75d57ca4e7ca0f7fc2fd84abd93595b0624ca", @@ -111,6 +130,11 @@ "branch": "master", "description": null, "homepage": null, + "mingw": { + "include": "source/include", + "lib": "build/Win64-MinGW-w64", + "make": "-C build/Win64-MinGW-w64 libsoftfloat3.a" + }, "owner": "urbit", "repo": "berkeley-softfloat-3", "rev": "ec4c7e31b32e07aad80e52f65ff46ac6d6aad986", diff --git a/pkg/urbit/Makefile b/pkg/urbit/Makefile index 872c5eb1d3..09e84d4a02 100644 --- a/pkg/urbit/Makefile +++ b/pkg/urbit/Makefile @@ -9,7 +9,17 @@ worker = $(wildcard worker/*.c) tests = $(wildcard tests/*.c) bench = $(wildcard bench/*.c) -common = $(jets) $(noun) $(ur) $(vere) +ifdef use_dumb_terminal +vere := $(filter-out vere/io/term.c,$(vere)) +else +vere := $(filter-out vere/io/dumb.c,$(vere)) +endif + +ifdef compat +compat := $(wildcard compat/$(compat)/*.c) +endif + +common = $(jets) $(noun) $(ur) $(vere) $(compat) headers = $(shell find include -type f) common_objs = $(shell echo $(common) | sed 's/\.c/.o/g') diff --git a/pkg/urbit/build-mingw b/pkg/urbit/build-mingw new file mode 100755 index 0000000000..b917052c7e --- /dev/null +++ b/pkg/urbit/build-mingw @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +function xxd() +{ + cch=0 + echo "unsigned char $2[] = {" + while IFS='' read line + do + for i in $line + do + echo -n " 0x$i," + cch=$((cch+1)) + done + echo + done < <(od -An -v -tx1 $1) + echo "};" + echo "unsigned int $2_len = $cch;" +} + +case $(uname -s|tr A-Z a-z) in + *mingw64*) + ;; + *) + echo This script works only on MingW64. + exit 1 + ;; +esac + +if [ ! -e config.mk ] +then + # ensure required mingw packages are installed + mpkgs=(cmake curl gcc jq libsigsegv libuv make wslay) + pacman -S --needed autoconf automake-wrapper libtool patch ${mpkgs[@]/#/mingw-w64-x86_64-} + + unset cdirs + unset ldirs + declare -a cdirs + declare -a ldirs + . <(jq -sr ' +add|to_entries|.[]|select(.value.mingw)|.key as $key|(if .value.url then " +mkdir -p ../\($key) +pushd ../\($key) +curl -L \(.value.url)|(tar --strip \(.value.mingw.strip+1) -xzf - || true)" + +("../urbit/compat/mingw/\($key).patch"|" +[ -e \(.) ] && patch -p 1 <\(.)") else " +pushd ../\($key)" end) + " +\(.value.mingw.prepare//"") +make \(.value.mingw.make//"") +popd +\(.value.mingw.include//"."|if type == "array" then .[] else . end|"cdirs+=(-I../\($key)/\(.)) +")\(.value.mingw.lib//"."|if type == "array" then .[] else . end|"ldirs+=(-L../\($key)/\(.)) +")"' ../../nix/sources.json ../../nix/sources-mingw.json) + + xxd /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt include_ca_bundle_crt >include/ca-bundle.h + xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h + + CFLAGS="${CFLAGS-} ${cdirs[@]}" LDFLAGS="${LDFLAGS-} ${ldirs[@]}" PKG_CONFIG=echo ./configure +fi + +make build/urbit build/urbit-worker diff --git a/pkg/urbit/compat/mingw/argon2u.patch b/pkg/urbit/compat/mingw/argon2u.patch new file mode 100755 index 0000000000..fdabbe630e --- /dev/null +++ b/pkg/urbit/compat/mingw/argon2u.patch @@ -0,0 +1,22 @@ +diff --git a/src/encoding.c b/src/encoding.c +index 73c36f5..753a1e1 100644 +--- a/src/encoding.c ++++ b/src/encoding.c +@@ -370,7 +370,7 @@ int decode_string(argon2_context *ctx, const char *str, argon2_type type) { + #undef BIN + } + +-void itoa(int i, char b[]){ ++static void encode_decimal(int i, char b[]){ + #ifdef ARGON2_JS + + // because this generates WASM error: +@@ -416,7 +416,7 @@ int encode_string(char *dst, size_t dst_len, argon2_context *ctx, + #define SX(x) \ + do { \ + char tmp[30]; \ +- itoa(x, tmp); \ ++ encode_decimal(x, tmp); \ + SS(tmp); \ + } while ((void)0, 0) + diff --git a/pkg/urbit/compat/mingw/compat.c b/pkg/urbit/compat/mingw/compat.c new file mode 100644 index 0000000000..1fe0c97254 --- /dev/null +++ b/pkg/urbit/compat/mingw/compat.c @@ -0,0 +1,436 @@ +#include "c/portable.h" +#include +#include + +// from https://github.com/git/git/blob/master/compat/mingw.c +// ----------------------------------------------------------------------- + +int err_win_to_posix(DWORD winerr) +{ + int error = ENOSYS; + switch(winerr) { + case ERROR_ACCESS_DENIED: error = EACCES; break; + case ERROR_ACCOUNT_DISABLED: error = EACCES; break; + case ERROR_ACCOUNT_RESTRICTION: error = EACCES; break; + case ERROR_ALREADY_ASSIGNED: error = EBUSY; break; + case ERROR_ALREADY_EXISTS: error = EEXIST; break; + case ERROR_ARITHMETIC_OVERFLOW: error = ERANGE; break; + case ERROR_BAD_COMMAND: error = EIO; break; + case ERROR_BAD_DEVICE: error = ENODEV; break; + case ERROR_BAD_DRIVER_LEVEL: error = ENXIO; break; + case ERROR_BAD_EXE_FORMAT: error = ENOEXEC; break; + case ERROR_BAD_FORMAT: error = ENOEXEC; break; + case ERROR_BAD_LENGTH: error = EINVAL; break; + case ERROR_BAD_PATHNAME: error = ENOENT; break; + case ERROR_BAD_PIPE: error = EPIPE; break; + case ERROR_BAD_UNIT: error = ENODEV; break; + case ERROR_BAD_USERNAME: error = EINVAL; break; + case ERROR_BROKEN_PIPE: error = EPIPE; break; + case ERROR_BUFFER_OVERFLOW: error = ENAMETOOLONG; break; + case ERROR_BUSY: error = EBUSY; break; + case ERROR_BUSY_DRIVE: error = EBUSY; break; + case ERROR_CALL_NOT_IMPLEMENTED: error = ENOSYS; break; + case ERROR_CANNOT_MAKE: error = EACCES; break; + case ERROR_CANTOPEN: error = EIO; break; + case ERROR_CANTREAD: error = EIO; break; + case ERROR_CANTWRITE: error = EIO; break; + case ERROR_CRC: error = EIO; break; + case ERROR_CURRENT_DIRECTORY: error = EACCES; break; + case ERROR_DEVICE_IN_USE: error = EBUSY; break; + case ERROR_DEV_NOT_EXIST: error = ENODEV; break; + case ERROR_DIRECTORY: error = EINVAL; break; + case ERROR_DIR_NOT_EMPTY: error = ENOTEMPTY; break; + case ERROR_DISK_CHANGE: error = EIO; break; + case ERROR_DISK_FULL: error = ENOSPC; break; + case ERROR_DRIVE_LOCKED: error = EBUSY; break; + case ERROR_ENVVAR_NOT_FOUND: error = EINVAL; break; + case ERROR_EXE_MARKED_INVALID: error = ENOEXEC; break; + case ERROR_FILENAME_EXCED_RANGE: error = ENAMETOOLONG; break; + case ERROR_FILE_EXISTS: error = EEXIST; break; + case ERROR_FILE_INVALID: error = ENODEV; break; + case ERROR_FILE_NOT_FOUND: error = ENOENT; break; + case ERROR_GEN_FAILURE: error = EIO; break; + case ERROR_HANDLE_DISK_FULL: error = ENOSPC; break; + case ERROR_INSUFFICIENT_BUFFER: error = ENOMEM; break; + case ERROR_INVALID_ACCESS: error = EACCES; break; + case ERROR_INVALID_ADDRESS: error = EFAULT; break; + case ERROR_INVALID_BLOCK: error = EFAULT; break; + case ERROR_INVALID_DATA: error = EINVAL; break; + case ERROR_INVALID_DRIVE: error = ENODEV; break; + case ERROR_INVALID_EXE_SIGNATURE: error = ENOEXEC; break; + case ERROR_INVALID_FLAGS: error = EINVAL; break; + case ERROR_INVALID_FUNCTION: error = ENOSYS; break; + case ERROR_INVALID_HANDLE: error = EBADF; break; + case ERROR_INVALID_LOGON_HOURS: error = EACCES; break; + case ERROR_INVALID_NAME: error = EINVAL; break; + case ERROR_INVALID_OWNER: error = EINVAL; break; + case ERROR_INVALID_PARAMETER: error = EINVAL; break; + case ERROR_INVALID_PASSWORD: error = EPERM; break; + case ERROR_INVALID_PRIMARY_GROUP: error = EINVAL; break; + case ERROR_INVALID_SIGNAL_NUMBER: error = EINVAL; break; + case ERROR_INVALID_TARGET_HANDLE: error = EIO; break; + case ERROR_INVALID_WORKSTATION: error = EACCES; break; + case ERROR_IO_DEVICE: error = EIO; break; + case ERROR_IO_INCOMPLETE: error = EINTR; break; + case ERROR_LOCKED: error = EBUSY; break; + case ERROR_LOCK_VIOLATION: error = EACCES; break; + case ERROR_LOGON_FAILURE: error = EACCES; break; + case ERROR_MAPPED_ALIGNMENT: error = EINVAL; break; + case ERROR_META_EXPANSION_TOO_LONG: error = E2BIG; break; + case ERROR_MORE_DATA: error = EPIPE; break; + case ERROR_NEGATIVE_SEEK: error = ESPIPE; break; + case ERROR_NOACCESS: error = EFAULT; break; + case ERROR_NONE_MAPPED: error = EINVAL; break; + case ERROR_NOT_ENOUGH_MEMORY: error = ENOMEM; break; + case ERROR_NOT_READY: error = EAGAIN; break; + case ERROR_NOT_SAME_DEVICE: error = EXDEV; break; + case ERROR_NO_DATA: error = EPIPE; break; + case ERROR_NO_MORE_SEARCH_HANDLES: error = EIO; break; + case ERROR_NO_PROC_SLOTS: error = EAGAIN; break; + case ERROR_NO_SUCH_PRIVILEGE: error = EACCES; break; + case ERROR_OPEN_FAILED: error = EIO; break; + case ERROR_OPEN_FILES: error = EBUSY; break; + case ERROR_OPERATION_ABORTED: error = EINTR; break; + case ERROR_OUTOFMEMORY: error = ENOMEM; break; + case ERROR_PASSWORD_EXPIRED: error = EACCES; break; + case ERROR_PATH_BUSY: error = EBUSY; break; + case ERROR_PATH_NOT_FOUND: error = ENOENT; break; + case ERROR_PIPE_BUSY: error = EBUSY; break; + case ERROR_PIPE_CONNECTED: error = EPIPE; break; + case ERROR_PIPE_LISTENING: error = EPIPE; break; + case ERROR_PIPE_NOT_CONNECTED: error = EPIPE; break; + case ERROR_PRIVILEGE_NOT_HELD: error = EACCES; break; + case ERROR_READ_FAULT: error = EIO; break; + case ERROR_SEEK: error = EIO; break; + case ERROR_SEEK_ON_DEVICE: error = ESPIPE; break; + case ERROR_SHARING_BUFFER_EXCEEDED: error = ENFILE; break; + case ERROR_SHARING_VIOLATION: error = EACCES; break; + case ERROR_STACK_OVERFLOW: error = ENOMEM; break; + case ERROR_SUCCESS: error = 0; break; + case ERROR_SWAPERROR: error = ENOENT; break; + case ERROR_TOO_MANY_MODULES: error = EMFILE; break; + case ERROR_TOO_MANY_OPEN_FILES: error = EMFILE; break; + case ERROR_UNRECOGNIZED_MEDIA: error = ENXIO; break; + case ERROR_UNRECOGNIZED_VOLUME: error = ENODEV; break; + case ERROR_WAIT_NO_CHILDREN: error = ECHILD; break; + case ERROR_WRITE_FAULT: error = EIO; break; + case ERROR_WRITE_PROTECT: error = EROFS; break; + } + return error; +} + +int kill(pid_t pid, int sig) +{ + if (pid > 0 && sig == SIGKILL) { + HANDLE h = OpenProcess(PROCESS_TERMINATE, FALSE, pid); + + if (TerminateProcess(h, -1)) { + CloseHandle(h); + return 0; + } + + errno = err_win_to_posix(GetLastError()); + CloseHandle(h); + return -1; + } + + // TODO: handle signals for self + // TODO: send SIGTERM as ctrl-c: https://stackoverflow.com/questions/813086/can-i-send-a-ctrl-c-sigint-to-an-application-on-windows + errno = EINVAL; + return -1; +} + +static HANDLE timer_event; +static HANDLE timer_thread; +static int timer_signal; +static int timer_interval; +static int one_shot; +static __p_sig_fn_t timer_fn = SIG_DFL, sigint_fn = SIG_DFL; + +/* The timer works like this: + * The thread, ticktack(), is a trivial routine that most of the time + * only waits to receive the signal to terminate. The main thread tells + * the thread to terminate by setting the timer_event to the signalled + * state. + * But ticktack() interrupts the wait state after the timer's interval + * length to call the signal handler. + */ + +static unsigned __stdcall ticktack(void *dummy) +{ + while (WaitForSingleObject(timer_event, timer_interval) == WAIT_TIMEOUT) { + raise(timer_signal); + if (one_shot) + break; + } + return 0; +} + +static int error(const char* s, ...) +{ + // TODO + va_list ap; + va_start(ap, s); + vfprintf(stderr, s, ap); + va_end(ap); + return -1; +} + +static int start_timer_thread(void) +{ + timer_event = CreateEvent(NULL, FALSE, FALSE, NULL); + if (timer_event) { + timer_thread = (HANDLE) _beginthreadex(NULL, 0, ticktack, NULL, 0, NULL); + if (!timer_thread ) + return errno = ENOMEM, + error("cannot start timer thread"); + } else + return errno = ENOMEM, + error("cannot allocate resources for timer"); + return 0; +} + +static void stop_timer_thread(void) +{ + if (timer_event) + SetEvent(timer_event); /* tell thread to terminate */ + if (timer_thread) { + int rc = WaitForSingleObject(timer_thread, 10000); + if (rc == WAIT_TIMEOUT) + error("timer thread did not terminate timely"); + else if (rc != WAIT_OBJECT_0) + error("waiting for timer thread failed: %lu", + GetLastError()); + CloseHandle(timer_thread); + } + if (timer_event) + CloseHandle(timer_event); + timer_event = NULL; + timer_thread = NULL; +} + +static inline int is_timeval_eq(const struct timeval *i1, const struct timeval *i2) +{ + return i1->tv_sec == i2->tv_sec && i1->tv_usec == i2->tv_usec; +} + +int setitimer(int type, struct itimerval *in, struct itimerval *out) +{ + static const struct timeval zero; + static int atexit_done; + + if (out != NULL) + return errno = EINVAL, + error("setitimer param 3 != NULL not implemented"); + if (!is_timeval_eq(&in->it_interval, &zero) && + !is_timeval_eq(&in->it_interval, &in->it_value)) + return errno = EINVAL, + error("setitimer: it_interval must be zero or eq it_value"); + + if (timer_thread) + stop_timer_thread(); + + if (is_timeval_eq(&in->it_value, &zero) && + is_timeval_eq(&in->it_interval, &zero)) + return 0; + + timer_signal = type == ITIMER_VIRTUAL ? SIGVTALRM : SIGALRM; + timer_interval = in->it_value.tv_sec * 1000 + in->it_value.tv_usec / 1000; + one_shot = is_timeval_eq(&in->it_interval, &zero); + if (!atexit_done) { + atexit(stop_timer_thread); + atexit_done = 1; + } + return start_timer_thread(); +} + +// from msys2 mingw-packages-dev patches +// ----------------------------------------------------------------------- + +static DWORD __map_mmap_prot_page(const int prot) +{ + DWORD protect = 0; + + if (prot == PROT_NONE) + return protect; + + if ((prot & PROT_EXEC) != 0) + { + protect = ((prot & PROT_WRITE) != 0) ? + PAGE_EXECUTE_READWRITE : PAGE_EXECUTE_READ; + } + else + { + protect = ((prot & PROT_WRITE) != 0) ? + PAGE_READWRITE : PAGE_READONLY; + } + + return protect; +} + +static DWORD __map_mmap_prot_file(const int prot) +{ + DWORD desiredAccess = 0; + + if (prot == PROT_NONE) + return desiredAccess; + + if ((prot & PROT_READ) != 0) + desiredAccess |= FILE_MAP_READ; + if ((prot & PROT_WRITE) != 0) + desiredAccess |= FILE_MAP_WRITE; + if ((prot & PROT_EXEC) != 0) + desiredAccess |= FILE_MAP_EXECUTE; + + return desiredAccess; +} + +void* mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off) +{ + HANDLE fm, h; + + void * map = MAP_FAILED; + + const DWORD dwFileOffsetLow = (sizeof(off_t) <= sizeof(DWORD)) ? + (DWORD)off : (DWORD)(off & 0xFFFFFFFFL); + const DWORD dwFileOffsetHigh = (sizeof(off_t) <= sizeof(DWORD)) ? + (DWORD)0 : (DWORD)((off >> 32) & 0xFFFFFFFFL); + const DWORD protect = __map_mmap_prot_page(prot); + const DWORD desiredAccess = __map_mmap_prot_file(prot); + + errno = 0; + + if (len == 0 + /* Usupported protection combinations */ + || prot == PROT_EXEC) + { + errno = EINVAL; + return MAP_FAILED; + } + + if ((flags & MAP_ANON) == 0) + { + h = (HANDLE)_get_osfhandle(fildes); + + if (h == INVALID_HANDLE_VALUE) + { + errno = EBADF; + return MAP_FAILED; + } + } + else h = INVALID_HANDLE_VALUE; + + fm = CreateFileMapping(h, NULL, protect, 0, len, NULL); + + if (fm == NULL) + { + errno = err_win_to_posix(GetLastError()); + return MAP_FAILED; + } + + map = MapViewOfFileEx(fm, desiredAccess, dwFileOffsetHigh, dwFileOffsetLow, len, addr); + errno = err_win_to_posix(GetLastError()); + + CloseHandle(fm); + + if (map == NULL) + return MAP_FAILED; + + if ((flags & MAP_FIXED) != 0 && map != addr) + { + UnmapViewOfFile(map); + errno = EEXIST; + return MAP_FAILED; + } + + return map; +} + +int munmap(void *addr, size_t len) +{ + if (UnmapViewOfFile(addr)) + return 0; + + errno = err_win_to_posix(GetLastError()); + return -1; +} + +int msync(void *addr, size_t len, int flags) +{ + if (FlushViewOfFile(addr, len)) + return 0; + + errno = err_win_to_posix(GetLastError()); + return -1; +} + +// ----------------------------------------------------------------------- + +// libgcc built for mingw has included an implementation of mprotect +// via VirtualProtect since olden days, but it takes int rather than size_t +// and therefore fails or does unexpected things for >2GB blocks on 64-bit +// https://github.com/gcc-mirror/gcc/blob/master/libgcc/libgcc2.c +int mprotect (void *addr, size_t len, int prot) +{ + DWORD np, op; + + if (prot == (PROT_READ | PROT_WRITE | PROT_EXEC)) + np = PAGE_EXECUTE_READWRITE; + else if (prot == (PROT_READ | PROT_EXEC)) + np = PAGE_EXECUTE_READ; + else if (prot == (PROT_EXEC)) + np = PAGE_EXECUTE; + else if (prot == (PROT_READ | PROT_WRITE)) + np = PAGE_READWRITE; + else if (prot == (PROT_READ)) + np = PAGE_READONLY; + else if (prot == 0) + np = PAGE_NOACCESS; + else + { + errno = EINVAL; + return -1; + } + + if (VirtualProtect (addr, len, np, &op)) + return 0; + + // NB: return code of ntdll!RtlGetLastNtStatus() is useful + // for diagnosing obscure VirtualProtect failures + errno = err_win_to_posix(GetLastError()); + return -1; +} + +int utimes(const char *path, const struct timeval times[2]) +{ + // TODO: implement in terms of utime + return -1; +} + +int fdatasync(int fildes) +{ + HANDLE h = (HANDLE)_get_osfhandle(fildes); + + if (h == INVALID_HANDLE_VALUE) + { + errno = EBADF; + return -1; + } + + if (FlushFileBuffers(h)) + { + errno = 0; + return 0; + } + else + { + errno = err_win_to_posix(GetLastError()); + return -1; + } +} + +char *realpath(const char *path, char *resolved_path) +{ + // TODO + return strdup(path); +} diff --git a/pkg/urbit/compat/mingw/compat.h b/pkg/urbit/compat/mingw/compat.h new file mode 100644 index 0000000000..b0f2737e23 --- /dev/null +++ b/pkg/urbit/compat/mingw/compat.h @@ -0,0 +1,41 @@ +#ifndef _MINGW_IO_H +#define _MINGW_IO_H + +// msvcrt setjmp/longjmp are broken on 64-bit systems, use gcc builtins +typedef struct jmp_buf { + intptr_t buffer[5]; + int retval; +} jmp_buf; + +#define _setjmp setjmp +#define _longjmp longjmp +#define longjmp(buf, val) {buf.retval = (val); __builtin_longjmp(buf.buffer, 1);} +#define setjmp(buf) (__builtin_setjmp(buf.buffer) ? (buf.retval) : 0) + +// no profiling on MingW means signal masks are not used +#define sigjmp_buf jmp_buf +#define siglongjmp longjmp +#define sigsetjmp(A, B) setjmp(A) + +#define mkdir(A, B) mkdir(A) + +char *realpath(const char *path, char *resolved_path); +int fdatasync(int fd); +int utimes(const char *path, const struct timeval times[2]); + +int kill(pid_t pid, int signum); + +#define SIGALRM 1233 +#define SIGVTALRM 1234 +#define SIGINFO 1235 +#define SIGUSR1 1236 +#define SIGTSTP 1238 + +#define ITIMER_REAL 0 +#define ITIMER_VIRTUAL 1 +struct itimerval { + struct timeval it_value, it_interval; +}; +int setitimer(int type, struct itimerval *in, struct itimerval *out); + +#endif//_MINGW_IO_H \ No newline at end of file diff --git a/pkg/urbit/compat/mingw/ed25519.patch b/pkg/urbit/compat/mingw/ed25519.patch new file mode 100644 index 0000000000..1dfa957f1d --- /dev/null +++ b/pkg/urbit/compat/mingw/ed25519.patch @@ -0,0 +1,13 @@ +diff --git a/Makefile b/Makefile +new file mode 100644 +--- /dev/null ++++ b/Makefile +@@ -0,0 +1,8 @@ ++.PHONY: all clean ++ ++all: *.c *.h ++ $(CC) -c -O3 -Wall -Werror *.c ++ $(AR) rcs libed25519.a *.o ++ ++clean: ++ rm -f *.o *.a diff --git a/pkg/urbit/compat/mingw/h2o.patch b/pkg/urbit/compat/mingw/h2o.patch new file mode 100755 index 0000000000..1bfc90e4ad --- /dev/null +++ b/pkg/urbit/compat/mingw/h2o.patch @@ -0,0 +1,1834 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 5a04e1426..9ddad60d1 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -69,6 +69,18 @@ OPTION(WITH_BUNDLED_SSL "whether or not to use the bundled libressl" ${WITH_BUND + + OPTION(WITHOUT_LIBS "skip building libs even when possible" OFF) + OPTION(BUILD_SHARED_LIBS "whether to build a shared library" OFF) ++OPTION(WITHOUT_FASTCGI "don't include fastcgi" OFF) ++OPTION(WITHOUT_MEMCACHED "don't include memcached" OFF) ++ ++IF (MINGW) ++ SET(WITHOUT_FASTCGI "ON") ++ SET(WITHOUT_MEMCACHED "ON") ++ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_POSIX_C_SOURCE -DO_CLOEXEC=0 -DH2O_NO_UNIX_SOCKETS") ++ENDIF (MINGW) ++ ++IF (WITHOUT_MEMCACHED) ++ SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DH2O_NO_MEMCACHED") ++ENDIF (WITHOUT_MEMCACHED) + + FIND_PROGRAM(RUBY ruby) + FIND_PROGRAM(BISON bison) +@@ -228,16 +240,7 @@ SET(PICOTLS_SOURCE_FILES + deps/picotls/lib/openssl.c) + + SET(LIB_SOURCE_FILES +- deps/cloexec/cloexec.c + deps/libgkc/gkc.c +- deps/libyrmcds/close.c +- deps/libyrmcds/connect.c +- deps/libyrmcds/recv.c +- deps/libyrmcds/send.c +- deps/libyrmcds/send_text.c +- deps/libyrmcds/socket.c +- deps/libyrmcds/strerror.c +- deps/libyrmcds/text_mode.c + deps/picohttpparser/picohttpparser.c + + lib/common/cache.c +@@ -245,7 +248,6 @@ SET(LIB_SOURCE_FILES + lib/common/filecache.c + lib/common/hostinfo.c + lib/common/http1client.c +- lib/common/memcached.c + lib/common/memory.c + lib/common/multithread.c + lib/common/serverutil.c +@@ -272,7 +274,6 @@ SET(LIB_SOURCE_FILES + lib/handler/compress/gzip.c + lib/handler/errordoc.c + lib/handler/expires.c +- lib/handler/fastcgi.c + lib/handler/file.c + lib/handler/headers.c + lib/handler/mimemap.c +@@ -290,7 +291,6 @@ SET(LIB_SOURCE_FILES + lib/handler/configurator/compress.c + lib/handler/configurator/errordoc.c + lib/handler/configurator/expires.c +- lib/handler/configurator/fastcgi.c + lib/handler/configurator/file.c + lib/handler/configurator/headers.c + lib/handler/configurator/proxy.c +@@ -314,6 +314,33 @@ SET(LIB_SOURCE_FILES + lib/http2/stream.c + lib/http2/http2_debug_state.c) + ++IF (NOT MINGW) ++ SET(LIB_SOURCE_FILES ++ ${LIB_SOURCE_FILES} ++ deps/cloexec/cloexec.c) ++ENDIF (NOT MINGW) ++ ++IF (NOT WITHOUT_FASTCGI) ++ SET(LIB_SOURCE_FILES ++ ${LIB_SOURCE_FILES} ++ lib/handler/fastcgi.c ++ lib/handler/configurator/fastcgi.c) ++ENDIF (NOT WITHOUT_FASTCGI) ++ ++IF (NOT WITHOUT_MEMCACHED) ++ SET(LIB_SOURCE_FILES ++ ${LIB_SOURCE_FILES} ++ deps/libyrmcds/close.c ++ deps/libyrmcds/connect.c ++ deps/libyrmcds/recv.c ++ deps/libyrmcds/send.c ++ deps/libyrmcds/send_text.c ++ deps/libyrmcds/socket.c ++ deps/libyrmcds/strerror.c ++ deps/libyrmcds/text_mode.c ++ lib/common/memcached.c) ++ENDIF (NOT WITHOUT_MEMCACHED) ++ + SET(UNIT_TEST_SOURCE_FILES + ${LIB_SOURCE_FILES} + ${LIBYAML_SOURCE_FILES} +diff --git a/examples/libh2o/http1client.c b/examples/libh2o/http1client.c +index bcf9b94b2..592376759 100644 +--- a/examples/libh2o/http1client.c ++++ b/examples/libh2o/http1client.c +@@ -134,6 +134,11 @@ h2o_http1client_head_cb on_connect(h2o_http1client_t *client, const char *errstr + + int main(int argc, char **argv) + { ++#ifdef _WIN32 ++ WSADATA wsaData; ++ WSAStartup(MAKEWORD(2, 0), &wsaData); ++#endif ++ + h2o_multithread_queue_t *queue; + h2o_multithread_receiver_t getaddr_receiver; + h2o_timeout_t io_timeout; +diff --git a/examples/libh2o/simple.c b/examples/libh2o/simple.c +index bb72bf60c..be7045d1c 100644 +--- a/examples/libh2o/simple.c ++++ b/examples/libh2o/simple.c +@@ -21,11 +21,15 @@ + */ + #include + #include +-#include + #include + #include + #include ++#ifdef _WIN32 ++#include ++#else ++#include + #include ++#endif + #include + #include "h2o.h" + #include "h2o/http1.h" +@@ -219,7 +223,12 @@ int main(int argc, char **argv) + { + h2o_hostconf_t *hostconf; + ++#ifdef _WIN32 ++ WSADATA wsaData; ++ WSAStartup(MAKEWORD(2, 0), &wsaData); ++#else + signal(SIGPIPE, SIG_IGN); ++#endif + + h2o_config_init(&config); + hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535); +diff --git a/include/h2o.h b/include/h2o.h +index 57877bd12..84822b792 100644 +--- a/include/h2o.h ++++ b/include/h2o.h +@@ -32,7 +32,11 @@ extern "C" { + #include + #include + #include ++#ifdef _WIN32 ++#include ++#else + #include ++#endif + #include + #include + #include +diff --git a/include/h2o/configurator.h b/include/h2o/configurator.h +index d1a2e2515..d59d070dd 100644 +--- a/include/h2o/configurator.h ++++ b/include/h2o/configurator.h +@@ -145,7 +145,7 @@ int h2o_configurator_apply_commands(h2o_configurator_context_t *ctx, yoml_t *nod + * emits configuration error + */ + void h2o_configurator_errprintf(h2o_configurator_command_t *cmd, yoml_t *node, const char *reason, ...) +- __attribute__((format(printf, 3, 4))); ++ __attribute__((format(gnu_printf, 3, 4))); + /** + * interprets the configuration value using sscanf, or prints an error upon failure + * @param configurator configurator +@@ -154,7 +154,7 @@ void h2o_configurator_errprintf(h2o_configurator_command_t *cmd, yoml_t *node, c + * @return 0 if successful, -1 if not + */ + int h2o_configurator_scanf(h2o_configurator_command_t *cmd, yoml_t *node, const char *fmt, ...) +- __attribute__((format(scanf, 3, 4))); ++ __attribute__((format(gnu_scanf, 3, 4))); + /** + * interprets the configuration value and returns the index of the matched string within the candidate strings, or prints an error + * upon failure +diff --git a/include/h2o/filecache.h b/include/h2o/filecache.h +index a000c4c6d..494f61fb1 100644 +--- a/include/h2o/filecache.h ++++ b/include/h2o/filecache.h +@@ -61,6 +61,7 @@ void h2o_filecache_destroy(h2o_filecache_t *cache); + void h2o_filecache_clear(h2o_filecache_t *cache); + + h2o_filecache_ref_t *h2o_filecache_open_file(h2o_filecache_t *cache, const char *path, int oflag); ++ssize_t h2o_filecache_read_file(h2o_filecache_ref_t *ref, void *buf, size_t count, off_t offset); + void h2o_filecache_close_file(h2o_filecache_ref_t *ref); + struct tm *h2o_filecache_get_last_modified(h2o_filecache_ref_t *ref, char *outbuf); + size_t h2o_filecache_get_etag(h2o_filecache_ref_t *ref, char *outbuf); +diff --git a/include/h2o/hostinfo.h b/include/h2o/hostinfo.h +index 14ac30c6c..b39fea520 100644 +--- a/include/h2o/hostinfo.h ++++ b/include/h2o/hostinfo.h +@@ -22,13 +22,16 @@ + #ifndef h2o__hostinfo_h + #define h2o__hostinfo_h + +-#include ++#ifdef _WIN32 ++#include ++#else + #include + #include + #include + #include + #include + #include ++#endif + #include "h2o/multithread.h" + + typedef struct st_h2o_hostinfo_getaddr_req_t h2o_hostinfo_getaddr_req_t; +diff --git a/include/h2o/memory.h b/include/h2o/memory.h +index 10c137c88..354fa4359 100644 +--- a/include/h2o/memory.h ++++ b/include/h2o/memory.h +@@ -78,8 +78,19 @@ typedef struct st_h2o_buffer_prototype_t h2o_buffer_prototype_t; + * buffer structure compatible with iovec + */ + typedef struct st_h2o_iovec_t { ++#ifdef __MINGW32__ ++#define H2O_IOVEC_NULL {0, NULL} ++#define H2O_IOVEC_EMPTY {0, ""} ++#define H2O_IOVEC_STRLIT(s) {sizeof(s) - 1, (s)} ++ unsigned int len; ++ char *base; ++#else ++#define H2O_IOVEC_NULL {NULL} ++#define H2O_IOVEC_EMPTY {"", 0} ++#define H2O_IOVEC_STRLIT(s) {(s), sizeof(s) - 1} + char *base; + size_t len; ++#endif + } h2o_iovec_t; + + typedef struct st_h2o_mem_recycle_t { +diff --git a/include/h2o/socket.h b/include/h2o/socket.h +index 58ada8509..268dd8a2e 100644 +--- a/include/h2o/socket.h ++++ b/include/h2o/socket.h +@@ -27,7 +27,11 @@ extern "C" { + #endif + + #include ++#ifdef _WIN32 ++#include ++#else + #include ++#endif + #include + #include "h2o/cache.h" + #include "h2o/memory.h" +diff --git a/include/h2o/socketpool.h b/include/h2o/socketpool.h +index cc4161df4..f1c4eb3c3 100644 +--- a/include/h2o/socketpool.h ++++ b/include/h2o/socketpool.h +@@ -26,8 +26,10 @@ + extern "C" { + #endif + ++#ifndef _WIN32 + #include + #include ++#endif + #include + #include "h2o/linklist.h" + #include "h2o/multithread.h" +diff --git a/include/h2o/url.h b/include/h2o/url.h +index 231c9a263..6a707a567 100644 +--- a/include/h2o/url.h ++++ b/include/h2o/url.h +@@ -22,7 +22,9 @@ + #ifndef h2o__url_h + #define h2o__url_h + ++#ifndef H2O_NO_UNIX_SOCKETS + #include ++#endif + #include "h2o/memory.h" + + typedef struct st_h2o_url_scheme_t { +@@ -92,11 +94,13 @@ static h2o_iovec_t h2o_url_stringify(h2o_mem_pool_t *pool, const h2o_url_t *url) + * copies a URL object (null-terminates all the string elements) + */ + void h2o_url_copy(h2o_mem_pool_t *pool, h2o_url_t *dest, const h2o_url_t *src); ++#ifndef H2O_NO_UNIX_SOCKETS + /** + * extracts sockaddr_un from host and returns NULL (or returns an error string if failed) + */ + const char *h2o_url_host_to_sun(h2o_iovec_t host, struct sockaddr_un *sa); + extern const char *h2o_url_host_to_sun_err_is_not_unix_socket; ++#endif + + /* inline definitions */ + +diff --git a/lib/common/file.c b/lib/common/file.c +index 3cf5ac5d1..d3e0048b6 100644 +--- a/lib/common/file.c ++++ b/lib/common/file.c +@@ -25,7 +25,6 @@ + #include + #include + #include +-#include + #include + #include "h2o/file.h" + +@@ -33,7 +32,7 @@ h2o_iovec_t h2o_file_read(const char *fn) + { + int fd; + struct stat st; +- h2o_iovec_t ret = {NULL}; ++ h2o_iovec_t ret = H2O_IOVEC_NULL; + + /* open */ + if ((fd = open(fn, O_RDONLY | O_CLOEXEC)) == -1) +@@ -64,5 +63,5 @@ Error: + if (fd != -1) + close(fd); + free(ret.base); +- return (h2o_iovec_t){NULL}; ++ return (h2o_iovec_t)H2O_IOVEC_NULL; + } +diff --git a/lib/common/filecache.c b/lib/common/filecache.c +index 747a1ffa6..e37e1f498 100644 +--- a/lib/common/filecache.c ++++ b/lib/common/filecache.c +@@ -136,6 +136,24 @@ Exit: + return ref; + } + ++ssize_t h2o_filecache_read_file(h2o_filecache_ref_t *ref, void *buf, size_t count, off_t offset) ++{ ++ #ifdef _WIN32 ++ // h2o reads from h2o_filecache_ref_t.fd only with pread (comment it out to verify) ++ // this allows me to ignore the existence of the file pointer position maintained ++ // by Windows I/O manager and use NtReadFile() without opening extra file handles ++ /* ++ #include ++ IO_STATUS_BLOCK sb; ++ LARGE_INTEGER li; ++ NTSTATUS s = NtReadFile(_get_osfhandle(fd), NULL, NULL, NULL, &sb, buf, count, &li, NULL); ++ */ ++ return -1; ++ #else ++ return pread(ref->fd, buf, count, offset); ++ #endif ++} ++ + void h2o_filecache_close_file(h2o_filecache_ref_t *ref) + { + if (--ref->_refcnt != 0) +diff --git a/lib/common/http1client.c b/lib/common/http1client.c +index 8547ea817..a9de3d67d 100644 +--- a/lib/common/http1client.c ++++ b/lib/common/http1client.c +@@ -19,12 +19,18 @@ + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ ++#ifdef _WIN32 ++#include ++#else + #include + #include + #include + #include ++#endif + #include ++#ifndef H2O_NO_UNIX_SOCKETS + #include ++#endif + #include "picohttpparser.h" + #include "h2o.h" + +@@ -535,6 +541,7 @@ void h2o_http1client_connect(h2o_http1client_t **_client, void *data, h2o_http1c + return; + } + } ++ #ifndef _WIN32 + { /* directly call connect(2) if `host` refers to an UNIX-domain socket */ + struct sockaddr_un sa; + const char *to_sa_err; +@@ -547,6 +554,7 @@ void h2o_http1client_connect(h2o_http1client_t **_client, void *data, h2o_http1c + return; + } + } ++ #endif + /* resolve destination and then connect */ + client->_getaddr_req = + h2o_hostinfo_getaddr(ctx->getaddr_receiver, host, h2o_iovec_init(serv, sprintf(serv, "%u", (unsigned)port)), AF_UNSPEC, +diff --git a/lib/common/memory.c b/lib/common/memory.c +index ba9f2dba2..3a505d8ea 100644 +--- a/lib/common/memory.c ++++ b/lib/common/memory.c +@@ -27,7 +27,9 @@ + #include + #include + #include ++#ifndef _WIN32 + #include ++#endif + #include + #include "h2o/memory.h" + +@@ -190,20 +192,24 @@ void h2o_mem_link_shared(h2o_mem_pool_t *pool, void *p) + link_shared(pool, H2O_STRUCT_FROM_MEMBER(struct st_h2o_mem_pool_shared_entry_t, bytes, p)); + } + ++#ifndef _WIN32 + static size_t topagesize(size_t capacity) + { + size_t pagesize = getpagesize(); + return (offsetof(h2o_buffer_t, _buf) + capacity + pagesize - 1) / pagesize * pagesize; + } ++#endif + + void h2o_buffer__do_free(h2o_buffer_t *buffer) + { + /* caller should assert that the buffer is not part of the prototype */ + if (buffer->capacity == buffer->_prototype->_initial_buf.capacity) { + h2o_mem_free_recycle(&buffer->_prototype->allocator, buffer); ++ #ifndef _WIN32 + } else if (buffer->_fd != -1) { + close(buffer->_fd); + munmap((void *)buffer, topagesize(buffer->capacity)); ++ #endif + } else { + free(buffer); + } +@@ -240,6 +246,7 @@ h2o_iovec_t h2o_buffer_reserve(h2o_buffer_t **_inbuf, size_t min_guarantee) + do { + new_capacity *= 2; + } while (new_capacity - inbuf->size < min_guarantee); ++ #ifndef _WIN32 + if (inbuf->_prototype->mmap_settings != NULL && inbuf->_prototype->mmap_settings->threshold <= new_capacity) { + size_t new_allocsize = topagesize(new_capacity); + int fd; +@@ -287,7 +294,9 @@ h2o_iovec_t h2o_buffer_reserve(h2o_buffer_t **_inbuf, size_t min_guarantee) + inbuf->capacity = new_capacity; + inbuf->bytes = newp->_buf + offset; + } +- } else { ++ } else ++ #endif ++ { + h2o_buffer_t *newp = h2o_mem_alloc(offsetof(h2o_buffer_t, _buf) + new_capacity); + newp->size = inbuf->size; + newp->bytes = newp->_buf; +@@ -306,10 +315,12 @@ h2o_iovec_t h2o_buffer_reserve(h2o_buffer_t **_inbuf, size_t min_guarantee) + + return ret; + ++#ifndef _WIN32 + MapError: + ret.base = NULL; + ret.len = 0; + return ret; ++#endif + } + + void h2o_buffer_consume(h2o_buffer_t **_inbuf, size_t delta) +diff --git a/lib/common/multithread.c b/lib/common/multithread.c +index b4e8ba836..9a3dbe7d4 100644 +--- a/lib/common/multithread.c ++++ b/lib/common/multithread.c +@@ -22,7 +22,6 @@ + */ + #include + #include +-#include "cloexec.h" + #include "h2o/multithread.h" + + struct st_h2o_multithread_queue_t { +@@ -73,6 +72,7 @@ pthread_mutex_t h2o_conn_id_mutex = PTHREAD_MUTEX_INITIALIZER; + #if H2O_USE_LIBUV + #else + ++#include "cloexec.h" + #include + #include + #include +diff --git a/lib/common/serverutil.c b/lib/common/serverutil.c +index 8226f6efc..a599e7412 100644 +--- a/lib/common/serverutil.c ++++ b/lib/common/serverutil.c +@@ -21,26 +21,30 @@ + */ + #include + #include ++#ifndef _WIN32 + #include +-#include + #include +-#include + #include ++#include ++#if !defined(_SC_NPROCESSORS_ONLN) ++#include ++#endif ++#endif ++#include ++#include + #include + #include + #include + #include +-#include + #include +-#if !defined(_SC_NPROCESSORS_ONLN) +-#include +-#endif +-#include "cloexec.h" + #include "h2o/memory.h" + #include "h2o/serverutil.h" + #include "h2o/socket.h" + #include "h2o/string_.h" +- ++#if !H2O_USE_LIBUV ++#include "cloexec.h" ++#endif ++#if 0 + void h2o_set_signal_handler(int signo, void (*cb)(int signo)) + { + struct sigaction action; +@@ -137,6 +141,7 @@ static char **build_spawn_env(void) + return newenv; + } + ++#ifndef _WIN32 + pid_t h2o_spawnp(const char *cmd, char *const *argv, const int *mapped_fds, int cloexec_mutex_is_locked) + { + #if defined(__linux__) +@@ -235,6 +240,7 @@ Error: + + #endif + } ++#endif + + int h2o_read_command(const char *cmd, char **argv, h2o_buffer_t **resp, int *child_status) + { +@@ -315,3 +321,4 @@ size_t h2o_numproc(void) + return 1; + #endif + } ++#endif +\ No newline at end of file +diff --git a/lib/common/socket.c b/lib/common/socket.c +index 5b1c37e04..86ebbd8b4 100644 +--- a/lib/common/socket.c ++++ b/lib/common/socket.c +@@ -23,11 +23,18 @@ + #include + #include + #include ++#ifdef _WIN32 ++#include ++#include ++#else + #include + #include + #include ++#endif + #include ++#ifndef H2O_NO_UNIX_SOCKETS + #include ++#endif + #include + #include + #if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) +@@ -861,12 +868,7 @@ int h2o_socket_compare_address(struct sockaddr *x, struct sockaddr *y) + + CMP(x->sa_family, y->sa_family); + +- if (x->sa_family == AF_UNIX) { +- struct sockaddr_un *xun = (void *)x, *yun = (void *)y; +- int r = strcmp(xun->sun_path, yun->sun_path); +- if (r != 0) +- return r; +- } else if (x->sa_family == AF_INET) { ++ if (x->sa_family == AF_INET) { + struct sockaddr_in *xin = (void *)x, *yin = (void *)y; + CMP(ntohl(xin->sin_addr.s_addr), ntohl(yin->sin_addr.s_addr)); + CMP(ntohs(xin->sin_port), ntohs(yin->sin_port)); +@@ -878,6 +880,13 @@ int h2o_socket_compare_address(struct sockaddr *x, struct sockaddr *y) + CMP(ntohs(xin6->sin6_port), ntohs(yin6->sin6_port)); + CMP(xin6->sin6_flowinfo, yin6->sin6_flowinfo); + CMP(xin6->sin6_scope_id, yin6->sin6_scope_id); ++ #ifndef H2O_NO_UNIX_SOCKETS ++ } else if (x->sa_family == AF_UNIX) { ++ struct sockaddr_un *xun = (void *)x, *yun = (void *)y; ++ int r = strcmp(xun->sun_path, yun->sun_path); ++ if (r != 0) ++ return r; ++ #endif + } else { + assert(!"unknown sa_family"); + } +@@ -1015,7 +1024,7 @@ static void on_handshake_complete(h2o_socket_t *sock, const char *err) + + static void proceed_handshake(h2o_socket_t *sock, const char *err) + { +- h2o_iovec_t first_input = {NULL}; ++ h2o_iovec_t first_input = H2O_IOVEC_NULL; + int ret = 0; + + sock->_cb.write = NULL; +diff --git a/lib/common/socketpool.c b/lib/common/socketpool.c +index da69933f7..730d3d22f 100644 +--- a/lib/common/socketpool.c ++++ b/lib/common/socketpool.c +@@ -21,12 +21,18 @@ + */ + #include + #include ++#ifdef _WIN32 ++#include ++#else + #include ++#include + #include + #include ++#endif + #include ++#ifndef H2O_NO_UNIX_SOCKETS + #include +-#include ++#endif + #include "h2o/hostinfo.h" + #include "h2o/linklist.h" + #include "h2o/socketpool.h" +@@ -110,11 +116,15 @@ void h2o_socketpool_init_by_address(h2o_socketpool_t *pool, struct sockaddr *sa, + assert(salen <= sizeof(pool->peer.sockaddr.bytes)); + + if ((host_len = h2o_socket_getnumerichost(sa, salen, host)) == SIZE_MAX) { ++ #ifndef H2O_NO_UNIX_SOCKETS + if (sa->sa_family != AF_UNIX) + h2o_fatal("failed to convert a non-unix socket address to a numerical representation"); + /* use the sockaddr_un::sun_path as the SNI indicator (is that the right thing to do?) */ + strcpy(host, ((struct sockaddr_un *)sa)->sun_path); + host_len = strlen(host); ++ #else ++ h2o_fatal("failed to convert a socket address to a numerical representation"); ++ #endif + } + + common_init(pool, H2O_SOCKETPOOL_TYPE_SOCKADDR, h2o_iovec_init(host, host_len), is_ssl, capacity); +diff --git a/lib/common/string.c b/lib/common/string.c +index 3c068f3ad..2c33624b7 100644 +--- a/lib/common/string.c ++++ b/lib/common/string.c +@@ -441,16 +441,21 @@ const char *h2o_next_token(h2o_iovec_t *iter, int separator, size_t *element_len + *iter = h2o_iovec_init(cur, end - cur); + *element_len = token_end - token_start; + if (value != NULL) +- *value = (h2o_iovec_t){NULL}; ++ *value = (h2o_iovec_t)H2O_IOVEC_NULL; + return token_start; + + FindValue: + *iter = h2o_iovec_init(cur, end - cur); + *element_len = token_end - token_start; +- if ((value->base = (char *)h2o_next_token(iter, separator, &value->len, NULL)) == NULL) { +- *value = (h2o_iovec_t){"", 0}; +- } else if (h2o_memis(value->base, value->len, H2O_STRLIT(","))) { +- *value = (h2o_iovec_t){"", 0}; ++ ++ size_t tmp; ++ if ((value->base = (char *)h2o_next_token(iter, separator, &tmp, NULL)) == NULL) { ++ *value = (h2o_iovec_t)H2O_IOVEC_EMPTY; ++ return token_start; ++ } ++ value->len = tmp; ++ if (h2o_memis(value->base, value->len, H2O_STRLIT(","))) { ++ *value = (h2o_iovec_t)H2O_IOVEC_EMPTY; + iter->base -= 1; + iter->len += 1; + } +@@ -499,7 +504,7 @@ h2o_iovec_t h2o_htmlescape(h2o_mem_pool_t *pool, const char *src, size_t len) + /* escape and return the result if necessary */ + if (add_size != 0) { + /* allocate buffer and fill in the chars that are known not to require escaping */ +- h2o_iovec_t escaped = {h2o_mem_alloc_pool(pool, len + add_size + 1), 0}; ++ h2o_iovec_t escaped = h2o_iovec_init(h2o_mem_alloc_pool(pool, len + add_size + 1), 0); + /* fill-in the rest */ + for (s = src; s != end; ++s) { + switch (*s) { +@@ -529,7 +534,7 @@ h2o_iovec_t h2o_htmlescape(h2o_mem_pool_t *pool, const char *src, size_t len) + + h2o_iovec_t h2o_concat_list(h2o_mem_pool_t *pool, h2o_iovec_t *list, size_t count) + { +- h2o_iovec_t ret = {NULL, 0}; ++ h2o_iovec_t ret = H2O_IOVEC_NULL; + size_t i; + + /* calc the length */ +diff --git a/lib/common/url.c b/lib/common/url.c +index d65d18fb5..9b9ec85bb 100644 +--- a/lib/common/url.c ++++ b/lib/common/url.c +@@ -19,15 +19,21 @@ + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ ++#ifdef _WIN32 ++#include ++#else + #include ++#endif + #include ++#ifndef H2O_NO_UNIX_SOCKETS + #include ++#endif + #include "h2o/memory.h" + #include "h2o/string_.h" + #include "h2o/url.h" + +-const h2o_url_scheme_t H2O_URL_SCHEME_HTTP = {{H2O_STRLIT("http")}, 80}; +-const h2o_url_scheme_t H2O_URL_SCHEME_HTTPS = {{H2O_STRLIT("https")}, 443}; ++const h2o_url_scheme_t H2O_URL_SCHEME_HTTP = {H2O_IOVEC_STRLIT("http"), 80}; ++const h2o_url_scheme_t H2O_URL_SCHEME_HTTPS = {H2O_IOVEC_STRLIT("https"), 443}; + + static int decode_hex(int ch) + { +@@ -277,8 +283,8 @@ int h2o_url_parse_relative(const char *url, size_t url_len, h2o_url_t *parsed) + return parse_authority_and_path(p + 2, url_end, parsed); + + /* reset authority, host, port, and set path */ +- parsed->authority = (h2o_iovec_t){NULL}; +- parsed->host = (h2o_iovec_t){NULL}; ++ parsed->authority = (h2o_iovec_t)H2O_IOVEC_NULL; ++ parsed->host = (h2o_iovec_t)H2O_IOVEC_NULL; + parsed->_port = 65535; + parsed->path = h2o_iovec_init(p, url_end - p); + +@@ -324,7 +330,7 @@ h2o_iovec_t h2o_url_resolve(h2o_mem_pool_t *pool, const h2o_url_t *base, const h + h2o_url_resolve_path(&base_path, &relative_path); + } else { + assert(relative->path.len == 0); +- relative_path = (h2o_iovec_t){NULL}; ++ relative_path = (h2o_iovec_t)H2O_IOVEC_NULL; + } + + Build: +@@ -388,6 +394,7 @@ void h2o_url_copy(h2o_mem_pool_t *pool, h2o_url_t *dest, const h2o_url_t *src) + dest->_port = src->_port; + } + ++#ifndef H2O_NO_UNIX_SOCKETS + const char *h2o_url_host_to_sun(h2o_iovec_t host, struct sockaddr_un *sa) + { + #define PREFIX "unix:" +@@ -407,3 +414,4 @@ const char *h2o_url_host_to_sun(h2o_iovec_t host, struct sockaddr_un *sa) + } + + const char *h2o_url_host_to_sun_err_is_not_unix_socket = "supplied name does not look like an unix-domain socket"; ++#endif +diff --git a/lib/core/config.c b/lib/core/config.c +index ce1d32018..267708723 100644 +--- a/lib/core/config.c ++++ b/lib/core/config.c +@@ -246,7 +246,7 @@ h2o_hostconf_t *h2o_config_register_host(h2o_globalconf_t *config, h2o_iovec_t h + /* create hostconf */ + hostconf = create_hostconf(config); + hostconf->authority.host = host_lc; +- host_lc = (h2o_iovec_t){NULL}; ++ host_lc = (h2o_iovec_t)H2O_IOVEC_NULL; + hostconf->authority.port = port; + if (hostconf->authority.port == 65535) { + hostconf->authority.hostport = hostconf->authority.host; +diff --git a/lib/core/context.c b/lib/core/context.c +index 8d1101381..f472adbe6 100644 +--- a/lib/core/context.c ++++ b/lib/core/context.c +@@ -23,7 +23,6 @@ + #include + #include + #include "h2o.h" +-#include "h2o/memcached.h" + + void h2o_context_init_pathconf_context(h2o_context_t *ctx, h2o_pathconf_t *pathconf) + { +@@ -194,7 +193,9 @@ void h2o_context_update_timestamp_cache(h2o_context_t *ctx) + if (ctx->_timestamp_cache.value != NULL) + h2o_mem_release_shared(ctx->_timestamp_cache.value); + ctx->_timestamp_cache.value = h2o_mem_alloc_shared(NULL, sizeof(h2o_timestamp_string_t), NULL); +- gmtime_r(&ctx->_timestamp_cache.tv_at.tv_sec, &gmt); ++ /* work around pointer type collision in MingW */ ++ time_t tmp = ctx->_timestamp_cache.tv_at.tv_sec; ++ gmtime_r(&tmp, &gmt); + h2o_time2str_rfc1123(ctx->_timestamp_cache.value->rfc1123, &gmt); + h2o_time2str_log(ctx->_timestamp_cache.value->log, ctx->_timestamp_cache.tv_at.tv_sec); + } +diff --git a/lib/core/logconf.c b/lib/core/logconf.c +index 4d79736cc..74c198ce7 100644 +--- a/lib/core/logconf.c ++++ b/lib/core/logconf.c +@@ -597,8 +597,11 @@ char *h2o_log_request(h2o_logconf_t *logconf, h2o_req_t *req, size_t *len, char + goto EmitNull; + { + size_t bufsz, len; +- if (localt.tm_year == 0) +- localtime_r(&req->processed_at.at.tv_sec, &localt); ++ if (localt.tm_year == 0) { ++ /* work around pointer type collision in MingW */ ++ time_t tmp = req->processed_at.at.tv_sec; ++ localtime_r(&tmp, &localt); ++ } + for (bufsz = 128;; bufsz *= 2) { + RESERVE(bufsz); + if ((len = strftime(pos, bufsz, element->data.name.base, &localt)) != 0) +diff --git a/lib/core/proxy.c b/lib/core/proxy.c +index edb4baf9d..448dc103d 100644 +--- a/lib/core/proxy.c ++++ b/lib/core/proxy.c +@@ -19,10 +19,14 @@ + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +-#include + #include + #include ++#ifdef _WIN32 ++#include ++#else ++#include + #include ++#endif + #include "picohttpparser.h" + #include "h2o.h" + #include "h2o/http1.h" +@@ -79,7 +83,7 @@ static h2o_iovec_t rewrite_location(h2o_mem_pool_t *pool, const char *location, + h2o_iovec_init(loc_parsed.path.base + match->path.len, loc_parsed.path.len - match->path.len)); + + NoRewrite: +- return (h2o_iovec_t){NULL}; ++ return (h2o_iovec_t)H2O_IOVEC_NULL; + } + + static h2o_iovec_t build_request_merge_headers(h2o_mem_pool_t *pool, h2o_iovec_t merged, h2o_iovec_t added, int seperator) +@@ -136,7 +140,7 @@ static h2o_iovec_t build_request(h2o_req_t *req, int keepalive, int is_websocket + char remote_addr[NI_MAXHOST]; + struct sockaddr_storage ss; + socklen_t sslen; +- h2o_iovec_t cookie_buf = {NULL}, xff_buf = {NULL}, via_buf = {NULL}; ++ h2o_iovec_t cookie_buf = H2O_IOVEC_NULL, xff_buf = H2O_IOVEC_NULL, via_buf = H2O_IOVEC_NULL; + int preserve_x_forwarded_proto = req->conn->ctx->globalconf->proxy.preserve_x_forwarded_proto; + int emit_x_forwarded_headers = req->conn->ctx->globalconf->proxy.emit_x_forwarded_headers; + int emit_via_header = req->conn->ctx->globalconf->proxy.emit_via_header; +@@ -202,7 +206,7 @@ static h2o_iovec_t build_request(h2o_req_t *req, int keepalive, int is_websocket + assert(offset <= buf.len); + if (req->entity.base != NULL || req_requires_content_length(req)) { + RESERVE(sizeof("content-length: " H2O_UINT64_LONGEST_STR) - 1); +- offset += sprintf(buf.base + offset, "content-length: %zu\r\n", req->entity.len); ++ offset += sprintf(buf.base + offset, "content-length: %zu\r\n", (size_t)req->entity.len); + } + + /* rewrite headers if necessary */ +diff --git a/lib/core/request.c b/lib/core/request.c +index 96aabb22d..65810ed91 100644 +--- a/lib/core/request.c ++++ b/lib/core/request.c +@@ -22,7 +22,9 @@ + #include + #include + #include ++#ifndef _WIN32 + #include ++#endif + #include "h2o.h" + + #ifndef IOV_MAX +@@ -595,10 +597,16 @@ void h2o_req_log_error(h2o_req_t *req, const char *module, const char *fmt, ...) + p += 3; + } + *p++ = ':'; ++ #ifdef _WIN32 ++ write(2, prefix, p - prefix); ++ write(2, errbuf, errlen); ++ write(2, "\n", 1); ++ #else + /* use writev(2) to emit error atomically */ + struct iovec vecs[] = {{prefix, p - prefix}, {errbuf, errlen}, {"\n", 1}}; + H2O_BUILD_ASSERT(sizeof(vecs) / sizeof(vecs[0]) < IOV_MAX); + writev(2, vecs, sizeof(vecs) / sizeof(vecs[0])); ++ #endif + } + } + +@@ -611,8 +619,8 @@ void h2o_send_redirect(h2o_req_t *req, int status, const char *reason, const cha + } + + static h2o_generator_t generator = {NULL, NULL}; +- static const h2o_iovec_t body_prefix = {H2O_STRLIT("Moved

The document has moved here")}; ++ static const h2o_iovec_t body_prefix = H2O_IOVEC_STRLIT("Moved

The document has moved here"); + + /* build and send response */ + h2o_iovec_t bufs[3]; +diff --git a/lib/core/token_table.h b/lib/core/token_table.h +index ae26aa6c4..fa74f1e08 100644 +--- a/lib/core/token_table.h ++++ b/lib/core/token_table.h +@@ -21,68 +21,68 @@ + */ + + /* DO NOT EDIT! generated by tokens.pl */ +-h2o_token_t h2o__tokens[] = {{{H2O_STRLIT(":authority")}, 1, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT(":method")}, 2, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT(":path")}, 4, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT(":scheme")}, 6, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT(":status")}, 8, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("accept")}, 19, 0, 0, 0, 0, 1, 0}, +- {{H2O_STRLIT("accept-charset")}, 15, 0, 0, 0, 0, 1, 0}, +- {{H2O_STRLIT("accept-encoding")}, 16, 0, 0, 0, 0, 1, 0}, +- {{H2O_STRLIT("accept-language")}, 17, 0, 0, 0, 0, 1, 0}, +- {{H2O_STRLIT("accept-ranges")}, 18, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("access-control-allow-origin")}, 20, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("age")}, 21, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("allow")}, 22, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("authorization")}, 23, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("cache-control")}, 24, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("cache-digest")}, 0, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("connection")}, 0, 1, 1, 0, 1, 0, 0}, +- {{H2O_STRLIT("content-disposition")}, 25, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("content-encoding")}, 26, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("content-language")}, 27, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("content-length")}, 28, 0, 0, 1, 0, 0, 0}, +- {{H2O_STRLIT("content-location")}, 29, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("content-range")}, 30, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("content-type")}, 31, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("cookie")}, 32, 0, 0, 0, 0, 0, 1}, +- {{H2O_STRLIT("date")}, 33, 0, 1, 0, 0, 0, 0}, +- {{H2O_STRLIT("etag")}, 34, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("expect")}, 35, 0, 0, 1, 0, 0, 0}, +- {{H2O_STRLIT("expires")}, 36, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("from")}, 37, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("host")}, 38, 0, 0, 1, 1, 0, 0}, +- {{H2O_STRLIT("http2-settings")}, 0, 1, 0, 0, 1, 0, 0}, +- {{H2O_STRLIT("if-match")}, 39, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("if-modified-since")}, 40, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("if-none-match")}, 41, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("if-range")}, 42, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("if-unmodified-since")}, 43, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("keep-alive")}, 0, 1, 1, 0, 0, 0, 0}, +- {{H2O_STRLIT("last-modified")}, 44, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("link")}, 45, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("location")}, 46, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("max-forwards")}, 47, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("proxy-authenticate")}, 48, 1, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("proxy-authorization")}, 49, 1, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("range")}, 50, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("referer")}, 51, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("refresh")}, 52, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("retry-after")}, 53, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("server")}, 54, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("set-cookie")}, 55, 0, 0, 0, 0, 0, 1}, +- {{H2O_STRLIT("strict-transport-security")}, 56, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("te")}, 0, 1, 0, 0, 1, 0, 0}, +- {{H2O_STRLIT("transfer-encoding")}, 57, 1, 1, 1, 1, 0, 0}, +- {{H2O_STRLIT("upgrade")}, 0, 1, 1, 1, 1, 0, 0}, +- {{H2O_STRLIT("user-agent")}, 58, 0, 0, 0, 0, 1, 0}, +- {{H2O_STRLIT("vary")}, 59, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("via")}, 60, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("www-authenticate")}, 61, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("x-compress-hint")}, 0, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("x-forwarded-for")}, 0, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("x-reproxy-url")}, 0, 0, 0, 0, 0, 0, 0}, +- {{H2O_STRLIT("x-traffic")}, 0, 0, 0, 0, 0, 0, 0}}; ++h2o_token_t h2o__tokens[] = {{H2O_IOVEC_STRLIT(":authority"), 1, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT(":method"), 2, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT(":path"), 4, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT(":scheme"), 6, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT(":status"), 8, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("accept"), 19, 0, 0, 0, 0, 1, 0}, ++ {H2O_IOVEC_STRLIT("accept-charset"), 15, 0, 0, 0, 0, 1, 0}, ++ {H2O_IOVEC_STRLIT("accept-encoding"), 16, 0, 0, 0, 0, 1, 0}, ++ {H2O_IOVEC_STRLIT("accept-language"), 17, 0, 0, 0, 0, 1, 0}, ++ {H2O_IOVEC_STRLIT("accept-ranges"), 18, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("access-control-allow-origin"), 20, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("age"), 21, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("allow"), 22, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("authorization"), 23, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("cache-control"), 24, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("cache-digest"), 0, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("connection"), 0, 1, 1, 0, 1, 0, 0}, ++ {H2O_IOVEC_STRLIT("content-disposition"), 25, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("content-encoding"), 26, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("content-language"), 27, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("content-length"), 28, 0, 0, 1, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("content-location"), 29, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("content-range"), 30, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("content-type"), 31, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("cookie"), 32, 0, 0, 0, 0, 0, 1}, ++ {H2O_IOVEC_STRLIT("date"), 33, 0, 1, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("etag"), 34, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("expect"), 35, 0, 0, 1, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("expires"), 36, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("from"), 37, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("host"), 38, 0, 0, 1, 1, 0, 0}, ++ {H2O_IOVEC_STRLIT("http2-settings"), 0, 1, 0, 0, 1, 0, 0}, ++ {H2O_IOVEC_STRLIT("if-match"), 39, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("if-modified-since"), 40, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("if-none-match"), 41, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("if-range"), 42, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("if-unmodified-since"), 43, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("keep-alive"), 0, 1, 1, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("last-modified"), 44, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("link"), 45, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("location"), 46, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("max-forwards"), 47, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("proxy-authenticate"), 48, 1, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("proxy-authorization"), 49, 1, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("range"), 50, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("referer"), 51, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("refresh"), 52, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("retry-after"), 53, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("server"), 54, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("set-cookie"), 55, 0, 0, 0, 0, 0, 1}, ++ {H2O_IOVEC_STRLIT("strict-transport-security"), 56, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("te"), 0, 1, 0, 0, 1, 0, 0}, ++ {H2O_IOVEC_STRLIT("transfer-encoding"), 57, 1, 1, 1, 1, 0, 0}, ++ {H2O_IOVEC_STRLIT("upgrade"), 0, 1, 1, 1, 1, 0, 0}, ++ {H2O_IOVEC_STRLIT("user-agent"), 58, 0, 0, 0, 0, 1, 0}, ++ {H2O_IOVEC_STRLIT("vary"), 59, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("via"), 60, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("www-authenticate"), 61, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("x-compress-hint"), 0, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("x-forwarded-for"), 0, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("x-reproxy-url"), 0, 0, 0, 0, 0, 0, 0}, ++ {H2O_IOVEC_STRLIT("x-traffic"), 0, 0, 0, 0, 0, 0, 0}}; + size_t h2o__num_tokens = 62; + + const h2o_token_t *h2o_lookup_token(const char *name, size_t len) +diff --git a/lib/core/util.c b/lib/core/util.c +index 50d2b2493..13c363ee1 100644 +--- a/lib/core/util.c ++++ b/lib/core/util.c +@@ -63,6 +63,7 @@ static void free_accept_data(struct st_h2o_accept_data_t *data) + free(data); + } + ++#ifndef H2O_NO_MEMCACHED + static struct { + h2o_memcached_context_t *memc; + unsigned expiration; +@@ -97,15 +98,18 @@ void h2o_accept_setup_async_ssl_resumption(h2o_memcached_context_t *memc, unsign + async_resumption_context.expiration = expiration; + h2o_socket_ssl_async_resumption_init(async_resumption_get, async_resumption_new); + } ++#endif + + void on_accept_timeout(h2o_timeout_entry_t *entry) + { + /* TODO log */ + struct st_h2o_accept_data_t *data = H2O_STRUCT_FROM_MEMBER(struct st_h2o_accept_data_t, timeout, entry); ++ #ifndef H2O_NO_MEMCACHED + if (data->async_resumption_get_req != NULL) { + h2o_memcached_cancel_get(async_resumption_context.memc, data->async_resumption_get_req); + data->async_resumption_get_req = NULL; + } ++ #endif + h2o_socket_t *sock = data->sock; + free_accept_data(data); + h2o_socket_close(sock); +@@ -158,7 +162,11 @@ static ssize_t parse_proxy_line(char *src, size_t len, struct sockaddr *sa, sock + + char *p = src, *end = p + len; + void *addr; ++ #if _WIN32 ++ u_short *port; ++ #else + in_port_t *port; ++ #endif + + /* "PROXY "*/ + EXPECT_CHAR('P'); +@@ -369,7 +377,7 @@ static void push_one_path(h2o_mem_pool_t *pool, h2o_iovec_vector_t *paths_to_pus + } + + /* check scheme and authority if given URL contains either of the two, or if base is specified */ +- h2o_url_t base = {input_scheme, input_authority, {NULL}, base_path, 65535}; ++ h2o_url_t base = {input_scheme, input_authority, H2O_IOVEC_NULL, base_path, 65535}; + if (base_scheme != NULL) { + base.scheme = base_scheme; + base.authority = *base_authority; +@@ -528,7 +536,7 @@ h2o_iovec_t h2o_build_destination(h2o_req_t *req, const char *prefix, size_t pre + if (req->path.base[0] != '/' && next_unnormalized == 1) { + next_unnormalized = 0; + } +- parts[num_parts++] = (h2o_iovec_t){req->path.base + next_unnormalized, req->path.len - next_unnormalized}; ++ parts[num_parts++] = h2o_iovec_init(req->path.base + next_unnormalized, req->path.len - next_unnormalized); + } + } + +@@ -536,10 +544,7 @@ h2o_iovec_t h2o_build_destination(h2o_req_t *req, const char *prefix, size_t pre + } + + /* h2-14 and h2-16 are kept for backwards compatibility, as they are often used */ +-#define ALPN_ENTRY(s) \ +- { \ +- H2O_STRLIT(s) \ +- } ++#define ALPN_ENTRY(s) H2O_IOVEC_STRLIT(s) + #define ALPN_PROTOCOLS_CORE ALPN_ENTRY("h2"), ALPN_ENTRY("h2-16"), ALPN_ENTRY("h2-14") + #define NPN_PROTOCOLS_CORE \ + "\x02" \ +@@ -549,10 +554,10 @@ h2o_iovec_t h2o_build_destination(h2o_req_t *req, const char *prefix, size_t pre + "\x05" \ + "h2-14" + +-static const h2o_iovec_t http2_alpn_protocols[] = {ALPN_PROTOCOLS_CORE, {NULL}}; ++static const h2o_iovec_t http2_alpn_protocols[] = {ALPN_PROTOCOLS_CORE, H2O_IOVEC_NULL}; + const h2o_iovec_t *h2o_http2_alpn_protocols = http2_alpn_protocols; + +-static const h2o_iovec_t alpn_protocols[] = {ALPN_PROTOCOLS_CORE, {H2O_STRLIT("http/1.1")}, {NULL}}; ++static const h2o_iovec_t alpn_protocols[] = {ALPN_PROTOCOLS_CORE, H2O_IOVEC_STRLIT("http/1.1"), H2O_IOVEC_NULL}; + const h2o_iovec_t *h2o_alpn_protocols = alpn_protocols; + + const char *h2o_http2_npn_protocols = NPN_PROTOCOLS_CORE; +diff --git a/lib/handler/access_log.c b/lib/handler/access_log.c +index 4a7704174..d56c21add 100644 +--- a/lib/handler/access_log.c ++++ b/lib/handler/access_log.c +@@ -22,12 +22,16 @@ + #include + #include + #include ++#include ++#include ++#ifdef _WIN32 ++#include ++#else + #include + #include + #include +-#include +-#include + #include ++#endif + #include + #include "h2o.h" + #include "h2o/serverutil.h" +@@ -73,6 +77,7 @@ int h2o_access_log_open_log(const char *path) + { + int fd; + ++ #ifndef _WIN32 + if (path[0] == '|') { + int pipefds[2]; + pid_t pid; +@@ -96,7 +101,9 @@ int h2o_access_log_open_log(const char *path) + /* close the read side of the pipefds and return the write side */ + close(pipefds[0]); + fd = pipefds[1]; +- } else { ++ } else ++ #endif ++ { + if ((fd = open(path, O_CREAT | O_WRONLY | O_APPEND | O_CLOEXEC, 0644)) == -1) { + fprintf(stderr, "failed to open log file:%s:%s\n", path, strerror(errno)); + return -1; +diff --git a/lib/handler/configurator/headers_util.c b/lib/handler/configurator/headers_util.c +index c05b9b7c2..56e47842e 100644 +--- a/lib/handler/configurator/headers_util.c ++++ b/lib/handler/configurator/headers_util.c +@@ -87,7 +87,7 @@ static int on_config_header_unset(h2o_configurator_command_t *cmd, h2o_configura + h2o_configurator_errprintf(cmd, node, "invalid header name"); + return -1; + } +- if (add_cmd(cmd, node, H2O_HEADERS_CMD_UNSET, name, (h2o_iovec_t){NULL}, self->get_commands(self->child)) != 0) { ++ if (add_cmd(cmd, node, H2O_HEADERS_CMD_UNSET, name, (h2o_iovec_t)H2O_IOVEC_NULL, self->get_commands(self->child)) != 0) { + if (!h2o_iovec_is_token(name)) + free(name->base); + return -1; +diff --git a/lib/handler/file.c b/lib/handler/file.c +index 5d7c7a2a4..c0bc486ec 100644 +--- a/lib/handler/file.c ++++ b/lib/handler/file.c +@@ -128,7 +128,7 @@ static void do_proceed(h2o_generator_t *_self, h2o_req_t *req) + rlen = self->bytesleft; + if (rlen > MAX_BUF_SIZE) + rlen = MAX_BUF_SIZE; +- while ((rret = pread(self->file.ref->fd, self->buf, rlen, self->file.off)) == -1 && errno == EINTR) ++ while ((rret = h2o_filecache_read_file(self->file.ref, self->buf, rlen, self->file.off)) == -1 && errno == EINTR) + ; + if (rret == -1) { + h2o_send(req, NULL, 0, H2O_SEND_STATE_ERROR); +@@ -177,7 +177,7 @@ static void do_multirange_proceed(h2o_generator_t *_self, h2o_req_t *req) + rlen = self->bytesleft; + if (rlen + used_buf > MAX_BUF_SIZE) + rlen = MAX_BUF_SIZE - used_buf; +- while ((rret = pread(self->file.ref->fd, self->buf + used_buf, rlen, self->file.off)) == -1 && errno == EINTR) ++ while ((rret = h2o_filecache_read_file(self->file.ref, self->buf + used_buf, rlen, self->file.off)) == -1 && errno == EINTR) + ; + if (rret == -1) + goto Error; +@@ -213,7 +213,7 @@ static h2o_send_state_t do_pull(h2o_generator_t *_self, h2o_req_t *req, h2o_iove + + if (self->bytesleft < buf->len) + buf->len = self->bytesleft; +- while ((rret = pread(self->file.ref->fd, buf->base, buf->len, self->file.off)) == -1 && errno == EINTR) ++ while ((rret = h2o_filecache_read_file(self->file.ref, buf->base, buf->len, self->file.off)) == -1 && errno == EINTR) + ; + if (rret <= 0) { + buf->len = 0; +@@ -237,7 +237,7 @@ static struct st_h2o_sendfile_generator_t *create_generator(h2o_req_t *req, cons + { + struct st_h2o_sendfile_generator_t *self; + h2o_filecache_ref_t *fileref; +- h2o_iovec_t content_encoding = (h2o_iovec_t){NULL}; ++ h2o_iovec_t content_encoding = H2O_IOVEC_NULL; + unsigned gunzip = 0; + + *is_dir = 0; +diff --git a/lib/handler/proxy.c b/lib/handler/proxy.c +index 1d87225e1..7de3c4ec5 100644 +--- a/lib/handler/proxy.c ++++ b/lib/handler/proxy.c +@@ -19,7 +19,9 @@ + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ ++#ifndef H2O_NO_UNIX_SOCKETS + #include ++#endif + #include "h2o.h" + #include "h2o/socketpool.h" + +@@ -136,29 +138,37 @@ static void on_handler_dispose(h2o_handler_t *_self) + + void h2o_proxy_register_reverse_proxy(h2o_pathconf_t *pathconf, h2o_url_t *upstream, h2o_proxy_config_vars_t *config) + { +- struct sockaddr_un sa; ++ #ifndef H2O_NO_UNIX_SOCKETS + const char *to_sa_err; ++ #endif + struct rp_handler_t *self = (void *)h2o_create_handler(pathconf, sizeof(*self)); + self->super.on_context_init = on_context_init; + self->super.on_context_dispose = on_context_dispose; + self->super.dispose = on_handler_dispose; + self->super.on_req = on_req; +- to_sa_err = h2o_url_host_to_sun(upstream->host, &sa); + if (config->keepalive_timeout != 0) { + self->sockpool = h2o_mem_alloc(sizeof(*self->sockpool)); + int is_ssl = upstream->scheme == &H2O_URL_SCHEME_HTTPS; ++ #ifndef H2O_NO_UNIX_SOCKETS ++ struct sockaddr_un sa; ++ to_sa_err = h2o_url_host_to_sun(upstream->host, &sa); + if (to_sa_err == h2o_url_host_to_sun_err_is_not_unix_socket) { ++ #endif + h2o_socketpool_init_by_hostport(self->sockpool, upstream->host, h2o_url_get_port(upstream), is_ssl, + SIZE_MAX /* FIXME */); ++ #ifndef H2O_NO_UNIX_SOCKETS + } else { + assert(to_sa_err == NULL); + h2o_socketpool_init_by_address(self->sockpool, (void *)&sa, sizeof(sa), is_ssl, SIZE_MAX /* FIXME */); + } ++ #endif + } + h2o_url_copy(NULL, &self->upstream, upstream); ++ #ifndef H2O_NO_UNIX_SOCKETS + if (to_sa_err) { + h2o_strtolower(self->upstream.host.base, self->upstream.host.len); + } ++ #endif + self->config = *config; + if (self->config.ssl_ctx != NULL) + SSL_CTX_up_ref(self->config.ssl_ctx); +diff --git a/lib/handler/redirect.c b/lib/handler/redirect.c +index c8b9c5086..9eb5d4552 100644 +--- a/lib/handler/redirect.c ++++ b/lib/handler/redirect.c +@@ -68,7 +68,7 @@ static void redirect_internally(h2o_redirect_handler_t *self, h2o_req_t *req, h2 + break; + default: + method = h2o_iovec_init(H2O_STRLIT("GET")); +- req->entity = (h2o_iovec_t){NULL}; ++ req->entity = (h2o_iovec_t)H2O_IOVEC_NULL; + break; + } + +diff --git a/lib/handler/reproxy.c b/lib/handler/reproxy.c +index 369386d27..2639045c6 100644 +--- a/lib/handler/reproxy.c ++++ b/lib/handler/reproxy.c +@@ -47,7 +47,7 @@ static void on_setup_ostream(h2o_filter_t *self, h2o_req_t *req, h2o_ostream_t * + break; + default: + method = h2o_iovec_init(H2O_STRLIT("GET")); +- req->entity = (h2o_iovec_t){NULL}; ++ req->entity = (h2o_iovec_t)H2O_IOVEC_NULL; + break; + } + +diff --git a/lib/handler/status.c b/lib/handler/status.c +index 93befed3b..95f9ce823 100644 +--- a/lib/handler/status.c ++++ b/lib/handler/status.c +@@ -95,7 +95,7 @@ static void send_response(struct st_h2o_status_collector_t *collector) + h2o_iovec_t resp[nr_resp]; + + memset(resp, 0, sizeof(resp[0]) * nr_resp); +- resp[cur_resp++] = (h2o_iovec_t){H2O_STRLIT("{\n")}; ++ resp[cur_resp++] = h2o_iovec_init(H2O_STRLIT("{\n")); + + int coma_removed = 0; + for (i = 0; i < req->conn->ctx->globalconf->statuses.size; i++) { +@@ -110,7 +110,7 @@ static void send_response(struct st_h2o_status_collector_t *collector) + coma_removed = 1; + } + } +- resp[cur_resp++] = (h2o_iovec_t){H2O_STRLIT("\n}\n")}; ++ resp[cur_resp++] = h2o_iovec_init(H2O_STRLIT("\n}\n")); + + req->res.status = 200; + h2o_add_header(&req->pool, &req->res.headers, H2O_TOKEN_CONTENT_TYPE, NULL, H2O_STRLIT("text/plain; charset=utf-8")); +@@ -213,7 +213,7 @@ static int on_req(h2o_handler_t *_self, h2o_req_t *req) + } else if (h2o_memis(local_path.base, local_path.len, H2O_STRLIT("/json"))) { + int ret; + /* "/json" maps to the JSON API */ +- h2o_iovec_t status_list = {NULL, 0}; /* NULL means we'll show all statuses */ ++ h2o_iovec_t status_list = H2O_IOVEC_NULL; /* NULL means we'll show all statuses */ + if (req->query_at != SIZE_MAX && (req->path.len - req->query_at > 6)) { + if (h2o_memis(&req->path.base[req->query_at], 6, "?show=", 6)) { + status_list = h2o_iovec_init(&req->path.base[req->query_at + 6], req->path.len - req->query_at - 6); +diff --git a/lib/handler/status/durations.c b/lib/handler/status/durations.c +index f011107bf..0e45e3eed 100644 +--- a/lib/handler/status/durations.c ++++ b/lib/handler/status/durations.c +@@ -203,5 +203,5 @@ void h2o_duration_stats_register(h2o_globalconf_t *conf) + } + + h2o_status_handler_t durations_status_handler = { +- {H2O_STRLIT("durations")}, durations_status_init, durations_status_per_thread, durations_status_final, ++ H2O_IOVEC_STRLIT("durations"), durations_status_init, durations_status_per_thread, durations_status_final, + }; +diff --git a/lib/handler/status/events.c b/lib/handler/status/events.c +index e6ed0b7c6..64fb6546f 100644 +--- a/lib/handler/status/events.c ++++ b/lib/handler/status/events.c +@@ -108,5 +108,5 @@ static h2o_iovec_t events_status_final(void *priv, h2o_globalconf_t *gconf, h2o_ + } + + h2o_status_handler_t events_status_handler = { +- {H2O_STRLIT("events")}, events_status_init, events_status_per_thread, events_status_final, ++ H2O_IOVEC_STRLIT("events"), events_status_init, events_status_per_thread, events_status_final, + }; +diff --git a/lib/handler/status/requests.c b/lib/handler/status/requests.c +index 4854e4a1f..56c28c36c 100644 +--- a/lib/handler/status/requests.c ++++ b/lib/handler/status/requests.c +@@ -123,7 +123,7 @@ static void *requests_status_init(void) + /* log format compilation error is an internal logic flaw, therefore we need not send the details to the client */ + fprintf(stderr, "[lib/handler/status/requests.c] failed to compile log format: %s", errbuf); + +- rsc->req_data = (h2o_iovec_t){NULL}; ++ rsc->req_data = (h2o_iovec_t)H2O_IOVEC_NULL; + pthread_mutex_init(&rsc->mutex, NULL); + + return rsc; +@@ -131,7 +131,7 @@ static void *requests_status_init(void) + + static h2o_iovec_t requests_status_final(void *priv, h2o_globalconf_t *gconf, h2o_req_t *req) + { +- h2o_iovec_t ret = {NULL}; ++ h2o_iovec_t ret = H2O_IOVEC_NULL; + struct st_requests_status_ctx_t *rsc = priv; + + if (rsc->logconf != NULL) { +@@ -147,5 +147,5 @@ static h2o_iovec_t requests_status_final(void *priv, h2o_globalconf_t *gconf, h2 + } + + h2o_status_handler_t requests_status_handler = { +- {H2O_STRLIT("requests")}, requests_status_init, requests_status_per_thread, requests_status_final, ++ H2O_IOVEC_STRLIT("requests"), requests_status_init, requests_status_per_thread, requests_status_final, + }; +diff --git a/lib/http1.c b/lib/http1.c +index 98c4e55ab..b765b8aae 100644 +--- a/lib/http1.c ++++ b/lib/http1.c +@@ -323,7 +323,7 @@ static ssize_t fixup_request(struct st_h2o_http1_conn_t *conn, struct phr_header + h2o_iovec_t *expect) + { + ssize_t entity_header_index; +- h2o_iovec_t connection = {NULL, 0}, host = {NULL, 0}, upgrade = {NULL, 0}; ++ h2o_iovec_t connection = H2O_IOVEC_NULL, host = H2O_IOVEC_NULL, upgrade = H2O_IOVEC_NULL; + + expect->base = NULL; + expect->len = 0; +@@ -406,12 +406,12 @@ static void send_bad_request_on_complete(h2o_socket_t *sock, const char *err) + + static void send_bad_request(struct st_h2o_http1_conn_t *conn) + { +- const static h2o_iovec_t resp = {H2O_STRLIT("HTTP/1.1 400 Bad Request\r\n" ++ const static h2o_iovec_t resp = H2O_IOVEC_STRLIT("HTTP/1.1 400 Bad Request\r\n" + "Content-Type: text/plain; charset=utf-8\r\n" + "Connection: close\r\n" + "Content-Length: 11\r\n" + "\r\n" +- "Bad Request")}; ++ "Bad Request"); + + assert(conn->req.version == 0 && "request has not been parsed successfully"); + assert(conn->req.http1_is_persistent == 0); +@@ -421,7 +421,7 @@ static void send_bad_request(struct st_h2o_http1_conn_t *conn) + + static void handle_incoming_request(struct st_h2o_http1_conn_t *conn) + { +- size_t inreqlen = conn->sock->input->size < H2O_MAX_REQLEN ? conn->sock->input->size : H2O_MAX_REQLEN; ++ size_t methodlen, pathlen, inreqlen = conn->sock->input->size < H2O_MAX_REQLEN ? conn->sock->input->size : H2O_MAX_REQLEN; + int reqlen, minor_version; + struct phr_header headers[H2O_MAX_HEADERS]; + size_t num_headers = H2O_MAX_HEADERS; +@@ -433,8 +433,10 @@ static void handle_incoming_request(struct st_h2o_http1_conn_t *conn) + conn->req.timestamps.request_begin_at = *h2o_get_timestamp(conn->super.ctx, NULL, NULL); + + reqlen = phr_parse_request(conn->sock->input->bytes, inreqlen, (const char **)&conn->req.input.method.base, +- &conn->req.input.method.len, (const char **)&conn->req.input.path.base, &conn->req.input.path.len, ++ &methodlen, (const char **)&conn->req.input.path.base, &pathlen, + &minor_version, headers, &num_headers, conn->_prevreqlen); ++ conn->req.input.method.len = methodlen; ++ conn->req.input.path.len = pathlen; + conn->_prevreqlen = inreqlen; + + switch (reqlen) { +@@ -455,7 +457,7 @@ static void handle_incoming_request(struct st_h2o_http1_conn_t *conn) + return; + } + if (expect.base != NULL) { +- static const h2o_iovec_t res = {H2O_STRLIT("HTTP/1.1 100 Continue\r\n\r\n")}; ++ static const h2o_iovec_t res = H2O_IOVEC_STRLIT("HTTP/1.1 100 Continue\r\n\r\n"); + h2o_socket_write(conn->sock, (void *)&res, 1, on_continue_sent); + /* processing of the incoming entity is postponed until the 100 response is sent */ + h2o_socket_read_stop(conn->sock); +@@ -477,7 +479,7 @@ static void handle_incoming_request(struct st_h2o_http1_conn_t *conn) + /* upgrade to HTTP/2 if the request starts with: PRI * HTTP/2 */ + if (conn->super.ctx->globalconf->http1.upgrade_to_http2) { + /* should check up to the first octet that phr_parse_request returns an error */ +- static const h2o_iovec_t HTTP2_SIG = {H2O_STRLIT("PRI * HTTP/2")}; ++ static const h2o_iovec_t HTTP2_SIG = H2O_IOVEC_STRLIT("PRI * HTTP/2"); + if (conn->sock->input->size >= HTTP2_SIG.len && memcmp(conn->sock->input->bytes, HTTP2_SIG.base, HTTP2_SIG.len) == 0) { + h2o_accept_ctx_t accept_ctx = {conn->super.ctx, conn->super.hosts}; + h2o_socket_t *sock = conn->sock; +@@ -638,7 +640,7 @@ static size_t flatten_headers(char *buf, h2o_req_t *req, const char *connection) + * - https://www.igvita.com/2013/05/01/deploying-webp-via-accept-content-negotiation/ + */ + if (is_msie(req)) { +- static h2o_header_t cache_control_private = {&H2O_TOKEN_CACHE_CONTROL->buf, NULL, {H2O_STRLIT("private")}}; ++ static h2o_header_t cache_control_private = {&H2O_TOKEN_CACHE_CONTROL->buf, NULL, H2O_IOVEC_STRLIT("private")}; + header = &cache_control_private; + } + } +@@ -660,11 +662,11 @@ static size_t flatten_headers(char *buf, h2o_req_t *req, const char *connection) + + static void proceed_pull(struct st_h2o_http1_conn_t *conn, size_t nfilled) + { +- h2o_iovec_t buf = {conn->_ostr_final.pull.buf, nfilled}; ++ h2o_iovec_t buf = h2o_iovec_init(conn->_ostr_final.pull.buf, nfilled); + h2o_send_state_t send_state; + + if (buf.len < MAX_PULL_BUF_SZ) { +- h2o_iovec_t cbuf = {buf.base + buf.len, MAX_PULL_BUF_SZ - buf.len}; ++ h2o_iovec_t cbuf = h2o_iovec_init(buf.base + buf.len, MAX_PULL_BUF_SZ - buf.len); + send_state = h2o_pull(&conn->req, conn->_ostr_final.pull.cb, &cbuf); + if (send_state == H2O_SEND_STATE_ERROR) { + conn->req.http1_is_persistent = 0; +diff --git a/lib/http2/casper.c b/lib/http2/casper.c +index 56e00d71f..e5c9a0ed7 100644 +--- a/lib/http2/casper.c ++++ b/lib/http2/casper.c +@@ -56,7 +56,7 @@ h2o_http2_casper_t *h2o_http2_casper_create(unsigned capacity_bits, unsigned rem + memset(&casper->keys, 0, sizeof(casper->keys)); + casper->capacity_bits = capacity_bits; + casper->remainder_bits = remainder_bits; +- casper->cookie_cache = (h2o_iovec_t){NULL}; ++ casper->cookie_cache = (h2o_iovec_t)H2O_IOVEC_NULL; + + return casper; + } +@@ -89,7 +89,7 @@ int h2o_http2_casper_lookup(h2o_http2_casper_t *casper, const char *path, size_t + + /* we need to set a new value */ + free(casper->cookie_cache.base); +- casper->cookie_cache = (h2o_iovec_t){NULL}; ++ casper->cookie_cache = (h2o_iovec_t)H2O_IOVEC_NULL; + h2o_vector_reserve(NULL, &casper->keys, casper->keys.size + 1); + memmove(casper->keys.entries + i + 1, casper->keys.entries + i, (casper->keys.size - i) * sizeof(casper->keys.entries[0])); + ++casper->keys.size; +@@ -99,7 +99,7 @@ int h2o_http2_casper_lookup(h2o_http2_casper_t *casper, const char *path, size_t + + void h2o_http2_casper_consume_cookie(h2o_http2_casper_t *casper, const char *cookie, size_t cookie_len) + { +- h2o_iovec_t binary = {NULL}; ++ h2o_iovec_t binary = H2O_IOVEC_NULL; + uint64_t tiny_keys_buf[128], *keys = tiny_keys_buf; + + /* check the name of the cookie */ +@@ -177,7 +177,7 @@ h2o_iovec_t h2o_http2_casper_get_cookie(h2o_http2_casper_t *casper) + return casper->cookie_cache; + + if (casper->keys.size == 0) +- return (h2o_iovec_t){NULL}; ++ return (h2o_iovec_t)H2O_IOVEC_NULL; + + /* encode as binary */ + char tiny_bin_buf[128], *bin_buf = tiny_bin_buf; +diff --git a/lib/http2/connection.c b/lib/http2/connection.c +index 2f8cad620..c7ba5d6b2 100644 +--- a/lib/http2/connection.c ++++ b/lib/http2/connection.c +@@ -27,7 +27,7 @@ + #include "h2o/http2.h" + #include "h2o/http2_internal.h" + +-static const h2o_iovec_t CONNECTION_PREFACE = {H2O_STRLIT("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n")}; ++static const h2o_iovec_t CONNECTION_PREFACE = H2O_IOVEC_STRLIT("PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"); + + const h2o_http2_priority_t h2o_http2_default_priority = { + 0, /* exclusive */ +@@ -43,7 +43,7 @@ const h2o_http2_settings_t H2O_HTTP2_SETTINGS_HOST = { + 16384 /* max_frame_size */ + }; + +-static const h2o_iovec_t SETTINGS_HOST_BIN = {H2O_STRLIT("\x00\x00\x0c" /* frame size */ ++static const h2o_iovec_t SETTINGS_HOST_BIN = H2O_IOVEC_STRLIT("\x00\x00\x0c"/* frame size */ + "\x04" /* settings frame */ + "\x00" /* no flags */ + "\x00\x00\x00\x00" /* stream id */ +@@ -51,7 +51,7 @@ static const h2o_iovec_t SETTINGS_HOST_BIN = {H2O_STRLIT("\x00\x00\x0c" /* f + "\x00\x00\x00\x64" /* max_concurrent_streams = 100 */ + "\x00\x04" + "\x01\x00\x00\x00" /* initial_window_size = 16777216 */ +- )}; ++ ); + + static __thread h2o_buffer_prototype_t wbuf_buffer_prototype = {{16}, {H2O_HTTP2_DEFAULT_OUTBUF_SIZE}}; + +@@ -103,7 +103,7 @@ static void graceful_shutdown_resend_goaway(h2o_timeout_entry_t *entry) + for (node = ctx->http2._conns.next; node != &ctx->http2._conns; node = node->next) { + h2o_http2_conn_t *conn = H2O_STRUCT_FROM_MEMBER(h2o_http2_conn_t, _conns, node); + if (conn->state < H2O_HTTP2_CONN_STATE_HALF_CLOSED) { +- enqueue_goaway(conn, H2O_HTTP2_ERROR_NONE, (h2o_iovec_t){NULL}); ++ enqueue_goaway(conn, H2O_HTTP2_ERROR_NONE, (h2o_iovec_t)H2O_IOVEC_NULL); + do_close_stragglers = 1; + } + } +@@ -135,7 +135,7 @@ static void initiate_graceful_shutdown(h2o_context_t *ctx) + h2o_http2_conn_t *conn = H2O_STRUCT_FROM_MEMBER(h2o_http2_conn_t, _conns, node); + if (conn->state < H2O_HTTP2_CONN_STATE_HALF_CLOSED) { + h2o_http2_encode_goaway_frame(&conn->_write.buf, INT32_MAX, H2O_HTTP2_ERROR_NONE, +- (h2o_iovec_t){H2O_STRLIT("graceful shutdown")}); ++ (h2o_iovec_t)H2O_IOVEC_STRLIT("graceful shutdown")); + h2o_http2_conn_request_write(conn); + } + } +@@ -876,7 +876,7 @@ static int parse_input(h2o_http2_conn_t *conn) + } else if (ret < 0) { + if (ret != H2O_HTTP2_ERROR_PROTOCOL_CLOSE_IMMEDIATELY) { + enqueue_goaway(conn, (int)ret, +- err_desc != NULL ? (h2o_iovec_t){(char *)err_desc, strlen(err_desc)} : (h2o_iovec_t){NULL}); ++ err_desc != NULL ? h2o_iovec_init((char *)err_desc, strlen(err_desc)) : (h2o_iovec_t)H2O_IOVEC_NULL); + } + return close_connection(conn); + } +@@ -1046,7 +1046,7 @@ void do_emit_writereq(h2o_http2_conn_t *conn) + + if (conn->_write.buf->size != 0) { + /* write and wait for completion */ +- h2o_iovec_t buf = {conn->_write.buf->bytes, conn->_write.buf->size}; ++ h2o_iovec_t buf = h2o_iovec_init(conn->_write.buf->bytes, conn->_write.buf->size); + h2o_socket_write(conn->sock, &buf, 1, on_write_complete); + conn->_write.buf_in_flight = conn->_write.buf; + h2o_buffer_init(&conn->_write.buf, &wbuf_buffer_prototype); +@@ -1290,7 +1290,7 @@ static void push_path(h2o_req_t *src_req, const char *abspath, size_t abspath_le + h2o_http2_stream_prepare_for_request(conn, stream); + + /* setup request */ +- stream->req.input.method = (h2o_iovec_t){H2O_STRLIT("GET")}; ++ stream->req.input.method = (h2o_iovec_t)H2O_IOVEC_STRLIT("GET"); + stream->req.input.scheme = src_stream->req.input.scheme; + stream->req.input.authority = + h2o_strdup(&stream->req.pool, src_stream->req.input.authority.base, src_stream->req.input.authority.len); +diff --git a/lib/http2/hpack.c b/lib/http2/hpack.c +index 4adb15cd7..f4e4679e2 100644 +--- a/lib/http2/hpack.c ++++ b/lib/http2/hpack.c +@@ -902,7 +902,7 @@ void h2o_hpack_flatten_response(h2o_buffer_t **buf, h2o_hpack_header_table_t *he + if (server_name->len) { + dst = encode_header(header_table, dst, &H2O_TOKEN_SERVER->buf, server_name); + } +- h2o_iovec_t date_value = {ts->str->rfc1123, H2O_TIMESTR_RFC1123_LEN}; ++ h2o_iovec_t date_value = h2o_iovec_init(ts->str->rfc1123, H2O_TIMESTR_RFC1123_LEN); + dst = encode_header(header_table, dst, &H2O_TOKEN_DATE->buf, &date_value); + #endif + size_t i; +diff --git a/lib/http2/hpack_static_table.h b/lib/http2/hpack_static_table.h +index 4c1243103..fae748537 100644 +--- a/lib/http2/hpack_static_table.h ++++ b/lib/http2/hpack_static_table.h +@@ -23,65 +23,65 @@ + /* automatically generated by tokens.pl */ + + static const struct st_h2o_hpack_static_table_entry_t h2o_hpack_static_table[61] = { +- { H2O_TOKEN_AUTHORITY, { H2O_STRLIT("") } }, +- { H2O_TOKEN_METHOD, { H2O_STRLIT("GET") } }, +- { H2O_TOKEN_METHOD, { H2O_STRLIT("POST") } }, +- { H2O_TOKEN_PATH, { H2O_STRLIT("/") } }, +- { H2O_TOKEN_PATH, { H2O_STRLIT("/index.html") } }, +- { H2O_TOKEN_SCHEME, { H2O_STRLIT("http") } }, +- { H2O_TOKEN_SCHEME, { H2O_STRLIT("https") } }, +- { H2O_TOKEN_STATUS, { H2O_STRLIT("200") } }, +- { H2O_TOKEN_STATUS, { H2O_STRLIT("204") } }, +- { H2O_TOKEN_STATUS, { H2O_STRLIT("206") } }, +- { H2O_TOKEN_STATUS, { H2O_STRLIT("304") } }, +- { H2O_TOKEN_STATUS, { H2O_STRLIT("400") } }, +- { H2O_TOKEN_STATUS, { H2O_STRLIT("404") } }, +- { H2O_TOKEN_STATUS, { H2O_STRLIT("500") } }, +- { H2O_TOKEN_ACCEPT_CHARSET, { H2O_STRLIT("") } }, +- { H2O_TOKEN_ACCEPT_ENCODING, { H2O_STRLIT("gzip, deflate") } }, +- { H2O_TOKEN_ACCEPT_LANGUAGE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_ACCEPT_RANGES, { H2O_STRLIT("") } }, +- { H2O_TOKEN_ACCEPT, { H2O_STRLIT("") } }, +- { H2O_TOKEN_ACCESS_CONTROL_ALLOW_ORIGIN, { H2O_STRLIT("") } }, +- { H2O_TOKEN_AGE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_ALLOW, { H2O_STRLIT("") } }, +- { H2O_TOKEN_AUTHORIZATION, { H2O_STRLIT("") } }, +- { H2O_TOKEN_CACHE_CONTROL, { H2O_STRLIT("") } }, +- { H2O_TOKEN_CONTENT_DISPOSITION, { H2O_STRLIT("") } }, +- { H2O_TOKEN_CONTENT_ENCODING, { H2O_STRLIT("") } }, +- { H2O_TOKEN_CONTENT_LANGUAGE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_CONTENT_LENGTH, { H2O_STRLIT("") } }, +- { H2O_TOKEN_CONTENT_LOCATION, { H2O_STRLIT("") } }, +- { H2O_TOKEN_CONTENT_RANGE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_CONTENT_TYPE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_COOKIE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_DATE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_ETAG, { H2O_STRLIT("") } }, +- { H2O_TOKEN_EXPECT, { H2O_STRLIT("") } }, +- { H2O_TOKEN_EXPIRES, { H2O_STRLIT("") } }, +- { H2O_TOKEN_FROM, { H2O_STRLIT("") } }, +- { H2O_TOKEN_HOST, { H2O_STRLIT("") } }, +- { H2O_TOKEN_IF_MATCH, { H2O_STRLIT("") } }, +- { H2O_TOKEN_IF_MODIFIED_SINCE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_IF_NONE_MATCH, { H2O_STRLIT("") } }, +- { H2O_TOKEN_IF_RANGE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_IF_UNMODIFIED_SINCE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_LAST_MODIFIED, { H2O_STRLIT("") } }, +- { H2O_TOKEN_LINK, { H2O_STRLIT("") } }, +- { H2O_TOKEN_LOCATION, { H2O_STRLIT("") } }, +- { H2O_TOKEN_MAX_FORWARDS, { H2O_STRLIT("") } }, +- { H2O_TOKEN_PROXY_AUTHENTICATE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_PROXY_AUTHORIZATION, { H2O_STRLIT("") } }, +- { H2O_TOKEN_RANGE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_REFERER, { H2O_STRLIT("") } }, +- { H2O_TOKEN_REFRESH, { H2O_STRLIT("") } }, +- { H2O_TOKEN_RETRY_AFTER, { H2O_STRLIT("") } }, +- { H2O_TOKEN_SERVER, { H2O_STRLIT("") } }, +- { H2O_TOKEN_SET_COOKIE, { H2O_STRLIT("") } }, +- { H2O_TOKEN_STRICT_TRANSPORT_SECURITY, { H2O_STRLIT("") } }, +- { H2O_TOKEN_TRANSFER_ENCODING, { H2O_STRLIT("") } }, +- { H2O_TOKEN_USER_AGENT, { H2O_STRLIT("") } }, +- { H2O_TOKEN_VARY, { H2O_STRLIT("") } }, +- { H2O_TOKEN_VIA, { H2O_STRLIT("") } }, +- { H2O_TOKEN_WWW_AUTHENTICATE, { H2O_STRLIT("") } } ++ { H2O_TOKEN_AUTHORITY, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_METHOD, H2O_IOVEC_STRLIT("GET") }, ++ { H2O_TOKEN_METHOD, H2O_IOVEC_STRLIT("POST") }, ++ { H2O_TOKEN_PATH, H2O_IOVEC_STRLIT("/") }, ++ { H2O_TOKEN_PATH, H2O_IOVEC_STRLIT("/index.html") }, ++ { H2O_TOKEN_SCHEME, H2O_IOVEC_STRLIT("http") }, ++ { H2O_TOKEN_SCHEME, H2O_IOVEC_STRLIT("https") }, ++ { H2O_TOKEN_STATUS, H2O_IOVEC_STRLIT("200") }, ++ { H2O_TOKEN_STATUS, H2O_IOVEC_STRLIT("204") }, ++ { H2O_TOKEN_STATUS, H2O_IOVEC_STRLIT("206") }, ++ { H2O_TOKEN_STATUS, H2O_IOVEC_STRLIT("304") }, ++ { H2O_TOKEN_STATUS, H2O_IOVEC_STRLIT("400") }, ++ { H2O_TOKEN_STATUS, H2O_IOVEC_STRLIT("404") }, ++ { H2O_TOKEN_STATUS, H2O_IOVEC_STRLIT("500") }, ++ { H2O_TOKEN_ACCEPT_CHARSET, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_ACCEPT_ENCODING, H2O_IOVEC_STRLIT("gzip, deflate") }, ++ { H2O_TOKEN_ACCEPT_LANGUAGE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_ACCEPT_RANGES, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_ACCEPT, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_ACCESS_CONTROL_ALLOW_ORIGIN, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_AGE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_ALLOW, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_AUTHORIZATION, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_CACHE_CONTROL, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_CONTENT_DISPOSITION, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_CONTENT_ENCODING, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_CONTENT_LANGUAGE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_CONTENT_LENGTH, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_CONTENT_LOCATION, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_CONTENT_RANGE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_CONTENT_TYPE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_COOKIE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_DATE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_ETAG, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_EXPECT, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_EXPIRES, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_FROM, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_HOST, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_IF_MATCH, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_IF_MODIFIED_SINCE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_IF_NONE_MATCH, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_IF_RANGE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_IF_UNMODIFIED_SINCE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_LAST_MODIFIED, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_LINK, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_LOCATION, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_MAX_FORWARDS, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_PROXY_AUTHENTICATE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_PROXY_AUTHORIZATION, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_RANGE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_REFERER, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_REFRESH, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_RETRY_AFTER, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_SERVER, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_SET_COOKIE, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_STRICT_TRANSPORT_SECURITY, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_TRANSFER_ENCODING, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_USER_AGENT, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_VARY, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_VIA, H2O_IOVEC_STRLIT("") }, ++ { H2O_TOKEN_WWW_AUTHENTICATE, H2O_IOVEC_STRLIT("") } + }; +diff --git a/lib/http2/http2_debug_state.c b/lib/http2/http2_debug_state.c +index 3ef8de37f..e9245ddcd 100644 +--- a/lib/http2/http2_debug_state.c ++++ b/lib/http2/http2_debug_state.c +@@ -62,7 +62,7 @@ static const char *get_debug_state_string(h2o_http2_stream_t *stream) + return NULL; + } + +-__attribute__((format(printf, 3, 4))) static void append_chunk(h2o_mem_pool_t *pool, h2o_iovec_vector_t *chunks, const char *fmt, ++__attribute__((format(gnu_printf, 3, 4))) static void append_chunk(h2o_mem_pool_t *pool, h2o_iovec_vector_t *chunks, const char *fmt, + ...) + { + va_list args; +diff --git a/misc/tokens.pl b/misc/tokens.pl +index 1fcf173fa..1407f6681 100755 +--- a/misc/tokens.pl ++++ b/misc/tokens.pl +@@ -94,7 +94,7 @@ print $fh render_mt(<< 'EOT', \@tokens, LICENSE)->as_string; + /* DO NOT EDIT! generated by tokens.pl */ + h2o_token_t h2o__tokens[] = { + ? for my $i (0..$#$tokens) { +- { { H2O_STRLIT("[$i][0] ?>") }, [$i][$_] } (1..$#{$tokens->[$i]})) ?> } ++ { H2O_IOVEC_STRLIT("[$i][0] ?>"), [$i][$_] } (1..$#{$tokens->[$i]})) ?> } + ? } + }; + size_t h2o__num_tokens = ; +@@ -135,7 +135,7 @@ print $fh render_mt(<< 'EOT', \@hpack, LICENSE)->as_string; + + static const struct st_h2o_hpack_static_table_entry_t h2o_hpack_static_table[] = { + ? for my $i (0..$#$entries) { +- { [$i][0]) ?>, { H2O_STRLIT("[$i][1] || "" ?>") } } ++ { [$i][0]) ?>, H2O_IOVEC_STRLIT("[$i][1] || "" ?>") } + ? } + }; + EOT +diff --git a/src/main.c b/src/main.c +index af0867f29..7c801755d 100644 +--- a/src/main.c ++++ b/src/main.c +@@ -43,7 +43,9 @@ + #include + #include + #include ++#ifndef H2O_NO_UNIX_SOCKETS + #include ++#endif + #include + #include + #include +@@ -886,6 +888,7 @@ Found: + return conf.server_starter.fds[i]; + } + ++#ifndef H2O_NO_UNIX_SOCKETS + static int open_unix_listener(h2o_configurator_command_t *cmd, yoml_t *node, struct sockaddr_un *sa) + { + struct stat st; +@@ -948,6 +951,7 @@ ErrorExit: + close(fd); + return -1; + } ++#endif + + static int open_tcp_listener(h2o_configurator_command_t *cmd, yoml_t *node, const char *hostname, const char *servname, int domain, + int type, int protocol, struct sockaddr *addr, socklen_t addrlen) +@@ -1067,6 +1071,7 @@ static int on_config_listen(h2o_configurator_command_t *cmd, h2o_configurator_co + return -1; + } + ++ #ifndef H2O_NO_UNIX_SOCKETS + if (strcmp(type, "unix") == 0) { + + /* unix socket */ +@@ -1110,7 +1115,9 @@ static int on_config_listen(h2o_configurator_command_t *cmd, h2o_configurator_co + if (listener->hosts != NULL && ctx->hostconf != NULL) + h2o_append_to_null_terminated_list((void *)&listener->hosts, ctx->hostconf); + +- } else if (strcmp(type, "tcp") == 0) { ++ } else ++ #endif ++ if (strcmp(type, "tcp") == 0) { + + /* TCP socket */ + struct addrinfo hints, *res, *ai; +@@ -1712,11 +1719,13 @@ static char **build_server_starter_argv(const char *h2o_cmd, const char *config_ + sprintf(newarg, "--port=%s:%s", host, serv); + } + } break; ++ #ifndef H2O_NO_UNIX_SOCKETS + case AF_UNIX: { + struct sockaddr_un *sa = (void *)&conf.listeners[i]->addr; + newarg = h2o_mem_alloc(sizeof("--path=") + strlen(sa->sun_path)); + sprintf(newarg, "--path=%s", sa->sun_path); + } break; ++ #endif + } + h2o_vector_reserve(NULL, &args, args.size + 1); + args.entries[args.size++] = newarg; diff --git a/pkg/urbit/compat/mingw/libscrypt.patch b/pkg/urbit/compat/mingw/libscrypt.patch new file mode 100755 index 0000000000..03e88629e8 --- /dev/null +++ b/pkg/urbit/compat/mingw/libscrypt.patch @@ -0,0 +1,17 @@ +diff --git a/Makefile b/Makefile +index 783c537..3156ee2 100644 +--- a/Makefile ++++ b/Makefile +@@ -12,10 +12,9 @@ LDFLAGS_EXTRA?=-Wl,-z,relro + + all: reference + +-OBJS= crypto_scrypt-nosse.o sha256.o crypto-mcf.o b64.o crypto-scrypt-saltgen.o crypto_scrypt-check.o crypto_scrypt-hash.o slowequals.o ++OBJS= crypto_scrypt-nosse.o sha256.o crypto-mcf.o b64.o slowequals.o + +-libscrypt.so.0: $(OBJS) +- $(CC) $(LDFLAGS) -shared -o libscrypt.so.0 $(OBJS) -lm -lc ++libscrypt.a: $(OBJS) + ar rcs libscrypt.a $(OBJS) + + reference: libscrypt.so.0 main.o crypto_scrypt-hexconvert.o diff --git a/pkg/urbit/compat/mingw/lmdb.patch b/pkg/urbit/compat/mingw/lmdb.patch new file mode 100644 index 0000000000..e023a71330 --- /dev/null +++ b/pkg/urbit/compat/mingw/lmdb.patch @@ -0,0 +1,74 @@ +diff --git a/mdb.c b/mdb.c +--- a/mdb.c ++++ b/mdb.c +@@ -1707,28 +1707,27 @@ static char *const mdb_errstr[] = { + "MDB_PROBLEM: Unexpected problem - txn should abort", + }; + +-char * +-mdb_strerror(int err) ++void ++mdb_logerror(FILE* f, int err, const char* fmt, ...) + { +-#ifdef _WIN32 +- /** HACK: pad 4KB on stack over the buf. Return system msgs in buf. +- * This works as long as no function between the call to mdb_strerror +- * and the actual use of the message uses more than 4K of stack. +- */ +-#define MSGSIZE 1024 +-#define PADSIZE 4096 +- char buf[MSGSIZE+PADSIZE], *ptr = buf; +-#endif ++ va_list ap; ++ va_start(ap, fmt); ++ vfprintf(f, fmt, ap); ++ va_end(ap); ++ + int i; + if (!err) +- return ("Successful return: 0"); ++ { ++ fprintf(stderr, ": %s\r\n", "Successful return: 0"); ++ return; ++ } + + if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) { + i = err - MDB_KEYEXIST; +- return mdb_errstr[i]; ++ fprintf(stderr, ": %s\r\n", mdb_errstr[i]); ++ return; + } + +-#ifdef _WIN32 + /* These are the C-runtime error codes we use. The comment indicates + * their numeric value, and the Win32 error they would correspond to + * if the error actually came from a Win32 API. A major mess, we should +@@ -1742,18 +1741,20 @@ mdb_strerror(int err) + case EBUSY: /* 16, CURRENT_DIRECTORY */ + case EINVAL: /* 22, BAD_COMMAND */ + case ENOSPC: /* 28, OUT_OF_PAPER */ +- return strerror(err); ++ fprintf(stderr, ": %s\r\n", strerror(err)); + default: + ; + } +- buf[0] = 0; +- FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | ++ LPSTR ptr; ++ if (FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | ++ FORMAT_MESSAGE_ALLOCATE_BUFFER | + FORMAT_MESSAGE_IGNORE_INSERTS, +- NULL, err, 0, ptr, MSGSIZE, (va_list *)buf+MSGSIZE); +- return ptr; +-#else +- return strerror(err); +-#endif ++ NULL, err, 0, (LPSTR)&ptr, sizeof (LPSTR), NULL)) ++ { ++ fprintf(stderr, ": %s\r\n", ptr); ++ LocalFree(ptr); ++ } else ++ fprintf(stderr, ": <%d>\r\n", err); + } + + /** assert(3) variant in cursor context */ diff --git a/pkg/urbit/compat/mingw/mman.h b/pkg/urbit/compat/mingw/mman.h new file mode 100644 index 0000000000..965fecfd8c --- /dev/null +++ b/pkg/urbit/compat/mingw/mman.h @@ -0,0 +1,26 @@ +#ifndef _SYS_MMAN_H +#define _SYS_MMAN_H + +void *mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset); +int munmap(void *addr, size_t length); +int msync(void *addr, size_t length, int flags); +int mprotect(void *addr, size_t len, int prot); + +#define PROT_NONE 0x00 /* No access. */ +#define PROT_READ 0x01 /* Pages can be read. */ +#define PROT_WRITE 0x02 /* Pages can be written. */ +#define PROT_EXEC 0x04 /* Pages can be executed. */ + +#define MAP_FILE 0x0001 /* Mapped from a file or device. */ +#define MAP_ANON 0x0002 /* Allocated from anonymous virtual memory. */ +#define MAP_TYPE 0x000f /* Mask for type field. */ +#define MAP_SHARED 0x0010 /* Share changes. */ +#define MAP_PRIVATE 0x0000 /* Changes private; copy pages on write. */ +#define MAP_FIXED 0x0100 /* Map address must be exactly as requested. */ +#define MAP_FAILED ((void *) -1) + +#define MS_ASYNC 1 /* Sync memory asynchronously. */ +#define MS_SYNC 0 /* Synchronous memory sync. */ +#define MS_INVALIDATE 2 /* Invalidate the caches. */ + +#endif//_SYS_MMAN_H diff --git a/pkg/urbit/compat/mingw/murmur3.patch b/pkg/urbit/compat/mingw/murmur3.patch new file mode 100644 index 0000000000..cd83e9fdda --- /dev/null +++ b/pkg/urbit/compat/mingw/murmur3.patch @@ -0,0 +1,13 @@ +diff --git a/makefile b/makefile +--- a/makefile ++++ b/makefile +@@ -12,5 +12,9 @@ shared: murmur3.c murmur3.h + $(CC) -fPIC -O3 -c murmur3.c + $(CC) -shared -Wl,--export-dynamic murmur3.o -o libmurmur3.so + ++static: murmur3.c murmur3.h ++ $(CC) -fPIC -O3 -c murmur3.c ++ $(AR) rcs libmurmur3.a murmur3.o ++ + clean: + rm -rf example *.o *.so diff --git a/pkg/urbit/compat/mingw/softfloat3.patch b/pkg/urbit/compat/mingw/softfloat3.patch new file mode 100644 index 0000000000..9f0c5c5e7b --- /dev/null +++ b/pkg/urbit/compat/mingw/softfloat3.patch @@ -0,0 +1,36 @@ +diff --git a/build/Win64-MinGW-w64/Makefile b/build/Win64-MinGW-w64/Makefile +--- a/build/Win64-MinGW-w64/Makefile ++++ b/build/Win64-MinGW-w64/Makefile +@@ -46,7 +46,8 @@ C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include + COMPILE_C = \ + x86_64-w64-mingw32-gcc -c -Werror-implicit-function-declaration \ + -DSOFTFLOAT_FAST_INT64 $(SOFTFLOAT_OPTS) $(C_INCLUDES) -O2 -o $@ +-MAKELIB = x86_64-w64-mingw32-ar crs $@ ++MAKELIB = x86_64-w64-mingw32-gcc-ar crs $@ ++LIBNAME = libsoftfloat3 + + OBJ = .o + LIB = .a +@@ -54,7 +55,7 @@ LIB = .a + OTHER_HEADERS = $(SOURCE_DIR)/include/opts-GCC.h + + .PHONY: all +-all: softfloat$(LIB) ++all: $(LIBNAME)$(LIB) + + OBJS_PRIMITIVES = \ + s_eq128$(OBJ) \ +@@ -380,11 +381,11 @@ $(OBJS_PRIMITIVES) $(OBJS_OTHERS): %$(OBJ): $(SOURCE_DIR)/%.c + $(OBJS_SPECIALIZE): %$(OBJ): $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/$*.c + +-softfloat$(LIB): $(OBJS_ALL) ++$(LIBNAME)$(LIB): $(OBJS_ALL) + $(DELETE) $@ + $(MAKELIB) $^ + + .PHONY: clean + clean: +- $(DELETE) $(OBJS_ALL) softfloat$(LIB) ++ $(DELETE) $(OBJS_ALL) $(LIBNAME)$(LIB) + diff --git a/pkg/urbit/compat/mingw/termios.h b/pkg/urbit/compat/mingw/termios.h new file mode 100644 index 0000000000..2fdb305a05 --- /dev/null +++ b/pkg/urbit/compat/mingw/termios.h @@ -0,0 +1,8 @@ +#ifndef _TERMIOS_H +#define _TERMIOS_H + +struct termios { + char _dummy; +}; + +#endif diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 3ea527fdd0..259640eb7a 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -6,7 +6,7 @@ URBIT_VERSION="$(cat ./version)" deps=" \ curl gmp sigsegv argon2 ed25519 ent h2o scrypt uv murmur3 secp256k1 \ - softfloat3 ssl crypto z lmdb ge-additions aes_siv pthread \ + softfloat3 aes_siv ssl crypto z lmdb ge-additions pthread \ " headers=" \ @@ -58,6 +58,15 @@ esac # TODO Determine if the target cpu is little or big endian. case $(tr A-Z a-z <<< $os) in + *mingw*) + defmacro U3_OS_mingw 1 + + # increase default thread stack size and link Windows implibs + ldextra="-Wl,--stack,67108864 -lbcrypt -lws2_32" + mkextra="use_dumb_terminal = 1" + vcompat="compat = mingw" + CFLAGS="${CFLAGS-} -DH2O_NO_UNIX_SOCKETS -Icompat/mingw" + ;; *linux*) defmacro U3_OS_linux 1 defmacro U3_OS_PROF 1 @@ -94,8 +103,10 @@ done cat >config.mk <&2 diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index da3d1b29f7..db3dcf484d 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -590,6 +590,10 @@ static void _on_boot_completed_cb() { static void _fork_into_background_process() { +#if defined(U3_OS_mingw) + fprintf(stderr, "Daemon mode is not yet supported on MingW\r\n"); + exit(1); +#else c3_i pipefd[2]; if ( 0 != pipe(pipefd) ) { @@ -618,6 +622,7 @@ _fork_into_background_process() c3_i status; wait(&status); exit(WEXITSTATUS(status)); +#endif } /* _stop_on_boot_completed_cb(): exit gracefully after boot is complete @@ -642,7 +647,12 @@ main(c3_i argc, // Set `u3_Host.wrk_c` to the worker executable path. c3_i worker_exe_len = 1 + strlen(argv[0]) + strlen("-worker"); u3_Host.wrk_c = c3_malloc(worker_exe_len); + #if defined(U3_OS_mingw) + strcpy(u3_Host.wrk_c, argv[0]); + strcpy(u3_Host.wrk_c + strlen(argv[0]) - 4, "-worker.exe"); + #else snprintf(u3_Host.wrk_c, worker_exe_len, "%s-worker", argv[0]); + #endif if ( c3y == u3_Host.ops_u.dem ) { _fork_into_background_process(); diff --git a/pkg/urbit/include/c/portable.h b/pkg/urbit/include/c/portable.h index fbe52b7632..fbcce1fde9 100644 --- a/pkg/urbit/include/c/portable.h +++ b/pkg/urbit/include/c/portable.h @@ -72,6 +72,23 @@ # include # include +# elif defined(U3_OS_mingw) +# define _POSIX +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include "mman.h" +# include "compat.h" + # else #error "port: headers" # endif @@ -101,19 +118,15 @@ # define U3_OS_LoomBase 0x36000000 # endif # define U3_OS_LoomBits 29 // ie, 2^29 words == 2GB -# elif defined(U3_OS_osx) +# elif defined(U3_OS_osx) || defined(U3_OS_bsd) # ifdef __LP64__ # define U3_OS_LoomBase 0x200000000 # else # define U3_OS_LoomBase 0x4000000 # endif # define U3_OS_LoomBits 29 // ie, 2^29 words == 2GB -# elif defined(U3_OS_bsd) -# ifdef __LP64__ -# define U3_OS_LoomBase 0x200000000 -# else -# define U3_OS_LoomBase 0x4000000 -# endif +# elif defined(U3_OS_mingw) +# define U3_OS_LoomBase 0x28000000000 # define U3_OS_LoomBits 29 // ie, 2^29 words == 2GB # else # error "port: LoomBase" @@ -157,7 +170,7 @@ /* Byte swapping. */ -# if defined(U3_OS_linux) || defined(U3_OS_bsd) +# if defined(U3_OS_linux) || defined(U3_OS_bsd) || defined(U3_OS_mingw) # define c3_bswap_16(x) bswap_16(x) # define c3_bswap_32(x) bswap_32(x) # define c3_bswap_64(x) bswap_64(x) @@ -171,7 +184,7 @@ /* Sync. */ -# if defined(U3_OS_linux) +# if defined(U3_OS_linux) || defined(U3_OS_mingw) # define c3_sync(fd) (fdatasync(fd)) # elif defined(U3_OS_osx) # define c3_sync(fd) (fcntl(fd, F_FULLFSYNC, 0)) @@ -186,7 +199,7 @@ # if defined(U3_OS_linux) # include # define c3_fpurge __fpurge -# elif defined(U3_OS_bsd) || defined(U3_OS_osx) +# elif defined(U3_OS_bsd) || defined(U3_OS_osx) || defined(U3_OS_mingw) # define c3_fpurge fpurge # else # error "port: fpurge" @@ -194,7 +207,7 @@ /* Stat. */ -# if defined(U3_OS_linux) +# if defined(U3_OS_linux) || defined(U3_OS_mingw) # define c3_stat_mtime(dp) (u3_time_t_in_ts((dp)->st_mtime)) # elif defined(U3_OS_osx) # define c3_stat_mtime(dp) (u3_time_in_ts(&((dp)->st_mtimespec))) diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index 23679c9b52..7ac5726f81 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -717,7 +717,7 @@ */ u3_atom u3_time_in_ts(struct timespec* tim_ts); -#if defined(U3_OS_linux) +#if defined(U3_OS_linux) || defined(U3_OS_mingw) /* u3_time_t_in_ts(): urbit time from time_t. */ u3_atom diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 922620f286..9c7abdcbce 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -1557,6 +1557,10 @@ u3m_wall(u3_noun wol) static void _cm_limits(void) { +# ifdef U3_OS_mingw + // Windows doesn't have rlimits. Default maximum thread + // stack size is set in the executable file header. +# else struct rlimit rlm; // Moar stack. @@ -1604,6 +1608,7 @@ _cm_limits(void) } } # endif +# endif } /* _cm_signals(): set up interrupts, etc. diff --git a/pkg/urbit/vere/db/lmdb.c b/pkg/urbit/vere/db/lmdb.c index ab9b92dbdf..63df171549 100644 --- a/pkg/urbit/vere/db/lmdb.c +++ b/pkg/urbit/vere/db/lmdb.c @@ -451,6 +451,7 @@ u3_lmdb_save_meta(MDB_env* env_u, return c3y; } +#if !defined(U3_OS_mingw) /* mdb_logerror(): writes an error message and lmdb error code to f. */ void mdb_logerror(FILE* f, int err, const char* fmt, ...) @@ -461,3 +462,4 @@ void mdb_logerror(FILE* f, int err, const char* fmt, ...) va_end(ap); fprintf(f, ": %s\r\n", mdb_strerror(err)); } +#endif diff --git a/pkg/urbit/vere/io/dumb.c b/pkg/urbit/vere/io/dumb.c new file mode 100644 index 0000000000..68ec963619 --- /dev/null +++ b/pkg/urbit/vere/io/dumb.c @@ -0,0 +1,171 @@ +/* vere/dumb.c +** +*/ +#include "all.h" +#include "vere/vere.h" + +/* u3_term_log_init(): initialize terminal for logging +*/ +void +u3_term_log_init(void) +{ +} + +/* u3_term_log_exit(): clean up terminal. +*/ +void +u3_term_log_exit(void) +{ +} + +/* u3_term_start_spinner(): prepare spinner state. RETAIN. +*/ +void +u3_term_start_spinner(u3_atom say, c3_o del_o) +{ +} + +/* u3_term_stop_spinner(): reset spinner state and restore input line. +*/ +void +u3_term_stop_spinner(void) +{ +} + +/* u3_term_get_blew(): return window size [columns rows]. +*/ +u3_noun +u3_term_get_blew(c3_l tid_l) +{ + return u3nc(80, 24); +} + +/* u3_term_ef_winc(): window change. Just console right now. +*/ +void +u3_term_ef_winc(void) +{ +} + +/* u3_term_ef_ctlc(): send ^C on console. +*/ +void +u3_term_ef_ctlc(void) +{ +} + +/* u3_term_io_hija(): hijack console for fprintf, returning FILE*. +*/ +FILE* +u3_term_io_hija(void) +{ + return stdout; +} + +/* u3_term_io_loja(): release console from fprintf. +*/ +void +u3_term_io_loja(int x) +{ + fflush(stdout); +} + +/* u3_term_it_log(): writes a log message +*/ +void +u3_term_io_log(c3_c* line) +{ + FILE* stream = u3_term_io_hija(); + u3_term_io_loja(fprintf(stream, "%s", line)); +} + +/* _term_ovum_plan(): plan term ovums, configuring spinner. +*/ +static u3_ovum* +_term_ovum_plan(u3_auto* car_u, u3_noun wir, u3_noun cad) +{ + u3_ovum* egg_u = u3_auto_plan(car_u, u3_ovum_init(0, c3__d, wir, cad)); + + // term events have no spinner label + // + u3z(egg_u->pin_u.lab); + egg_u->pin_u.lab = u3_blip; + + return egg_u; +} + +/* _term_io_talk(): +*/ +static void +_term_io_talk(u3_auto* car_u) +{ + // XX groace hardcoded terminal number + // + u3_noun wir = u3nt(c3__term, '1', u3_nul); + u3_noun cad; + + // send terminal dimensions + // + { + cad = u3nc(c3__blew, u3_term_get_blew(1)); + _term_ovum_plan(car_u, u3k(wir), cad); + } + + // NB, term.c used to also start :dojo + // + // u3nq(c3__flow, c3__seat, c3__dojo, u3_nul) + + // refresh terminal state + // + { + cad = u3nc(c3__hail, u3_nul); + _term_ovum_plan(car_u, wir, cad); + } +} + +/* _term_io_kick(): apply effects. +*/ +static c3_o +_term_io_kick(u3_auto* car_u, u3_noun wir, u3_noun cad) +{ + u3_noun tag, dat, i_wir, t_wir; + c3_o ret_o; + + if ( (c3n == u3r_cell(wir, &i_wir, &t_wir)) + || (c3n == u3r_cell(cad, &tag, &dat)) + || (c3__term != i_wir) ) + { + ret_o = c3n; + } + else { + // eat everything + ret_o = c3y; + } + + u3z(wir); u3z(cad); + return ret_o; +} + +/* _term_io_exit(): clean up terminal. +*/ +static void +_term_io_exit(u3_auto* car_u) +{ + c3_free(car_u); +} + +/* u3_term_io_init(): initialize terminal +*/ +u3_auto* +u3_term_io_init(u3_pier* pir_u) +{ + u3_auto* car_u = c3_calloc(sizeof(*car_u)); + + car_u->nam_m = c3__term; + car_u->liv_o = c3y; + car_u->io.talk_f = _term_io_talk; + car_u->io.kick_f = _term_io_kick; + car_u->io.exit_f = _term_io_exit; + + return car_u; +} diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c index ad1dc14304..bf6adea698 100644 --- a/pkg/urbit/vere/king.c +++ b/pkg/urbit/vere/king.c @@ -749,12 +749,16 @@ u3_king_commence() u3C.sign_move_f = _king_sign_move; // Ignore SIGPIPE signals. + #ifndef U3_OS_mingw { struct sigaction sig_s = {{0}}; sigemptyset(&(sig_s.sa_mask)); sig_s.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sig_s, 0); } + #else + signal(SIGPIPE, SIG_IGN); + #endif // boot the ivory pill // @@ -762,6 +766,7 @@ u3_king_commence() // disable core dumps (due to lmdb size) // + #ifndef U3_OS_mingw { struct rlimit rlm; @@ -773,6 +778,7 @@ u3_king_commence() exit(1); } } + #endif // run the loop // diff --git a/pkg/urbit/vere/time.c b/pkg/urbit/vere/time.c index 993a04a9f3..6e41c1e56a 100644 --- a/pkg/urbit/vere/time.c +++ b/pkg/urbit/vere/time.c @@ -98,7 +98,7 @@ u3_time_in_ts(struct timespec* tim_ts) return u3_time_in_tv(&tim_tv); } -#if defined(U3_OS_linux) +#if defined(U3_OS_linux) || defined(U3_OS_mingw) /* u3_time_t_in_ts(): urbit time from time_t. */ u3_atom @@ -111,7 +111,7 @@ u3_time_t_in_ts(time_t tim) return u3_time_in_tv(&tim_tv); } -#endif // defined(U3_OS_linux) +#endif // defined(U3_OS_linux) || defined(U3_OS_mingw) /* u3_time_out_ts(): struct timespec from urbit time. */ diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index 5377ce90d9..c1cff83ea0 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -195,12 +195,16 @@ _cw_serf_commence(c3_i argc, c3_c* argv[]) // Ignore SIGPIPE signals. // + #ifndef U3_OS_mingw { struct sigaction sig_s = {{0}}; sigemptyset(&(sig_s.sa_mask)); sig_s.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sig_s, 0); } + #else + signal(SIGPIPE, SIG_IGN); + #endif // configure pipe to daemon process // From 00bd79704f48ef08d6404ac6bb6e4d6ffff3c8c3 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 29 Mar 2021 11:54:04 +0000 Subject: [PATCH 028/136] vere: select dumb terminal with #define rather than Makefile conditional --- pkg/urbit/Makefile | 6 -- pkg/urbit/configure | 3 +- pkg/urbit/vere/io/dumb.c | 171 --------------------------------------- pkg/urbit/vere/io/term.c | 58 ++++++++++++- 4 files changed, 57 insertions(+), 181 deletions(-) delete mode 100644 pkg/urbit/vere/io/dumb.c diff --git a/pkg/urbit/Makefile b/pkg/urbit/Makefile index 09e84d4a02..bb37b8c20b 100644 --- a/pkg/urbit/Makefile +++ b/pkg/urbit/Makefile @@ -9,12 +9,6 @@ worker = $(wildcard worker/*.c) tests = $(wildcard tests/*.c) bench = $(wildcard bench/*.c) -ifdef use_dumb_terminal -vere := $(filter-out vere/io/term.c,$(vere)) -else -vere := $(filter-out vere/io/dumb.c,$(vere)) -endif - ifdef compat compat := $(wildcard compat/$(compat)/*.c) endif diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 259640eb7a..718522c2ef 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -60,10 +60,10 @@ esac case $(tr A-Z a-z <<< $os) in *mingw*) defmacro U3_OS_mingw 1 + defmacro U3_OS_TERM_dumb 1 # increase default thread stack size and link Windows implibs ldextra="-Wl,--stack,67108864 -lbcrypt -lws2_32" - mkextra="use_dumb_terminal = 1" vcompat="compat = mingw" CFLAGS="${CFLAGS-} -DH2O_NO_UNIX_SOCKETS -Icompat/mingw" ;; @@ -106,7 +106,6 @@ CFLAGS := ${CFLAGS-} -funsigned-char -ffast-math -fcommon -std=gnu99 LDFLAGS := $LDFLAGS ${ldextra-} CC := ${CC-cc} ${vcompat-} -${mkextra-} EOF echo == config.mk == >&2 diff --git a/pkg/urbit/vere/io/dumb.c b/pkg/urbit/vere/io/dumb.c deleted file mode 100644 index 68ec963619..0000000000 --- a/pkg/urbit/vere/io/dumb.c +++ /dev/null @@ -1,171 +0,0 @@ -/* vere/dumb.c -** -*/ -#include "all.h" -#include "vere/vere.h" - -/* u3_term_log_init(): initialize terminal for logging -*/ -void -u3_term_log_init(void) -{ -} - -/* u3_term_log_exit(): clean up terminal. -*/ -void -u3_term_log_exit(void) -{ -} - -/* u3_term_start_spinner(): prepare spinner state. RETAIN. -*/ -void -u3_term_start_spinner(u3_atom say, c3_o del_o) -{ -} - -/* u3_term_stop_spinner(): reset spinner state and restore input line. -*/ -void -u3_term_stop_spinner(void) -{ -} - -/* u3_term_get_blew(): return window size [columns rows]. -*/ -u3_noun -u3_term_get_blew(c3_l tid_l) -{ - return u3nc(80, 24); -} - -/* u3_term_ef_winc(): window change. Just console right now. -*/ -void -u3_term_ef_winc(void) -{ -} - -/* u3_term_ef_ctlc(): send ^C on console. -*/ -void -u3_term_ef_ctlc(void) -{ -} - -/* u3_term_io_hija(): hijack console for fprintf, returning FILE*. -*/ -FILE* -u3_term_io_hija(void) -{ - return stdout; -} - -/* u3_term_io_loja(): release console from fprintf. -*/ -void -u3_term_io_loja(int x) -{ - fflush(stdout); -} - -/* u3_term_it_log(): writes a log message -*/ -void -u3_term_io_log(c3_c* line) -{ - FILE* stream = u3_term_io_hija(); - u3_term_io_loja(fprintf(stream, "%s", line)); -} - -/* _term_ovum_plan(): plan term ovums, configuring spinner. -*/ -static u3_ovum* -_term_ovum_plan(u3_auto* car_u, u3_noun wir, u3_noun cad) -{ - u3_ovum* egg_u = u3_auto_plan(car_u, u3_ovum_init(0, c3__d, wir, cad)); - - // term events have no spinner label - // - u3z(egg_u->pin_u.lab); - egg_u->pin_u.lab = u3_blip; - - return egg_u; -} - -/* _term_io_talk(): -*/ -static void -_term_io_talk(u3_auto* car_u) -{ - // XX groace hardcoded terminal number - // - u3_noun wir = u3nt(c3__term, '1', u3_nul); - u3_noun cad; - - // send terminal dimensions - // - { - cad = u3nc(c3__blew, u3_term_get_blew(1)); - _term_ovum_plan(car_u, u3k(wir), cad); - } - - // NB, term.c used to also start :dojo - // - // u3nq(c3__flow, c3__seat, c3__dojo, u3_nul) - - // refresh terminal state - // - { - cad = u3nc(c3__hail, u3_nul); - _term_ovum_plan(car_u, wir, cad); - } -} - -/* _term_io_kick(): apply effects. -*/ -static c3_o -_term_io_kick(u3_auto* car_u, u3_noun wir, u3_noun cad) -{ - u3_noun tag, dat, i_wir, t_wir; - c3_o ret_o; - - if ( (c3n == u3r_cell(wir, &i_wir, &t_wir)) - || (c3n == u3r_cell(cad, &tag, &dat)) - || (c3__term != i_wir) ) - { - ret_o = c3n; - } - else { - // eat everything - ret_o = c3y; - } - - u3z(wir); u3z(cad); - return ret_o; -} - -/* _term_io_exit(): clean up terminal. -*/ -static void -_term_io_exit(u3_auto* car_u) -{ - c3_free(car_u); -} - -/* u3_term_io_init(): initialize terminal -*/ -u3_auto* -u3_term_io_init(u3_pier* pir_u) -{ - u3_auto* car_u = c3_calloc(sizeof(*car_u)); - - car_u->nam_m = c3__term; - car_u->liv_o = c3y; - car_u->io.talk_f = _term_io_talk; - car_u->io.kick_f = _term_io_kick; - car_u->io.exit_f = _term_io_exit; - - return car_u; -} diff --git a/pkg/urbit/vere/io/term.c b/pkg/urbit/vere/io/term.c index 8c769e43cc..ce1c898b9f 100644 --- a/pkg/urbit/vere/io/term.c +++ b/pkg/urbit/vere/io/term.c @@ -3,6 +3,8 @@ */ #include "all.h" #include "vere/vere.h" + +#if !defined(U3_OS_TERM_dumb) #include #include #include @@ -125,12 +127,14 @@ _term_close_cb(uv_handle_t* han_t) c3_free(tty_u); } #endif +#endif // U3_OS_TERM_dumb /* u3_term_log_init(): initialize terminal for logging */ void u3_term_log_init(void) { + #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = c3_calloc(sizeof(u3_utty)); if ( c3y == u3_Host.ops_u.tem ) { @@ -273,6 +277,7 @@ u3_term_log_init(void) uty_u->tat_u.sun_u.tim_u.data = uty_u; } } + #endif // U3_OS_TERM_dumb } /* u3_term_log_exit(): clean up terminal. @@ -280,6 +285,7 @@ u3_term_log_init(void) void u3_term_log_exit(void) { + #if !defined(U3_OS_TERM_dumb) if ( c3n == u3_Host.ops_u.tem ) { u3_utty* uty_u; @@ -298,8 +304,11 @@ u3_term_log_exit(void) if ( u3_Host.uty_u ) { uv_close((uv_handle_t*)&u3_Host.uty_u->pop_u, 0); } + #endif // U3_OS_TERM_dumb } +#if !defined(U3_OS_TERM_dumb) + /* _term_tcsetattr(): tcsetattr w/retry on EINTR. */ static c3_i @@ -670,6 +679,8 @@ _term_it_save(u3_noun pax, u3_noun pad) c3_free(bas_c); } +#endif // U3_OS_TERM_dumb + /* _term_ovum_plan(): plan term ovums, configuring spinner. */ static u3_ovum* @@ -685,6 +696,8 @@ _term_ovum_plan(u3_auto* car_u, u3_noun wir, u3_noun cad) return egg_u; } +#if !defined(U3_OS_TERM_dumb) + /* _term_io_belt(): send belt. */ static void @@ -975,11 +988,14 @@ _term_spin_timer_cb(uv_timer_t* tim_u) #define _SPIN_RATE_US 250UL // spinner rate (ms/frame) #define _SPIN_IDLE_US 500UL // spinner cools down if stopped this long +#endif // U3_OS_TERM_dumb + /* u3_term_start_spinner(): prepare spinner state. RETAIN. */ void u3_term_start_spinner(u3_atom say, c3_o del_o) { + #if !defined(U3_OS_TERM_dumb) if ( c3n == u3_Host.ops_u.tem ) { u3_utty* uty_u = _term_main(); u3_utat* tat_u = &uty_u->tat_u; @@ -1003,6 +1019,7 @@ u3_term_start_spinner(u3_atom say, c3_o del_o) wen_d, _SPIN_RATE_US); } } + #endif // U3_OS_TERM_dumb } /* u3_term_stop_spinner(): reset spinner state and restore input line. @@ -1010,6 +1027,7 @@ u3_term_start_spinner(u3_atom say, c3_o del_o) void u3_term_stop_spinner(void) { + #if !defined(U3_OS_TERM_dumb) if ( c3n == u3_Host.ops_u.tem ) { u3_utty* uty_u = _term_main(); u3_utat* tat_u = &uty_u->tat_u; @@ -1027,8 +1045,11 @@ u3_term_stop_spinner(void) tat_u->sun_u.end_d = 0; } } + #endif // U3_OS_TERM_dumb } +#if !defined(U3_OS_TERM_dumb) + /* _term_main(): return main or console terminal. */ static u3_utty* @@ -1061,11 +1082,14 @@ _term_ef_get(c3_l tid_l) return _term_main(); } +#endif // U3_OS_TERM_dumb + /* u3_term_get_blew(): return window size [columns rows]. */ u3_noun u3_term_get_blew(c3_l tid_l) { + #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = _term_ef_get(tid_l); c3_l col_l, row_l; @@ -1087,6 +1111,9 @@ u3_term_get_blew(c3_l tid_l) } return u3nc(col_l, row_l); + #else + return u3nc(80, 24); + #endif // U3_OS_TERM_dumb } /* u3_term_ef_winc(): window change. Just console right now. @@ -1094,6 +1121,7 @@ u3_term_get_blew(c3_l tid_l) void u3_term_ef_winc(void) { + #if !defined(U3_OS_TERM_dumb) // XX groace, this should be a global handler sent to each pier // if ( u3_Host.uty_u->car_u ) { @@ -1104,6 +1132,7 @@ u3_term_ef_winc(void) _term_ovum_plan(u3_Host.uty_u->car_u, wir, cad); } + #endif } /* u3_term_ef_ctlc(): send ^C on console. @@ -1111,6 +1140,7 @@ u3_term_ef_winc(void) void u3_term_ef_ctlc(void) { + #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = _term_main(); { @@ -1124,8 +1154,11 @@ u3_term_ef_ctlc(void) } _term_it_refresh_line(uty_u); + #endif } +#if !defined(U3_OS_TERM_dumb) + /* _term_it_put_value(): put numeric color value on lin_w. */ static c3_w @@ -1442,11 +1475,14 @@ _term_ef_blit(u3_utty* uty_u, u3z(blt); } +#endif // U3_OS_TERM_dumb + /* u3_term_io_hija(): hijack console for fprintf, returning FILE*. */ FILE* u3_term_io_hija(void) { + #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = _term_main(); if ( uty_u ) { @@ -1480,10 +1516,10 @@ u3_term_io_hija(void) _write(uty_u->fid_i, buf_u->base, buf_u->len); } } - return stdout; } } - else return stdout; + #endif // U3_OS_TERM_dumb + return stdout; } /* u3_term_io_loja(): release console from fprintf. @@ -1491,6 +1527,7 @@ u3_term_io_hija(void) void u3_term_io_loja(int x) { + #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = _term_main(); if ( uty_u ) { @@ -1525,6 +1562,9 @@ u3_term_io_loja(int x) } } } + #else + fflush(stdout); + #endif // U3_OS_TERM_dumb } /* u3_term_it_log(): writes a log message @@ -1536,6 +1576,8 @@ u3_term_io_log(c3_c* line) u3_term_io_loja(fprintf(stream, "%s", line)); } +#if !defined(U3_OS_TERM_dumb) + /* u3_term_tape_to(): dump a tape to a file. */ void @@ -1589,11 +1631,14 @@ u3_term_wall(u3_noun wol) u3z(wol); } +#endif // U3_OS_TERM_dumb + /* _term_io_talk(): */ static void _term_io_talk(u3_auto* car_u) { + #if !defined(U3_OS_TERM_dumb) if ( c3n == u3_Host.ops_u.tem ) { u3_utty* uty_u = _term_main(); @@ -1601,6 +1646,7 @@ _term_io_talk(u3_auto* car_u) _term_alloc, _term_read_cb); } + #endif // XX groace hardcoded terminal number // @@ -1688,6 +1734,7 @@ _term_io_kick(u3_auto* car_u, u3_noun wir, u3_noun cad) case c3__blit: { ret_o = c3y; + #if !defined(U3_OS_TERM_dumb) { u3_utty* uty_u = _term_ef_get(tid_l); if ( 0 == uty_u ) { @@ -1703,6 +1750,7 @@ _term_io_kick(u3_auto* car_u, u3_noun wir, u3_noun cad) } } } + #endif } break; // XX obsolete %ames @@ -1756,6 +1804,7 @@ _term_io_exit_cb(uv_handle_t* han_u) static void _term_io_exit(u3_auto* car_u) { + #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = _term_main(); // NB, closed in u3_term_log_exit() @@ -1771,6 +1820,9 @@ _term_io_exit(u3_auto* car_u) else { c3_free(car_u); } + #else + c3_free(car_u); + #endif } /* u3_term_io_init(): initialize terminal @@ -1780,8 +1832,10 @@ u3_term_io_init(u3_pier* pir_u) { u3_auto* car_u = c3_calloc(sizeof(*car_u)); + #if !defined(U3_OS_TERM_dumb) c3_assert( u3_Host.uty_u ); u3_Host.uty_u->car_u = car_u; + #endif car_u->nam_m = c3__term; car_u->liv_o = c3y; From 19fc2bcf7c86e507bade929c46c1436e36b025e2 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 29 Mar 2021 12:14:59 +0000 Subject: [PATCH 029/136] vere: export term.c:_write() as u3_write_fd(), use in dprintf() replacement --- pkg/urbit/include/vere/vere.h | 4 +++ pkg/urbit/vere/io/http.c | 2 +- pkg/urbit/vere/io/term.c | 54 ++++++++++++++++------------------- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index 7ac5726f81..e15ba9e4ad 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -1372,6 +1372,10 @@ void u3_king_grab(void* ptr_v); + /* u3_write_fd(): retry interrupts, continue partial writes, assert errors. + */ + void + u3_write_fd(c3_i fid_i, const void* buf_v, size_t len_i); c3_w u3_readdir_r(DIR *dirp, struct dirent *entry, struct dirent **result); diff --git a/pkg/urbit/vere/io/http.c b/pkg/urbit/vere/io/http.c index 8a26a35ae9..e9a7c7d9e4 100644 --- a/pkg/urbit/vere/io/http.c +++ b/pkg/urbit/vere/io/http.c @@ -1608,7 +1608,7 @@ _http_write_ports_file(u3_httd* htd_u, c3_c *pax_c) c3_c temp[32]; while ( 0 != htp_u ) { if ( 0 < htp_u->por_s ) { - write(por_i, temp, snprintf(temp, 32, "%u %s %s\n", htp_u->por_s, + u3_write_fd(por_i, temp, snprintf(temp, 32, "%u %s %s\n", htp_u->por_s, (c3y == htp_u->sec) ? "secure" : "insecure", (c3y == htp_u->lop) ? "loopback" : "public")); } diff --git a/pkg/urbit/vere/io/term.c b/pkg/urbit/vere/io/term.c index ce1c898b9f..9d2e0fdd38 100644 --- a/pkg/urbit/vere/io/term.c +++ b/pkg/urbit/vere/io/term.c @@ -4,33 +4,10 @@ #include "all.h" #include "vere/vere.h" -#if !defined(U3_OS_TERM_dumb) -#include -#include -#include -#include -#include -#include -#include - -// macros for string literal args/buffers -// -// since (sizeof(s) - 1) is used for vector length, parameters -// must be appropriately typed. use with care! -// -#define TERM_LIT(s) sizeof(s) - 1, (const c3_y*)(s) -#define TERM_LIT_BUF(s) uv_buf_init(s, sizeof(s) - 1) - -static u3_utty* _term_main(); -static void _term_read_cb(uv_stream_t* tcp_u, - ssize_t siz_i, - const uv_buf_t* buf_u); -static c3_i _term_tcsetattr(c3_i, c3_i, const struct termios*); - -/* _write(): retry interrupts, continue partial writes, assert errors. +/* u3_write_fd(): retry interrupts, continue partial writes, assert errors. */ -static void -_write(c3_i fid_i, const void* buf_v, size_t len_i) +void +u3_write_fd(c3_i fid_i, const void* buf_v, size_t len_i) { ssize_t ret_i; @@ -54,7 +31,7 @@ _write(c3_i fid_i, const void* buf_v, size_t len_i) // assert on true errors // - // NB: can't call u3l_log here or we would re-enter _write() + // NB: can't call u3l_log here or we would re-enter u3_write_fd() // if ( ret_i < 0 ) { fprintf(stderr, "term: write failed %s\r\n", strerror(errno)); @@ -69,6 +46,23 @@ _write(c3_i fid_i, const void* buf_v, size_t len_i) } } +#if !defined(U3_OS_TERM_dumb) +#include + +// macros for string literal args/buffers +// +// since (sizeof(s) - 1) is used for vector length, parameters +// must be appropriately typed. use with care! +// +#define TERM_LIT(s) sizeof(s) - 1, (const c3_y*)(s) +#define TERM_LIT_BUF(s) uv_buf_init(s, sizeof(s) - 1) + +static u3_utty* _term_main(); +static void _term_read_cb(uv_stream_t* tcp_u, + ssize_t siz_i, + const uv_buf_t* buf_u); +static c3_i _term_tcsetattr(c3_i, c3_i, const struct termios*); + /* _term_msc_out_host(): unix microseconds from current host time. */ static c3_d @@ -297,7 +291,7 @@ u3_term_log_exit(void) if ( -1 == fcntl(uty_u->fid_i, F_SETFL, uty_u->cug_i) ) { c3_assert(!"exit-fcntl"); } - _write(uty_u->fid_i, "\r\n", 2); + u3_write_fd(uty_u->fid_i, "\r\n", 2); } } @@ -1510,10 +1504,10 @@ u3_term_io_hija(void) perror("hija-fcntl-0"); c3_assert(!"hija-fcntl"); } - _write(uty_u->fid_i, "\r", 1); + u3_write_fd(uty_u->fid_i, "\r", 1); { uv_buf_t* buf_u = &uty_u->ufo_u.out.el_u; - _write(uty_u->fid_i, buf_u->base, buf_u->len); + u3_write_fd(uty_u->fid_i, buf_u->base, buf_u->len); } } } From d03e48702ab2a7426df5b39b885bdf00fc149b01 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 29 Mar 2021 12:21:29 +0000 Subject: [PATCH 030/136] compat: fix .patch file modes --- pkg/urbit/compat/mingw/argon2u.patch | 0 pkg/urbit/compat/mingw/h2o.patch | 0 pkg/urbit/compat/mingw/libscrypt.patch | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 pkg/urbit/compat/mingw/argon2u.patch mode change 100755 => 100644 pkg/urbit/compat/mingw/h2o.patch mode change 100755 => 100644 pkg/urbit/compat/mingw/libscrypt.patch diff --git a/pkg/urbit/compat/mingw/argon2u.patch b/pkg/urbit/compat/mingw/argon2u.patch old mode 100755 new mode 100644 diff --git a/pkg/urbit/compat/mingw/h2o.patch b/pkg/urbit/compat/mingw/h2o.patch old mode 100755 new mode 100644 diff --git a/pkg/urbit/compat/mingw/libscrypt.patch b/pkg/urbit/compat/mingw/libscrypt.patch old mode 100755 new mode 100644 From 2b5f00075da018b0cf8a53ee289eae79c13e5808 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 29 Mar 2021 13:27:27 +0000 Subject: [PATCH 031/136] compat: move build-mingw script into configure --- nix/sources-mingw.json | 3 +- pkg/urbit/build-mingw | 61 ----------------------------------------- pkg/urbit/configure | 62 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 58 insertions(+), 68 deletions(-) delete mode 100755 pkg/urbit/build-mingw diff --git a/nix/sources-mingw.json b/nix/sources-mingw.json index 9fbb3f5f07..53dcecb3ce 100644 --- a/nix/sources-mingw.json +++ b/nix/sources-mingw.json @@ -21,7 +21,8 @@ "mingw": { "include": "include", "lib": ".libs", - "prepare": "./autogen.sh && ./configure --enable-module-recovery" + "prepare": "./autogen.sh && ./configure --enable-module-recovery && make", + "make": "libsecp256k1.la" }, "owner": "bitcoin-core", "repo": "secp256k1", diff --git a/pkg/urbit/build-mingw b/pkg/urbit/build-mingw deleted file mode 100755 index b917052c7e..0000000000 --- a/pkg/urbit/build-mingw +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -function xxd() -{ - cch=0 - echo "unsigned char $2[] = {" - while IFS='' read line - do - for i in $line - do - echo -n " 0x$i," - cch=$((cch+1)) - done - echo - done < <(od -An -v -tx1 $1) - echo "};" - echo "unsigned int $2_len = $cch;" -} - -case $(uname -s|tr A-Z a-z) in - *mingw64*) - ;; - *) - echo This script works only on MingW64. - exit 1 - ;; -esac - -if [ ! -e config.mk ] -then - # ensure required mingw packages are installed - mpkgs=(cmake curl gcc jq libsigsegv libuv make wslay) - pacman -S --needed autoconf automake-wrapper libtool patch ${mpkgs[@]/#/mingw-w64-x86_64-} - - unset cdirs - unset ldirs - declare -a cdirs - declare -a ldirs - . <(jq -sr ' -add|to_entries|.[]|select(.value.mingw)|.key as $key|(if .value.url then " -mkdir -p ../\($key) -pushd ../\($key) -curl -L \(.value.url)|(tar --strip \(.value.mingw.strip+1) -xzf - || true)" + -("../urbit/compat/mingw/\($key).patch"|" -[ -e \(.) ] && patch -p 1 <\(.)") else " -pushd ../\($key)" end) + " -\(.value.mingw.prepare//"") -make \(.value.mingw.make//"") -popd -\(.value.mingw.include//"."|if type == "array" then .[] else . end|"cdirs+=(-I../\($key)/\(.)) -")\(.value.mingw.lib//"."|if type == "array" then .[] else . end|"ldirs+=(-L../\($key)/\(.)) -")"' ../../nix/sources.json ../../nix/sources-mingw.json) - - xxd /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt include_ca_bundle_crt >include/ca-bundle.h - xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h - - CFLAGS="${CFLAGS-} ${cdirs[@]}" LDFLAGS="${LDFLAGS-} ${ldirs[@]}" PKG_CONFIG=echo ./configure -fi - -make build/urbit build/urbit-worker diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 718522c2ef..1ae1bd8cca 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e +set -euo pipefail URBIT_VERSION="$(cat ./version)" @@ -19,12 +19,28 @@ defmacro () { echo "#define $1 $2" >>include/config.h } +xxd () { + cch=0 + echo "unsigned char $2[] = {" + while IFS='' read line + do + for i in $line + do + echo -n " 0x$i," + cch=$((cch+1)) + done + echo + done < <(od -An -v -tx1 $1) + echo "};" + echo "unsigned int $2_len = $cch;" +} + defmacro URBIT_VERSION "\"$URBIT_VERSION\"" -[ -n "$MEMORY_DEBUG" ] && defmacro U3_MEMORY_DEBUG 1 -[ -n "$MEMORY_LOG" ] && defmacro U3_MEMORY_LOG 1 -[ -n "$CPU_DEBUG" ] && defmacro U3_CPU_DEBUG 1 -[ -n "$EVENT_TIME_DEBUG" ] && defmacro U3_EVENT_TIME_DEBUG 1 +[ -n "${MEMORY_DEBUG-}" ] && defmacro U3_MEMORY_DEBUG 1 +[ -n "${MEMORY_LOG-}" ] && defmacro U3_MEMORY_LOG 1 +[ -n "${CPU_DEBUG-}" ] && defmacro U3_CPU_DEBUG 1 +[ -n "${EVENT_TIME_DEBUG-}" ] && defmacro U3_EVENT_TIME_DEBUG 1 if [ -n "${HOST-}" ] then os=$(sed 's$^[^-]*-\([^-]*\)-.*$\1$' <<< "$HOST") @@ -59,13 +75,47 @@ esac # TODO Determine if the target cpu is little or big endian. case $(tr A-Z a-z <<< $os) in *mingw*) + # ensure required mingw packages are installed + mpkgs=(cmake curl gcc jq libsigsegv libuv make wslay) + pacman -S --needed autoconf automake-wrapper libtool patch ${mpkgs[@]/#/mingw-w64-x86_64-} + + # poor man's nix-shell + declare -a cdirs + declare -a ldirs + . <(jq -sr ' +add|to_entries|.[]|select(.value.mingw)|.key as $key|"../\(.key)" as $dir|(if .value.url then " +if [ ! -d \($dir) ] +then + mkdir -p \($dir) + pushd \($dir) + curl -L \(.value.url)|(tar --strip \(.value.mingw.strip+1) -xzf - || true) + popd +fi" else "" end) + " +pushd \($dir) +if [ ! -e .mingw~ ] +then" + ("../urbit/compat/mingw/\($key).patch"|" + [ -e \(.) ] && patch -p 1 <\(.)") + " + \(.value.mingw.prepare//"") + touch .mingw~ +fi +make \(.value.mingw.make//"") +popd +\(.value.mingw.include//"."|if type == "array" then .[] else . end|"cdirs+=(-I../\($key)/\(.)) +")\(.value.mingw.lib//"."|if type == "array" then .[] else . end|"ldirs+=(-L../\($key)/\(.)) +")"' ../../nix/sources.json ../../nix/sources-mingw.json) + + [ -e include/ca-bundle.h ] || xxd /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt include_ca_bundle_crt >include/ca-bundle.h + [ -e include/ivory.h ] || xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h + defmacro U3_OS_mingw 1 defmacro U3_OS_TERM_dumb 1 # increase default thread stack size and link Windows implibs ldextra="-Wl,--stack,67108864 -lbcrypt -lws2_32" vcompat="compat = mingw" - CFLAGS="${CFLAGS-} -DH2O_NO_UNIX_SOCKETS -Icompat/mingw" + CFLAGS="${CFLAGS-} ${cdirs[@]} -DH2O_NO_UNIX_SOCKETS -Icompat/mingw" + LDFLAGS="${LDFLAGS-} ${ldirs[@]}" + PKG_CONFIG="echo" ;; *linux*) defmacro U3_OS_linux 1 From 201f1f36f2d6eb00b6a58aaa04237856f9648d7f Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 29 Mar 2021 15:42:15 +0000 Subject: [PATCH 032/136] compat: support build from a tarball that does not contain binary pills --- pkg/urbit/configure | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 1ae1bd8cca..41ee361a94 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -79,6 +79,9 @@ case $(tr A-Z a-z <<< $os) in mpkgs=(cmake curl gcc jq libsigsegv libuv make wslay) pacman -S --needed autoconf automake-wrapper libtool patch ${mpkgs[@]/#/mingw-w64-x86_64-} + # support running off a tarball that doesn't contain binary pills + (( $(stat -c %s ../../bin/ivory.pill) > 512 )) || curl -L https://github.com/urbit/urbit/raw/urbit-v$URBIT_VERSION/bin/ivory.pill > ../../bin/ivory.pill + # poor man's nix-shell declare -a cdirs declare -a ldirs From c690b571240ad6d57ed9e5911bc4be7dd8d93d78 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 29 Mar 2021 20:49:00 +0000 Subject: [PATCH 033/136] vere: fix strict builds --- pkg/urbit/daemon/main.c | 9 +++++++-- pkg/urbit/include/c/portable.h | 1 - pkg/urbit/include/noun/allocate.h | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index db3dcf484d..6cea5c4276 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -5,6 +5,9 @@ #define C3_GLOBAL #include "all.h" #include "vere/vere.h" +#if !defined(U3_OS_mingw) +#include +#endif #include #include #include @@ -372,8 +375,9 @@ _setup_cert_store() /* _setup_ssl_x509(): adds embedded CA certificates to a X509_STORE */ static void -_setup_ssl_x509(X509_STORE* cts) +_setup_ssl_x509(void* arg) { + X509_STORE* cts = arg; int i; for ( i = 0; i < sk_X509_INFO_num(_cert_store); i++ ) { X509_INFO *itmp = sk_X509_INFO_value(_cert_store, i); @@ -402,8 +406,9 @@ _curl_ssl_ctx_cb(CURL* curl, SSL_CTX* sslctx, void* param) /* _setup_ssl_curl(): adds embedded CA certificates to a curl context */ static void -_setup_ssl_curl(CURL* curl) +_setup_ssl_curl(void* arg) { + CURL* curl = arg; curl_easy_setopt(curl, CURLOPT_CAINFO, NULL); curl_easy_setopt(curl, CURLOPT_CAPATH, NULL); curl_easy_setopt(curl, CURLOPT_SSL_CTX_FUNCTION, _curl_ssl_ctx_cb); diff --git a/pkg/urbit/include/c/portable.h b/pkg/urbit/include/c/portable.h index fbcce1fde9..3700a8065a 100644 --- a/pkg/urbit/include/c/portable.h +++ b/pkg/urbit/include/c/portable.h @@ -74,7 +74,6 @@ # elif defined(U3_OS_mingw) # define _POSIX -# include # include # include # include diff --git a/pkg/urbit/include/noun/allocate.h b/pkg/urbit/include/noun/allocate.h index 564b69dbab..b8f367d863 100644 --- a/pkg/urbit/include/noun/allocate.h +++ b/pkg/urbit/include/noun/allocate.h @@ -2,6 +2,9 @@ ** ** This file is in the public domain. */ + +#include + /** Constants. **/ /* u3a_bits: number of bits in word-addressed pointer. 29 == 2GB. From 1c58db28de33d2652d657adcb37a31f34c14da5d Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 29 Mar 2021 20:49:41 +0000 Subject: [PATCH 034/136] meta: add mingw build action --- .github/workflows/build.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7b9542271..67629bedca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -94,3 +94,21 @@ jobs: - run: nix-build -A hs.urbit-king.components.exes.urbit-king --arg enableStatic true - run: nix-build -A hs-checks - run: nix-build shell.nix + + mingw: + runs-on: windows-latest + defaults: + run: + shell: C:\msys64\msys2_shell.cmd -mingw64 -defterm -no-start -here -c ". <(cygpath '{0}')" + + steps: + - uses: actions/checkout@v2 + with: + lfs: true + + # echo suppresses pacman prompt + - run: echo|./configure + working-directory: ./pkg/urbit + + - run: make build/urbit build/urbit-worker + working-directory: ./pkg/urbit From 0d9adffb25d9d2171c120cfd7ebaa416015c9669 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 29 Mar 2021 23:35:12 +0000 Subject: [PATCH 035/136] vere: mingw: build static executables --- nix/sources-mingw.json | 17 +++++++++++++++++ pkg/urbit/configure | 8 ++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/nix/sources-mingw.json b/nix/sources-mingw.json index 53dcecb3ce..59e74f2c9b 100644 --- a/nix/sources-mingw.json +++ b/nix/sources-mingw.json @@ -1,4 +1,21 @@ { + "curl": { + "branch": "master", + "description": "A command line tool and library for transferring data with URL syntax", + "homepage": "http://curl.se/", + "mingw": { + "include": "include", + "lib": "lib/.libs", + "prepare": "autoreconf -vfi && ./configure --disable-shared --disable-ldap --disable-rtsp --without-brotli --without-libidn2 --without-libpsl --without-nghttp2", + "make": "-C lib libcurl.la" + }, + "owner": "curl", + "repo": "curl", + "rev": "curl-7_75_0", + "type": "tarball", + "url": "https://github.com/curl/curl/archive/curl-7_75_0.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "lmdb": { "branch": "mdb.master", "description": "LMDB library", diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 41ee361a94..52809d6cd8 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -115,9 +115,13 @@ popd # increase default thread stack size and link Windows implibs ldextra="-Wl,--stack,67108864 -lbcrypt -lws2_32" + # libcurl implibs + ldextra="$ldextra -lzstd" + # libuv implibs + ldextra="$ldextra -luserenv -liphlpapi -lpsapi" vcompat="compat = mingw" - CFLAGS="${CFLAGS-} ${cdirs[@]} -DH2O_NO_UNIX_SOCKETS -Icompat/mingw" - LDFLAGS="${LDFLAGS-} ${ldirs[@]}" + CFLAGS="${CFLAGS-} ${cdirs[@]} -DCURL_STATICLIB -DH2O_NO_UNIX_SOCKETS -Icompat/mingw" + LDFLAGS="${LDFLAGS-} -static ${ldirs[@]}" PKG_CONFIG="echo" ;; *linux*) From a36193c967b3ffe5a5eb47513adcb22495a82ef2 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Tue, 30 Mar 2021 07:30:40 +0000 Subject: [PATCH 036/136] compat: mingw: don't export secp256k1 symbols from exe --- nix/sources-mingw.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/sources-mingw.json b/nix/sources-mingw.json index 59e74f2c9b..39ade85a56 100644 --- a/nix/sources-mingw.json +++ b/nix/sources-mingw.json @@ -38,7 +38,7 @@ "mingw": { "include": "include", "lib": ".libs", - "prepare": "./autogen.sh && ./configure --enable-module-recovery && make", + "prepare": "./autogen.sh && ./configure --disable-shared --enable-module-recovery CFLAGS=-DSECP256K1_API=", "make": "libsecp256k1.la" }, "owner": "bitcoin-core", From 85030ab72d40413e52573abd2c0505c24674ce24 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Tue, 30 Mar 2021 10:14:22 +0000 Subject: [PATCH 037/136] vere: revert d554c18dc35253e0158353932944f8563c34b788, set mingw binary mode using _fmode --- pkg/urbit/compat/mingw/compat.c | 6 ++++++ pkg/urbit/include/c/portable.h | 4 ---- pkg/urbit/noun/events.c | 10 +++++----- pkg/urbit/noun/manage.c | 2 +- pkg/urbit/noun/trace.c | 2 +- pkg/urbit/noun/urth.c | 6 +++--- pkg/urbit/vere/dawn.c | 2 +- pkg/urbit/vere/io/http.c | 2 +- pkg/urbit/vere/io/unix.c | 14 +++++++------- pkg/urbit/vere/king.c | 2 +- pkg/urbit/vere/walk.c | 6 +++--- pkg/urbit/worker/main.c | 2 +- pkg/urbit/worker/serf.c | 4 ++-- 13 files changed, 32 insertions(+), 30 deletions(-) diff --git a/pkg/urbit/compat/mingw/compat.c b/pkg/urbit/compat/mingw/compat.c index 1fe0c97254..3ab9029b67 100644 --- a/pkg/urbit/compat/mingw/compat.c +++ b/pkg/urbit/compat/mingw/compat.c @@ -1,7 +1,13 @@ #include "c/portable.h" +#include #include #include +// set default CRT file mode to binary +// note that mingw binmode.o does nothing +#undef _fmode +int _fmode = _O_BINARY; + // from https://github.com/git/git/blob/master/compat/mingw.c // ----------------------------------------------------------------------- diff --git a/pkg/urbit/include/c/portable.h b/pkg/urbit/include/c/portable.h index 3700a8065a..5c675a926d 100644 --- a/pkg/urbit/include/c/portable.h +++ b/pkg/urbit/include/c/portable.h @@ -92,10 +92,6 @@ #error "port: headers" # endif -# ifndef O_BINARY -# define O_BINARY 0 -# endif - # ifndef __has_feature # define __has_feature(x) 0 # endif diff --git a/pkg/urbit/noun/events.c b/pkg/urbit/noun/events.c index 0085df8ed7..224eb86d9c 100644 --- a/pkg/urbit/noun/events.c +++ b/pkg/urbit/noun/events.c @@ -170,7 +170,7 @@ u3e_fault(void* adr_v, c3_i ser_i) static c3_o _ce_image_open(u3e_image* img_u) { - c3_i mod_i = O_RDWR | O_BINARY | O_CREAT; + c3_i mod_i = O_RDWR | O_CREAT; c3_c ful_c[8193]; snprintf(ful_c, 8192, "%s", u3P.dir_c); @@ -274,13 +274,13 @@ _ce_patch_create(u3_ce_patch* pat_u) mkdir(ful_c, 0700); snprintf(ful_c, 8192, "%s/.urb/chk/control.bin", u3P.dir_c); - if ( -1 == (pat_u->ctl_i = open(ful_c, O_RDWR | O_BINARY | O_CREAT | O_EXCL, 0600)) ) { + if ( -1 == (pat_u->ctl_i = open(ful_c, O_RDWR | O_CREAT | O_EXCL, 0600)) ) { fprintf(stderr, "loom: patch open control.bin: %s\r\n", strerror(errno)); c3_assert(0); } snprintf(ful_c, 8192, "%s/.urb/chk/memory.bin", u3P.dir_c); - if ( -1 == (pat_u->mem_i = open(ful_c, O_RDWR | O_BINARY | O_CREAT | O_EXCL, 0600)) ) { + if ( -1 == (pat_u->mem_i = open(ful_c, O_RDWR | O_CREAT | O_EXCL, 0600)) ) { fprintf(stderr, "loom: patch open memory.bin: %s\r\n", strerror(errno)); c3_assert(0); } @@ -376,12 +376,12 @@ _ce_patch_open(void) mkdir(ful_c, 0700); snprintf(ful_c, 8192, "%s/.urb/chk/control.bin", u3P.dir_c); - if ( -1 == (ctl_i = open(ful_c, O_RDWR | O_BINARY)) ) { + if ( -1 == (ctl_i = open(ful_c, O_RDWR)) ) { return 0; } snprintf(ful_c, 8192, "%s/.urb/chk/memory.bin", u3P.dir_c); - if ( -1 == (mem_i = open(ful_c, O_RDWR | O_BINARY)) ) { + if ( -1 == (mem_i = open(ful_c, O_RDWR)) ) { close(ctl_i); _ce_patch_delete(); diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 9c7abdcbce..5fd1c82aea 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -411,7 +411,7 @@ u3_noun u3m_file(c3_c* pas_c) { struct stat buf_b; - c3_i fid_i = open(pas_c, O_RDONLY | O_BINARY, 0644); + c3_i fid_i = open(pas_c, O_RDONLY, 0644); c3_w fln_w, red_w; c3_y* pad_y; diff --git a/pkg/urbit/noun/trace.c b/pkg/urbit/noun/trace.c index 2b53ae9388..1ee8168c70 100644 --- a/pkg/urbit/noun/trace.c +++ b/pkg/urbit/noun/trace.c @@ -288,7 +288,7 @@ u3t_trace_open(c3_c* dir_c) c3_c lif_c[2056]; snprintf(lif_c, 2056, "%s/%d.json", fil_c, u3_Host.tra_u.fun_w); - u3_Host.tra_u.fil_u = fopen(lif_c, "wb"); + u3_Host.tra_u.fil_u = fopen(lif_c, "w"); u3_Host.tra_u.nid_w = (int)getpid(); fprintf(u3_Host.tra_u.fil_u, "[ "); diff --git a/pkg/urbit/noun/urth.c b/pkg/urbit/noun/urth.c index 8fc0b33145..c92a7d3dad 100644 --- a/pkg/urbit/noun/urth.c +++ b/pkg/urbit/noun/urth.c @@ -549,7 +549,7 @@ _cu_rock_save(c3_c* dir_c, c3_d eve_d, c3_d len_d, c3_y* byt_y) return c3n; } - if ( -1 == (fid_i = open(nam_c, O_RDWR | O_BINARY | O_CREAT | O_TRUNC, 0644)) ) { + if ( -1 == (fid_i = open(nam_c, O_RDWR | O_CREAT | O_TRUNC, 0644)) ) { fprintf(stderr, "rock: open failed (%s, %" PRIu64 "): %s\r\n", dir_c, eve_d, strerror(errno)); c3_free(nam_c); @@ -688,7 +688,7 @@ u3u_mmap_read(c3_c* cap_c, c3_c* pat_c, c3_d* out_d, c3_y** out_y) // open file // - if ( -1 == (fid_i = open(pat_c, O_RDONLY | O_BINARY, 0644)) ) { + if ( -1 == (fid_i = open(pat_c, O_RDONLY, 0644)) ) { fprintf(stderr, "%s: open failed (%s): %s\r\n", cap_c, pat_c, strerror(errno)); return c3n; @@ -741,7 +741,7 @@ u3u_mmap(c3_c* cap_c, c3_c* pat_c, c3_d len_d, c3_y** out_y) // open file // - if ( -1 == (fid_i = open(pat_c, O_RDWR | O_BINARY | O_CREAT | O_TRUNC, 0644)) ) { + if ( -1 == (fid_i = open(pat_c, O_RDWR | O_CREAT | O_TRUNC, 0644)) ) { fprintf(stderr, "%s: open failed (%s): %s\r\n", cap_c, pat_c, strerror(errno)); return c3n; diff --git a/pkg/urbit/vere/dawn.c b/pkg/urbit/vere/dawn.c index a7663235b3..349999102a 100644 --- a/pkg/urbit/vere/dawn.c +++ b/pkg/urbit/vere/dawn.c @@ -520,7 +520,7 @@ _dawn_come(u3_noun stars) c3_c pat_c[64]; snprintf(pat_c, 64, "%s.key", who_c + 1); - FILE* fil_u = fopen(pat_c, "wb"); + FILE* fil_u = fopen(pat_c, "w"); fprintf(fil_u, "%s\n", key_c); fclose(fil_u); } diff --git a/pkg/urbit/vere/io/http.c b/pkg/urbit/vere/io/http.c index e9a7c7d9e4..4e8e2e6d43 100644 --- a/pkg/urbit/vere/io/http.c +++ b/pkg/urbit/vere/io/http.c @@ -1600,7 +1600,7 @@ _http_write_ports_file(u3_httd* htd_u, c3_c *pax_c) c3_c* paf_c = c3_malloc(len_w); snprintf(paf_c, len_w, "%s/%s", pax_c, nam_c); - c3_i por_i = open(paf_c, O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, 0666); + c3_i por_i = open(paf_c, O_WRONLY | O_CREAT | O_TRUNC, 0666); c3_free(paf_c); u3_http* htp_u = htd_u->htp_u; diff --git a/pkg/urbit/vere/io/unix.c b/pkg/urbit/vere/io/unix.c index 2885686c4a..a485d5115b 100644 --- a/pkg/urbit/vere/io/unix.c +++ b/pkg/urbit/vere/io/unix.c @@ -225,7 +225,7 @@ _unix_mkdir(c3_c* pax_c) static c3_w _unix_write_file_hard(c3_c* pax_c, u3_noun mim) { - c3_i fid_i = open(pax_c, O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, 0666); + c3_i fid_i = open(pax_c, O_WRONLY | O_CREAT | O_TRUNC, 0666); c3_w len_w, rit_w, siz_w, mug_w = 0; c3_y* dat_y; @@ -268,7 +268,7 @@ static void _unix_write_file_soft(u3_ufil* fil_u, u3_noun mim) { struct stat buf_u; - c3_i fid_i = open(fil_u->pax_c, O_RDONLY | O_BINARY, 0644); + c3_i fid_i = open(fil_u->pax_c, O_RDONLY, 0644); c3_ws len_ws, red_ws; c3_w old_w; c3_y* old_y; @@ -692,7 +692,7 @@ _unix_update_file(u3_unix* unx_u, u3_ufil* fil_u) fil_u->dry = c3n; struct stat buf_u; - c3_i fid_i = open(fil_u->pax_c, O_RDONLY | O_BINARY, 0644); + c3_i fid_i = open(fil_u->pax_c, O_RDONLY, 0644); c3_ws len_ws, red_ws; c3_y* dat_y; @@ -795,7 +795,7 @@ _unix_update_dir(u3_unix* unx_u, u3_udir* dir_u) } else { struct stat buf_u; - c3_i fid_i = open(nod_u->pax_c, O_RDONLY | O_BINARY, 0644); + c3_i fid_i = open(nod_u->pax_c, O_RDONLY, 0644); if ( (fid_i < 0) || (fstat(fid_i, &buf_u) < 0) ) { if ( ENOENT != errno ) { @@ -965,7 +965,7 @@ static u3_noun _unix_initial_update_file(c3_c* pax_c, c3_c* bas_c) { struct stat buf_u; - c3_i fid_i = open(pax_c, O_RDONLY | O_BINARY, 0644); + c3_i fid_i = open(pax_c, O_RDONLY, 0644); c3_ws len_ws, red_ws; c3_y* dat_y; @@ -1281,7 +1281,7 @@ u3_unix_acquire(c3_c* pax_c) c3_w pid_w; FILE* loq_u; - if ( NULL != (loq_u = fopen(paf_c, "rb")) ) { + if ( NULL != (loq_u = fopen(paf_c, "r")) ) { if ( 1 != fscanf(loq_u, "%" SCNu32, &pid_w) ) { u3l_log("lockfile %s is corrupt!\n", paf_c); kill(getpid(), SIGTERM); @@ -1319,7 +1319,7 @@ u3_unix_acquire(c3_c* pax_c) unlink(paf_c); } - if ( NULL == (loq_u = fopen(paf_c, "wb")) ) { + if ( NULL == (loq_u = fopen(paf_c, "w")) ) { u3l_log("unix: unable to open %s\n", paf_c); c3_assert(0); } diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c index bf6adea698..bc65fc7e85 100644 --- a/pkg/urbit/vere/king.c +++ b/pkg/urbit/vere/king.c @@ -916,7 +916,7 @@ u3_king_grab(void* vod_p) c3_c man_c[2048]; snprintf(man_c, 2048, "%s/%s-daemon.txt", nam_c, wen_c); - fil_u = fopen(man_c, "wb"); + fil_u = fopen(man_c, "w"); fprintf(fil_u, "%s\r\n", wen_c); c3_free(wen_c); diff --git a/pkg/urbit/vere/walk.c b/pkg/urbit/vere/walk.c index 4d03f12bdd..5c113cbac7 100644 --- a/pkg/urbit/vere/walk.c +++ b/pkg/urbit/vere/walk.c @@ -32,7 +32,7 @@ u3_noun u3_walk_safe(c3_c* pas_c) { struct stat buf_b; - c3_i fid_i = open(pas_c, O_RDONLY | O_BINARY, 0644); + c3_i fid_i = open(pas_c, O_RDONLY, 0644); c3_w fln_w, red_w; c3_y* pad_y; @@ -64,7 +64,7 @@ u3_noun u3_walk_load(c3_c* pas_c) { struct stat buf_b; - c3_i fid_i = open(pas_c, O_RDONLY | O_BINARY, 0644); + c3_i fid_i = open(pas_c, O_RDONLY, 0644); c3_w fln_w, red_w; c3_y* pad_y; @@ -130,7 +130,7 @@ _walk_mkdirp(c3_c* bas_c, u3_noun pax) void u3_walk_save(c3_c* pas_c, u3_noun tim, u3_atom pad, c3_c* bas_c, u3_noun pax) { - c3_i fid_i = open(pas_c, O_WRONLY | O_BINARY | O_CREAT | O_TRUNC, 0666); + c3_i fid_i = open(pas_c, O_WRONLY | O_CREAT | O_TRUNC, 0666); c3_w fln_w, rit_w; c3_y* pad_y; diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index c1cff83ea0..8c4001d8d8 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -130,7 +130,7 @@ _cw_serf_stdio(c3_i* inn_i, c3_i* out_i) // we replace [FD 0] (stdin) with a fd pointing to /dev/null // we replace [FD 1] (stdout) with a dup of [FD 2] (stderr) // - c3_i nul_i = open("/dev/null", O_RDWR | O_BINARY, 0); + c3_i nul_i = open("/dev/null", O_RDWR, 0); *inn_i = dup(0); *out_i = dup(1); diff --git a/pkg/urbit/worker/serf.c b/pkg/urbit/worker/serf.c index 2376d8e48a..8ed603b90f 100644 --- a/pkg/urbit/worker/serf.c +++ b/pkg/urbit/worker/serf.c @@ -210,7 +210,7 @@ _serf_grab(u3_serf* sef_u) c3_c man_c[2054]; snprintf(man_c, 2053, "%s/%s-serf.txt", nam_c, wen_c); - fil_u = fopen(man_c, "wb"); + fil_u = fopen(man_c, "w"); fprintf(fil_u, "%s\r\n", wen_c); c3_free(wen_c); @@ -809,7 +809,7 @@ _serf_writ_live_exit(u3_serf* sef_u, c3_w cod_w) c3_c man_c[2054]; snprintf(man_c, 2053, "%s/%s.txt", nam_c, wen_c); - fil_u = fopen(man_c, "wb"); + fil_u = fopen(man_c, "w"); c3_free(wen_c); u3z(wen); From 8df8f654b2699152d3d54991166d50230856ec92 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Wed, 31 Mar 2021 10:29:45 +0000 Subject: [PATCH 038/136] compat: make Makefile more flexible --- pkg/urbit/Makefile | 4 +--- pkg/urbit/configure | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/urbit/Makefile b/pkg/urbit/Makefile index bb37b8c20b..b82e936b55 100644 --- a/pkg/urbit/Makefile +++ b/pkg/urbit/Makefile @@ -9,9 +9,7 @@ worker = $(wildcard worker/*.c) tests = $(wildcard tests/*.c) bench = $(wildcard bench/*.c) -ifdef compat -compat := $(wildcard compat/$(compat)/*.c) -endif +compat := $(foreach dir,$(compat),$(wildcard compat/$(dir)/*.c)) common = $(jets) $(noun) $(ur) $(vere) $(compat) headers = $(shell find include -type f) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 52809d6cd8..00b486ace2 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -119,7 +119,7 @@ popd ldextra="$ldextra -lzstd" # libuv implibs ldextra="$ldextra -luserenv -liphlpapi -lpsapi" - vcompat="compat = mingw" + vcompat=mingw CFLAGS="${CFLAGS-} ${cdirs[@]} -DCURL_STATICLIB -DH2O_NO_UNIX_SOCKETS -Icompat/mingw" LDFLAGS="${LDFLAGS-} -static ${ldirs[@]}" PKG_CONFIG="echo" @@ -162,7 +162,7 @@ cat >config.mk <&2 From d86fb8ddd26ed08c7743700e1a69cfaf8b3f704b Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Wed, 31 Mar 2021 10:39:40 +0000 Subject: [PATCH 039/136] vere: terminal compatibility --- pkg/urbit/compat/mingw/ptty.c | 152 +++++++++++++++++++++ pkg/urbit/compat/mingw/termios.h | 8 -- pkg/urbit/compat/posix/ptty.c | 189 +++++++++++++++++++++++++ pkg/urbit/configure | 2 +- pkg/urbit/include/vere/vere.h | 40 ++++-- pkg/urbit/vere/io/term.c | 227 +++++-------------------------- 6 files changed, 410 insertions(+), 208 deletions(-) create mode 100644 pkg/urbit/compat/mingw/ptty.c delete mode 100644 pkg/urbit/compat/mingw/termios.h create mode 100644 pkg/urbit/compat/posix/ptty.c diff --git a/pkg/urbit/compat/mingw/ptty.c b/pkg/urbit/compat/mingw/ptty.c new file mode 100644 index 0000000000..a65d9370d6 --- /dev/null +++ b/pkg/urbit/compat/mingw/ptty.c @@ -0,0 +1,152 @@ +/* compat/mingw/ptty.c +** +*/ +#include "all.h" +#include "vere/vere.h" +#include + +/* _ptty_get_type(): detects tty type. +*/ +static DWORD +_ptty_get_type(int fd) +{ + HANDLE h = (HANDLE)_get_osfhandle(fd); + if (h == INVALID_HANDLE_VALUE) + return FILE_TYPE_UNKNOWN; + + DWORD t = GetFileType(h); + if (t != FILE_TYPE_PIPE) + return t ; + + // https://github.com/fusesource/jansi-native/commit/461068c67a38647d2890e96250636fc0117074f5 + ULONG result; + BYTE buffer[1024]; + POBJECT_NAME_INFORMATION nameinfo = (POBJECT_NAME_INFORMATION) buffer; + PWSTR name; + + /* get pipe name */ + if (!NT_SUCCESS(NtQueryObject(h, ObjectNameInformation, buffer, sizeof(buffer) - sizeof(WCHAR), &result))) + return FILE_TYPE_UNKNOWN; + + name = nameinfo->Name.Buffer; + name[nameinfo->Name.Length] = 0; + + // check for popular terminal emulators + // that use named pipes to communicate with subprocesses + if (wcsstr(name, L"\\ConEmu") || + (wcsstr(name, L"msys-") || wcsstr(name, L"cygwin-")) && wcsstr(name, L"-pty")) + return FILE_TYPE_PIPE; + + return FILE_TYPE_UNKNOWN; +} + +/* _ttyf_nop(): stub function. +*/ +static c3_o +_ttyf_nop(u3_utty* uty_u) +{ + return c3y; +} + +/* _ttyf_start_raw_input(): ends raw input on the tty. +*/ +static c3_o +_ttyf_set_normal(u3_utty* uty_u) +{ + c3_i e; + if ( 0 != (e = uv_tty_set_mode(&uty_u->pin_u.tty_u, UV_TTY_MODE_NORMAL)) ) { + fprintf(stderr, "uv_tty_set_mode(UV_TTY_MODE_NORMAL) -> %d\r\n", e); + return c3n; + } + return c3y; +} + +/* _ttyf_start_raw_input(): sets the tty to raw input. +*/ +static c3_o +_ttyf_set_raw(u3_utty* uty_u) +{ + c3_i e; + if ( 0 != (e = uv_tty_set_mode(&uty_u->pin_u.tty_u, UV_TTY_MODE_RAW)) ) { + fprintf(stderr, "uv_tty_set_mode(UV_TTY_MODE_RAW) -> %d\r\n", e); + return c3n; + } + return c3y; +} + +/* _ttyf_get_winsize(): gets the tty window size. +*/ +static c3_o +_ttyf_get_winsize(u3_utty* uty_u, c3_l* col_l, c3_l* row_l) +{ + c3_i col_i, row_i; + if ( 0 != uv_tty_get_winsize(&uty_u->pop_u.tty_u, &col_i, &row_i) ) { + return c3n; + } + + *col_l = col_i; + *row_l = row_i; + return c3y; +} + +/* _ttyf_get_winsize(): gets the tty window size. +*/ +static c3_o +_ttyf_nop_winsize(u3_utty* uty_u, c3_l* col_l, c3_l* row_l) +{ + return c3n; +} + +/* u3_ptty_init(): initialize platform-specific tty. +*/ +u3_utty* +u3_ptty_init(uv_loop_t* lup_u, const c3_c** err_c) +{ + DWORD pip_l = _ptty_get_type(0); + DWORD pop_l = _ptty_get_type(1); + if ( pip_l == FILE_TYPE_UNKNOWN || pop_l == FILE_TYPE_UNKNOWN) { + *err_c = "not a tty"; + return NULL; + } + + if ( pip_l != pop_l ) { + *err_c = "partly redirected"; + return NULL; + } + + c3_i e; + u3_utty* uty_u = c3_calloc(sizeof(u3_utty)); + if ( pip_l == FILE_TYPE_CHAR ) { + if ( 0 == (e = uv_tty_init(lup_u, &uty_u->pin_u.tty_u, 0, 0)) && + 0 == (e = uv_tty_init(lup_u, &uty_u->pop_u.tty_u, 1, 0)) ) + { + SetConsoleOutputCP(CP_UTF8); + uty_u->sta_f = _ttyf_set_raw; + uty_u->sto_f = _ttyf_set_normal; + uty_u->wsz_f = _ttyf_get_winsize; + } + } else { + if ( 0 == (e = uv_pipe_init(lup_u, &uty_u->pin_u.pop_u, 0)) && + 0 == (e = uv_pipe_init(lup_u, &uty_u->pop_u.pop_u, 0)) && + 0 == (e = uv_pipe_open(&uty_u->pin_u.pop_u, 0)) && + 0 == (e = uv_pipe_open(&uty_u->pop_u.pop_u, 1)) ) + { + fprintf(stderr, "vere: running interactive in a terminal emulator is in alpha\r\n" + " use -t to disable interactivity or use native Windows console\r\n") ; + uty_u->sta_f = _ttyf_nop; + uty_u->sto_f = _ttyf_nop; + uty_u->wsz_f = _ttyf_nop_winsize; + } + } + + if ( e ) { + *err_c = uv_strerror(e); + c3_free(uty_u); + return NULL; + } + + uty_u->fid_i = 1; + uty_u->hij_f = _ttyf_nop; + uty_u->loj_f = _ttyf_nop; + return uty_u; +} diff --git a/pkg/urbit/compat/mingw/termios.h b/pkg/urbit/compat/mingw/termios.h deleted file mode 100644 index 2fdb305a05..0000000000 --- a/pkg/urbit/compat/mingw/termios.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef _TERMIOS_H -#define _TERMIOS_H - -struct termios { - char _dummy; -}; - -#endif diff --git a/pkg/urbit/compat/posix/ptty.c b/pkg/urbit/compat/posix/ptty.c new file mode 100644 index 0000000000..d7d2af91ca --- /dev/null +++ b/pkg/urbit/compat/posix/ptty.c @@ -0,0 +1,189 @@ +/* compat/posix/ptty.c +** +*/ +#include "all.h" +#include "vere/vere.h" +#include +#include + +/* u3_ptty: POSIX terminal extension to u3_utty. +*/ +typedef struct { + u3_utty tty_u; // common tty structure + c3_i cug_i; // blocking fcntl flags + c3_i nob_i; // nonblocking fcntl flags + struct termios bak_u; // cooked terminal state + struct termios raw_u; // raw terminal state +} u3_ptty; + +/* _term_tcsetattr(): tcsetattr w/retry on EINTR. +*/ +static c3_i +_term_tcsetattr(c3_i fil_i, c3_i act_i, const struct termios* tms_u) +{ + c3_i ret_i = 0; + c3_w len_w = 0; + + do { + // abort pathological retry loop + // + if ( 100 == ++len_w ) { + fprintf(stderr, "term: tcsetattr loop: %s\r\n", strerror(errno)); + return -1; + } + ret_i = tcsetattr(fil_i, act_i, tms_u); + } while ( (-1 == ret_i) && (EINTR == errno) ); + + return ret_i; +} + +/* _ttyf_start_raw_input(): sets the tty to raw input. +*/ +static c3_o +_ttyf_start_raw_input(u3_utty* uty_u) +{ + u3_ptty* pty_u = (u3_ptty*)uty_u; + if ( 0 != _term_tcsetattr(uty_u->fid_i, TCSADRAIN, &pty_u->raw_u) ) { + return c3n; + } + if ( -1 == fcntl(uty_u->fid_i, F_SETFL, pty_u->nob_i) ) { + c3_assert(!"init-fcntl"); + } + return c3y; +} + +/* _ttyf_start_raw_input(): ends raw input on the tty. +*/ +static c3_o +_ttyf_end_raw_input(u3_utty* uty_u) +{ + u3_ptty* pty_u = (u3_ptty*)uty_u; + if ( 0 != _term_tcsetattr(uty_u->fid_i, TCSADRAIN, &pty_u->bak_u) ) { + return c3n; + } + if ( -1 == fcntl(uty_u->fid_i, F_SETFL, pty_u->cug_i) ) { + c3_assert(!"exit-fcntl"); + } + return c3y; +} + +/* _ttyf_hija(): hijacks the tty for cooked output. +*/ +static c3_o +_ttyf_hija(u3_utty* uty_u) +{ + u3_ptty* pty_u = (u3_ptty*)uty_u; + if ( 0 != _term_tcsetattr(1, TCSADRAIN, &pty_u->bak_u) ) { + perror("hija-tcsetattr-1"); + c3_assert(!"hija-tcsetattr"); + } + if ( -1 == fcntl(1, F_SETFL, pty_u->cug_i) ) { + perror("hija-fcntl-1"); + c3_assert(!"hija-fcntl"); + } + if ( 0 != _term_tcsetattr(0, TCSADRAIN, &pty_u->bak_u) ) { + perror("hija-tcsetattr-0"); + c3_assert(!"hija-tcsetattr"); + } + if ( -1 == fcntl(0, F_SETFL, pty_u->cug_i) ) { + perror("hija-fcntl-0"); + c3_assert(!"hija-fcntl"); + } + return c3y; +} + +/* _ttyf_loja(): releases the tty from cooked output. +*/ +static c3_o +_ttyf_loja(u3_utty* uty_u) +{ + u3_ptty* pty_u = (u3_ptty*)uty_u; + if ( 0 != _term_tcsetattr(1, TCSADRAIN, &pty_u->raw_u) ) { + perror("loja-tcsetattr-1"); + c3_assert(!"loja-tcsetattr"); + } + if ( -1 == fcntl(1, F_SETFL, pty_u->nob_i) ) { + perror("hija-fcntl-1"); + c3_assert(!"loja-fcntl"); + } + if ( 0 != _term_tcsetattr(0, TCSADRAIN, &pty_u->raw_u) ) { + perror("loja-tcsetattr-0"); + c3_assert(!"loja-tcsetattr"); + } + if ( -1 == fcntl(0, F_SETFL, pty_u->nob_i) ) { + perror("hija-fcntl-0"); + c3_assert(!"loja-fcntl"); + } + return c3y; +} + +/* _ttyf_get_winsize(): gets the tty window size. +*/ +static c3_o +_ttyf_get_winsize(u3_utty* uty_u, c3_l* col_l, c3_l* row_l) +{ + struct winsize siz_u; + if ( 0 == ioctl(uty_u->fid_i, TIOCGWINSZ, &siz_u) ) + { + *col_l = siz_u.ws_col; + *row_l = siz_u.ws_row; + return c3y; + } else { + return c3n; + } +} + +/* u3_ptty_init(): initialize platform-specific tty. +*/ +u3_utty* +u3_ptty_init(uv_loop_t* lup_u, const c3_c** err_c) +{ + u3_ptty* pty_u = c3_calloc(sizeof(u3_ptty)); + u3_utty* uty_u = &pty_u->tty_u; + + if ( !isatty(0) || !isatty(1) ) { + *err_c = "not a tty"; + c3_free(pty_u); + return NULL; + } + + uv_pipe_init(lup_u, &uty_u->pin_u.pop_u, 0); + uv_pipe_init(lup_u, &uty_u->pop_u.pop_u, 0); + uv_pipe_open(&uty_u->pin_u.pop_u, 0); + uv_pipe_open(&uty_u->pop_u.pop_u, 1); + + // Load old terminal state to restore. + // + { + if ( 0 != tcgetattr(uty_u->fid_i, &pty_u->bak_u) ) { + c3_assert(!"init-tcgetattr"); + } + if ( -1 == fcntl(uty_u->fid_i, F_GETFL, &pty_u->cug_i) ) { + c3_assert(!"init-fcntl"); + } + pty_u->cug_i &= ~O_NONBLOCK; // could fix? + pty_u->nob_i = pty_u->cug_i | O_NONBLOCK; // O_NDELAY on older unix + } + + // Construct raw termios configuration. + // + { + pty_u->raw_u = pty_u->bak_u; + + pty_u->raw_u.c_lflag &= ~(ECHO | ECHONL | ICANON | IEXTEN); + pty_u->raw_u.c_iflag &= ~(ICRNL | INPCK | ISTRIP); + pty_u->raw_u.c_cflag &= ~(CSIZE | PARENB); + pty_u->raw_u.c_cflag |= CS8; + pty_u->raw_u.c_oflag &= ~(OPOST); + pty_u->raw_u.c_cc[VMIN] = 0; + pty_u->raw_u.c_cc[VTIME] = 0; + } + + uty_u->fid_i = 1; + uty_u->sta_f = _ttyf_start_raw_input; + uty_u->sto_f = _ttyf_end_raw_input; + uty_u->hij_f = _ttyf_hija; + uty_u->loj_f = _ttyf_loja; + uty_u->wsz_f = _ttyf_get_winsize; + return uty_u; +} diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 00b486ace2..1c9f2921a2 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -114,7 +114,7 @@ popd defmacro U3_OS_TERM_dumb 1 # increase default thread stack size and link Windows implibs - ldextra="-Wl,--stack,67108864 -lbcrypt -lws2_32" + ldextra="-Wl,--stack,67108864 -lbcrypt -lntdll -lws2_32" # libcurl implibs ldextra="$ldextra -lzstd" # libuv implibs diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index e15ba9e4ad..c2dad6f1d2 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -2,7 +2,6 @@ */ #include -#include /** Quasi-tunable parameters. **/ @@ -223,22 +222,40 @@ }; #endif + /* u3_utty: unix tty. + */ + struct _u3_utty; + + /* u3_ustm: uv stream. + */ + typedef union _u3_ustm { + uv_pipe_t pop_u; + uv_tcp_t wax_u; + uv_tty_t tty_u; + } u3_ustm; + + /* u3_ttyf: simple unix tty function. + */ + typedef c3_o (*u3_ttyf)(struct _u3_utty* uty_u); + /* u3_utty: unix tty. */ typedef struct _u3_utty { - union { - uv_pipe_t pop_u; - uv_tcp_t wax_u; - }; + u3_ustm pin_u; // input stream + u3_ustm pop_u; // output stream struct _u3_utty* nex_u; // next in host list + u3_ttyf sta_f; // start tty + u3_ttyf sto_f; // clean up tty + u3_ttyf hij_f; // hijack tty for cooked print + u3_ttyf loj_f; // release tty from cooked print + c3_o (*wsz_f) + (struct _u3_utty* uty_u, + c3_l* col_l, + c3_l* row_l); // return tty window size c3_i fid_i; // file descriptor c3_w tid_l; // terminal identity number u3_utfo ufo_u; // terminfo strings - c3_i cug_i; // blocking fcntl flags - c3_i nob_i; // nonblocking fcntl flags u3_utat tat_u; // control state - struct termios bak_u; // cooked terminal state - struct termios raw_u; // raw terminal state struct _u3_auto* car_u; // driver hack } u3_utty; @@ -1104,6 +1121,11 @@ void u3_term_log_exit(void); + /* u3_ptty_init(): initialize platform-specific tty. + */ + u3_utty* + u3_ptty_init(uv_loop_t* lup_u, const c3_c** err_c); + /** Ames, packet networking. **/ diff --git a/pkg/urbit/vere/io/term.c b/pkg/urbit/vere/io/term.c index 9d2e0fdd38..b5b789cc55 100644 --- a/pkg/urbit/vere/io/term.c +++ b/pkg/urbit/vere/io/term.c @@ -4,6 +4,19 @@ #include "all.h" #include "vere/vere.h" +// macros for string literal args/buffers +// +// since (sizeof(s) - 1) is used for vector length, parameters +// must be appropriately typed. use with care! +// +#define TERM_LIT(s) sizeof(s) - 1, (const c3_y*)(s) +#define TERM_LIT_BUF(s) uv_buf_init(s, sizeof(s) - 1) + +static u3_utty* _term_main(); +static void _term_read_cb(uv_stream_t* tcp_u, + ssize_t siz_i, + const uv_buf_t* buf_u); + /* u3_write_fd(): retry interrupts, continue partial writes, assert errors. */ void @@ -46,23 +59,6 @@ u3_write_fd(c3_i fid_i, const void* buf_v, size_t len_i) } } -#if !defined(U3_OS_TERM_dumb) -#include - -// macros for string literal args/buffers -// -// since (sizeof(s) - 1) is used for vector length, parameters -// must be appropriately typed. use with care! -// -#define TERM_LIT(s) sizeof(s) - 1, (const c3_y*)(s) -#define TERM_LIT_BUF(s) uv_buf_init(s, sizeof(s) - 1) - -static u3_utty* _term_main(); -static void _term_read_cb(uv_stream_t* tcp_u, - ssize_t siz_i, - const uv_buf_t* buf_u); -static c3_i _term_tcsetattr(c3_i, c3_i, const struct termios*); - /* _term_msc_out_host(): unix microseconds from current host time. */ static c3_d @@ -121,36 +117,29 @@ _term_close_cb(uv_handle_t* han_t) c3_free(tty_u); } #endif -#endif // U3_OS_TERM_dumb /* u3_term_log_init(): initialize terminal for logging */ void u3_term_log_init(void) { - #if !defined(U3_OS_TERM_dumb) - u3_utty* uty_u = c3_calloc(sizeof(u3_utty)); + u3_utty* uty_u; if ( c3y == u3_Host.ops_u.tem ) { + uty_u = c3_calloc(sizeof(u3_utty)); uty_u->fid_i = 1; - uv_pipe_init(u3L, &(uty_u->pop_u), 0); - uv_pipe_open(&(uty_u->pop_u), uty_u->fid_i); + uv_pipe_init(u3L, &(uty_u->pop_u.pop_u), 0); + uv_pipe_open(&(uty_u->pop_u.pop_u), uty_u->fid_i); } else { // Initialize event processing. Rawdog it. // - { - uty_u->fid_i = 0; // stdin, yes we write to it... - - if ( !isatty(uty_u->fid_i) ) { - fprintf(stderr, "vere: unable to initialize terminal (not a tty)\r\n" - " use -t to disable interactivity\r\n"); - u3_king_bail(); - } - - uv_pipe_init(u3L, &(uty_u->pop_u), 0); - uv_pipe_open(&(uty_u->pop_u), uty_u->fid_i); + const c3_c* err_c; + if ( NULL == (uty_u = u3_ptty_init(u3L, &err_c)) ) { + fprintf(stderr, "vere: unable to initialize terminal (%s)\r\n" + " use -t to disable interactivity\r\n", err_c); + u3_king_bail(); } // configure output escape sequences @@ -183,33 +172,6 @@ u3_term_log_init(void) // uty_u->ufo_u.inn.kcub1_u = TERM_LIT_BUF("\033[D"); // terminfo reports "\033OD" // } - // Load old terminal state to restore. - // - { - if ( 0 != tcgetattr(uty_u->fid_i, &uty_u->bak_u) ) { - c3_assert(!"init-tcgetattr"); - } - if ( -1 == fcntl(uty_u->fid_i, F_GETFL, &uty_u->cug_i) ) { - c3_assert(!"init-fcntl"); - } - uty_u->cug_i &= ~O_NONBLOCK; // could fix? - uty_u->nob_i = uty_u->cug_i | O_NONBLOCK; // O_NDELAY on older unix - } - - // Construct raw termios configuration. - // - { - uty_u->raw_u = uty_u->bak_u; - - uty_u->raw_u.c_lflag &= ~(ECHO | ECHONL | ICANON | IEXTEN); - uty_u->raw_u.c_iflag &= ~(ICRNL | INPCK | ISTRIP); - uty_u->raw_u.c_cflag &= ~(CSIZE | PARENB); - uty_u->raw_u.c_cflag |= CS8; - uty_u->raw_u.c_oflag &= ~(OPOST); - uty_u->raw_u.c_cc[VMIN] = 0; - uty_u->raw_u.c_cc[VTIME] = 0; - } - // Initialize mirror and accumulator state. // { @@ -255,13 +217,8 @@ u3_term_log_init(void) if ( c3n == u3_Host.ops_u.tem ) { // Start raw input. // - { - if ( 0 != _term_tcsetattr(uty_u->fid_i, TCSADRAIN, &uty_u->raw_u) ) { - c3_assert(!"init-tcsetattr"); - } - if ( -1 == fcntl(uty_u->fid_i, F_SETFL, uty_u->nob_i) ) { - c3_assert(!"init-fcntl"); - } + if ( c3n == uty_u->sta_f(uty_u) ) { + c3_assert(!"init-tcsetattr"); } // initialize spinner timeout @@ -271,7 +228,6 @@ u3_term_log_init(void) uty_u->tat_u.sun_u.tim_u.data = uty_u; } } - #endif // U3_OS_TERM_dumb } /* u3_term_log_exit(): clean up terminal. @@ -279,49 +235,22 @@ u3_term_log_init(void) void u3_term_log_exit(void) { - #if !defined(U3_OS_TERM_dumb) if ( c3n == u3_Host.ops_u.tem ) { u3_utty* uty_u; for ( uty_u = u3_Host.uty_u; uty_u; uty_u = uty_u->nex_u ) { if ( uty_u->fid_i == -1 ) { continue; } - if ( 0 != _term_tcsetattr(uty_u->fid_i, TCSADRAIN, &uty_u->bak_u) ) { + if ( c3n == uty_u->sto_f(uty_u) ) { c3_assert(!"exit-tcsetattr"); } - if ( -1 == fcntl(uty_u->fid_i, F_SETFL, uty_u->cug_i) ) { - c3_assert(!"exit-fcntl"); - } u3_write_fd(uty_u->fid_i, "\r\n", 2); } } if ( u3_Host.uty_u ) { + uv_close((uv_handle_t*)&u3_Host.uty_u->pin_u, 0); uv_close((uv_handle_t*)&u3_Host.uty_u->pop_u, 0); } - #endif // U3_OS_TERM_dumb -} - -#if !defined(U3_OS_TERM_dumb) - -/* _term_tcsetattr(): tcsetattr w/retry on EINTR. -*/ -static c3_i -_term_tcsetattr(c3_i fil_i, c3_i act_i, const struct termios* tms_u) -{ - c3_i ret_i = 0; - c3_w len_w = 0; - - do { - // abort pathological retry loop - // - if ( 100 == ++len_w ) { - fprintf(stderr, "term: tcsetattr loop: %s\r\n", strerror(errno)); - return -1; - } - ret_i = tcsetattr(fil_i, act_i, tms_u); - } while ( (-1 == ret_i) && (EINTR == errno) ); - - return ret_i; } /* _term_it_write_cb(): general write callback. @@ -673,8 +602,6 @@ _term_it_save(u3_noun pax, u3_noun pad) c3_free(bas_c); } -#endif // U3_OS_TERM_dumb - /* _term_ovum_plan(): plan term ovums, configuring spinner. */ static u3_ovum* @@ -690,8 +617,6 @@ _term_ovum_plan(u3_auto* car_u, u3_noun wir, u3_noun cad) return egg_u; } -#if !defined(U3_OS_TERM_dumb) - /* _term_io_belt(): send belt. */ static void @@ -931,11 +856,7 @@ _term_spin_step(u3_utty* uty_u) // { uv_buf_t lef_u = uty_u->ufo_u.out.cub1_u; - c3_i fid_i; - - if ( uv_fileno((uv_handle_t*)&uty_u->pop_u, &fid_i) ) { - return; - } + c3_i fid_i = uty_u->fid_i; // One-time cursor backoff. // @@ -982,14 +903,11 @@ _term_spin_timer_cb(uv_timer_t* tim_u) #define _SPIN_RATE_US 250UL // spinner rate (ms/frame) #define _SPIN_IDLE_US 500UL // spinner cools down if stopped this long -#endif // U3_OS_TERM_dumb - /* u3_term_start_spinner(): prepare spinner state. RETAIN. */ void u3_term_start_spinner(u3_atom say, c3_o del_o) { - #if !defined(U3_OS_TERM_dumb) if ( c3n == u3_Host.ops_u.tem ) { u3_utty* uty_u = _term_main(); u3_utat* tat_u = &uty_u->tat_u; @@ -1013,7 +931,6 @@ u3_term_start_spinner(u3_atom say, c3_o del_o) wen_d, _SPIN_RATE_US); } } - #endif // U3_OS_TERM_dumb } /* u3_term_stop_spinner(): reset spinner state and restore input line. @@ -1021,7 +938,6 @@ u3_term_start_spinner(u3_atom say, c3_o del_o) void u3_term_stop_spinner(void) { - #if !defined(U3_OS_TERM_dumb) if ( c3n == u3_Host.ops_u.tem ) { u3_utty* uty_u = _term_main(); u3_utat* tat_u = &uty_u->tat_u; @@ -1039,11 +955,8 @@ u3_term_stop_spinner(void) tat_u->sun_u.end_d = 0; } } - #endif // U3_OS_TERM_dumb } -#if !defined(U3_OS_TERM_dumb) - /* _term_main(): return main or console terminal. */ static u3_utty* @@ -1076,25 +989,17 @@ _term_ef_get(c3_l tid_l) return _term_main(); } -#endif // U3_OS_TERM_dumb - /* u3_term_get_blew(): return window size [columns rows]. */ u3_noun u3_term_get_blew(c3_l tid_l) { - #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = _term_ef_get(tid_l); c3_l col_l, row_l; - struct winsize siz_u; - if ( (c3n == u3_Host.ops_u.tem) && - uty_u && (0 == ioctl(uty_u->fid_i, TIOCGWINSZ, &siz_u)) ) + if ( (c3y == u3_Host.ops_u.tem) || !uty_u || + (c3y != uty_u->wsz_f(uty_u, &col_l, &row_l)) ) { - col_l = siz_u.ws_col; - row_l = siz_u.ws_row; - } - else { col_l = 80; row_l = 24; } @@ -1105,9 +1010,6 @@ u3_term_get_blew(c3_l tid_l) } return u3nc(col_l, row_l); - #else - return u3nc(80, 24); - #endif // U3_OS_TERM_dumb } /* u3_term_ef_winc(): window change. Just console right now. @@ -1115,7 +1017,6 @@ u3_term_get_blew(c3_l tid_l) void u3_term_ef_winc(void) { - #if !defined(U3_OS_TERM_dumb) // XX groace, this should be a global handler sent to each pier // if ( u3_Host.uty_u->car_u ) { @@ -1126,7 +1027,6 @@ u3_term_ef_winc(void) _term_ovum_plan(u3_Host.uty_u->car_u, wir, cad); } - #endif } /* u3_term_ef_ctlc(): send ^C on console. @@ -1134,7 +1034,6 @@ u3_term_ef_winc(void) void u3_term_ef_ctlc(void) { - #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = _term_main(); { @@ -1148,11 +1047,8 @@ u3_term_ef_ctlc(void) } _term_it_refresh_line(uty_u); - #endif } -#if !defined(U3_OS_TERM_dumb) - /* _term_it_put_value(): put numeric color value on lin_w. */ static c3_w @@ -1469,14 +1365,11 @@ _term_ef_blit(u3_utty* uty_u, u3z(blt); } -#endif // U3_OS_TERM_dumb - /* u3_term_io_hija(): hijack console for fprintf, returning FILE*. */ FILE* u3_term_io_hija(void) { - #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = _term_main(); if ( uty_u ) { @@ -1488,22 +1381,9 @@ u3_term_io_hija(void) } else { if ( c3n == u3_Host.ops_u.tem ) { - if ( 0 != _term_tcsetattr(1, TCSADRAIN, &uty_u->bak_u) ) { - perror("hija-tcsetattr-1"); + if ( c3y != uty_u->hij_f(uty_u) ) { c3_assert(!"hija-tcsetattr"); } - if ( -1 == fcntl(1, F_SETFL, uty_u->cug_i) ) { - perror("hija-fcntl-1"); - c3_assert(!"hija-fcntl"); - } - if ( 0 != _term_tcsetattr(0, TCSADRAIN, &uty_u->bak_u) ) { - perror("hija-tcsetattr-0"); - c3_assert(!"hija-tcsetattr"); - } - if ( -1 == fcntl(0, F_SETFL, uty_u->cug_i) ) { - perror("hija-fcntl-0"); - c3_assert(!"hija-fcntl"); - } u3_write_fd(uty_u->fid_i, "\r", 1); { uv_buf_t* buf_u = &uty_u->ufo_u.out.el_u; @@ -1512,7 +1392,6 @@ u3_term_io_hija(void) } } } - #endif // U3_OS_TERM_dumb return stdout; } @@ -1521,7 +1400,6 @@ u3_term_io_hija(void) void u3_term_io_loja(int x) { - #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = _term_main(); if ( uty_u ) { @@ -1534,31 +1412,14 @@ u3_term_io_loja(int x) else { if ( c3y == u3_Host.ops_u.tem ) { fflush(stdout); - } - else { - if ( 0 != _term_tcsetattr(1, TCSADRAIN, &uty_u->raw_u) ) { - perror("loja-tcsetattr-1"); + } else { + if ( c3y != uty_u->loj_f(uty_u) ) { c3_assert(!"loja-tcsetattr"); } - if ( -1 == fcntl(1, F_SETFL, uty_u->nob_i) ) { - perror("hija-fcntl-1"); - c3_assert(!"loja-fcntl"); - } - if ( 0 != _term_tcsetattr(0, TCSADRAIN, &uty_u->raw_u) ) { - perror("loja-tcsetattr-0"); - c3_assert(!"loja-tcsetattr"); - } - if ( -1 == fcntl(0, F_SETFL, uty_u->nob_i) ) { - perror("hija-fcntl-0"); - c3_assert(!"loja-fcntl"); - } _term_it_refresh_line(uty_u); } } } - #else - fflush(stdout); - #endif // U3_OS_TERM_dumb } /* u3_term_it_log(): writes a log message @@ -1570,8 +1431,6 @@ u3_term_io_log(c3_c* line) u3_term_io_loja(fprintf(stream, "%s", line)); } -#if !defined(U3_OS_TERM_dumb) - /* u3_term_tape_to(): dump a tape to a file. */ void @@ -1625,22 +1484,18 @@ u3_term_wall(u3_noun wol) u3z(wol); } -#endif // U3_OS_TERM_dumb - /* _term_io_talk(): */ static void _term_io_talk(u3_auto* car_u) { - #if !defined(U3_OS_TERM_dumb) if ( c3n == u3_Host.ops_u.tem ) { u3_utty* uty_u = _term_main(); - uv_read_start((uv_stream_t*)&(uty_u->pop_u), + uv_read_start((uv_stream_t*)&(uty_u->pin_u), _term_alloc, _term_read_cb); } - #endif // XX groace hardcoded terminal number // @@ -1728,7 +1583,6 @@ _term_io_kick(u3_auto* car_u, u3_noun wir, u3_noun cad) case c3__blit: { ret_o = c3y; - #if !defined(U3_OS_TERM_dumb) { u3_utty* uty_u = _term_ef_get(tid_l); if ( 0 == uty_u ) { @@ -1744,7 +1598,6 @@ _term_io_kick(u3_auto* car_u, u3_noun wir, u3_noun cad) } } } - #endif } break; // XX obsolete %ames @@ -1798,14 +1651,13 @@ _term_io_exit_cb(uv_handle_t* han_u) static void _term_io_exit(u3_auto* car_u) { - #if !defined(U3_OS_TERM_dumb) u3_utty* uty_u = _term_main(); - // NB, closed in u3_term_log_exit() - // - uv_read_stop((uv_stream_t*)&(uty_u->pop_u)); - if ( c3n == u3_Host.ops_u.tem ) { + // NB, closed in u3_term_log_exit() + // + uv_read_stop((uv_stream_t*)&(uty_u->pin_u)); + uv_timer_t* han_u = &(uty_u->tat_u.sun_u.tim_u); han_u->data = car_u; @@ -1814,9 +1666,6 @@ _term_io_exit(u3_auto* car_u) else { c3_free(car_u); } - #else - c3_free(car_u); - #endif } /* u3_term_io_init(): initialize terminal @@ -1826,10 +1675,8 @@ u3_term_io_init(u3_pier* pir_u) { u3_auto* car_u = c3_calloc(sizeof(*car_u)); - #if !defined(U3_OS_TERM_dumb) c3_assert( u3_Host.uty_u ); u3_Host.uty_u->car_u = car_u; - #endif car_u->nam_m = c3__term; car_u->liv_o = c3y; From 56b62742dbf4791992255e4c365912734b21d4ce Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Wed, 31 Mar 2021 10:52:03 +0000 Subject: [PATCH 040/136] vere: terminal compatibility --- pkg/urbit/configure | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 1c9f2921a2..54649bb228 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -111,7 +111,6 @@ popd [ -e include/ivory.h ] || xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h defmacro U3_OS_mingw 1 - defmacro U3_OS_TERM_dumb 1 # increase default thread stack size and link Windows implibs ldextra="-Wl,--stack,67108864 -lbcrypt -lntdll -lws2_32" From 24fde4b0d5642e8f72bb1a10959f6ef4dff2b259 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Wed, 31 Mar 2021 11:28:23 +0000 Subject: [PATCH 041/136] vere: allow enable/disable optimizations from ./configure --- pkg/urbit/configure | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 54649bb228..60a3cf508e 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -37,6 +37,24 @@ xxd () { defmacro URBIT_VERSION "\"$URBIT_VERSION\"" +while test $# != 0 +do + case $1 in + --enable-debug) + CFLAGS="${CFLAGS-} -O0 -g" + MEMORY_DEBUG=1 + CPU_DEBUG=1 + ;; + --disable-debug) + CFLAGS="${CFLAGS-} -O3 -g" + ;; + *) + echo "unrecognized option: $1" + ;; + esac + shift +done + [ -n "${MEMORY_DEBUG-}" ] && defmacro U3_MEMORY_DEBUG 1 [ -n "${MEMORY_LOG-}" ] && defmacro U3_MEMORY_LOG 1 [ -n "${CPU_DEBUG-}" ] && defmacro U3_CPU_DEBUG 1 @@ -120,8 +138,8 @@ popd ldextra="$ldextra -luserenv -liphlpapi -lpsapi" vcompat=mingw CFLAGS="${CFLAGS-} ${cdirs[@]} -DCURL_STATICLIB -DH2O_NO_UNIX_SOCKETS -Icompat/mingw" - LDFLAGS="${LDFLAGS-} -static ${ldirs[@]}" - PKG_CONFIG="echo" + LDFLAGS="${LDFLAGS-} ${ldirs[@]} -static" + PKG_CONFIG=">/dev/null echo" ;; *linux*) defmacro U3_OS_linux 1 From 06faf0b13cfc931d8851d1e9fb94e438107bf136 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 31 Mar 2021 14:54:28 -0400 Subject: [PATCH 042/136] gall: print on missing request for ack --- pkg/arvo/sys/vane/gall.hoon | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/arvo/sys/vane/gall.hoon b/pkg/arvo/sys/vane/gall.hoon index a481307874..690fa615eb 100644 --- a/pkg/arvo/sys/vane/gall.hoon +++ b/pkg/arvo/sys/vane/gall.hoon @@ -646,6 +646,7 @@ :: cleared queue in +load 3-to-4 or +load-4-to-5 :: =? stand ?=(~ stand) + ~& [%gall-missing wire hen] (~(put to *(qeu remote-request)) %missing) ~| [full-wire=full-wire hen=hen stand=stand] =^ rr stand ~(get to stand) From b4a768c50637eb0f6eab9a13449ae94f720b72b0 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 1 Apr 2021 13:14:02 +0000 Subject: [PATCH 043/136] vere: more file portability --- pkg/urbit/compat/mingw/compat.c | 9 +++++++++ pkg/urbit/include/c/portable.h | 10 ++++++++++ pkg/urbit/worker/main.c | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/pkg/urbit/compat/mingw/compat.c b/pkg/urbit/compat/mingw/compat.c index 3ab9029b67..4cd1eb3863 100644 --- a/pkg/urbit/compat/mingw/compat.c +++ b/pkg/urbit/compat/mingw/compat.c @@ -8,6 +8,15 @@ #undef _fmode int _fmode = _O_BINARY; +// set standard I/O fds to binary too, because +// MSVCRT creates them before MingW sets _fmode +static void __attribute__ ((constructor)) _set_stdio_to_binary() +{ + _setmode(0, _O_BINARY); + _setmode(1, _O_BINARY); + _setmode(2, _O_BINARY); +} + // from https://github.com/git/git/blob/master/compat/mingw.c // ----------------------------------------------------------------------- diff --git a/pkg/urbit/include/c/portable.h b/pkg/urbit/include/c/portable.h index 5c675a926d..71bb7e5dd9 100644 --- a/pkg/urbit/include/c/portable.h +++ b/pkg/urbit/include/c/portable.h @@ -212,6 +212,16 @@ # define lseek64 lseek # else # error "port: timeconvert" +# endif + + /* Null. + */ +# if defined(U3_OS_linux) || defined(U3_OS_bsd) || defined(U3_OS_osx) +# define c3_dev_null "/dev/null" +# elif defined(U3_OS_mingw) +# define c3_dev_null "nul" +# else +# error "port: /dev/null" # endif /* Static assertion. diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index 8c4001d8d8..d31ea8e116 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -130,7 +130,7 @@ _cw_serf_stdio(c3_i* inn_i, c3_i* out_i) // we replace [FD 0] (stdin) with a fd pointing to /dev/null // we replace [FD 1] (stdout) with a dup of [FD 2] (stderr) // - c3_i nul_i = open("/dev/null", O_RDWR, 0); + c3_i nul_i = open(c3_dev_null, O_RDWR, 0); *inn_i = dup(0); *out_i = dup(1); From 6ab51c7bafa3633d20145cfd17b8c908afae2e83 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 1 Apr 2021 13:15:26 +0000 Subject: [PATCH 044/136] vere: fix no-tty mode startup --- pkg/urbit/vere/io/term.c | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/urbit/vere/io/term.c b/pkg/urbit/vere/io/term.c index b5b789cc55..d757ecc70d 100644 --- a/pkg/urbit/vere/io/term.c +++ b/pkg/urbit/vere/io/term.c @@ -129,6 +129,7 @@ u3_term_log_init(void) uty_u = c3_calloc(sizeof(u3_utty)); uty_u->fid_i = 1; + uv_pipe_init(u3L, &(uty_u->pin_u.pop_u), 0); uv_pipe_init(u3L, &(uty_u->pop_u.pop_u), 0); uv_pipe_open(&(uty_u->pop_u.pop_u), uty_u->fid_i); } From 33d1bf5d7deb0ec92a9b18f704f2321150e51fbc Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 1 Apr 2021 13:18:38 +0000 Subject: [PATCH 045/136] vere: pipe serf stderr back to lord When uv_spawn creates a process and any standard fds are set to UV_INHERIT_FD, it ignores UV_PROCESS_WINDOWS_HIDE flags, and worker console pops up in -d mode. --- pkg/urbit/include/vere/vere.h | 1 + pkg/urbit/vere/lord.c | 47 +++++++++++++++++++++++++++++++++-- pkg/urbit/worker/main.c | 5 ++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index c2dad6f1d2..69f8e0de19 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -486,6 +486,7 @@ time_t wen_t; // process creation time u3_mojo inn_u; // client's stdin u3_moat out_u; // client's stdout + uv_pipe_t err_u; // client's stderr c3_w wag_w; // config flags c3_c* bin_c; // binary path c3_c* pax_c; // directory diff --git a/pkg/urbit/vere/lord.c b/pkg/urbit/vere/lord.c index 501427d3d7..e8fb65d8a8 100644 --- a/pkg/urbit/vere/lord.c +++ b/pkg/urbit/vere/lord.c @@ -141,6 +141,8 @@ _lord_stop(u3_lord* god_u) u3_newt_moat_stop(&god_u->out_u, _lord_stop_cb); u3_newt_mojo_stop(&god_u->inn_u, _lord_bail_noop); + uv_read_stop((uv_stream_t*)&(god_u->err_u)); + uv_close((uv_handle_t*)&god_u->cub_u, 0); #if defined(LORD_TRACE_JAM) || defined(LORD_TRACE_CUE) @@ -1028,6 +1030,42 @@ u3_lord_halt(u3_lord* god_u) _lord_stop(god_u); } +/* _lord_serf_err_alloc(): libuv buffer allocator. +*/ +static void +_lord_serf_err_alloc(uv_handle_t* had_u, size_t len_i, uv_buf_t* buf) +{ + // error/info messages as a rule don't exceed one line + // + *buf = uv_buf_init(c3_malloc(80), 80); +} + +/* _lord_on_serf_err_cb(): subprocess stderr callback. +*/ +static void +_lord_on_serf_err_cb(uv_stream_t* pyp_u, + ssize_t siz_i, + const uv_buf_t* buf_u) +{ + if ( siz_i >= 0 ) { + // serf used to write to 2 directly + // this can't be any worse than that + // + u3_write_fd(2, buf_u->base, siz_i); + } else { + uv_read_stop(pyp_u); + + if ( siz_i != UV_EOF ) { + u3l_log("lord: serf stderr: %s\r\n", uv_strerror(siz_i)); + } + } + + if ( buf_u->base != NULL ) { + c3_free(buf_u->base); + } +} + + /* _lord_on_serf_exit(): handle subprocess exit. */ static void @@ -1140,6 +1178,7 @@ u3_lord_init(c3_c* pax_c, c3_w wag_w, c3_d key_d[4], u3_lord_cb cb_u) uv_pipe_init(u3L, &god_u->inn_u.pyp_u, 0); uv_timer_init(u3L, &god_u->out_u.tim_u); uv_pipe_init(u3L, &god_u->out_u.pyp_u, 0); + uv_pipe_init(u3L, &god_u->err_u, 0); god_u->cod_u[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE; god_u->cod_u[0].data.stream = (uv_stream_t *)&god_u->inn_u; @@ -1147,12 +1186,14 @@ u3_lord_init(c3_c* pax_c, c3_w wag_w, c3_d key_d[4], u3_lord_cb cb_u) god_u->cod_u[1].flags = UV_CREATE_PIPE | UV_WRITABLE_PIPE; god_u->cod_u[1].data.stream = (uv_stream_t *)&god_u->out_u; - god_u->cod_u[2].flags = UV_INHERIT_FD; - god_u->cod_u[2].data.fd = 2; + god_u->cod_u[2].flags = UV_CREATE_PIPE | UV_WRITABLE_PIPE; + god_u->cod_u[2].data.stream = (uv_stream_t *)&god_u->err_u; god_u->ops_u.stdio = god_u->cod_u; god_u->ops_u.stdio_count = 3; + // if any fds are inherited, libuv ignores UV_PROCESS_WINDOWS_HIDE* + god_u->ops_u.flags = UV_PROCESS_WINDOWS_HIDE; god_u->ops_u.exit_cb = _lord_on_serf_exit; god_u->ops_u.file = arg_c[0]; god_u->ops_u.args = arg_c; @@ -1162,6 +1203,8 @@ u3_lord_init(c3_c* pax_c, c3_w wag_w, c3_d key_d[4], u3_lord_cb cb_u) return 0; } + + uv_read_start((uv_stream_t *)&god_u->err_u, _lord_serf_err_alloc, _lord_on_serf_err_cb); } #if defined(LORD_TRACE_JAM) || defined(LORD_TRACE_CUE) diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index d31ea8e116..461aa9d5cf 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -139,6 +139,11 @@ _cw_serf_stdio(c3_i* inn_i, c3_i* out_i) dup2(2, 1); close(nul_i); + + // set stream I/O to unbuffered because it's now a pipe not a console + // + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); } /* _cw_serf_stdio(): cleanup on serf exit. From 7d031f4ca7fc9320611cfbde55a8f6f2905f2b3b Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 1 Apr 2021 13:22:35 +0000 Subject: [PATCH 046/136] vere: mingw: terminal compatibility --- nix/sources-mingw.json | 17 +++++++++++++++++ pkg/urbit/compat/mingw/uv.patch | 17 +++++++++++++++++ pkg/urbit/vere/io/term.c | 13 ++++++++++++- 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 pkg/urbit/compat/mingw/uv.patch diff --git a/nix/sources-mingw.json b/nix/sources-mingw.json index 39ade85a56..ae3a2d59f0 100644 --- a/nix/sources-mingw.json +++ b/nix/sources-mingw.json @@ -48,6 +48,23 @@ "url": "https://github.com/bitcoin-core/secp256k1/archive/26de4dfeb1f1436dae1fcf17f57bdaa43540f940.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, + "uv": { + "branch": "v1.x", + "description": "Cross-platform asynchronous I/O", + "homepage": "http://libuv.org/", + "mingw": { + "include": "include", + "lib": ".libs", + "prepare": "./autogen.sh && ./configure --disable-shared", + "make": "libuv.la" + }, + "owner": "libuv", + "repo": "libuv", + "rev": "v1.40.0", + "type": "tarball", + "url": "https://github.com/libuv/libuv/archive/v1.40.0.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, "ent": { "mingw": { "prepare": "./configure" diff --git a/pkg/urbit/compat/mingw/uv.patch b/pkg/urbit/compat/mingw/uv.patch new file mode 100644 index 0000000000..b64cfd4739 --- /dev/null +++ b/pkg/urbit/compat/mingw/uv.patch @@ -0,0 +1,17 @@ +diff --git a/src/win/pipe.c b/src/win/pipe.c +index 0f2bb869b..f81245ec6 100644 +--- a/src/win/pipe.c ++++ b/src/win/pipe.c +@@ -270,6 +270,12 @@ static int uv_set_pipe_handle(uv_loop_t* loop, + + if (!SetNamedPipeHandleState(pipeHandle, &mode, NULL, NULL)) { + err = GetLastError(); ++ if ((err == ERROR_INVALID_FUNCTION || err == ERROR_INVALID_PARAMETER) && (duplex_flags & UV_HANDLE_WRITABLE)) { ++ /* ++ * it's not a pipe, but simple writes should be fine ++ * let's trust callers to know what they're doing ++ */ ++ } else + if (err == ERROR_ACCESS_DENIED) { + /* + * SetNamedPipeHandleState can fail if the handle doesn't have either diff --git a/pkg/urbit/vere/io/term.c b/pkg/urbit/vere/io/term.c index d757ecc70d..bfd6d2fc48 100644 --- a/pkg/urbit/vere/io/term.c +++ b/pkg/urbit/vere/io/term.c @@ -213,6 +213,17 @@ u3_term_log_init(void) u3_Host.uty_u = uty_u; } + // Disable I/O buffering on terminal streams. + // This is not necessary if output is a tty, + // but helps when output is redirected. + // + { + fflush(stdout); + fflush(stderr); + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + } + // if terminal/tty is enabled // if ( c3n == u3_Host.ops_u.tem ) { @@ -711,7 +722,7 @@ _term_io_suck_char(u3_utty* uty_u, c3_y cay_y) else if ( 8 == cay_y || 127 == cay_y ) { _term_io_belt(uty_u, u3nc(c3__bac, u3_nul)); } - else if ( 13 == cay_y ) { + else if ( 13 == cay_y || 10 == cay_y ) { _term_io_belt(uty_u, u3nc(c3__ret, u3_nul)); } #if 0 From 578841d527e6b0f3955dc4d58f88a8f0b0c0234c Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 1 Apr 2021 15:19:10 +0000 Subject: [PATCH 047/136] compat: mingw: implement utimes --- pkg/urbit/compat/mingw/compat.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/compat/mingw/compat.c b/pkg/urbit/compat/mingw/compat.c index 4cd1eb3863..10e5305862 100644 --- a/pkg/urbit/compat/mingw/compat.c +++ b/pkg/urbit/compat/mingw/compat.c @@ -418,8 +418,8 @@ int mprotect (void *addr, size_t len, int prot) int utimes(const char *path, const struct timeval times[2]) { - // TODO: implement in terms of utime - return -1; + struct _utimbuf utb = {.actime = times[0].tv_sec, .modtime = times[1].tv_sec}; + return _utime(path, &utb); } int fdatasync(int fildes) From 24d22e9e1101247491c2074d473fa1aed832c6db Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 1 Apr 2021 15:21:37 +0000 Subject: [PATCH 048/136] vere: mingw: basic daemon mode support --- pkg/urbit/compat/mingw/ptty.c | 2 +- pkg/urbit/daemon/main.c | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pkg/urbit/compat/mingw/ptty.c b/pkg/urbit/compat/mingw/ptty.c index a65d9370d6..063db564c5 100644 --- a/pkg/urbit/compat/mingw/ptty.c +++ b/pkg/urbit/compat/mingw/ptty.c @@ -131,7 +131,7 @@ u3_ptty_init(uv_loop_t* lup_u, const c3_c** err_c) 0 == (e = uv_pipe_open(&uty_u->pin_u.pop_u, 0)) && 0 == (e = uv_pipe_open(&uty_u->pop_u.pop_u, 1)) ) { - fprintf(stderr, "vere: running interactive in a terminal emulator is in alpha\r\n" + fprintf(stderr, "vere: running interactive in a terminal emulator is experimental\r\n" " use -t to disable interactivity or use native Windows console\r\n") ; uty_u->sta_f = _ttyf_nop; uty_u->sto_f = _ttyf_nop; diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 6cea5c4276..323383b53b 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -569,6 +569,17 @@ static void _on_boot_completed_cb() { _child_process_booted_signal_fd = -1; } +/* _fix_std_handle(): replaces given stdio fd with a handle to null. +*/ +#if defined(U3_OS_mingw) +static void +_fix_std_handle(c3_i fid_i, DWORD typ_u) +{ + dup2(open(c3_dev_null, O_RDWR, 0), fid_i); // this closes the system-provided handle + SetStdHandle(typ_u, (HANDLE)_get_osfhandle(fid_i)); +} +#endif + /* In daemon mode, run the urbit as a background process, but don't exit from the parent process until the ship is finished booting. @@ -596,8 +607,13 @@ static void _fork_into_background_process() { #if defined(U3_OS_mingw) - fprintf(stderr, "Daemon mode is not yet supported on MingW\r\n"); - exit(1); + // detect stdout/stderr redirection + DWORD dum_u; + BOOL do1_u = GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &dum_u); + BOOL do2_u = GetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), &dum_u); + FreeConsole(); + if ( do1_u ) _fix_std_handle(1, STD_OUTPUT_HANDLE); + if ( do2_u ) _fix_std_handle(2, STD_ERROR_HANDLE); #else c3_i pipefd[2]; From 755d73bc3a5cc6ae869a393836f6c54b50a0aa88 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 1 Apr 2021 15:22:04 +0000 Subject: [PATCH 049/136] vere: mingw: support invocation w/o .exe suffix --- pkg/urbit/daemon/main.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 323383b53b..77e038439b 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -666,11 +666,15 @@ main(c3_i argc, } // Set `u3_Host.wrk_c` to the worker executable path. - c3_i worker_exe_len = 1 + strlen(argv[0]) + strlen("-worker"); + c3_i urbit_exe_len = strlen(argv[0]); + c3_i worker_exe_len = 1 + urbit_exe_len + strlen("-worker"); u3_Host.wrk_c = c3_malloc(worker_exe_len); #if defined(U3_OS_mingw) - strcpy(u3_Host.wrk_c, argv[0]); - strcpy(u3_Host.wrk_c + strlen(argv[0]) - 4, "-worker.exe"); + if ( urbit_exe_len >= 4 && !strcmp(argv[0] + urbit_exe_len - 4, ".exe")) { + snprintf(u3_Host.wrk_c, worker_exe_len, "%.*s-worker.exe", urbit_exe_len - 4, argv[0]); + } else { + snprintf(u3_Host.wrk_c, worker_exe_len, "%s-worker", argv[0]); + } #else snprintf(u3_Host.wrk_c, worker_exe_len, "%s-worker", argv[0]); #endif From d7f48903a439dcb34d81ee42c70067602c91ba99 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Fri, 2 Apr 2021 19:52:27 +0000 Subject: [PATCH 050/136] vere: portability fixes after urbit-v1.3 merge --- pkg/urbit/compat/mingw/compat.c | 7 +++++++ pkg/urbit/daemon/main.c | 2 +- pkg/urbit/vere/db/lmdb.c | 24 ++++++++++++++++++------ pkg/urbit/vere/io/http.c | 2 +- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pkg/urbit/compat/mingw/compat.c b/pkg/urbit/compat/mingw/compat.c index 10e5305862..f9b229cf7e 100644 --- a/pkg/urbit/compat/mingw/compat.c +++ b/pkg/urbit/compat/mingw/compat.c @@ -444,6 +444,13 @@ int fdatasync(int fildes) } } +intmax_t mdb_get_filesize(HANDLE han_u) +{ + LARGE_INTEGER li; + GetFileSizeEx(han_u, &li); + return li.QuadPart; +} + char *realpath(const char *path, char *resolved_path) { // TODO diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 5648a82fba..42d7857434 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -298,7 +298,7 @@ _main_getopt(c3_i argc, c3_c** argv) } struct sockaddr_in t; - if ( u3_Host.ops_u.bin_c != 0 && inet_aton(u3_Host.ops_u.bin_c, &t.sin_addr) == 0 ) { + if ( u3_Host.ops_u.bin_c != 0 && inet_pton(AF_INET, u3_Host.ops_u.bin_c, &t.sin_addr) == 0 ) { fprintf(stderr, "-b invalid IP address\n"); return c3n; } diff --git a/pkg/urbit/vere/db/lmdb.c b/pkg/urbit/vere/db/lmdb.c index f75e8ee321..a6acbba6d5 100644 --- a/pkg/urbit/vere/db/lmdb.c +++ b/pkg/urbit/vere/db/lmdb.c @@ -10,6 +10,10 @@ */ void mdb_logerror(FILE* f, int err, const char* fmt, ...); +/* mdb_get_filesize(): gets the size of a lmdb database file on disk. +*/ +intmax_t mdb_get_filesize(mdb_filehandle_t han_u); + // lmdb api wrapper // // this module implements a simple persistence api on top of lmdb. @@ -105,18 +109,17 @@ u3_lmdb_stat(MDB_env* env_u, FILE* fil_u) } { - c3_i fid_i; - struct stat sat_u; + mdb_filehandle_t han_u; + mdb_env_get_fd(env_u, &han_u); - mdb_env_get_fd(env_u, &fid_i); - fstat(fid_i, &sat_u); + intmax_t dis_i = mdb_get_filesize(han_u); - if ( siz_i != sat_u.st_size ) { + if ( siz_i != dis_i ) { fprintf(fil_u, "MISMATCH:\n"); } fprintf(fil_u, " file size (page): %zu\n", siz_i); - fprintf(fil_u, " file size (stat): %jd\n", (intmax_t)sat_u.st_size); + fprintf(fil_u, " file size (stat): %jd\n", dis_i); } } @@ -504,4 +507,13 @@ void mdb_logerror(FILE* f, int err, const char* fmt, ...) va_end(ap); fprintf(f, ": %s\r\n", mdb_strerror(err)); } + +/* mdb_get_filesize(): gets the size of a lmdb database file on disk. +*/ +intmax_t mdb_get_filesize(mdb_filehandle_t han_u) +{ + struct stat sat_u; + fstat(han_u, &sat_u); + return (intmax_t)sat_u.st_size; +} #endif diff --git a/pkg/urbit/vere/io/http.c b/pkg/urbit/vere/io/http.c index a9a8f4b8ae..f0d2f0ea23 100644 --- a/pkg/urbit/vere/io/http.c +++ b/pkg/urbit/vere/io/http.c @@ -1459,7 +1459,7 @@ _http_serv_start(u3_http* htp_u) INADDR_ANY; if ( 0 != u3_Host.ops_u.bin_c && c3n == htp_u->lop ) { - inet_aton(u3_Host.ops_u.bin_c, &adr_u.sin_addr); + inet_pton(AF_INET, u3_Host.ops_u.bin_c, &adr_u.sin_addr); } uv_tcp_init(u3L, &htp_u->wax_u); From 1ec5e5acfb5dc733886f0241558ffcd7456a07df Mon Sep 17 00:00:00 2001 From: fang Date: Wed, 7 Apr 2021 21:14:13 +0200 Subject: [PATCH 051/136] hoon: add +stap, path parser And cleans up a bunch of locally hand-written implementations of it. --- pkg/arvo/lib/language-server/parser.hoon | 4 ++-- pkg/arvo/sys/hoon.hoon | 5 ++--- pkg/arvo/sys/vane/clay.hoon | 4 ++-- pkg/arvo/sys/vane/eyre.hoon | 2 +- pkg/arvo/sys/zuse.hoon | 2 +- 5 files changed, 8 insertions(+), 9 deletions(-) diff --git a/pkg/arvo/lib/language-server/parser.hoon b/pkg/arvo/lib/language-server/parser.hoon index d4f5002fb6..58e4b91b7b 100644 --- a/pkg/arvo/lib/language-server/parser.hoon +++ b/pkg/arvo/lib/language-server/parser.hoon @@ -21,7 +21,7 @@ (most ;~(plug com gaw) taut-rule) :: %+ rune tis - ;~(plug sym ;~(pfix gap fas (more fas urs:ab))) + ;~(plug sym ;~(pfix gap stap)) :: %+ rune cen ;~(plug sym ;~(pfix gap ;~(pfix cen sym))) @@ -37,7 +37,7 @@ ;~ (glue gap) sym ;~(pfix cen sym) - ;~(pfix fas (more fas urs:ab)) + stap == :: %+ stag %tssg diff --git a/pkg/arvo/sys/hoon.hoon b/pkg/arvo/sys/hoon.hoon index 834cbcf390..d40777ca7a 100644 --- a/pkg/arvo/sys/hoon.hoon +++ b/pkg/arvo/sys/hoon.hoon @@ -5965,9 +5965,8 @@ :: ++ spat |=(pax=path (crip (spud pax))) :: render path to cord ++ spud |=(pax=path ~(ram re (smyt pax))) :: render path to tape -++ stab :: parse cord to path - =+ fel=;~(pfix fas (more fas urs:ab)) - |=(zep=@t `path`(rash zep fel)) +++ stab |=(zep=@t `path`(rash zep stap)) :: parse cord to path +++ stap ;~(pfix fas (more fas urs:ab)) :: path parser :: :::: 4n: virtualization :: diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index af2f653037..5458e0026f 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -869,7 +869,7 @@ (most ;~(plug com gaw) taut-rule) :: %+ rune tis - ;~(plug sym ;~(pfix gap fas (more fas urs:ab))) + ;~(plug sym ;~(pfix gap stap)) :: %+ rune cen ;~(plug sym ;~(pfix gap ;~(pfix cen sym))) @@ -885,7 +885,7 @@ ;~ (glue gap) sym ;~(pfix cen sym) - ;~(pfix fas (more fas urs:ab)) + ;~(pfix stap) == :: %+ stag %tssg diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index b509e71314..c83dac2763 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -215,7 +215,7 @@ ?: =('subscribe' u.maybe-key) %. item %+ pe %subscribe - (ot id+ni ship+(su fed:ag) app+so path+(su ;~(pfix fas (more fas urs:ab))) ~) + (ot id+ni ship+(su fed:ag) app+so path+(su stap) ~) ?: =('unsubscribe' u.maybe-key) %. item %+ pe %unsubscribe diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 14c34390d6..9bdff87188 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -3458,7 +3458,7 @@ [(rash a fel) b] :: :: ++pa:dejs:format ++ pa :: string as path - (su ;~(pfix fas (more fas urs:ab))) + (su stap) :: :: ++pe:dejs:format ++ pe :: prefix |* [pre=* wit=fist] From 62d7281156dbe1b16db15551bfc431fa57b824b4 Mon Sep 17 00:00:00 2001 From: fang Date: Fri, 9 Apr 2021 01:49:14 +0200 Subject: [PATCH 052/136] clay: add /~ ford rune, as /= for directories Hoon files may want to import nouns from all files in a given directory. /~ lets you do so, importing as a (map @ta *) (but with typed values). Note the description as "directories" here, instead of "path prefix". The behavior, as implemented, will not include /path/hoon for /~ /path, instead only including /path/more/hoon and more deeply nested files. This seems to be, generally, the behavior you want, for example when importing from /app/myapp/* for /app/myapp/hoon. Actually using the resulting map requires some manual casting, which is not ideal. Some code style improvement work remains to be done as well. --- pkg/arvo/lib/language-server/parser.hoon | 3 + pkg/arvo/sys/lull.hoon | 2 + pkg/arvo/sys/vane/clay.hoon | 133 +++++++++++++++++++---- 3 files changed, 119 insertions(+), 19 deletions(-) diff --git a/pkg/arvo/lib/language-server/parser.hoon b/pkg/arvo/lib/language-server/parser.hoon index 58e4b91b7b..d52e87219b 100644 --- a/pkg/arvo/lib/language-server/parser.hoon +++ b/pkg/arvo/lib/language-server/parser.hoon @@ -22,6 +22,9 @@ :: %+ rune tis ;~(plug sym ;~(pfix gap stap)) + :: + %+ rune sig + ;~(plug sym ;~(pfix gap stap)) :: %+ rune cen ;~(plug sym ;~(pfix gap ;~(pfix cen sym))) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 23d08ae5bf..c6b2eb1141 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -928,6 +928,7 @@ :: /- sur-file :: surface imports from /sur :: /+ lib-file :: library imports from /lib :: /= face /path :: imports built hoon file at path + :: /~ face /path :: imports built hoon files from directory :: /% face %mark :: imports mark definition from /mar :: /$ face %from %to :: imports mark converter from /mar :: /* face %mark /path :: unbuilt file imports, as mark @@ -936,6 +937,7 @@ $: sur=(list taut) lib=(list taut) raw=(list [face=term =path]) + raz=(list [face=term =path]) maz=(list [face=term =mark]) caz=(list [face=term =mars]) bar=(list [face=term =mark =path]) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 5458e0026f..406c17e219 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -118,11 +118,11 @@ :: Ford cache :: +$ ford-cache - $: files=(map path [res=vase dez=(set path)]) - naves=(map mark [res=vase dez=(set path)]) - marks=(map mark [res=dais dez=(set path)]) - casts=(map mars [res=vase dez=(set path)]) - tubes=(map mars [res=tube dez=(set path)]) + $: files=(map path [res=vase dez=(set [dir=? =path])]) + naves=(map mark [res=vase dez=(set [dir=? =path])]) + marks=(map mark [res=dais dez=(set [dir=? =path])]) + casts=(map mars [res=vase dez=(set [dir=? =path])]) + tubes=(map mars [res=tube dez=(set [dir=? =path])]) == :: $reef-cache: built system files :: @@ -428,18 +428,23 @@ :: ++ an |_ nak=ankh + :: +dug: produce ankh at path + :: + ++ dug + |= =path + ^- (unit ankh) + ?~ path `nak + ?~ kid=(~(get by dir.nak) i.path) + ~ + $(nak u.kid, path t.path) :: +get: produce file at path :: ++ get |= =path ^- (unit cage) - ?~ path - ?~ fil.nak - ~ - `q.u.fil.nak - ?~ kid=(~(get by dir.nak) i.path) - ~ - $(nak u.kid, path t.path) + ?~ nik=(dug path) ~ + ?~ fil.u.nik ~ + `q.u.fil.u.nik -- ++ with-face |=([face=@tas =vase] vase(p [%face face p.vase])) ++ with-faces @@ -472,7 +477,7 @@ +$ state $: baked=(map path cage) cache=ford-cache - stack=(list (set path)) + stack=(list (set [dir=? =path])) cycle=(set build) == +$ args @@ -493,8 +498,8 @@ :: +pop-stack: pop build stack, copying deps downward :: ++ pop-stack - ^- [(set path) _stack.nub] - =^ top=(set path) stack.nub stack.nub + ^- [(set [dir=? =path]) _stack.nub] + =^ top=(set [dir=? =path]) stack.nub stack.nub =? stack.nub ?=(^ stack.nub) stack.nub(i (~(uni in i.stack.nub) top)) [top stack.nub] @@ -524,6 +529,29 @@ ?< (~(has in deletes) path) ~| %file-not-found^path :_(nub (need (~(get an ankh) path))) + :: +list-directory: retrieve directory listing + :: + :: this excludes files directly at /path/mark, + :: instead only including files in the unix-style directory at /path, + :: such as /path/more/mark + :: + ++ list-directory + |= =path + ^- (list ^path) + =/ nuk=(unit _ankh) (~(dug an ankh) path) + ?~ nuk ~ + =| dep=@ud + |- + =; dir=(list ^path) + ?: |((lte dep 1) ?=(~ fil.u.nuk) (~(has in deletes) path)) + dir + [path dir] + %- zing + %+ turn + %+ sort ~(tap by dir.u.nuk) + |=([[a=@ *] [b=@ *]] (aor a b)) + |= [nom=@ta nak=_ankh] + ^$(u.nuk nak, path (snoc path nom), dep +(dep)) :: +build-nave: build a statically typed mark core :: ++ build-nave @@ -816,7 +844,7 @@ ?: (~(has in cycle.nub) file+path) ~|(cycle+file+path^stack.nub !!) =. cycle.nub (~(put in cycle.nub) file+path) - =. stack.nub [(sy path ~) stack.nub] + =. stack.nub [(sy [| path] ~) stack.nub] =^ cag=cage nub (read-file path) ?> =(%hoon p.cag) =/ tex=tape (trip !<(@t q.cag)) @@ -826,11 +854,43 @@ =. files.cache.nub (~(put by files.cache.nub) path [res top]) [res nub] :: + ++ build-directory + |= =path + ^- [(map ^path vase) state] + =/ paz=(list ^path) (list-directory path) + =| rez=(map ^path vase) + |- + ?~ paz + [rez nub] + =* pax i.paz + ?. =(%hoon (rear pax)) + $(paz t.paz) + :: + ::TODO deduplicate with +build-file + ?^ got=(~(get by files.cache.nub) pax) + =? stack.nub ?=(^ stack.nub) + stack.nub(i (~(uni in i.stack.nub) dez.u.got)) + $(paz t.paz, rez (~(put by rez) pax res.u.got)) + ?: (~(has in cycle.nub) file+pax) + ~|(cycle+file+pax^stack.nub !!) + =. cycle.nub (~(put in cycle.nub) file+pax) + =. stack.nub [(sy [& path] ~) stack.nub] + =^ cag=cage nub (read-file pax) + ?> =(%hoon p.cag) + =/ tex=tape (trip !<(@t q.cag)) + =/ =pile (parse-pile pax tex) + =^ res=vase nub (run-pile pile) + =^ top stack.nub pop-stack + =. files.cache.nub (~(put by files.cache.nub) pax [res top]) + :: + $(paz t.paz, rez (~(put by rez) pax res)) + :: ++ run-pile |= =pile =^ sut=vase nub (run-tauts bud %sur sur.pile) =^ sut=vase nub (run-tauts sut %lib lib.pile) =^ sut=vase nub (run-raw sut raw.pile) + =^ sut=vase nub (run-raz sut raz.pile) =^ sut=vase nub (run-maz sut maz.pile) =^ sut=vase nub (run-caz sut caz.pile) =^ sut=vase nub (run-bar sut bar.pile) @@ -870,6 +930,9 @@ :: %+ rune tis ;~(plug sym ;~(pfix gap stap)) + :: + %+ rune sig + ;~(plug sym ;~(pfix gap stap)) :: %+ rune cen ;~(plug sym ;~(pfix gap ;~(pfix cen sym))) @@ -931,6 +994,30 @@ =. p.pin [%face face.i.raw p.pin] $(sut (slop pin sut), raw t.raw) :: + ++ run-raz + |= [sut=vase raz=(list [face=term =path])] + ^- [vase state] + ?~ raz [sut nub] + =^ res=(map path vase) nub + (build-directory path.i.raz) + =; pin=vase + =. p.pin [%face face.i.raz p.pin] + $(sut (slop pin sut), raz t.raz) + :: convert the (map path vase) into a vase of (map @ta *), + :: with paths hyphenated into names and the .hoon cut off + :: + =/ lap=@ud (lent path.i.raz) + |- ^- vase + ?~ res [[%atom %n `0] 0] + %+ slop + %+ slop + :- [%atom %ta ~] + %+ rap 3 + %+ join '-' + (snip (slag lap p.n.res)) + q.n.res + (slop $(res l.res) $(res r.res)) + :: ++ run-maz |= [sut=vase maz=(list [face=term =mark])] ^- [vase state] @@ -1582,12 +1669,20 @@ :: ++ invalidate |* [key=mold value=mold] - |= [cache=(map key [value dez=(set path)]) invalid=(set path)] - =/ builds=(list [key value dez=(set path)]) ~(tap by cache) + |= [cache=(map key [value dez=(set [dir=? =path])]) invalid=(set path)] + =/ builds=(list [key value dez=(set [dir=? =path])]) + ~(tap by cache) |- ^+ cache ?~ builds ~ - ?: ?=(^ (~(int in dez.i.builds) invalid)) + ?: %+ lien ~(tap in dez.i.builds) + |= [dir=? =path] + ?. dir (~(has in invalid) path) + =+ l=(lent path) + %+ lien ~(tap in invalid) + |= i=^path + =+ j=(lent i) + &((gth j +(l)) =(path (scag l i))) $(builds t.builds) (~(put by $(builds t.builds)) i.builds) :: From 1411e08944aec2b505f2bc88459823b8ed71405d Mon Sep 17 00:00:00 2001 From: fang Date: Sat, 10 Apr 2021 01:59:26 +0200 Subject: [PATCH 053/136] clay: specify target type for /~ results Clay will nest-check the results and put a homogenous map into the subject. --- pkg/arvo/lib/language-server/parser.hoon | 2 +- pkg/arvo/sys/lull.hoon | 4 +-- pkg/arvo/sys/vane/clay.hoon | 31 +++++++++++++----------- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/pkg/arvo/lib/language-server/parser.hoon b/pkg/arvo/lib/language-server/parser.hoon index d52e87219b..cf57790926 100644 --- a/pkg/arvo/lib/language-server/parser.hoon +++ b/pkg/arvo/lib/language-server/parser.hoon @@ -24,7 +24,7 @@ ;~(plug sym ;~(pfix gap stap)) :: %+ rune sig - ;~(plug sym ;~(pfix gap stap)) + ;~((glue gap) sym wyde:vast stap) :: %+ rune cen ;~(plug sym ;~(pfix gap ;~(pfix cen sym))) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index c6b2eb1141..67854e6b15 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -928,7 +928,7 @@ :: /- sur-file :: surface imports from /sur :: /+ lib-file :: library imports from /lib :: /= face /path :: imports built hoon file at path - :: /~ face /path :: imports built hoon files from directory + :: /~ face type /path :: imports built hoon files from directory :: /% face %mark :: imports mark definition from /mar :: /$ face %from %to :: imports mark converter from /mar :: /* face %mark /path :: unbuilt file imports, as mark @@ -937,7 +937,7 @@ $: sur=(list taut) lib=(list taut) raw=(list [face=term =path]) - raz=(list [face=term =path]) + raz=(list [face=term =spec =path]) maz=(list [face=term =mark]) caz=(list [face=term =mars]) bar=(list [face=term =mark =path]) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 406c17e219..82f3d6fb04 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -932,7 +932,7 @@ ;~(plug sym ;~(pfix gap stap)) :: %+ rune sig - ;~(plug sym ;~(pfix gap stap)) + ;~((glue gap) sym wyde:vast stap) :: %+ rune cen ;~(plug sym ;~(pfix gap ;~(pfix cen sym))) @@ -995,7 +995,7 @@ $(sut (slop pin sut), raw t.raw) :: ++ run-raz - |= [sut=vase raz=(list [face=term =path])] + |= [sut=vase raz=(list [face=term =spec =path])] ^- [vase state] ?~ raz [sut nub] =^ res=(map path vase) nub @@ -1003,20 +1003,23 @@ =; pin=vase =. p.pin [%face face.i.raz p.pin] $(sut (slop pin sut), raz t.raz) - :: convert the (map path vase) into a vase of (map @ta *), - :: with paths hyphenated into names and the .hoon cut off :: =/ lap=@ud (lent path.i.raz) - |- ^- vase - ?~ res [[%atom %n `0] 0] - %+ slop - %+ slop - :- [%atom %ta ~] - %+ rap 3 - %+ join '-' - (snip (slag lap p.n.res)) - q.n.res - (slop $(res l.res) $(res r.res)) + =/ =type (~(play ut p.sut) [%kttr spec.i.raz]) + :: ensure results nest in the specified type, + :: and produce a homogenous map containing that type. + :: + :- %- ~(play ut p.sut) + [%kttr %make [%wing ~[%map]] ~[[%base %atom %ta] spec.i.raz]] + |- + ?~ res ~ + ~| [%nest-fail path.i.raz p.n.res] + ?> (~(nest ut type) | p.q.n.res) + :_ [$(res l.res) $(res r.res)] + :_ q.q.n.res + %+ rap 3 + %+ join '-' + (snip (slag lap p.n.res)) :: ++ run-maz |= [sut=vase maz=(list [face=term =mark])] From a45179412b8c166f07de2840273dd79e5f7ae423 Mon Sep 17 00:00:00 2001 From: fang Date: Sat, 10 Apr 2021 02:26:32 +0200 Subject: [PATCH 054/136] clay: for /~, only build top-level files This is semantically cleaner than arbitrary-depth recursion, and lets us stop worrying about hyphenated path conflicts. --- pkg/arvo/sys/vane/clay.hoon | 63 ++++++++++++++----------------------- 1 file changed, 24 insertions(+), 39 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 82f3d6fb04..25a906410c 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -529,29 +529,6 @@ ?< (~(has in deletes) path) ~| %file-not-found^path :_(nub (need (~(get an ankh) path))) - :: +list-directory: retrieve directory listing - :: - :: this excludes files directly at /path/mark, - :: instead only including files in the unix-style directory at /path, - :: such as /path/more/mark - :: - ++ list-directory - |= =path - ^- (list ^path) - =/ nuk=(unit _ankh) (~(dug an ankh) path) - ?~ nuk ~ - =| dep=@ud - |- - =; dir=(list ^path) - ?: |((lte dep 1) ?=(~ fil.u.nuk) (~(has in deletes) path)) - dir - [path dir] - %- zing - %+ turn - %+ sort ~(tap by dir.u.nuk) - |=([[a=@ *] [b=@ *]] (aor a b)) - |= [nom=@ta nak=_ankh] - ^$(u.nuk nak, path (snoc path nom), dep +(dep)) :: +build-nave: build a statically typed mark core :: ++ build-nave @@ -853,24 +830,36 @@ =^ top stack.nub pop-stack =. files.cache.nub (~(put by files.cache.nub) path [res top]) [res nub] + :: +build-directory: builds files in top level of a directory + :: + :: this excludes files directly at /path/hoon, + :: instead only including files in the unix-style directory at /path, + :: such as /path/file/hoon, but not /path/more/file/hoon. :: ++ build-directory |= =path - ^- [(map ^path vase) state] - =/ paz=(list ^path) (list-directory path) - =| rez=(map ^path vase) + ^- [(map @ta vase) state] + =/ fiz=(list @ta) + =/ nuk=(unit _ankh) (~(dug an ankh) path) + ?~ nuk ~ + %+ murn + ~(tap by dir.u.nuk) + |= [nom=@ta nak=_ankh] + ?. ?=([~ [~ *] *] (~(get by dir.nak) %hoon)) ~ + `nom + :: + =| rez=(map @ta vase) |- - ?~ paz + ?~ fiz [rez nub] - =* pax i.paz - ?. =(%hoon (rear pax)) - $(paz t.paz) + =* nom=@ta i.fiz + =/ pax=^path (weld path nom %hoon ~) :: ::TODO deduplicate with +build-file ?^ got=(~(get by files.cache.nub) pax) =? stack.nub ?=(^ stack.nub) stack.nub(i (~(uni in i.stack.nub) dez.u.got)) - $(paz t.paz, rez (~(put by rez) pax res.u.got)) + $(fiz t.fiz, rez (~(put by rez) nom res.u.got)) ?: (~(has in cycle.nub) file+pax) ~|(cycle+file+pax^stack.nub !!) =. cycle.nub (~(put in cycle.nub) file+pax) @@ -883,7 +872,7 @@ =^ top stack.nub pop-stack =. files.cache.nub (~(put by files.cache.nub) pax [res top]) :: - $(paz t.paz, rez (~(put by rez) pax res)) + $(fiz t.fiz, rez (~(put by rez) nom res)) :: ++ run-pile |= =pile @@ -998,13 +987,12 @@ |= [sut=vase raz=(list [face=term =spec =path])] ^- [vase state] ?~ raz [sut nub] - =^ res=(map path vase) nub + =^ res=(map @ta vase) nub (build-directory path.i.raz) =; pin=vase =. p.pin [%face face.i.raz p.pin] $(sut (slop pin sut), raz t.raz) :: - =/ lap=@ud (lent path.i.raz) =/ =type (~(play ut p.sut) [%kttr spec.i.raz]) :: ensure results nest in the specified type, :: and produce a homogenous map containing that type. @@ -1015,11 +1003,8 @@ ?~ res ~ ~| [%nest-fail path.i.raz p.n.res] ?> (~(nest ut type) | p.q.n.res) - :_ [$(res l.res) $(res r.res)] - :_ q.q.n.res - %+ rap 3 - %+ join '-' - (snip (slag lap p.n.res)) + :- [p.n.res q.q.n.res] + [$(res l.res) $(res r.res)] :: ++ run-maz |= [sut=vase maz=(list [face=term =mark])] From 8b4dba28fae87067b54f6db63d099d056d89ba73 Mon Sep 17 00:00:00 2001 From: fang Date: Sat, 10 Apr 2021 16:22:52 +0200 Subject: [PATCH 055/136] clay: update dependency logic for /~ imports To match recent changes to the build logic, where it only includes top-level hoon files. --- pkg/arvo/sys/vane/clay.hoon | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 25a906410c..a0493f7585 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -1669,8 +1669,7 @@ =+ l=(lent path) %+ lien ~(tap in invalid) |= i=^path - =+ j=(lent i) - &((gth j +(l)) =(path (scag l i))) + &(=(path (scag l i)) ?=([@ %hoon ~] (slag l i))) $(builds t.builds) (~(put by $(builds t.builds)) i.builds) :: From 0570fb5b768057ad19f664e24fe0b7ad5eb85e6d Mon Sep 17 00:00:00 2001 From: fang Date: Mon, 12 Apr 2021 13:10:36 +0200 Subject: [PATCH 056/136] clay: share logic between build-file and directory By factoring their shared logic out into +build-dependency, which gets passed the relevant details about how to track the file being built in the dependency stack. --- pkg/arvo/sys/vane/clay.hoon | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index a0493f7585..3ca9057d02 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -810,9 +810,11 @@ =^ res=vase nub (run-pile pile) res :: - ++ build-file - |= =path + ++ build-dependency + |= dep=(each [dir=path fil=path] path) ^- [vase state] + =/ =path + ?:(?=(%| -.dep) p.dep fil.p.dep) ~| %error-building^path ?^ got=(~(get by files.cache.nub) path) =? stack.nub ?=(^ stack.nub) @@ -821,7 +823,9 @@ ?: (~(has in cycle.nub) file+path) ~|(cycle+file+path^stack.nub !!) =. cycle.nub (~(put in cycle.nub) file+path) - =. stack.nub [(sy [| path] ~) stack.nub] + =. stack.nub + =- [(sy - ~) stack.nub] + ?:(?=(%| -.dep) dep [& dir.p.dep]) =^ cag=cage nub (read-file path) ?> =(%hoon p.cag) =/ tex=tape (trip !<(@t q.cag)) @@ -830,6 +834,10 @@ =^ top stack.nub pop-stack =. files.cache.nub (~(put by files.cache.nub) path [res top]) [res nub] + :: + ++ build-file + |= =path + (build-dependency |+path) :: +build-directory: builds files in top level of a directory :: :: this excludes files directly at /path/hoon, @@ -854,24 +862,7 @@ [rez nub] =* nom=@ta i.fiz =/ pax=^path (weld path nom %hoon ~) - :: - ::TODO deduplicate with +build-file - ?^ got=(~(get by files.cache.nub) pax) - =? stack.nub ?=(^ stack.nub) - stack.nub(i (~(uni in i.stack.nub) dez.u.got)) - $(fiz t.fiz, rez (~(put by rez) nom res.u.got)) - ?: (~(has in cycle.nub) file+pax) - ~|(cycle+file+pax^stack.nub !!) - =. cycle.nub (~(put in cycle.nub) file+pax) - =. stack.nub [(sy [& path] ~) stack.nub] - =^ cag=cage nub (read-file pax) - ?> =(%hoon p.cag) - =/ tex=tape (trip !<(@t q.cag)) - =/ =pile (parse-pile pax tex) - =^ res=vase nub (run-pile pile) - =^ top stack.nub pop-stack - =. files.cache.nub (~(put by files.cache.nub) pax [res top]) - :: + =^ res nub (build-dependency &+[path pax]) $(fiz t.fiz, rez (~(put by rez) nom res)) :: ++ run-pile From 2f0ed36101f9e89714f1adc59e6a34b83b65fefa Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Mon, 19 Apr 2021 15:09:52 -0500 Subject: [PATCH 057/136] zuse: add +all and +run to ++ordered-map --- pkg/arvo/sys/zuse.hoon | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 9bdff87188..f826cc1e74 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5296,6 +5296,23 @@ ?~ a b :: $(a r.a, b [n.a $(a l.a)]) + :: +run: apply gate to transform all values in place + :: without modifying their type + :: + ++ run + |= [a=(tree item) b=$-(val val)] + |- ^+ a + ?~ a a + [n=[key.n.a (b val.n.a)] l=$(a l.a) r=$(a r.a)] + :: +all: apply logical AND boolean test on all values + :: + ++ all + |= [a=(tree item) b=$-(item ?)] + ^- ? + |- + ?~ a + & + ?&((b n.a) $(a l.a) $(a r.a)) :: +gas: put a list of items :: ++ gas From b2e1008d9add7e5c2b263e247c9814469ca96351 Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Mon, 19 Apr 2021 23:46:46 -0400 Subject: [PATCH 058/136] clay: add %fuse --- .gitignore | 7 +- Makefile | 3 + bin/brass.pill | 4 +- bin/ivory.pill | 4 +- bin/solid.pill | 4 +- pkg/arvo/gen/hood/fuse.hoon | 12 ++ pkg/arvo/gen/hood/fuse/help.txt | 8 + pkg/arvo/lib/hood/kiln.hoon | 9 + pkg/arvo/sys/lull.hoon | 4 + pkg/arvo/sys/vane/clay.hoon | 311 +++++++++++++++++++++++++++----- 10 files changed, 315 insertions(+), 51 deletions(-) create mode 100644 pkg/arvo/gen/hood/fuse.hoon create mode 100644 pkg/arvo/gen/hood/fuse/help.txt diff --git a/.gitignore b/.gitignore index e79759dc67..d2c66c9451 100644 --- a/.gitignore +++ b/.gitignore @@ -76,4 +76,9 @@ pkg/interface/link-webext/web-ext-artifacts *.xz # Logs -*.log \ No newline at end of file +*.log + +backup* +notes +bel +wex diff --git a/Makefile b/Makefile index a06eac1c8a..fd3eaf12bb 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,9 @@ pills: sh/update-brass-pill sh/update-ivory-pill +solid: + sh/update-solid-pill + ropsten-pills: sh/create-ropsten-pills diff --git a/bin/brass.pill b/bin/brass.pill index f549dac587..4f5af448ab 100644 --- a/bin/brass.pill +++ b/bin/brass.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:61e583dd7db795dac4a7c31bfd3ee8b240e679bb882e35d4e7d1acb5f9f2f3d6 -size 8270131 +oid sha256:2d45514e6ebb8345911137a03980e709c8ea0c28b14b770a24a51190db7c8ed2 +size 8215365 diff --git a/bin/ivory.pill b/bin/ivory.pill index 7c2e8e4746..ae03a26b96 100644 --- a/bin/ivory.pill +++ b/bin/ivory.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:185ea5e76dc48695e55efc543377e0682e485f81b16e3b443f9be881d026d4f2 -size 2616564 +oid sha256:b9807f664c9b4f6d6ce92c10f2959f28ad6e48a24a880556f828d3cf0cad2ee9 +size 4537425 diff --git a/bin/solid.pill b/bin/solid.pill index 92802fcddc..aaeb0a4d27 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bb221ce7316346bfaf1ddd953927c0f2afe7eaf5b160bed545f67ec7e5ccb005 -size 9410487 +oid sha256:1ddeef9bf39394ff9c19a0d66c9fc45d218e4891dbd152f8f7696f8c0fa9747e +size 9877817 diff --git a/pkg/arvo/gen/hood/fuse.hoon b/pkg/arvo/gen/hood/fuse.hoon new file mode 100644 index 0000000000..39bf9f74c8 --- /dev/null +++ b/pkg/arvo/gen/hood/fuse.hoon @@ -0,0 +1,12 @@ +:: Kiln: Fuse local desk from (optionally-)foreign sources +:: +:::: /hoon/fuse/hood/gen + :: +/* help-text %txt /gen/hood/fuse/help/txt +=, clay +:: +:::: + :: +:- %say +|= [[now=@da eny=@uvJ bec=beak] [des=desk srcs=(list [beak germ]) ~] ~] +[%kiln-fuse des srcs] diff --git a/pkg/arvo/gen/hood/fuse/help.txt b/pkg/arvo/gen/hood/fuse/help.txt new file mode 100644 index 0000000000..32431f5cd0 --- /dev/null +++ b/pkg/arvo/gen/hood/fuse/help.txt @@ -0,0 +1,8 @@ +Usage: + + |fuse %destination-desk ~[[source-beak %some-germ] [another-beak %another-germ]] + +A fuse replaces the contents of %destination-desk with the merge of the +specified beaks according to their merge strategies. This has no dependence +on the previous state of %destination-desk so any commits/work there will +be overwritten. diff --git a/pkg/arvo/lib/hood/kiln.hoon b/pkg/arvo/lib/hood/kiln.hoon index 0e7f719eb2..167750360f 100644 --- a/pkg/arvo/lib/hood/kiln.hoon +++ b/pkg/arvo/lib/hood/kiln.hoon @@ -55,6 +55,10 @@ cas=case :: gim=?(%auto germ) :: == ++$ kiln-fuse + $: syd=desk + srcs=(list [beak germ]) + == -- |= [bowl:gall state] ?> =(src our) @@ -381,6 +385,10 @@ ?~ +< abet abet:abet:(merge:(work syd) ali sud cas gim) :: +++ poke-fuse + |= k=kiln-fuse + abet:(emit [%pass /kiln/fuse/[syd.k] %arvo %c [%fuse syd.k srcs.k]]) +:: ++ poke-cancel |= a=@tas abet:(emit %pass /cancel %arvo %c [%drop a]) @@ -430,6 +438,7 @@ %kiln-info =;(f (f !<(_+<.f vase)) poke-info) %kiln-label =;(f (f !<(_+<.f vase)) poke-label) %kiln-merge =;(f (f !<(_+<.f vase)) poke-merge) + %kiln-fuse =;(f (f !<(_+<.f vase)) poke-fuse) %kiln-mount =;(f (f !<(_+<.f vase)) poke-mount) %kiln-ota =;(f (f !<(_+<.f vase)) poke:update) %kiln-ota-info =;(f (f !<(_+<.f vase)) poke-ota-info) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 23d08ae5bf..e91d889352 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -762,6 +762,10 @@ her=@p dem=desk cas=case :: source how=germ :: method == :: + $: %fuse :: mega merge + des=desk :: target desk + srcs=(list [beak germ]) :: sources + == [%mont pot=term bem=beam] :: mount to unix [%dirk des=desk] :: mark mount dirty [%ogre pot=$@(desk beam)] :: delete mount point diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index af2f653037..09130e8151 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -58,6 +58,9 @@ :: when the request is filled/updated. :: +$ cult (jug wove duct) +:: State for ongoing %fuse merges - the list maintains the ordering +:: and the map stores the data needed to merge ++$ melt [(list [beak germ]) (map [beak germ] (unit dome:clay))] :: :: Domestic desk state. :: @@ -69,6 +72,7 @@ dom=dome :: desk state per=regs :: read perms per path pew=regs :: write perms per path + fiz=melt :: state for mega merges == :: :: Desk state. @@ -212,6 +216,7 @@ dom=dome :: revision state per=regs :: read perms per path pew=regs :: write perms per path + fiz=melt :: domestic mega merges == :: :: :: Foreign request manager. @@ -303,6 +308,7 @@ $: %c :: to %clay $> $? %info :: internal edit %merg :: merge desks + %fuse :: mega merge %pork :: %warp :: %werp :: @@ -1043,12 +1049,12 @@ ~ =/ rus rus:(~(gut by hoy.ruf) her *rung) %+ ~(gut by rus) syd - [lim=~2000.1.1 ref=`*rind qyx=~ dom=*dome per=~ pew=~] + [lim=~2000.1.1 ref=`*rind qyx=~ dom=*dome per=~ pew=~ fiz=[~ ~]] :: administrative duct, domestic +rede :: :+ ~ `hun.rom.ruf =/ jod (~(gut by dos.rom.ruf) syd *dojo) - [lim=now ref=~ [qyx dom per pew]:jod] + [lim=now ref=~ [qyx dom per pew fiz]:jod] :: =* red=rede ->+ |% @@ -1065,7 +1071,7 @@ :: %= ruf hun.rom (need hun) - dos.rom (~(put by dos.rom.ruf) syd [qyx dom per pew]:red) + dos.rom (~(put by dos.rom.ruf) syd [qyx dom per pew fiz]:red) == :: :: Handle `%sing` requests @@ -1962,32 +1968,167 @@ =/ =wire /merge/[syd]/(scot %p ali-ship)/[ali-desk]/[germ] (emit hen %pass wire %c %warp ali-ship ali-desk `[%sing %v case /]) :: + ++ make-melt + |= srcs=(list [beak germ]) + ^- melt + :- srcs + %- ~(gas by *(map [beak germ] (unit dome:clay))) + (turn srcs |=(a=[beak germ] [a *(unit dome:clay)])) + :: + ++ start-fuse + |= [srcs=(list [beak germ])] + ^+ ..start-fuse + :: use emil here to queue a list of moves + :: syd is current desk (target desk) + :: what is hen? it's the current duct... can this be reused? + =/ moves=(list move) + %+ turn + srcs + |= [bec=beak g=germ] + ^- move + =/ wir=wire /fuse/[syd]/(scot %p p.bec)/[q.bec]/(scot r.bec)/[g] + [hen %pass wir %c %warp p.bec q.bec `[%sing %v r.bec /]] + :: we also want to clear the state (fiz) associated with this + :: merge and print a warning if it's non trivial i.e. we're + :: starting a new fuse before the previous one terminated. + ~& ?~ -.fiz + [%starting-fuse srcs] + :^ %starting-fuse srcs %discarding-state + :: we don't want to ~& an entire dome + %- ~(run by +.fiz) + |=(v=(unit dome:clay) ?~(v %not-received %recieved)) + =. fiz (make-melt srcs) + (emil moves) + :: + ++ take-fuse + |= [[bec=beak g=germ] =riot] + ^+ ..take-fuse + ?~ riot + ~& [%fuse-for syd %missing bec] + :: by setting fiz to *melt the merge is aborted - any further + :: responses we get for the merge will cause take-fuse to crash + =. fiz *melt + ..take-fuse + ?> (~(has by +.fiz) [bec g]) + =. fiz + :- -.fiz + (~(put by +.fiz) [bec g] `!<(dome:clay q.r.u.riot)) + =/ all-done=flag + %- ~(all by +.fiz) + |= res=(unit dome:clay) + ^- flag + !=(res ~) + ?. all-done + ..take-fuse + :: do the merge + =/ merges=(list [beak germ]) -.fiz + :: there's no point to an empty fuse and a single element fuse + :: should just be an %only-that merge. + ?> (gte (lent merges) 2) + =| rag=rang + =/ clean-hut-ran hut.ran + =/ initial-dome=dome:clay (need (~(got by +.fiz) (snag 0 merges))) + =/ continuation-yaki=yaki + (~(got by hut.ran) (~(got by hit.initial-dome) let.initial-dome)) + =/ parents=(list tako) ~[(~(got by hit.initial-dome) let.initial-dome)] + =. merges (slag 1 merges) + |- + ^+ ..take-fuse + ?~ merges + :: reset fiz before potentially parking - we hope to + :: succeed so we don't need this state anymore. + =. fiz *melt + =. hut.ran clean-hut-ran + (park | [%| continuation-yaki(p (flop parents))] rag) + =/ [bec=beak g=germ] i.merges + =/ ali-dom=dome:clay (need (~(got by +.fiz) bec g)) + =/ result (merge-helper p.bec q.bec g ali-dom `continuation-yaki) + ?- -.result + %| + :: merge failed + (mean %fuse-merge-failed p.result) + :: + %& + :: now we have a unit merge-result + =/ merge-result=(unit merge-result) +.result + ?~ merge-result + :: this merge was a no-op, just continue + $(merges t.merges) + ?~ conflicts.u.merge-result + :: no merge conflicts + =/ merged-yaki=yaki + ?- -.new.u.merge-result + %| + +.new.u.merge-result + :: + %& + :: convert the yuki to yaki + =/ yuk=yuki +.new.u.merge-result + =/ lobes=(map path lobe) + %- ~(run by q.yuk) + |= val=(each page lobe) + ^- lobe + ?- -.val + %& (page-to-lobe +.val) + %| +.val + == + (make-yaki p.yuk lobes now) + == + %= $ + continuation-yaki merged-yaki + merges t.merges + hut.ran (~(put by hut.ran) r.merged-yaki merged-yaki) + lat.rag (~(uni by lat.rag) lat.u.merge-result) + parents [(~(got by hit.ali-dom) let.ali-dom) parents] + == + :: if there are merge conflicts send the error and abort the merge + =. fiz *melt + (done %& conflicts.u.merge-result) + == + :: + ++ done + |= result=(each (set path) (pair term tang)) + ^+ ..merge + (emit hen %give %mere result) + :: ++ merge |= [=ali=ship =ali=desk =germ =riot] ^+ ..merge - |^ ?~ riot - (done %| %ali-unavailable >[ali-ship ali-desk germ]< ~) + (done %| %ali-unavailable ~[>[ali-ship ali-desk germ]<]) =/ ali-dome=dome:clay !<(dome:clay q.r.u.riot) + =/ result=(each (unit merge-result) (pair term tang)) + (merge-helper ali-ship ali-desk germ ali-dome ~) + ?- -.result + %| + (done %| +.result) + :: + %& + =/ mr=(unit merge-result) +.result + ?~ mr + (done %& ~) + =. ..merge (done %& conflicts.u.mr) + (park | new.u.mr ~ lat.u.mr) + == + :: + +$ merge-result [conflicts=(set path) new=yoki lat=(map lobe blob)] + :: + ++ merge-helper + |= [=ali=ship =ali=desk =germ ali-dome=dome:clay continuation-yaki=(unit yaki)] + ^- (each (unit merge-result) [term tang]) + |^ + :: term tang is the error, merge result being ~ means the merge was + :: a no-op + :: (each a b) is [| b] or [& a] + ^- (each (unit merge-result) [term tang]) =/ ali-yaki=yaki (~(got by hut.ran) (~(got by hit.ali-dome) let.ali-dome)) =/ bob-yaki=(unit yaki) - ?~ let.dom - ~ - (~(get by hut.ran) (~(got by hit.dom) let.dom)) - =/ merge-result (merge-by-germ ali-yaki bob-yaki) - ?: ?=(%| -.merge-result) - (done %| p.merge-result) - ?~ p.merge-result - (done %& ~) - =. ..merge (done %& conflicts.u.p.merge-result) - (park | new.u.p.merge-result ~ lat.u.p.merge-result) - :: - ++ done - |= result=(each (set path) (pair term tang)) - ^+ ..merge - (emit hen %give %mere result) - :: - +$ merge-result [conflicts=(set path) new=yoki lat=(map lobe blob)] + ?~ continuation-yaki + ?~ let.dom + ~ + (~(get by hut.ran) (~(got by hit.dom) let.dom)) + continuation-yaki + (merge-by-germ ali-yaki bob-yaki) ++ merge-by-germ |= [=ali=yaki bob-yaki=(unit yaki)] ^- (each (unit merge-result) [term tang]) @@ -2005,16 +2146,13 @@ ?- germ :: :: If this is a %only-this merge, we check to see if ali's and bob's - :: commits are the same, in which case we're done. Otherwise, we - :: check to see if ali's commit is in the ancestry of bob's, in - :: which case we're done. Otherwise, we create a new commit with - :: bob's data plus ali and bob as parents. + :: commits are the same, in which case we're done. + :: Otherwise, we create a new commit with bob's data plus ali and + :: bob as parents. :: %only-this ?: =(r.ali-yaki r.bob-yaki) &+~ - ?: (~(has in (reachable-takos:ze r.bob-yaki)) r.ali-yaki) - &+~ :* %& ~ conflicts=~ new=&+[[r.bob-yaki r.ali-yaki ~] (to-yuki q.bob-yaki)] @@ -2042,8 +2180,6 @@ %take-this ?: =(r.ali-yaki r.bob-yaki) &+~ - ?: (~(has in (reachable-takos:ze r.bob-yaki)) r.ali-yaki) - &+~ =/ new-data (~(uni by q.ali-yaki) q.bob-yaki) :* %& ~ conflicts=~ @@ -3384,6 +3520,7 @@ :: :: Creates a nako of all the changes between a and b. :: + :: ++ make-nako |= [ver=@ud a=aeon b=aeon] ^- nako @@ -3391,12 +3528,29 @@ |- ?: =(b let.dom) hit.dom + :: del everything after b $(hit.dom (~(del by hit.dom) let.dom), let.dom (dec let.dom)) b ?: =(0 b) [~ ~] - (data-twixt-takos =(0 ver) (~(get by hit.dom) a) (aeon-to-tako b)) - :: + =/ excludes=(set tako) + =| acc=(set tako) + =/ lower=@ud 1 + |- + :: a should be excluded, so wait until we're past it + ?: =(lower +(a)) + acc + =/ res=(set tako) (reachable-takos (~(got by hit.dom) lower)) + $(acc (~(uni in acc) res), lower +(lower)) + =/ includes=(set tako) + =| acc=(set tako) + =/ upper=@ud b + |- + ?: =(upper a) + acc + =/ res=(set tako) (reachable-takos (~(got by hit.dom) upper)) + $(acc (~(uni in acc) res), upper (dec upper)) + [(~(run in (~(dif in includes) excludes)) tako-to-yaki) ~] :: Traverse parentage and find all ancestor hashes :: ++ reachable-takos :: reachable @@ -3429,15 +3583,9 @@ |= [plops=? a=(unit tako) b=tako] ^- [(set yaki) (set plop)] =+ old=?~(a ~ (reachable-takos u.a)) - =/ yal=(set tako) - %- silt - %+ skip - ~(tap in (reachable-takos b)) - |=(tak=tako (~(has in old) tak)) + =/ yal=(set tako) (~(dif in (reachable-takos b)) old) :- (silt (turn ~(tap in yal) tako-to-yaki)) - ?. plops - ~ - (silt (turn ~(tap in (new-lobes (new-lobes ~ old) yal)) lobe-to-blob)) + ~ :: :: Get all the lobes that are referenced in `a` except those that are :: already in `b`. @@ -4074,6 +4222,14 @@ =/ den ((de now rof hen ruf) our des.req) abet:(start-merge:den her.req dem.req cas.req how.req) [mos ..^$] + :: + %fuse + ?: =(%$ des.req) + ~&(%fuse-no-desk !!) + =^ mos ruf + =/ den ((de now rof hen ruf) our des.req) + abet:(start-fuse:den srcs.req) + [mos ..^$] :: %mont =. hez.ruf ?^(hez.ruf hez.ruf `[[%$ %sync ~] ~]) @@ -4202,11 +4358,43 @@ ++ load => |% +$ raft-any - $% [%7 raft-7] + $% [%8 raft-8] + [%7 raft-7] [%6 raft-6] == - +$ raft-7 raft - +$ dojo-7 dojo + +$ raft-8 raft + +$ raft-7 + $: rom=room-7 + hoy=(map ship rung-7) + ran=rang + mon=(map term beam) + hez=(unit duct) + cez=(map @ta crew) + pud=(unit [=desk =yoki]) + == + +$ room-7 + $: hun=duct + dos=(map desk dojo-7) + == + + +$ rung-7 + $: rus=(map desk rede-7) + == + +$ dojo-7 + $: qyx=cult + dom=dome + per=regs + pew=regs + == + +$ rede-7 + $: lim=@da + ref=(unit rind) + qyx=cult + dom=dome + per=regs + pew=regs + == + +$ ford-cache-7 ford-cache +$ raft-6 $: rom=room-6 :: domestic @@ -4250,7 +4438,7 @@ |^ =? old ?=(%6 -.old) 7+(raft-6-to-7 +.old) ?> ?=(%7 -.old) - ..^^$(ruf +.old) + ..^^$(ruf (raft-7-to-8 +.old)) :: +raft-6-to-7: delete stale ford caches (they could all be invalid) :: ++ raft-6-to-7 @@ -4270,6 +4458,26 @@ |= =rede-6 rede-6(dom dom.rede-6(fod *ford-cache-7)) == + :: +raft-7-to-8: create bunted melts in each dojo/rede + :: + ++ raft-7-to-8 + |= raf=raft-7 + ^- raft-8 + %= raf + dos.rom + %- ~(run by dos.rom.raf) + |= doj=dojo-7 + ^- dojo + [qyx.doj dom.doj per.doj pew.doj *melt] + :: + hoy + %- ~(run by hoy.raf) + |= =rung-7 + %- ~(run by rus.rung-7) + |= r=rede-7 + ^- rede + [lim.r ref.r qyx.r dom.r per.r pew.r *melt] + == -- :: ++ scry :: inspect @@ -4320,6 +4528,7 @@ dos.rom %- ~(run by dos.rom.ruf) |= =dojo + ::dojo dojo(fod.dom [~ ~ ~ ~ ~]) :: hoy @@ -4329,6 +4538,7 @@ rus %- ~(run by rus.rung) |= =rede + ::rede rede(fod.dom [~ ~ ~ ~ ~]) == == @@ -4350,6 +4560,19 @@ abet:(merge:den ali-ship ali-desk germ p.hin) [mos ..^$] :: + ?: ?=([%fuse @ @ @ @ @ ~] tea) + ?> ?=(%writ +<.hin) + =* syd i.t.tea + =/ ali-ship=@p (slav %p i.t.t.tea) + =* ali-desk=desk i.t.t.t.tea + =/ ali-case (rash i.t.t.t.t.tea nuck:so) + ?> ?=([%$ *] ali-case) + =/ germ (germ i.t.t.t.t.t.tea) + =^ mos ruf + =/ den ((de now rof hen ruf) our i.t.tea) + abet:(take-fuse:den [[ali-ship ali-desk (case +.ali-case)] germ] p.hin) + [mos ..^$] + :: ?: ?=([%foreign-warp *] tea) ?> ?=(%writ +<.hin) :_ ..^$ From fd6f6b3bd04e5469574e00eb73d6ab0ffa470871 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 20 Apr 2021 12:04:59 -0500 Subject: [PATCH 059/136] zuse: added jet hints to all +ordered-map arms, reordered them alphabetically as is done in +in and +by --- pkg/arvo/sys/zuse.hoon | 344 ++++++++++++----------- pkg/arvo/tests/sys/zuse/ordered-map.hoon | 2 +- 2 files changed, 180 insertions(+), 166 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index f826cc1e74..d6f4cae0e5 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5070,7 +5070,7 @@ |= ord=$-([key key] ?) |= a=* =/ b ;;((tree [key=key val=value]) a) - ?> (check-balance:((ordered-map key value) ord) b) + ?> (apt:((ordered-map key value) ord) b) b :: :: $mk-item: constructor for +ordered-map item type @@ -5094,9 +5094,20 @@ :: |= compare=$-([key key] ?) |% - :: +check-balance: verify horizontal and vertical orderings + :: +all: apply logical AND boolean test on all values :: - ++ check-balance + ++ all + ~/ %all + |= [a=(tree item) b=$-(item ?)] + ^- ? + |- + ?~ a + & + ?&((b n.a) $(a l.a) $(a r.a)) + :: +apt: verify horizontal and vertical orderings + :: + ++ apt + ~/ %apt =| [l=(unit key) r=(unit key)] |= a=(tree item) ^- ? @@ -5122,9 +5133,107 @@ :: ?~(r.a %.y &((mor key.n.a key.n.r.a) $(a r.a, r `key.n.a))) == + :: +bap: convert to list, largest to smallest + :: + ++ bap + ~/ %bap + |= a=(tree item) + ^- (list item) + :: + =| b=(list item) + |- ^+ b + ?~ a b + :: + $(a r.a, b [n.a $(a l.a)]) + :: +del: delete .key from .a if it exists, producing value iff deleted + :: + ++ del + ~/ %del + |= [a=(tree item) =key] + ^- [(unit val) (tree item)] + :: + ?~ a [~ ~] + :: we found .key at the root; delete and rebalance + :: + ?: =(key key.n.a) + [`val.n.a (nip a)] + :: recurse left or right to find .key + :: + ?: (compare key key.n.a) + =+ [found lef]=$(a l.a) + [found a(l lef)] + =+ [found rig]=$(a r.a) + [found a(r rig)] + :: +gas: put a list of items + :: + ++ gas + ~/ %gas + |= [a=(tree item) b=(list item)] + ^- (tree item) + :: + ?~ b a + $(b t.b, a (put a i.b)) + :: + :: +get: get val at key or return ~ + :: + ++ get + ~/ %get + |= [a=(tree item) b=key] + ^- (unit val) + ?~ a ~ + ?: =(b key.n.a) + `val.n.a + ?: (compare b key.n.a) + $(a l.a) + $(a r.a) + :: +nip: remove root; for internal use + :: + ++ nip + ~/ %nip + |= a=(tree item) + ^- (tree item) + :: + ?> ?=(^ a) + :: delete .n.a; merge and balance .l.a and .r.a + :: + |- ^- (tree item) + ?~ l.a r.a + ?~ r.a l.a + ?: (mor key.n.l.a key.n.r.a) + l.a(r $(l.a r.l.a)) + r.a(l $(r.a l.r.a)) + :: +peek: produce head (smallest item) or null + :: + ++ peek + ~/ %peek + |= a=(tree item) + ^- (unit item) + :: + ?~ a ~ + ?~ l.a `n.a + $(a l.a) + :: + :: +pop: produce .head (smallest item) and .rest or crash if empty + :: + ++ pop + ~/ %pop + |= a=(tree item) + ^- [head=item rest=(tree item)] + :: + ?~ a !! + ?~ l.a [n.a r.a] + :: + =/ l $(a l.a) + :- head.l + :: load .rest.l back into .a and rebalance + :: + ?: |(?=(~ rest.l) (mor key.n.a key.n.rest.l)) + a(l rest.l) + rest.l(r a(r r.rest.l)) :: +put: ordered item insert :: ++ put + ~/ %put |= [a=(tree item) =key =val] ^- (tree item) :: base case: replace null with single-item tree @@ -5148,65 +5257,77 @@ ?: (mor key.n.a key.n.r) a(r r) r(l a(r l.r)) - :: +peek: produce head (smallest item) or null + :: +run: apply gate to transform all values in place :: - ++ peek - |= a=(tree item) - ^- (unit item) - :: - ?~ a ~ - ?~ l.a `n.a - $(a l.a) + ++ run + ~/ %run + |= [a=(tree item) b=$-(val val)] + |- + ?~ a a + [n=[key.n.a (b val.n.a)] l=$(a l.a) r=$(a r.a)] :: - :: +pop: produce .head (smallest item) and .rest or crash if empty + :: +subset: take a range excluding start and/or end and all elements + :: outside the range :: - ++ pop - |= a=(tree item) - ^- [head=item rest=(tree item)] - :: - ?~ a !! - ?~ l.a [n.a r.a] - :: - =/ l $(a l.a) - :- head.l - :: load .rest.l back into .a and rebalance - :: - ?: |(?=(~ rest.l) (mor key.n.a key.n.rest.l)) - a(l rest.l) - rest.l(r a(r r.rest.l)) - :: +del: delete .key from .a if it exists, producing value iff deleted - :: - ++ del - |= [a=(tree item) =key] - ^- [(unit val) (tree item)] - :: - ?~ a [~ ~] - :: we found .key at the root; delete and rebalance - :: - ?: =(key key.n.a) - [`val.n.a (nip a)] - :: recurse left or right to find .key - :: - ?: (compare key key.n.a) - =+ [found lef]=$(a l.a) - [found a(l lef)] - =+ [found rig]=$(a r.a) - [found a(r rig)] - :: +nip: remove root; for internal use - :: - ++ nip - |= a=(tree item) + ++ subset + ~/ %subset + |= $: tre=(tree item) + start=(unit key) + end=(unit key) + == ^- (tree item) + |^ + ?: ?&(?=(~ start) ?=(~ end)) + tre + ?~ start + (del-span tre %end end) + ?~ end + (del-span tre %start start) + ?> (compare u.start u.end) + =. tre (del-span tre %start start) + (del-span tre %end end) :: - ?> ?=(^ a) - :: delete .n.a; merge and balance .l.a and .r.a + ++ del-span + |= [a=(tree item) b=?(%start %end) c=(unit key)] + ^- (tree item) + ?~ a a + ?~ c a + ?- b + %start + :: found key + ?: =(key.n.a u.c) + (nip a(l ~)) + :: traverse to find key + ?: (compare key.n.a u.c) + :: found key to the left of start + $(a (nip a(l ~))) + :: found key to the right of start + a(l $(a l.a)) + :: + %end + :: found key + ?: =(u.c key.n.a) + (nip a(r ~)) + :: traverse to find key + ?: (compare key.n.a u.c) + :: found key to the left of end + a(r $(a r.a)) + :: found key to the right of end + $(a (nip a(r ~))) + == + -- + :: +tap: convert to list, smallest to largest + :: + ++ tap + ~/ %tap + |= a=(tree item) + ^- (list item) :: - |- ^- (tree item) - ?~ l.a r.a - ?~ r.a l.a - ?: (mor key.n.l.a key.n.r.a) - l.a(r $(l.a r.l.a)) - r.a(l $(r.a l.r.a)) + =| b=(list item) + |- ^+ b + ?~ a b + :: + $(a l.a, b [n.a $(a r.a)]) :: +traverse: stateful partial inorder traversal :: :: Mutates .state on each run of .f. Starts at .start key, or if @@ -5216,6 +5337,7 @@ :: item. :: ++ traverse + ~/ %traverse |* state=mold |= $: a=(tree item) =state @@ -5274,58 +5396,12 @@ =/ rig main(a r.a) rig(a a(r a.rig)) -- - :: +tap: convert to list, smallest to largest - :: - ++ tap - |= a=(tree item) - ^- (list item) - :: - =| b=(list item) - |- ^+ b - ?~ a b - :: - $(a l.a, b [n.a $(a r.a)]) - :: +bap: convert to list, largest to smallest - :: - ++ bap - |= a=(tree item) - ^- (list item) - :: - =| b=(list item) - |- ^+ b - ?~ a b - :: - $(a r.a, b [n.a $(a l.a)]) - :: +run: apply gate to transform all values in place - :: without modifying their type - :: - ++ run - |= [a=(tree item) b=$-(val val)] - |- ^+ a - ?~ a a - [n=[key.n.a (b val.n.a)] l=$(a l.a) r=$(a r.a)] - :: +all: apply logical AND boolean test on all values - :: - ++ all - |= [a=(tree item) b=$-(item ?)] - ^- ? - |- - ?~ a - & - ?&((b n.a) $(a l.a) $(a r.a)) - :: +gas: put a list of items - :: - ++ gas - |= [a=(tree item) b=(list item)] - ^- (tree item) - :: - ?~ b a - $(b t.b, a (put a i.b)) :: +uni: unify two ordered maps :: :: .b takes precedence over .a if keys overlap. :: ++ uni + ~/ %uni |= [a=(tree item) b=(tree item)] ^- (tree item) :: @@ -5344,68 +5420,6 @@ ?: (compare key.n.a key.n.b) $(l.b $(b l.b, r.a ~), a r.a) $(r.b $(b r.b, l.a ~), a l.a) - :: - :: +get: get val at key or return ~ - :: - ++ get - |= [a=(tree item) b=key] - ^- (unit val) - ?~ a ~ - ?: =(b key.n.a) - `val.n.a - ?: (compare b key.n.a) - $(a l.a) - $(a r.a) - :: - :: +subset: take a range excluding start and/or end and all elements - :: outside the range - :: - ++ subset - |= $: tre=(tree item) - start=(unit key) - end=(unit key) - == - ^- (tree item) - |^ - ?: ?&(?=(~ start) ?=(~ end)) - tre - ?~ start - (del-span tre %end end) - ?~ end - (del-span tre %start start) - ?> (compare u.start u.end) - =. tre (del-span tre %start start) - (del-span tre %end end) - :: - ++ del-span - |= [a=(tree item) b=?(%start %end) c=(unit key)] - ^- (tree item) - ?~ a a - ?~ c a - ?- b - %start - :: found key - ?: =(key.n.a u.c) - (nip a(l ~)) - :: traverse to find key - ?: (compare key.n.a u.c) - :: found key to the left of start - $(a (nip a(l ~))) - :: found key to the right of start - a(l $(a l.a)) - :: - %end - :: found key - ?: =(u.c key.n.a) - (nip a(r ~)) - :: traverse to find key - ?: (compare key.n.a u.c) - :: found key to the left of end - a(r $(a r.a)) - :: found key to the right of end - $(a (nip a(r ~))) - == - -- -- :: :: :::: ++userlib :: (2u) non-vane utils diff --git a/pkg/arvo/tests/sys/zuse/ordered-map.hoon b/pkg/arvo/tests/sys/zuse/ordered-map.hoon index 261b8f0851..bc5943a4f9 100644 --- a/pkg/arvo/tests/sys/zuse/ordered-map.hoon +++ b/pkg/arvo/tests/sys/zuse/ordered-map.hoon @@ -17,7 +17,7 @@ :: %+ expect-eq !> %.y - !> (check-balance:atom-map a) + !> (apt:atom-map a) :: ++ test-ordered-map-tap ^- tang :: From 2e302818d3976e8f26c10fc87c04e56141913cf5 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 20 Apr 2021 12:05:24 -0500 Subject: [PATCH 060/136] zuse: added +any to +ordered-map --- pkg/arvo/sys/zuse.hoon | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index d6f4cae0e5..7160a150a0 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5104,6 +5104,15 @@ ?~ a & ?&((b n.a) $(a l.a) $(a r.a)) + :: +any: apply logical OR boolean test on all values + :: + ++ any + ~/ %any + |= [a=(tree item) b=$-(item ?)] + |- ^- ? + ?~ a + | + ?|((b n.a) $(a l.a) $(a r.a)) :: +apt: verify horizontal and vertical orderings :: ++ apt From ae2aefed2a22dc5937bc770bf45f3d34a2f8428a Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 20 Apr 2021 12:06:03 -0500 Subject: [PATCH 061/136] zuse: made +run allow type transform to match +by and +in --- pkg/arvo/sys/zuse.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 7160a150a0..2e5d1158be 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5270,7 +5270,7 @@ :: ++ run ~/ %run - |= [a=(tree item) b=$-(val val)] + |= [a=(tree item) b=$-(val *)] |- ?~ a a [n=[key.n.a (b val.n.a)] l=$(a l.a) r=$(a r.a)] From 9247fc247c4f980e4ea151b3fe60a024f53a4a38 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 20 Apr 2021 12:09:18 -0500 Subject: [PATCH 062/136] zuse: add +got and +has arms to +ordered-map --- pkg/arvo/sys/zuse.hoon | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 2e5d1158be..89fae22c17 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5182,7 +5182,6 @@ :: ?~ b a $(b t.b, a (put a i.b)) - :: :: +get: get val at key or return ~ :: ++ get @@ -5195,6 +5194,19 @@ ?: (compare b key.n.a) $(a l.a) $(a r.a) + :: +got: need value at key + :: + ++ got + |= [a=(tree item) b=key] + ^- val + (need (get a b)) + :: +has: check for key existence + :: + ++ has + ~/ %has + |= [a=(tree item) b=key] + ^- ? + !=(~ (get a b)) :: +nip: remove root; for internal use :: ++ nip From f0ba5afe1cbaaf869e14080fa36a20c8cb51c61f Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 20 Apr 2021 12:56:24 -0500 Subject: [PATCH 063/136] zuse: respond to joe's requests to remove mk-item and add fast hint for ordered-map --- pkg/arvo/sys/zuse.hoon | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 89fae22c17..8bc9d3000f 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5072,10 +5072,6 @@ =/ b ;;((tree [key=key val=value]) a) ?> (apt:((ordered-map key value) ord) b) b -:: -:: $mk-item: constructor for +ordered-map item type -:: -++ mk-item |$ [key val] [key=key val=val] :: +ordered-map: treap with user-specified horizontal order :: :: Conceptually smaller items go on the left, so the item with the @@ -5088,11 +5084,12 @@ ++ ordered-map |* [key=mold val=mold] => |% - +$ item (mk-item key val) + +$ item [key=key val=val] -- :: +compare: item comparator for horizontal order :: |= compare=$-([key key] ?) + ~% %ordered-map ..part ~ |% :: +all: apply logical AND boolean test on all values :: @@ -5286,7 +5283,6 @@ |- ?~ a a [n=[key.n.a (b val.n.a)] l=$(a l.a) r=$(a r.a)] - :: :: +subset: take a range excluding start and/or end and all elements :: outside the range :: From b5a8e7a40b44daa4c21b991ab2946056f42cc372 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 20 Apr 2021 15:03:57 -0500 Subject: [PATCH 064/136] zuse: change name of +ordered-map to +on and add alias, change name of +traverse to +dip as per talk with ted --- pkg/arvo/sys/zuse.hoon | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 8bc9d3000f..163da70948 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5072,7 +5072,8 @@ =/ b ;;((tree [key=key val=value]) a) ?> (apt:((ordered-map key value) ord) b) b -:: +ordered-map: treap with user-specified horizontal order +++ ordered-map on +:: +on: treap with user-specified horizontal order, ordered-map :: :: Conceptually smaller items go on the left, so the item with the :: smallest key can be popped off the head. If $key is `@` and @@ -5081,15 +5082,17 @@ :: WARNING: ordered-map will not work properly if two keys can be :: unequal under noun equality but equal via the compare gate :: -++ ordered-map +++ on + ~/ %on |* [key=mold val=mold] => |% +$ item [key=key val=val] -- :: +compare: item comparator for horizontal order :: + ~% %comp +>+ ~ |= compare=$-([key key] ?) - ~% %ordered-map ..part ~ + ~% %core + ~ |% :: +all: apply logical AND boolean test on all values :: @@ -5345,7 +5348,7 @@ ?~ a b :: $(a l.a, b [n.a $(a r.a)]) - :: +traverse: stateful partial inorder traversal + :: +dip: stateful partial inorder traversal :: :: Mutates .state on each run of .f. Starts at .start key, or if :: .start is ~, starts at the head (item with smallest key). Stops @@ -5353,8 +5356,8 @@ :: keys. Each run of .f can replace an item's value or delete the :: item. :: - ++ traverse - ~/ %traverse + ++ dip + ~/ %dip |* state=mold |= $: a=(tree item) =state From 6b0dc989529bd6f9da1f3bb89c985f7716db2378 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 20 Apr 2021 15:04:28 -0500 Subject: [PATCH 065/136] ames: reflect update of name +traverse -> +dip --- pkg/arvo/sys/vane/ames.hoon | 8 ++++---- pkg/arvo/tests/sys/zuse/ordered-map.hoon | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index ccf31d062f..79d84446f6 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2603,7 +2603,7 @@ =| acc=(unit static-fragment) ^+ [static-fragment=acc live=live.state] :: - %^ (traverse:packet-queue _acc) live.state acc + %^ (dip:packet-queue _acc) live.state acc |= $: acc=_acc key=live-packet-key val=live-packet-val @@ -2681,7 +2681,7 @@ =/ acc resends=*(list static-fragment) :: - %^ (traverse:packet-queue _acc) live.state acc + %^ (dip:packet-queue _acc) live.state acc |= $: acc=_acc key=live-packet-key val=live-packet-val @@ -2734,7 +2734,7 @@ :: ^+ [acc live=live.state] :: - %^ (traverse:packet-queue _acc) live.state acc + %^ (dip:packet-queue _acc) live.state acc |= $: acc=_acc key=live-packet-key val=live-packet-val @@ -2781,7 +2781,7 @@ :: ^+ [metrics=metrics.state live=live.state] :: - %^ (traverse:packet-queue pump-metrics) live.state acc=metrics.state + %^ (dip:packet-queue pump-metrics) live.state acc=metrics.state |= $: metrics=pump-metrics key=live-packet-key val=live-packet-val diff --git a/pkg/arvo/tests/sys/zuse/ordered-map.hoon b/pkg/arvo/tests/sys/zuse/ordered-map.hoon index bc5943a4f9..94b4628b32 100644 --- a/pkg/arvo/tests/sys/zuse/ordered-map.hoon +++ b/pkg/arvo/tests/sys/zuse/ordered-map.hoon @@ -103,11 +103,11 @@ !> (gas:atom-map ~ ~[[1^%b]]) !> b :: -++ test-ordered-map-traverse ^- tang +++ test-ordered-map-dip ^- tang :: =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) :: - =/ b %- (traverse:atom-map ,(list [@ud @tas])) + =/ b %- (dip:atom-map ,(list [@ud @tas])) :* a state=~ :: @@ -129,11 +129,11 @@ !> -.b == :: -++ test-ordered-map-traverse-delete-all ^- tang +++ test-ordered-map-dip-delete-all ^- tang ;: weld =/ q ((ordered-map ,@ ,~) lte) =/ o (gas:q ~ ~[1/~ 2/~ 3/~]) - =/ b ((traverse:q ,~) o ~ |=([~ key=@ ~] [~ %| ~])) + =/ b ((dip:q ,~) o ~ |=([~ key=@ ~] [~ %| ~])) %+ expect-eq !> [~ ~] !> b @@ -147,7 +147,7 @@ ?:((lth aa ba) %.y ?:((gth aa ba) %.n (lte ab bb))) =/ q ((ordered-map ,[@ @] ,~) compare) =/ o (gas:q ~ c) - =/ b ((traverse:q ,~) o ~ |=([~ key=[@ @] ~] [~ %| ~])) + =/ b ((dip:q ,~) o ~ |=([~ key=[@ @] ~] [~ %| ~])) %+ expect-eq !> [~ ~] !> b From 5b7b7e6cb76030a1345a6c298842a7c2d97807c2 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 20 Apr 2021 15:21:07 -0500 Subject: [PATCH 066/136] zuse: reorder dip alphabetically --- pkg/arvo/sys/zuse.hoon | 139 ++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 70 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 163da70948..57cd2bac19 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5070,7 +5070,7 @@ |= ord=$-([key key] ?) |= a=* =/ b ;;((tree [key=key val=value]) a) - ?> (apt:((ordered-map key value) ord) b) + ?> (apt:((on key value) ord) b) b ++ ordered-map on :: +on: treap with user-specified horizontal order, ordered-map @@ -5173,13 +5173,80 @@ [found a(l lef)] =+ [found rig]=$(a r.a) [found a(r rig)] + :: +dip: stateful partial inorder traversal + :: + :: Mutates .state on each run of .f. Starts at .start key, or if + :: .start is ~, starts at the head (item with smallest key). Stops + :: when .f produces .stop=%.y. Traverses from smaller to larger + :: keys. Each run of .f can replace an item's value or delete the + :: item. + :: + ++ dip + ~/ %dip + |* state=mold + |= $: a=(tree item) + =state + f=$-([state item] [(unit val) ? state]) + == + ^+ [state a] + :: acc: accumulator + :: + :: .stop: set to %.y by .f when done traversing + :: .state: threaded through each run of .f and produced by +abet + :: + =/ acc [stop=`?`%.n state=state] + =< abet =< main + |% + ++ this . + ++ abet [state.acc a] + :: +main: main recursive loop; performs a partial inorder traversal + :: + ++ main + ^+ this + :: stop if empty or we've been told to stop + :: + ?: =(~ a) this + ?: stop.acc this + :: inorder traversal: left -> node -> right, until .f sets .stop + :: + =. this left + ?: stop.acc this + =^ del this node + =? this !stop.acc right + =? a del (nip a) + this + :: +node: run .f on .n.a, updating .a, .state, and .stop + :: + ++ node + ^+ [del=*? this] + :: run .f on node, updating .stop.acc and .state.acc + :: + ?> ?=(^ a) + =^ res acc (f state.acc n.a) + ?~ res + [del=& this] + [del=| this(val.n.a u.res)] + :: +left: recurse on left subtree, copying mutant back into .l.a + :: + ++ left + ^+ this + ?~ a this + =/ lef main(a l.a) + lef(a a(l a.lef)) + :: +right: recurse on right subtree, copying mutant back into .r.a + :: + ++ right + ^+ this + ?~ a this + =/ rig main(a r.a) + rig(a a(r a.rig)) + -- :: +gas: put a list of items :: ++ gas ~/ %gas |= [a=(tree item) b=(list item)] ^- (tree item) - :: ?~ b a $(b t.b, a (put a i.b)) :: +get: get val at key or return ~ @@ -5348,74 +5415,6 @@ ?~ a b :: $(a l.a, b [n.a $(a r.a)]) - :: +dip: stateful partial inorder traversal - :: - :: Mutates .state on each run of .f. Starts at .start key, or if - :: .start is ~, starts at the head (item with smallest key). Stops - :: when .f produces .stop=%.y. Traverses from smaller to larger - :: keys. Each run of .f can replace an item's value or delete the - :: item. - :: - ++ dip - ~/ %dip - |* state=mold - |= $: a=(tree item) - =state - f=$-([state item] [(unit val) ? state]) - == - ^+ [state a] - :: acc: accumulator - :: - :: .stop: set to %.y by .f when done traversing - :: .state: threaded through each run of .f and produced by +abet - :: - =/ acc [stop=`?`%.n state=state] - =< abet =< main - |% - ++ this . - ++ abet [state.acc a] - :: +main: main recursive loop; performs a partial inorder traversal - :: - ++ main - ^+ this - :: stop if empty or we've been told to stop - :: - ?: =(~ a) this - ?: stop.acc this - :: inorder traversal: left -> node -> right, until .f sets .stop - :: - =. this left - ?: stop.acc this - =^ del this node - =? this !stop.acc right - =? a del (nip a) - this - :: +node: run .f on .n.a, updating .a, .state, and .stop - :: - ++ node - ^+ [del=*? this] - :: run .f on node, updating .stop.acc and .state.acc - :: - ?> ?=(^ a) - =^ res acc (f state.acc n.a) - ?~ res - [del=& this] - [del=| this(val.n.a u.res)] - :: +left: recurse on left subtree, copying mutant back into .l.a - :: - ++ left - ^+ this - ?~ a this - =/ lef main(a l.a) - lef(a a(l a.lef)) - :: +right: recurse on right subtree, copying mutant back into .r.a - :: - ++ right - ^+ this - ?~ a this - =/ rig main(a r.a) - rig(a a(r a.rig)) - -- :: +uni: unify two ordered maps :: :: .b takes precedence over .a if keys overlap. From cef4e6a7261bc30d1ebd52c484d306e8797483e3 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 20 Apr 2021 16:04:48 -0500 Subject: [PATCH 067/136] zuse: fix +apt jet parent not found --- pkg/arvo/sys/zuse.hoon | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 57cd2bac19..c925941835 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5072,6 +5072,8 @@ =/ b ;;((tree [key=key val=value]) a) ?> (apt:((on key value) ord) b) b +:: +:: ++ ordered-map on :: +on: treap with user-specified horizontal order, ordered-map :: @@ -5117,9 +5119,9 @@ :: ++ apt ~/ %apt - =| [l=(unit key) r=(unit key)] |= a=(tree item) - ^- ? + =| [l=(unit key) r=(unit key)] + |- ^- ? :: empty tree is valid :: ?~ a %.y From 6917aabde57493d7d7d5b9d2b409736c247a48e8 Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Tue, 20 Apr 2021 21:20:39 -0400 Subject: [PATCH 068/136] repo: fix gitignore/makefile --- .gitignore | 7 +------ Makefile | 3 --- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index d2c66c9451..e79759dc67 100644 --- a/.gitignore +++ b/.gitignore @@ -76,9 +76,4 @@ pkg/interface/link-webext/web-ext-artifacts *.xz # Logs -*.log - -backup* -notes -bel -wex +*.log \ No newline at end of file diff --git a/Makefile b/Makefile index fd3eaf12bb..a06eac1c8a 100644 --- a/Makefile +++ b/Makefile @@ -17,9 +17,6 @@ pills: sh/update-brass-pill sh/update-ivory-pill -solid: - sh/update-solid-pill - ropsten-pills: sh/create-ropsten-pills From ca9f6180d8199a66760eb712176852618f089cb5 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Thu, 22 Apr 2021 10:46:48 -0500 Subject: [PATCH 069/136] zuse: make ordered-map's +run wet --- pkg/arvo/sys/zuse.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index c925941835..2e5f4b98e6 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5351,7 +5351,7 @@ :: ++ run ~/ %run - |= [a=(tree item) b=$-(val *)] + |* [a=(tree item) b=$-(val *)] |- ?~ a a [n=[key.n.a (b val.n.a)] l=$(a l.a) r=$(a r.a)] From dcdf695991bc6cdccdc42186874da8fbc6d92af5 Mon Sep 17 00:00:00 2001 From: fang Date: Fri, 23 Apr 2021 22:56:24 +0200 Subject: [PATCH 070/136] clay: insert %nest-fail into trace conditionally Previously, we would stack on the ~| for every file we had previously processed, even though we only want it in the trace for actual failures. --- pkg/arvo/sys/vane/clay.hoon | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 3ca9057d02..d11fdedfa5 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -992,8 +992,9 @@ [%kttr %make [%wing ~[%map]] ~[[%base %atom %ta] spec.i.raz]] |- ?~ res ~ - ~| [%nest-fail path.i.raz p.n.res] - ?> (~(nest ut type) | p.q.n.res) + ?. (~(nest ut type) | p.q.n.res) + ~| [%nest-fail path.i.raz p.n.res] + !! :- [p.n.res q.q.n.res] [$(res l.res) $(res r.res)] :: From 550dee32dab0a898d46702d908804dd6ee931ec7 Mon Sep 17 00:00:00 2001 From: fang Date: Fri, 23 Apr 2021 23:05:11 +0200 Subject: [PATCH 071/136] clay: +any:in in place of +lien over +tap:in --- pkg/arvo/sys/vane/clay.hoon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index d11fdedfa5..3ed87130f8 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -1655,11 +1655,11 @@ |- ^+ cache ?~ builds ~ - ?: %+ lien ~(tap in dez.i.builds) + ?: %- ~(any in dez.i.builds) |= [dir=? =path] ?. dir (~(has in invalid) path) =+ l=(lent path) - %+ lien ~(tap in invalid) + %- ~(any in invalid) |= i=^path &(=(path (scag l i)) ?=([@ %hoon ~] (slag l i))) $(builds t.builds) From 9d7afd10ee1aaddf9f58341b1a8efc294204e8d3 Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Fri, 23 Apr 2021 17:33:49 -0400 Subject: [PATCH 072/136] clay: change fuse request --- pkg/arvo/sys/lull.hoon | 3 +- pkg/arvo/sys/vane/clay.hoon | 81 ++++++++++++++++++++----------------- 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index e91d889352..bc8c552874 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -764,7 +764,8 @@ == :: $: %fuse :: mega merge des=desk :: target desk - srcs=(list [beak germ]) :: sources + ful=(list beak) :: source desks + gim=(list germ) :: strategies == [%mont pot=term bem=beam] :: mount to unix [%dirk des=desk] :: mark mount dirty diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 09130e8151..5aff319995 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -60,7 +60,7 @@ +$ cult (jug wove duct) :: State for ongoing %fuse merges - the list maintains the ordering :: and the map stores the data needed to merge -+$ melt [(list [beak germ]) (map [beak germ] (unit dome:clay))] ++$ melt [bas=beak con=(list [beak germ]) sto=(map beak (unit dome:clay))] :: :: Domestic desk state. :: @@ -1049,7 +1049,7 @@ ~ =/ rus rus:(~(gut by hoy.ruf) her *rung) %+ ~(gut by rus) syd - [lim=~2000.1.1 ref=`*rind qyx=~ dom=*dome per=~ pew=~ fiz=[~ ~]] + [lim=~2000.1.1 ref=`*rind qyx=~ dom=*dome per=~ pew=~ fiz=*melt] :: administrative duct, domestic +rede :: :+ ~ `hun.rom.ruf @@ -1969,39 +1969,54 @@ (emit hen %pass wire %c %warp ali-ship ali-desk `[%sing %v case /]) :: ++ make-melt - |= srcs=(list [beak germ]) + |= [bas=beak con=(list [beak germ])] ^- melt - :- srcs - %- ~(gas by *(map [beak germ] (unit dome:clay))) - (turn srcs |=(a=[beak germ] [a *(unit dome:clay)])) + :+ bas con + %- ~(gas by (~(put by *(map beak (unit dome:clay))) bas *(unit dome:clay))) + (turn con |=(a=[beak germ] [-.a *(unit dome:clay)])) :: ++ start-fuse - |= [srcs=(list [beak germ])] + |= [ful=(list beak) gim=(list germ)] ^+ ..start-fuse - :: use emil here to queue a list of moves - :: syd is current desk (target desk) - :: what is hen? it's the current duct... can this be reused? + :: must have n sources and n-1 gems + ?> =((lent ful) +((lent gim))) =/ moves=(list move) %+ turn - srcs - |= [bec=beak g=germ] + ful + |= bec=beak ^- move - =/ wir=wire /fuse/[syd]/(scot %p p.bec)/[q.bec]/(scot r.bec)/[g] + =/ wir=wire /fuse/[syd]/(scot %p p.bec)/[q.bec]/(scot r.bec) [hen %pass wir %c %warp p.bec q.bec `[%sing %v r.bec /]] + ?~ ful + !! + =/ base=beak i.ful + =/ ful=(list beak) t.ful + =/ con=(list [beak germ]) + =| acc=(list [beak germ]) + |- + ?~ ful + acc + ?~ gim + !! + $(acc [[i.ful i.gim] acc], ful t.ful, gim t.gim) :: we also want to clear the state (fiz) associated with this :: merge and print a warning if it's non trivial i.e. we're :: starting a new fuse before the previous one terminated. - ~& ?~ -.fiz - [%starting-fuse srcs] - :^ %starting-fuse srcs %discarding-state - :: we don't want to ~& an entire dome - %- ~(run by +.fiz) - |=(v=(unit dome:clay) ?~(v %not-received %recieved)) - =. fiz (make-melt srcs) + ~& ?~ con.fiz + [%starting-fuse base con] + :* %starting-fuse + base + con + %discarding-state + :: we don't want to ~& an entire dome + %- ~(run by sto.fiz) + |=(v=(unit dome:clay) ?~(v %not-received %recieved)) + == + =. fiz (make-melt base con) (emil moves) :: ++ take-fuse - |= [[bec=beak g=germ] =riot] + |= [bec=beak =riot] ^+ ..take-fuse ?~ riot ~& [%fuse-for syd %missing bec] @@ -2009,29 +2024,24 @@ :: responses we get for the merge will cause take-fuse to crash =. fiz *melt ..take-fuse - ?> (~(has by +.fiz) [bec g]) + ?> (~(has by sto.fiz) bec) =. fiz - :- -.fiz - (~(put by +.fiz) [bec g] `!<(dome:clay q.r.u.riot)) + :+ bas.fiz con.fiz + (~(put by sto.fiz) bec `!<(dome:clay q.r.u.riot)) =/ all-done=flag - %- ~(all by +.fiz) + %- ~(all by sto.fiz) |= res=(unit dome:clay) ^- flag !=(res ~) ?. all-done ..take-fuse - :: do the merge - =/ merges=(list [beak germ]) -.fiz - :: there's no point to an empty fuse and a single element fuse - :: should just be an %only-that merge. - ?> (gte (lent merges) 2) =| rag=rang =/ clean-hut-ran hut.ran - =/ initial-dome=dome:clay (need (~(got by +.fiz) (snag 0 merges))) + =/ initial-dome=dome:clay (need (~(got by sto.fiz) bas.fiz)) =/ continuation-yaki=yaki (~(got by hut.ran) (~(got by hit.initial-dome) let.initial-dome)) =/ parents=(list tako) ~[(~(got by hit.initial-dome) let.initial-dome)] - =. merges (slag 1 merges) + =/ merges con.fiz |- ^+ ..take-fuse ?~ merges @@ -2041,7 +2051,7 @@ =. hut.ran clean-hut-ran (park | [%| continuation-yaki(p (flop parents))] rag) =/ [bec=beak g=germ] i.merges - =/ ali-dom=dome:clay (need (~(got by +.fiz) bec g)) + =/ ali-dom=dome:clay (need (~(got by sto.fiz) bec)) =/ result (merge-helper p.bec q.bec g ali-dom `continuation-yaki) ?- -.result %| @@ -4228,7 +4238,7 @@ ~&(%fuse-no-desk !!) =^ mos ruf =/ den ((de now rof hen ruf) our des.req) - abet:(start-fuse:den srcs.req) + abet:(start-fuse:den ful.req gim.req) [mos ..^$] :: %mont @@ -4567,10 +4577,9 @@ =* ali-desk=desk i.t.t.t.tea =/ ali-case (rash i.t.t.t.t.tea nuck:so) ?> ?=([%$ *] ali-case) - =/ germ (germ i.t.t.t.t.t.tea) =^ mos ruf =/ den ((de now rof hen ruf) our i.t.tea) - abet:(take-fuse:den [[ali-ship ali-desk (case +.ali-case)] germ] p.hin) + abet:(take-fuse:den [ali-ship ali-desk (case +.ali-case)] p.hin) [mos ..^$] :: ?: ?=([%foreign-warp *] tea) From 201c3db19300255e481e3a846b0e4c70554094b9 Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Sat, 24 Apr 2021 20:50:50 -0400 Subject: [PATCH 073/136] clay: finalize fuse request - update |fuse --- pkg/arvo/gen/hood/fuse.hoon | 4 ++-- pkg/arvo/lib/hood/kiln.hoon | 5 +++-- pkg/arvo/sys/lull.hoon | 4 ++-- pkg/arvo/sys/vane/clay.hoon | 30 ++++++++---------------------- 4 files changed, 15 insertions(+), 28 deletions(-) diff --git a/pkg/arvo/gen/hood/fuse.hoon b/pkg/arvo/gen/hood/fuse.hoon index 39bf9f74c8..03effe5c89 100644 --- a/pkg/arvo/gen/hood/fuse.hoon +++ b/pkg/arvo/gen/hood/fuse.hoon @@ -8,5 +8,5 @@ :::: :: :- %say -|= [[now=@da eny=@uvJ bec=beak] [des=desk srcs=(list [beak germ]) ~] ~] -[%kiln-fuse des srcs] +|= [[now=@da eny=@uvJ bec=beak] [des=desk bas=beak con=(list [beak germ]) ~] ~] +[%kiln-fuse des bas con] diff --git a/pkg/arvo/lib/hood/kiln.hoon b/pkg/arvo/lib/hood/kiln.hoon index 167750360f..8700670bc8 100644 --- a/pkg/arvo/lib/hood/kiln.hoon +++ b/pkg/arvo/lib/hood/kiln.hoon @@ -57,7 +57,8 @@ == +$ kiln-fuse $: syd=desk - srcs=(list [beak germ]) + bas=beak + con=(list [beak germ]) == -- |= [bowl:gall state] @@ -387,7 +388,7 @@ :: ++ poke-fuse |= k=kiln-fuse - abet:(emit [%pass /kiln/fuse/[syd.k] %arvo %c [%fuse syd.k srcs.k]]) + abet:(emit [%pass /kiln/fuse/[syd.k] %arvo %c [%fuse syd.k bas.k con.k]]) :: ++ poke-cancel |= a=@tas diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index bc8c552874..1cac3043b5 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -764,8 +764,8 @@ == :: $: %fuse :: mega merge des=desk :: target desk - ful=(list beak) :: source desks - gim=(list germ) :: strategies + bas=beak :: base desk + con=(list [beak germ]) :: merges == [%mont pot=term bem=beam] :: mount to unix [%dirk des=desk] :: mark mount dirty diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 5aff319995..086ed02493 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -1976,43 +1976,29 @@ (turn con |=(a=[beak germ] [-.a *(unit dome:clay)])) :: ++ start-fuse - |= [ful=(list beak) gim=(list germ)] + |= [bas=beak con=(list [beak germ])] ^+ ..start-fuse - :: must have n sources and n-1 gems - ?> =((lent ful) +((lent gim))) =/ moves=(list move) %+ turn - ful - |= bec=beak + [[bas *germ] con] + |= [bec=beak germ] ^- move =/ wir=wire /fuse/[syd]/(scot %p p.bec)/[q.bec]/(scot r.bec) [hen %pass wir %c %warp p.bec q.bec `[%sing %v r.bec /]] - ?~ ful - !! - =/ base=beak i.ful - =/ ful=(list beak) t.ful - =/ con=(list [beak germ]) - =| acc=(list [beak germ]) - |- - ?~ ful - acc - ?~ gim - !! - $(acc [[i.ful i.gim] acc], ful t.ful, gim t.gim) :: we also want to clear the state (fiz) associated with this :: merge and print a warning if it's non trivial i.e. we're :: starting a new fuse before the previous one terminated. ~& ?~ con.fiz - [%starting-fuse base con] + [%starting-fuse bas con] :* %starting-fuse - base + bas con %discarding-state :: we don't want to ~& an entire dome %- ~(run by sto.fiz) |=(v=(unit dome:clay) ?~(v %not-received %recieved)) == - =. fiz (make-melt base con) + =. fiz (make-melt bas con) (emil moves) :: ++ take-fuse @@ -4238,7 +4224,7 @@ ~&(%fuse-no-desk !!) =^ mos ruf =/ den ((de now rof hen ruf) our des.req) - abet:(start-fuse:den ful.req gim.req) + abet:(start-fuse:den bas.req con.req) [mos ..^$] :: %mont @@ -4570,7 +4556,7 @@ abet:(merge:den ali-ship ali-desk germ p.hin) [mos ..^$] :: - ?: ?=([%fuse @ @ @ @ @ ~] tea) + ?: ?=([%fuse @ @ @ @ ~] tea) ?> ?=(%writ +<.hin) =* syd i.t.tea =/ ali-ship=@p (slav %p i.t.t.tea) From 8cbacb2545b1126b3d55797537364f71d764a4af Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Sun, 25 Apr 2021 10:49:30 -0400 Subject: [PATCH 074/136] clay: allow successive upgrades --- pkg/arvo/sys/vane/clay.hoon | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 086ed02493..eb36d55e5c 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -4102,7 +4102,7 @@ :: :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: =| :: instrument state - $: ver=%7 :: vane version + $: ver=%8 :: vane version ruf=raft :: revision tree == :: |= [now=@da eny=@uvJ rof=roof] :: current invocation @@ -4433,8 +4433,9 @@ |= old=raft-any |^ =? old ?=(%6 -.old) 7+(raft-6-to-7 +.old) - ?> ?=(%7 -.old) - ..^^$(ruf (raft-7-to-8 +.old)) + =? old ?=(%7 -.old) 8+(raft-7-to-8 +.old) + ?> ?=(%8 -.old) + ..^^$(ruf +.old) :: +raft-6-to-7: delete stale ford caches (they could all be invalid) :: ++ raft-6-to-7 From 919450b0c62fa140cea2f2067b25144915bb1f6d Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Sun, 25 Apr 2021 14:23:06 -0400 Subject: [PATCH 075/136] fuse: update help text --- pkg/arvo/gen/hood/fuse/help.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/gen/hood/fuse/help.txt b/pkg/arvo/gen/hood/fuse/help.txt index 32431f5cd0..9e18eb80e9 100644 --- a/pkg/arvo/gen/hood/fuse/help.txt +++ b/pkg/arvo/gen/hood/fuse/help.txt @@ -1,6 +1,6 @@ Usage: - |fuse %destination-desk ~[[source-beak %some-germ] [another-beak %another-germ]] + |fuse %destination-desk base-beak ~[[source-beak %some-germ] [another-beak %another-germ]] A fuse replaces the contents of %destination-desk with the merge of the specified beaks according to their merge strategies. This has no dependence From 0a71b75550aae8309cab8e56f5dc82d3e7a67b2d Mon Sep 17 00:00:00 2001 From: fang Date: Mon, 26 Apr 2021 19:36:43 +0200 Subject: [PATCH 076/136] tests: update clay tests to account for #4745 It had changed the shape of some data structures, but hadn't updated the tests to reflect that. --- pkg/arvo/tests/sys/vane/clay.hoon | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/arvo/tests/sys/vane/clay.hoon b/pkg/arvo/tests/sys/vane/clay.hoon index 636efd3a35..7f0e96542f 100644 --- a/pkg/arvo/tests/sys/vane/clay.hoon +++ b/pkg/arvo/tests/sys/vane/clay.hoon @@ -23,7 +23,7 @@ =/ src "." %+ expect-eq !> ^- pile:fusion - :* ~ ~ ~ ~ ~ ~ + :* ~ ~ ~ ~ ~ ~ ~ tssg+[%dbug [/sur/foo/hoon [[1 1] [1 2]]] [%cnts ~[[%.y 1]] ~]]~ == !> (parse-pile:(ford):fusion /sur/foo/hoon src) @@ -32,7 +32,7 @@ =/ src "/% moo %mime\0a." %+ expect-eq !> ^- pile:fusion - :* sur=~ lib=~ raw=~ + :* sur=~ lib=~ raw=~ raz=~ maz=[face=%moo mark=%mime]~ caz=~ bar=~ tssg+[%dbug [/sur/foo/hoon [[2 1] [2 2]]] [%cnts ~[[%.y 1]] ~]]~ @@ -43,7 +43,7 @@ =/ src "/$ goo %mime %txt\0a." %+ expect-eq !> ^- pile:fusion - :* sur=~ lib=~ raw=~ maz=~ + :* sur=~ lib=~ raw=~ raz=~ maz=~ caz=[face=%goo from=%mime to=%txt]~ bar=~ tssg+[%dbug [/sur/foo/hoon [[2 1] [2 2]]] [%cnts ~[[%.y 1]] ~]]~ @@ -74,7 +74,7 @@ [`%hood-drum %hood-drum] [`%hood-write %hood-write] == - raw=~ maz=~ caz=~ bar=~ + raw=~ raz=~ maz=~ caz=~ bar=~ tssg+[%dbug [/sur/foo/hoon [[10 1] [10 2]]] [%cnts ~[[%.y 1]] ~]]~ == !> (parse-pile:(ford):fusion /sur/foo/hoon src) @@ -115,7 +115,7 @@ (slap res limb/%bunt) :: %+ expect-eq - !> (~(gas in *(set path)) /mar/mime/hoon ~) + !> (~(gas in *(set [? path])) |^/mar/mime/hoon ~) !> dez:(~(got by files.cache.nub) /mar/mime/hoon) == :: @@ -142,7 +142,7 @@ (slap res limb/%bunt) :: %+ expect-eq - !> (~(gas in *(set path)) /mar/udon/hoon /lib/cram/hoon ~) + !> (~(gas in *(set [? path])) |^/mar/udon/hoon |^/lib/cram/hoon ~) !> dez:(~(got by files.cache.nub) /mar/udon/hoon) == :: @@ -224,7 +224,7 @@ (slap res (ream '(+ [*^ [%bob ~] ~])')) :: %+ expect-eq - !> (~(gas in *(set path)) /gen/hello/hoon ~) + !> (~(gas in *(set [? path])) |^/gen/hello/hoon ~) !> dez:(~(got by files.cache.nub) /gen/hello/hoon) == :: @@ -249,10 +249,10 @@ !>((slab %read %get-our -.res)) :: %+ expect-eq - !> %- ~(gas in *(set path)) - :~ /lib/strandio/hoon - /lib/strand/hoon - /sur/spider/hoon + !> %- ~(gas in *(set [? path])) + :~ [| /lib/strandio/hoon] + [| /lib/strand/hoon] + [| /sur/spider/hoon] == !> dez:(~(got by files.cache.nub) /lib/strandio/hoon) == From d95696f29cd4927e2ccdb028810d468950823600 Mon Sep 17 00:00:00 2001 From: fang Date: Mon, 26 Apr 2021 19:40:26 +0200 Subject: [PATCH 077/136] tests: update format tests to account for #4677 It had changed ship encoding to fit into a json %n, but hadn't updated the tests to reflect this. --- pkg/arvo/tests/sys/zuse/format.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/tests/sys/zuse/format.hoon b/pkg/arvo/tests/sys/zuse/format.hoon index a8ee826a4a..dddc9ef2e4 100644 --- a/pkg/arvo/tests/sys/zuse/format.hoon +++ b/pkg/arvo/tests/sys/zuse/format.hoon @@ -187,7 +187,7 @@ :: ship - store ship identity as a string :: %+ expect-eq - !> [%s 'zod'] + !> [%n '"zod"'] !> (ship ~zod) == :: dejs - recursive processing of `json` values From dd6eeccdae2ff96bc3f81bfa051e662fe56156b7 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Wed, 28 Apr 2021 16:12:46 -0500 Subject: [PATCH 078/136] zuse: update names of +subset to +lot, +peek to +pry, remove unnecessary comments --- pkg/arvo/app/graph-store.hoon | 10 +- pkg/arvo/app/hark-graph-hook.hoon | 2 +- pkg/arvo/gen/tally.hoon | 6 +- pkg/arvo/sys/vane/ames.hoon | 6 +- pkg/arvo/sys/vane/behn.hoon | 4 +- pkg/arvo/sys/zuse.hoon | 170 +++++++++++------------ pkg/arvo/tests/sys/zuse/ordered-map.hoon | 24 ++-- 7 files changed, 104 insertions(+), 118 deletions(-) diff --git a/pkg/arvo/app/graph-store.hoon b/pkg/arvo/app/graph-store.hoon index 87f4644803..961df66603 100644 --- a/pkg/arvo/app/graph-store.hoon +++ b/pkg/arvo/app/graph-store.hoon @@ -918,7 +918,7 @@ :+ %add-nodes [ship term] %- ~(gas by *(map index:store node:store)) - %+ turn (tap:orm `graph:store`(subset:orm p.u.graph start end)) + %+ turn (tap:orm `graph:store`(lot:orm p.u.graph start end)) |= [=atom =node:store] ^- [index:store node:store] [~[atom] node] @@ -971,7 +971,7 @@ %+ turn =- ?.(older (slag (safe-sub (lent -) count) -) (scag count -)) %- tap:orm - %+ subset:orm u.graph + %+ lot:orm u.graph =/ idx (snag (dec (lent index)) index) ?:(older [`idx ~] [~ `idx]) @@ -1024,7 +1024,7 @@ :+ %add-nodes [ship term] %- ~(gas by *(map index:store node:store)) - %+ turn (tap:orm `graph:store`(subset:orm p.children.u.node end start)) + %+ turn (tap:orm `graph:store`(lot:orm p.children.u.node end start)) |= [=atom =node:store] ^- [index:store node:store] [(snoc index atom) node] @@ -1038,7 +1038,7 @@ =/ update-log=(unit update-log:store) (~(get by update-logs) [ship term]) ?~ update-log [~ ~] :: orm-log is ordered backwards, so swap start and end - ``noun+!>((subset:orm-log u.update-log end start)) + ``noun+!>((lot:orm-log u.update-log end start)) :: [%x %update-log @ @ ~] =/ =ship (slav %p i.t.t.path) @@ -1056,7 +1056,7 @@ %+ biff m-update-log |= =update-log:store =/ result=(unit [=time =update:store]) - (peek:orm-log:store update-log) + (pry:orm-log:store update-log) (bind result |=([=time update:store] time)) == :: diff --git a/pkg/arvo/app/hark-graph-hook.hoon b/pkg/arvo/app/hark-graph-hook.hoon index 3e779a8bcf..3796339086 100644 --- a/pkg/arvo/app/hark-graph-hook.hoon +++ b/pkg/arvo/app/hark-graph-hook.hoon @@ -256,7 +256,7 @@ =/ graph=graph:graph-store :: graph in subscription is bunted (get-graph-mop:gra rid) =/ node=(unit node:graph-store) - (bind (peek:orm:graph-store graph) |=([@ =node:graph-store] node)) + (bind (pry:orm:graph-store graph) |=([@ =node:graph-store] node)) =^ cards state (check-nodes (drop node) rid) ?. (should-watch:ha rid) diff --git a/pkg/arvo/gen/tally.hoon b/pkg/arvo/gen/tally.hoon index fbaf04e0a9..b7289c2c1c 100644 --- a/pkg/arvo/gen/tally.hoon +++ b/pkg/arvo/gen/tally.hoon @@ -1,5 +1,5 @@ /- gr=group, md=metadata-store, ga=graph-store -/+ re=resource +/+ re=resource, graph=graph-store !: :- %say |= $: [now=@da eny=@uvJ =beak] @@ -85,9 +85,9 @@ %+ scry update:ga [%x %graph-store /graph/(scot %p entity.r)/[name.r]/noun] ?> ?=(%add-graph -.q.upd) - =/ mo ((ordered-map atom node:ga) gth) + =* mo orm:graph =/ week=(list [@da node:ga]) - (tap:mo (subset:mo graph.q.upd ~ `(sub now ~d7))) + (tap:mo (lot:mo graph.q.upd ~ `(sub now ~d7))) :- (lent week) %~ wyt in %+ roll week diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 79d84446f6..b8b12f0913 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2535,7 +2535,7 @@ ++ assert ^+ message-pump =/ top-live - (peek:packet-queue:*make-packet-pump live.packet-pump-state.state) + (pry:packet-queue:*make-packet-pump live.packet-pump-state.state) ?. |(?=(~ top-live) (lte current.state message-num.key.u.top-live)) ~| [%strange-current current=current.state key.u.top-live] !! @@ -2804,10 +2804,10 @@ :: ++ set-wake ^+ packet-pump - :: if nonempty .live, peek at head to get next wake time + :: if nonempty .live, pry at head to get next wake time :: =/ new-wake=(unit @da) - ?~ head=(peek:packet-queue live.state) + ?~ head=(pry:packet-queue live.state) ~ `(next-expiry:gauge u.head) :: no-op if no change diff --git a/pkg/arvo/sys/vane/behn.hoon b/pkg/arvo/sys/vane/behn.hoon index 3933c22e62..afff958eae 100644 --- a/pkg/arvo/sys/vane/behn.hoon +++ b/pkg/arvo/sys/vane/behn.hoon @@ -186,7 +186,7 @@ =* timers timers.state :: if no timers, cancel existing wakeup timer or no-op :: - =/ first=(unit [date=@da *]) (peek:timer-map timers.state) + =/ first=(unit [date=@da *]) (pry:timer-map timers.state) ?~ first ?~ next-wake event-core @@ -351,7 +351,7 @@ [%timers %next ~] :^ ~ ~ %noun !> ^- (unit @da) - (bind (peek:timer-map timers) head) + (bind (pry:timer-map timers) head) :: [%timers @ ~] ?~ til=(slaw %da i.t.tyl) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 2e5f4b98e6..a843d4ed04 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5150,11 +5150,9 @@ ~/ %bap |= a=(tree item) ^- (list item) - :: =| b=(list item) |- ^+ b ?~ a b - :: $(a r.a, b [n.a $(a l.a)]) :: +del: delete .key from .a if it exists, producing value iff deleted :: @@ -5162,7 +5160,6 @@ ~/ %del |= [a=(tree item) =key] ^- [(unit val) (tree item)] - :: ?~ a [~ ~] :: we found .key at the root; delete and rebalance :: @@ -5276,90 +5273,11 @@ |= [a=(tree item) b=key] ^- ? !=(~ (get a b)) - :: +nip: remove root; for internal use - :: - ++ nip - ~/ %nip - |= a=(tree item) - ^- (tree item) - :: - ?> ?=(^ a) - :: delete .n.a; merge and balance .l.a and .r.a - :: - |- ^- (tree item) - ?~ l.a r.a - ?~ r.a l.a - ?: (mor key.n.l.a key.n.r.a) - l.a(r $(l.a r.l.a)) - r.a(l $(r.a l.r.a)) - :: +peek: produce head (smallest item) or null - :: - ++ peek - ~/ %peek - |= a=(tree item) - ^- (unit item) - :: - ?~ a ~ - ?~ l.a `n.a - $(a l.a) - :: - :: +pop: produce .head (smallest item) and .rest or crash if empty - :: - ++ pop - ~/ %pop - |= a=(tree item) - ^- [head=item rest=(tree item)] - :: - ?~ a !! - ?~ l.a [n.a r.a] - :: - =/ l $(a l.a) - :- head.l - :: load .rest.l back into .a and rebalance - :: - ?: |(?=(~ rest.l) (mor key.n.a key.n.rest.l)) - a(l rest.l) - rest.l(r a(r r.rest.l)) - :: +put: ordered item insert - :: - ++ put - ~/ %put - |= [a=(tree item) =key =val] - ^- (tree item) - :: base case: replace null with single-item tree - :: - ?~ a [n=[key val] l=~ r=~] - :: base case: overwrite existing .key with new .val - :: - ?: =(key.n.a key) a(val.n val) - :: if item goes on left, recurse left then rebalance vertical order - :: - ?: (compare key key.n.a) - =/ l $(a l.a) - ?> ?=(^ l) - ?: (mor key.n.a key.n.l) - a(l l) - l(r a(l r.l)) - :: item goes on right; recurse right then rebalance vertical order - :: - =/ r $(a r.a) - ?> ?=(^ r) - ?: (mor key.n.a key.n.r) - a(r r) - r(l a(r l.r)) - :: +run: apply gate to transform all values in place - :: - ++ run - ~/ %run - |* [a=(tree item) b=$-(val *)] - |- - ?~ a a - [n=[key.n.a (b val.n.a)] l=$(a l.a) r=$(a r.a)] - :: +subset: take a range excluding start and/or end and all elements + :: +lot: take a subset range excluding start and/or end and all elements :: outside the range :: - ++ subset - ~/ %subset + ++ lot + ~/ %lot |= $: tre=(tree item) start=(unit key) end=(unit key) @@ -5405,17 +5323,90 @@ $(a (nip a(r ~))) == -- + :: +nip: remove root; for internal use + :: + ++ nip + ~/ %nip + |= a=(tree item) + ^- (tree item) + ?> ?=(^ a) + :: delete .n.a; merge and balance .l.a and .r.a + :: + |- ^- (tree item) + ?~ l.a r.a + ?~ r.a l.a + ?: (mor key.n.l.a key.n.r.a) + l.a(r $(l.a r.l.a)) + r.a(l $(r.a l.r.a)) + :: + :: +pop: produce .head (smallest item) and .rest or crash if empty + :: + ++ pop + ~/ %pop + |= a=(tree item) + ^- [head=item rest=(tree item)] + ?~ a !! + ?~ l.a [n.a r.a] + =/ l $(a l.a) + :- head.l + :: load .rest.l back into .a and rebalance + :: + ?: |(?=(~ rest.l) (mor key.n.a key.n.rest.l)) + a(l rest.l) + rest.l(r a(r r.rest.l)) + :: +pry: produce head (smallest item) or null + :: + ++ pry + ~/ %pry + |= a=(tree item) + ^- (unit item) + ?~ a ~ + ?~ l.a `n.a + $(a l.a) + :: +put: ordered item insert + :: + ++ put + ~/ %put + |= [a=(tree item) =key =val] + ^- (tree item) + :: base case: replace null with single-item tree + :: + ?~ a [n=[key val] l=~ r=~] + :: base case: overwrite existing .key with new .val + :: + ?: =(key.n.a key) a(val.n val) + :: if item goes on left, recurse left then rebalance vertical order + :: + ?: (compare key key.n.a) + =/ l $(a l.a) + ?> ?=(^ l) + ?: (mor key.n.a key.n.l) + a(l l) + l(r a(l r.l)) + :: item goes on right; recurse right then rebalance vertical order + :: + =/ r $(a r.a) + ?> ?=(^ r) + ?: (mor key.n.a key.n.r) + a(r r) + r(l a(r l.r)) + :: +run: apply gate to transform all values in place + :: + ++ run + ~/ %run + |* [a=(tree item) b=$-(val *)] + |- + ?~ a a + [n=[key.n.a (b val.n.a)] l=$(a l.a) r=$(a r.a)] :: +tap: convert to list, smallest to largest :: ++ tap ~/ %tap |= a=(tree item) ^- (list item) - :: =| b=(list item) |- ^+ b ?~ a b - :: $(a l.a, b [n.a $(a r.a)]) :: +uni: unify two ordered maps :: @@ -5425,19 +5416,14 @@ ~/ %uni |= [a=(tree item) b=(tree item)] ^- (tree item) - :: ?~ b a ?~ a b ?: =(key.n.a key.n.b) - :: [n=n.b l=$(a l.a, b l.b) r=$(a r.a, b r.b)] - :: ?: (mor key.n.a key.n.b) - :: ?: (compare key.n.b key.n.a) $(l.a $(a l.a, r.b ~), b r.b) $(r.a $(a r.a, l.b ~), b l.b) - :: ?: (compare key.n.a key.n.b) $(l.b $(b l.b, r.a ~), a r.a) $(r.b $(b r.b, l.a ~), a l.a) diff --git a/pkg/arvo/tests/sys/zuse/ordered-map.hoon b/pkg/arvo/tests/sys/zuse/ordered-map.hoon index 94b4628b32..4191d3e7ee 100644 --- a/pkg/arvo/tests/sys/zuse/ordered-map.hoon +++ b/pkg/arvo/tests/sys/zuse/ordered-map.hoon @@ -35,13 +35,13 @@ !> [[0 %a] (gas:atom-map ~ (items-from-keys (gulf 1 6)))] !> (pop:atom-map a) :: -++ test-ordered-map-peek ^- tang +++ test-ordered-map-pry ^- tang :: =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) :: %+ expect-eq !> `[0 %a] - !> (peek:atom-map a) + !> (pry:atom-map a) :: ++ test-ordered-map-nip ^- tang :: @@ -53,51 +53,51 @@ !> (gas:atom-map ~ ~[[0^%a] [1^%b] [2^%c] [3^%d] [4^%e] [5^%f]]) !> b :: -++ test-ordered-map-subset ^- tang +++ test-ordered-map-lot ^- tang :: =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) :: - =/ b (subset:atom-map a `0 `4) + =/ b (lot:atom-map a `0 `4) :: %+ expect-eq !> (gas:atom-map ~ ~[[1^%b] [2^%c] [3^%d]]) !> b :: -++ test-ordered-map-null-start-subset ^- tang +++ test-ordered-map-null-start-lot ^- tang :: =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) :: - =/ b (subset:atom-map a ~ `5) + =/ b (lot:atom-map a ~ `5) :: %+ expect-eq !> (gas:atom-map ~ ~[[0^%a] [1^%b] [2^%c] [3^%d] [4^%e]]) !> b :: -++ test-ordered-map-null-end-subset ^- tang +++ test-ordered-map-null-end-lot ^- tang :: =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) :: - =/ b (subset:atom-map a `1 ~) + =/ b (lot:atom-map a `1 ~) :: %+ expect-eq !> (gas:atom-map ~ ~[[2^%c] [3^%d] [4^%e] [5^%f] [6^%g]]) !> b :: -++ test-ordered-map-double-null-subset ^- tang +++ test-ordered-map-double-null-lot ^- tang :: =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) :: - =/ b (subset:atom-map a ~ ~) + =/ b (lot:atom-map a ~ ~) :: %+ expect-eq !> (gas:atom-map ~ ~[[0^%a] [1^%b] [2^%c] [3^%d] [4^%e] [5^%f] [6^%g]]) !> b :: -++ test-ordered-map-not-found-start-subset ^- tang +++ test-ordered-map-not-found-start-lot ^- tang :: =/ a=(tree [@ud @tas]) (gas:atom-map ~ ~[[1^%b]]) :: - =/ b (subset:atom-map a `0 ~) + =/ b (lot:atom-map a `0 ~) :: %+ expect-eq !> (gas:atom-map ~ ~[[1^%b]]) From 7b7e9e310491508d2f601af0836b7ee2b9264174 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Wed, 28 Apr 2021 23:12:57 -0700 Subject: [PATCH 079/136] clay: fix requests for strange cares over ames Four changes: - implement +validate-u to allow %u requests over the network - make +validate-x use our local marks to make %x requests generally work over the network - in +start-request, if a foreign ship is making a request that we shouldn't send over the network, ignore it. This closes a DOS vector. - in +duce, if we're about to make a request to a foreign ship which they won't be able to answer, crash the event. Combined, these fix many of the common cases of weirdness around foreign clay requests. Notably absent is a fix for reading `%a` across the network, which I still maintain should happen against the foreign hoon/zuse. fixes #4834 see also #4307 --- pkg/arvo/sys/vane/clay.hoon | 46 +++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index af2f653037..355b504d9e 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -1256,6 +1256,24 @@ =/ =path [%question desk (scot %ud index) ~] (emit duct %pass wire %a %plea ship %c path `riff-any`[%1 riff]) :: + ++ foreign-capable + |= =rave + |^ + ?- -.rave + %many & + %sing (good-care care.mood.rave) + %next (good-care care.mood.rave) + %mult + %- ~(all in paths.mool.rave) + |= [=care =path] + (good-care care) + == + :: + ++ good-care + |= =care + (~(has in ^~((silt `(list ^care)`~[%u %w %x %y %z]))) care) + -- + :: :: Create a request that cannot be filled immediately. :: :: If it's a local request, we just put in in `qyx`, setting a timer if it's @@ -1275,6 +1293,10 @@ =. rave ?. ?=([%sing %v *] rave) rave [%many %| [%ud let.dom] case.mood.rave path.mood.rave] + :: + ?. (foreign-capable rave) + ~|([%clay-bad-foreign-request-care rave] !!) + :: =+ inx=nix.u.ref =. +>+.$ =< ?>(?=(^ ref) .) @@ -2665,6 +2687,9 @@ ++ start-request |= [for=(unit [ship @ud]) rav=rave] ^+ ..start-request + ?: &(?=(^ for) !(foreign-capable rav)) + ~& [%bad-foreign-request-care from=for rav] + ..start-request =^ [new-sub=(unit rove) sub-results=(list sub-result)] fod.dom (try-fill-sub for (rave-to-rove rav)) =. ..start-request (send-sub-results sub-results [hen ~ ~]) @@ -2721,14 +2746,23 @@ %r ~| %no-cages-please-they-are-just-way-too-big !! %s ~| %please-dont-get-your-takos-over-a-network !! %t ~| %requesting-foreign-directory-is-vaporware !! - %u ~| %prolly-poor-idea-to-get-rang-over-network !! %v ~| %weird-shouldnt-get-v-request-from-network !! - %z `(validate-z r.rand) + %u `(validate-u r.rand) %w `(validate-w r.rand) %x (validate-x [p.p q.p q r]:rand) %y `[p.r.rand !>(;;(arch q.r.rand))] + %z `(validate-z r.rand) == :: + :: Make sure the incoming data is a %u response + :: + ++ validate-u + |= =page + ^- cage + ?> ?=(%flag p.page) + :- p.page + !> ;;(? q.page) + :: :: Make sure the incoming data is a %w response :: ++ validate-w @@ -2749,7 +2783,11 @@ =/ vale-result %- mule |. %- wrap:fusion - (page-to-cage:(ford:fusion static-ford-args) peg) + :: Use %home's marks to validate, so we don't have to build the + :: foreign hoon/zuse + :: + =/ args %*(static-ford-args . dom dom:(~(got by dos.rom) %home)) + (page-to-cage:(ford:fusion args) peg) ?: ?=(%| -.vale-result) %- (slog >%validate-x-failed< p.vale-result) ~ @@ -2762,7 +2800,7 @@ ^- cage ?> ?=(%uvi p.page) :- p.page - !>(;;(@uvI q.page)) + !> ;;(@uvI q.page) -- :: :: Respond to backfill request From c6f296ee95bf1238939c5c13a6805b5d45e76057 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Fri, 30 Apr 2021 11:51:07 +0000 Subject: [PATCH 080/136] vere: move daemon mode code into compat/*/daemon.c --- pkg/urbit/compat/mingw/daemon.c | 25 +++++++ pkg/urbit/compat/posix/daemon.c | 91 ++++++++++++++++++++++++ pkg/urbit/daemon/main.c | 119 ++------------------------------ pkg/urbit/include/vere/vere.h | 5 ++ 4 files changed, 125 insertions(+), 115 deletions(-) create mode 100644 pkg/urbit/compat/mingw/daemon.c create mode 100644 pkg/urbit/compat/posix/daemon.c diff --git a/pkg/urbit/compat/mingw/daemon.c b/pkg/urbit/compat/mingw/daemon.c new file mode 100644 index 0000000000..aca9888b54 --- /dev/null +++ b/pkg/urbit/compat/mingw/daemon.c @@ -0,0 +1,25 @@ +#include "all.h" +#include "vere/vere.h" + +/* _fix_std_handle(): replaces given stdio fd with a handle to null. +*/ +static void +_fix_std_handle(c3_i fid_i, DWORD typ_u) +{ + dup2(open(c3_dev_null, O_RDWR, 0), fid_i); // this closes the system-provided handle + SetStdHandle(typ_u, (HANDLE)_get_osfhandle(fid_i)); +} + +/* u3_daemon_init(): platform-specific daemon mode initialization. +*/ +void +u3_daemon_init() +{ + // detect stdout/stderr redirection + DWORD dum_u; + BOOL do1_u = GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &dum_u); + BOOL do2_u = GetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), &dum_u); + FreeConsole(); + if ( do1_u ) _fix_std_handle(1, STD_OUTPUT_HANDLE); + if ( do2_u ) _fix_std_handle(2, STD_ERROR_HANDLE); +} diff --git a/pkg/urbit/compat/posix/daemon.c b/pkg/urbit/compat/posix/daemon.c new file mode 100644 index 0000000000..5391fda469 --- /dev/null +++ b/pkg/urbit/compat/posix/daemon.c @@ -0,0 +1,91 @@ +#include "all.h" +#include "vere/vere.h" +#include + +/* + This is set to the the write-end of a pipe when Urbit is started in + daemon mode. It's meant to be used as a signal to the parent process + that the child process has finished booting. +*/ +static c3_i _child_process_booted_signal_fd = -1; + +/* + This should be called whenever the ship has been booted enough to + handle commands from automation code. Specifically, once the Eyre's + `chis` interface is up and running. + + In daemon mode, this signals to the parent process that it can + exit. Otherwise, it does nothing. + + Once we've sent a signal with `write`, we close the file descriptor + and overwrite the global to make it impossible to accidentally do + this twice. +*/ +static void _on_boot_completed_cb() { + c3_c buf[2] = {0,0}; + + if ( -1 == _child_process_booted_signal_fd ) { + return; + } + + if ( 0 == write(_child_process_booted_signal_fd, buf, 1) ) { + c3_assert(!"_on_boot_completed_cb: Can't write to parent FD"); + } + + close(_child_process_booted_signal_fd); + _child_process_booted_signal_fd = -1; +} + +/* u3_daemon_init(): platform-specific daemon mode initialization. + + We use a pipe to communicate between the child and the parent. The + parent waits for the child to write something to the pipe and + then exits. If the pipe is closed with nothing written to it, get + the exit status from the child process and also exit with that status. + + We want the child to write to the pipe once it's booted, so we put + `_on_boot_completed_cb` into `u3_Host.bot_f`, which is NULL in + non-daemon mode. That gets called once the `chis` service is + available. + + In both processes, we are good fork() citizens, and close all unused + file descriptors. Closing `pipefd[1]` in the parent process is + especially important, since the pipe needs to be closed if the child + process dies. When the pipe is closed, the read fails, and that's + how we know that something went wrong. + + There are some edge cases around `WEXITSTATUS` that are not handled + here, but I don't think it matters. +*/ +void +u3_daemon_init() +{ + c3_i pipefd[2]; + + if ( 0 != pipe(pipefd) ) { + c3_assert(!"Failed to create pipe"); + } + + pid_t childpid = fork(); + + if ( 0 == childpid ) { + close(pipefd[0]); + _child_process_booted_signal_fd = pipefd[1]; + u3_Host.bot_f = _on_boot_completed_cb; + return; + } + + close(pipefd[1]); + close(0); + close(1); + close(2); + + c3_c buf[2] = {0,0}; + if ( 1 == read(pipefd[0], buf, 1) ) { + exit(0); + } + + c3_i status; + wait(&status); + exit(WEXITSTATUS(status)); +} diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 42d7857434..372bd28bc1 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -5,9 +5,6 @@ #define C3_GLOBAL #include "all.h" #include "vere/vere.h" -#if !defined(U3_OS_mingw) -#include -#endif #include #include #include @@ -546,117 +543,6 @@ _stop_exit(c3_i int_i) u3_king_bail(); } -/* - This is set to the the write-end of a pipe when Urbit is started in - daemon mode. It's meant to be used as a signal to the parent process - that the child process has finished booting. -*/ -static c3_i _child_process_booted_signal_fd = -1; - -/* - This should be called whenever the ship has been booted enough to - handle commands from automation code. Specifically, once the Eyre's - `chis` interface is up and running. - - In daemon mode, this signals to the parent process that it can - exit. Otherwise, it does nothing. - - Once we've sent a signal with `write`, we close the file descriptor - and overwrite the global to make it impossible to accidentally do - this twice. -*/ -static void _on_boot_completed_cb() { - c3_c buf[2] = {0,0}; - - if ( -1 == _child_process_booted_signal_fd ) { - return; - } - - if ( 0 == write(_child_process_booted_signal_fd, buf, 1) ) { - c3_assert(!"_on_boot_completed_cb: Can't write to parent FD"); - } - - close(_child_process_booted_signal_fd); - _child_process_booted_signal_fd = -1; -} - -/* _fix_std_handle(): replaces given stdio fd with a handle to null. -*/ -#if defined(U3_OS_mingw) -static void -_fix_std_handle(c3_i fid_i, DWORD typ_u) -{ - dup2(open(c3_dev_null, O_RDWR, 0), fid_i); // this closes the system-provided handle - SetStdHandle(typ_u, (HANDLE)_get_osfhandle(fid_i)); -} -#endif - -/* - In daemon mode, run the urbit as a background process, but don't - exit from the parent process until the ship is finished booting. - - We use a pipe to communicate between the child and the parent. The - parent waits for the child to write something to the pipe and - then exits. If the pipe is closed with nothing written to it, get - the exit status from the child process and also exit with that status. - - We want the child to write to the pipe once it's booted, so we put - `_on_boot_completed_cb` into `u3_Host.bot_f`, which is NULL in - non-daemon mode. That gets called once the `chis` service is - available. - - In both processes, we are good fork() citizens, and close all unused - file descriptors. Closing `pipefd[1]` in the parent process is - especially important, since the pipe needs to be closed if the child - process dies. When the pipe is closed, the read fails, and that's - how we know that something went wrong. - - There are some edge cases around `WEXITSTATUS` that are not handled - here, but I don't think it matters. -*/ -static void -_fork_into_background_process() -{ -#if defined(U3_OS_mingw) - // detect stdout/stderr redirection - DWORD dum_u; - BOOL do1_u = GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &dum_u); - BOOL do2_u = GetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), &dum_u); - FreeConsole(); - if ( do1_u ) _fix_std_handle(1, STD_OUTPUT_HANDLE); - if ( do2_u ) _fix_std_handle(2, STD_ERROR_HANDLE); -#else - c3_i pipefd[2]; - - if ( 0 != pipe(pipefd) ) { - c3_assert(!"Failed to create pipe"); - } - - pid_t childpid = fork(); - - if ( 0 == childpid ) { - close(pipefd[0]); - _child_process_booted_signal_fd = pipefd[1]; - u3_Host.bot_f = _on_boot_completed_cb; - return; - } - - close(pipefd[1]); - close(0); - close(1); - close(2); - - c3_c buf[2] = {0,0}; - if ( 1 == read(pipefd[0], buf, 1) ) { - exit(0); - } - - c3_i status; - wait(&status); - exit(WEXITSTATUS(status)); -#endif -} - /* _stop_on_boot_completed_cb(): exit gracefully after boot is complete */ static void @@ -723,7 +609,10 @@ main(c3_i argc, #endif if ( c3y == u3_Host.ops_u.dem ) { - _fork_into_background_process(); + // In daemon mode, run the urbit as a background process, but don't + // exit from the parent process until the ship is finished booting. + // + u3_daemon_init(); } if ( c3y == u3_Host.ops_u.rep ) { diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index 94cd4d6942..56edece23a 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -1396,6 +1396,11 @@ void u3_king_grab(void* ptr_v); + /* u3_daemon_init(): platform-specific daemon mode initialization. + */ + void + u3_daemon_init(); + /* u3_write_fd(): retry interrupts, continue partial writes, assert errors. */ void From a8275777da32c9a1f9dfd080c9d17a8f0f50e24f Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Fri, 30 Apr 2021 21:29:21 +0300 Subject: [PATCH 081/136] vere: boot notification in mingw daemon mode --- pkg/urbit/compat/mingw/daemon.c | 115 ++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 12 deletions(-) diff --git a/pkg/urbit/compat/mingw/daemon.c b/pkg/urbit/compat/mingw/daemon.c index aca9888b54..0220cb75ae 100644 --- a/pkg/urbit/compat/mingw/daemon.c +++ b/pkg/urbit/compat/mingw/daemon.c @@ -1,13 +1,32 @@ #include "all.h" #include "vere/vere.h" -/* _fix_std_handle(): replaces given stdio fd with a handle to null. +/* _dup_std_handle(): creates an inheritable duplicate of a standard handle. */ -static void -_fix_std_handle(c3_i fid_i, DWORD typ_u) +static BOOL +_dup_std_handle(HANDLE* new_u, DWORD typ_u) { - dup2(open(c3_dev_null, O_RDWR, 0), fid_i); // this closes the system-provided handle - SetStdHandle(typ_u, (HANDLE)_get_osfhandle(fid_i)); + DWORD dum_u; + HANDLE han_u = GetStdHandle(typ_u); + BOOL con_u = GetConsoleMode(han_u, &dum_u); + if ( con_u ) { + han_u = (HANDLE)_get_osfhandle(open(c3_dev_null, O_RDWR, 0)); + } + + if ( !DuplicateHandle(GetCurrentProcess(), han_u, GetCurrentProcess(), new_u, 0, TRUE, DUPLICATE_SAME_ACCESS) ) { + fprintf(stderr, "vere: DuplicateHandle(%d): %d\r\n", typ_u, GetLastError()); + exit(1); + } + + return con_u; +} + +/* _on_boot_completed_cb: invoked when the ship has finished booting. +*/ +static void _on_boot_completed_cb() { + HANDLE hin_u = GetStdHandle(STD_INPUT_HANDLE); + SetEvent(hin_u); + CloseHandle(hin_u); } /* u3_daemon_init(): platform-specific daemon mode initialization. @@ -15,11 +34,83 @@ _fix_std_handle(c3_i fid_i, DWORD typ_u) void u3_daemon_init() { - // detect stdout/stderr redirection - DWORD dum_u; - BOOL do1_u = GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &dum_u); - BOOL do2_u = GetConsoleMode(GetStdHandle(STD_ERROR_HANDLE), &dum_u); - FreeConsole(); - if ( do1_u ) _fix_std_handle(1, STD_OUTPUT_HANDLE); - if ( do2_u ) _fix_std_handle(2, STD_ERROR_HANDLE); + // detect if this process is the child daemon process + // + if ( ResetEvent(GetStdHandle(STD_INPUT_HANDLE)) ) { + u3_Host.bot_f = _on_boot_completed_cb; + return; + } + + STARTUPINFOW psi_u; + ZeroMemory(&psi_u, sizeof(psi_u)); + psi_u.cb = sizeof(psi_u); + psi_u.dwFlags = STARTF_USESTDHANDLES; + + // duplicate standard output and error handles for the child process, + // replacing any raw console handles with handles to /dev/null + // print a warning if raw console output detected + // + // On Windows, console handles become invalid once the console is + // detached. This will cause urbit terminal output to fail. libuv + // provides no way of changing the handle of an open uv_pipe_handle, + // and Windows has no equivalent of dup2() for handles, so I cannot + // substitute a /dev/null handle once the terminal is initialized. + // It is possible to create an anonymous pipe and have the child + // process take over its drain end after it signals that the ship + // has booted, but -d is intended for background operation anyway + // and does not seem to warrant the added complexity. + // + if ( _dup_std_handle(&psi_u.hStdOutput, STD_OUTPUT_HANDLE) | + _dup_std_handle(&psi_u.hStdError, STD_ERROR_HANDLE) ) + { + fprintf(stderr, "vere: -d used from a Windows console without redirection\r\n" + " no output from the daemon process will be visible\r\n"); + fflush(stderr); + } + + // create an event for the child process to signal + // the parent that the ship has finished booting + // pass the handle as "stdin" (otherwise unused with -d) + // + SECURITY_ATTRIBUTES sa_u = {sizeof (SECURITY_ATTRIBUTES), NULL, TRUE}; + if ( !(psi_u.hStdInput = CreateEvent(&sa_u, TRUE, FALSE, NULL)) ) { + fprintf(stderr, "vere: CreateEvent: %d\r\n", GetLastError()); + exit(1); + } + + // create the child process with the same command line as parent + // it will start, re-parse the command line, and call u3_daemon_init + // + PROCESS_INFORMATION ppi_u; + if ( !CreateProcessW(NULL, _wcsdup(GetCommandLineW()), NULL, NULL, TRUE, DETACHED_PROCESS, NULL, NULL, &psi_u, &ppi_u) ) { + fprintf(stderr, "vere: CreateProcess: %d\r\n", GetLastError()); + exit(1); + } + + CloseHandle(ppi_u.hThread); + + // wait for the child process to exit or to signal the event + // + DWORD exi_u; + HANDLE han_u[2] = {ppi_u.hProcess, psi_u.hStdInput}; + switch ( WaitForMultipleObjects(2, han_u, FALSE, INFINITE) ) { + case WAIT_OBJECT_0: + // the child process exited prematurely, propagate its exit code + // + if ( GetExitCodeProcess(ppi_u.hProcess, &exi_u) ) { + exit(exi_u); + } + + fprintf(stderr, "vere: GetExitCodeProcess: %d\r\n", GetLastError()); + exit(1); + + case WAIT_OBJECT_0 + 1: + // the child process has finished booting, exit normally + // + exit(0); + + default: + fprintf(stderr, "vere: WaitForMultipleObjects: %d\r\n", GetLastError()); + exit(1); + } } From 90f9c68c0983d56748eaa471e6ddf2fbc31e108b Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Fri, 30 Apr 2021 22:25:59 +0300 Subject: [PATCH 082/136] vere: mingw: add smoke test to build action --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cdd3fecffc..51b8650730 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -134,3 +134,6 @@ jobs: - run: make build/urbit build/urbit-worker working-directory: ./pkg/urbit + + - run: build/urbit -d -B ../../bin/brass.pill -F zod && curl -f --data '{"source":{"dojo":"+hood/exit"},"sink":{"app":"hood"}}' http://localhost:12321 + working-directory: ./pkg/urbit From baa06904f69a1c8e22048f47f38ad68e9b7d556e Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Sat, 1 May 2021 23:04:19 -0400 Subject: [PATCH 083/136] fuse: print help text in generator --- pkg/arvo/gen/hood/fuse.hoon | 7 +++++-- pkg/arvo/lib/hood/kiln.hoon | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/gen/hood/fuse.hoon b/pkg/arvo/gen/hood/fuse.hoon index 03effe5c89..870b38d2aa 100644 --- a/pkg/arvo/gen/hood/fuse.hoon +++ b/pkg/arvo/gen/hood/fuse.hoon @@ -8,5 +8,8 @@ :::: :: :- %say -|= [[now=@da eny=@uvJ bec=beak] [des=desk bas=beak con=(list [beak germ]) ~] ~] -[%kiln-fuse des bas con] +|= [[now=@da eny=@uvJ bec=beak] [arg=[?(~ [des=desk bas=beak con=(list [beak germ]) ~])]] ~] +:- %kiln-fuse +?~ arg + ((slog (turn `wain`help-text |=(=@t leaf+(trip t)))) ~) +[des bas con]:arg diff --git a/pkg/arvo/lib/hood/kiln.hoon b/pkg/arvo/lib/hood/kiln.hoon index 8700670bc8..9f541bacdb 100644 --- a/pkg/arvo/lib/hood/kiln.hoon +++ b/pkg/arvo/lib/hood/kiln.hoon @@ -56,6 +56,7 @@ gim=?(%auto germ) :: == +$ kiln-fuse + $@ ~ $: syd=desk bas=beak con=(list [beak germ]) @@ -388,6 +389,7 @@ :: ++ poke-fuse |= k=kiln-fuse + ?~ k abet abet:(emit [%pass /kiln/fuse/[syd.k] %arvo %c [%fuse syd.k bas.k con.k]]) :: ++ poke-cancel From 75ed2cc0af8612747174feff32b5a610bb75c107 Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Sun, 2 May 2021 13:44:18 -0400 Subject: [PATCH 084/136] fuse: c/r --- pkg/arvo/sys/lull.hoon | 2 +- pkg/arvo/sys/vane/clay.hoon | 151 +++++++++++++++++------------------- 2 files changed, 73 insertions(+), 80 deletions(-) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 1cac3043b5..806eadbc40 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -762,7 +762,7 @@ her=@p dem=desk cas=case :: source how=germ :: method == :: - $: %fuse :: mega merge + $: %fuse :: merge many des=desk :: target desk bas=beak :: base desk con=(list [beak germ]) :: merges diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index eb36d55e5c..35e94c6570 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -1972,104 +1972,116 @@ |= [bas=beak con=(list [beak germ])] ^- melt :+ bas con - %- ~(gas by (~(put by *(map beak (unit dome:clay))) bas *(unit dome:clay))) + %- ~(gas by *(map beak (unit dome:clay))) + :- [bas *(unit dome:clay)] (turn con |=(a=[beak germ] [-.a *(unit dome:clay)])) :: ++ start-fuse |= [bas=beak con=(list [beak germ])] ^+ ..start-fuse =/ moves=(list move) - %+ turn - [[bas *germ] con] - |= [bec=beak germ] - ^- move - =/ wir=wire /fuse/[syd]/(scot %p p.bec)/[q.bec]/(scot r.bec) - [hen %pass wir %c %warp p.bec q.bec `[%sing %v r.bec /]] + %+ turn + [[bas *germ] con] + |= [bec=beak germ] + ^- move + =/ wir=wire /fuse/[syd]/(scot %p p.bec)/[q.bec]/(scot r.bec) + [hen %pass wir %c %warp p.bec q.bec `[%sing %v r.bec /]] + :: :: we also want to clear the state (fiz) associated with this :: merge and print a warning if it's non trivial i.e. we're :: starting a new fuse before the previous one terminated. - ~& ?~ con.fiz - [%starting-fuse bas con] - :* %starting-fuse - bas - con - %discarding-state - :: we don't want to ~& an entire dome - %- ~(run by sto.fiz) - |=(v=(unit dome:clay) ?~(v %not-received %recieved)) - == + :: + =/ err=tang + ?~ con.fiz + ~ + =/ discarded=tang + %+ turn + ~(tap in sto.fiz) + |= [k=beak v=(unit dome:clay)] + ^- tank + =/ received=tape ?~(v "missing" "received") + leaf+"{} {received}" + :- leaf+"fusing into {} from {} {} - overwriting prior fuse" + discarded =. fiz (make-melt bas con) - (emil moves) + ((slog err) (emil moves)) :: ++ take-fuse |= [bec=beak =riot] ^+ ..take-fuse ?~ riot - ~& [%fuse-for syd %missing bec] + :: :: by setting fiz to *melt the merge is aborted - any further :: responses we get for the merge will cause take-fuse to crash + :: =. fiz *melt - ..take-fuse + ((slog [leaf+"clay: fuse failed, missing {}"]~) ..take-fuse) ?> (~(has by sto.fiz) bec) =. fiz :+ bas.fiz con.fiz (~(put by sto.fiz) bec `!<(dome:clay q.r.u.riot)) =/ all-done=flag - %- ~(all by sto.fiz) - |= res=(unit dome:clay) - ^- flag - !=(res ~) + %- ~(all by sto.fiz) + |= res=(unit dome:clay) + ^- flag + !=(res ~) ?. all-done ..take-fuse =| rag=rang =/ clean-hut-ran hut.ran =/ initial-dome=dome:clay (need (~(got by sto.fiz) bas.fiz)) =/ continuation-yaki=yaki - (~(got by hut.ran) (~(got by hit.initial-dome) let.initial-dome)) + (~(got by hut.ran) (~(got by hit.initial-dome) let.initial-dome)) =/ parents=(list tako) ~[(~(got by hit.initial-dome) let.initial-dome)] =/ merges con.fiz + :: + :: We must ensure that we don't leave garbage around. + :: In all cases we want to clear fiz and restore hut.ran + :: to clean-hut-ran. + :: |- ^+ ..take-fuse ?~ merges - :: reset fiz before potentially parking - we hope to - :: succeed so we don't need this state anymore. + =/ t=tang [leaf+"{} fused from {} {}" ~] =. fiz *melt =. hut.ran clean-hut-ran - (park | [%| continuation-yaki(p (flop parents))] rag) + =. ..take-fuse (park | [%| continuation-yaki(p (flop parents))] rag) + (done %| %fuse-succeded t) =/ [bec=beak g=germ] i.merges =/ ali-dom=dome:clay (need (~(got by sto.fiz) bec)) =/ result (merge-helper p.bec q.bec g ali-dom `continuation-yaki) - ?- -.result + ?- -.result %| - :: merge failed - (mean %fuse-merge-failed p.result) + =. fiz *melt + =. hut.ran clean-hut-ran + (done %| %fuse-merge-failed p.result) :: %& - :: now we have a unit merge-result =/ merge-result=(unit merge-result) +.result ?~ merge-result :: this merge was a no-op, just continue $(merges t.merges) ?~ conflicts.u.merge-result - :: no merge conflicts =/ merged-yaki=yaki - ?- -.new.u.merge-result - %| - +.new.u.merge-result - :: - %& - :: convert the yuki to yaki - =/ yuk=yuki +.new.u.merge-result - =/ lobes=(map path lobe) - %- ~(run by q.yuk) - |= val=(each page lobe) - ^- lobe - ?- -.val - %& (page-to-lobe +.val) - %| +.val - == - (make-yaki p.yuk lobes now) - == + ?- -.new.u.merge-result + %| + +.new.u.merge-result + :: + %& + :: + :: convert the yuki to yaki + :: + =/ yuk=yuki +.new.u.merge-result + =/ lobes=(map path lobe) + %- ~(run by q.yuk) + |= val=(each page lobe) + ^- lobe + ?- -.val + %& (page-to-lobe +.val) + %| +.val + == + (make-yaki p.yuk lobes now) + == %= $ continuation-yaki merged-yaki merges t.merges @@ -2077,8 +2089,11 @@ lat.rag (~(uni by lat.rag) lat.u.merge-result) parents [(~(got by hit.ali-dom) let.ali-dom) parents] == + :: :: if there are merge conflicts send the error and abort the merge + :: =. fiz *melt + =. hut.ran clean-hut-ran (done %& conflicts.u.merge-result) == :: @@ -2094,8 +2109,8 @@ (done %| %ali-unavailable ~[>[ali-ship ali-desk germ]<]) =/ ali-dome=dome:clay !<(dome:clay q.r.u.riot) =/ result=(each (unit merge-result) (pair term tang)) - (merge-helper ali-ship ali-desk germ ali-dome ~) - ?- -.result + (merge-helper ali-ship ali-desk germ ali-dome ~) + ?- -.result %| (done %| +.result) :: @@ -2119,11 +2134,11 @@ ^- (each (unit merge-result) [term tang]) =/ ali-yaki=yaki (~(got by hut.ran) (~(got by hit.ali-dome) let.ali-dome)) =/ bob-yaki=(unit yaki) - ?~ continuation-yaki - ?~ let.dom - ~ - (~(get by hut.ran) (~(got by hit.dom) let.dom)) - continuation-yaki + ?~ continuation-yaki + ?~ let.dom + ~ + (~(get by hut.ran) (~(got by hit.dom) let.dom)) + continuation-yaki (merge-by-germ ali-yaki bob-yaki) ++ merge-by-germ |= [=ali=yaki bob-yaki=(unit yaki)] @@ -3516,7 +3531,6 @@ :: :: Creates a nako of all the changes between a and b. :: - :: ++ make-nako |= [ver=@ud a=aeon b=aeon] ^- nako @@ -3565,24 +3579,6 @@ =. s ^$(p i.p.y) $(p.y t.p.y) :: - :: Gets the data between two commit hashes, assuming the first is an - :: ancestor of the second. - :: - :: Get all the takos before `a`, then get all takos before `b` except the - :: ones we found before `a`. Then convert the takos to yakis and also get - :: all the data in all the yakis. - :: - :: What happens if you run an %init merge on a desk that already - :: had a commit? - :: - ++ data-twixt-takos - |= [plops=? a=(unit tako) b=tako] - ^- [(set yaki) (set plop)] - =+ old=?~(a ~ (reachable-takos u.a)) - =/ yal=(set tako) (~(dif in (reachable-takos b)) old) - :- (silt (turn ~(tap in yal) tako-to-yaki)) - ~ - :: :: Get all the lobes that are referenced in `a` except those that are :: already in `b`. :: @@ -4372,7 +4368,6 @@ $: hun=duct dos=(map desk dojo-7) == - +$ rung-7 $: rus=(map desk rede-7) == @@ -4525,7 +4520,6 @@ dos.rom %- ~(run by dos.rom.ruf) |= =dojo - ::dojo dojo(fod.dom [~ ~ ~ ~ ~]) :: hoy @@ -4535,7 +4529,6 @@ rus %- ~(run by rus.rung) |= =rede - ::rede rede(fod.dom [~ ~ ~ ~ ~]) == == From 08d3aa24856830e90c300d2f5cb6cf1c9212e45c Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Sun, 2 May 2021 14:04:03 -0400 Subject: [PATCH 085/136] fuse: c/r --- pkg/arvo/sys/vane/clay.hoon | 70 +++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 35e94c6570..01df534705 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -2059,42 +2059,44 @@ %& =/ merge-result=(unit merge-result) +.result ?~ merge-result - :: this merge was a no-op, just continue - $(merges t.merges) - ?~ conflicts.u.merge-result - =/ merged-yaki=yaki - ?- -.new.u.merge-result - %| - +.new.u.merge-result :: - %& - :: - :: convert the yuki to yaki - :: - =/ yuk=yuki +.new.u.merge-result - =/ lobes=(map path lobe) - %- ~(run by q.yuk) - |= val=(each page lobe) - ^- lobe - ?- -.val - %& (page-to-lobe +.val) - %| +.val - == - (make-yaki p.yuk lobes now) - == - %= $ - continuation-yaki merged-yaki - merges t.merges - hut.ran (~(put by hut.ran) r.merged-yaki merged-yaki) - lat.rag (~(uni by lat.rag) lat.u.merge-result) - parents [(~(got by hit.ali-dom) let.ali-dom) parents] - == + :: this merge was a no-op, just continue + :: + $(merges t.merges) + ?^ conflicts.u.merge-result + :: + :: if there are merge conflicts send the error and abort the merge + :: + =. fiz *melt + =. hut.ran clean-hut-ran + (done %& conflicts.u.merge-result) + =/ merged-yaki=yaki + ?- -.new.u.merge-result + %| + +.new.u.merge-result :: - :: if there are merge conflicts send the error and abort the merge - :: - =. fiz *melt - =. hut.ran clean-hut-ran - (done %& conflicts.u.merge-result) + %& + :: + :: convert the yuki to yaki + :: + =/ yuk=yuki +.new.u.merge-result + =/ lobes=(map path lobe) + %- ~(run by q.yuk) + |= val=(each page lobe) + ^- lobe + ?- -.val + %& (page-to-lobe +.val) + %| +.val + == + (make-yaki p.yuk lobes now) + == + %= $ + continuation-yaki merged-yaki + merges t.merges + hut.ran (~(put by hut.ran) r.merged-yaki merged-yaki) + lat.rag (~(uni by lat.rag) lat.u.merge-result) + parents [(~(got by hit.ali-dom) let.ali-dom) parents] + == == :: ++ done From 20fb39bba72fb92d567079f8d03c3abb02d975da Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Sun, 2 May 2021 14:15:03 -0400 Subject: [PATCH 086/136] fuse: fix comments --- pkg/arvo/sys/vane/clay.hoon | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 01df534705..75205eeb35 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -58,8 +58,11 @@ :: when the request is filled/updated. :: +$ cult (jug wove duct) -:: State for ongoing %fuse merges - the list maintains the ordering -:: and the map stores the data needed to merge +:: +:: State for ongoing %fuse merges. `con` maintains the ordering, +:: `sto` stores the data needed to merge, and `bas` is the base +:: beak for the merge. +:: +$ melt [bas=beak con=(list [beak germ]) sto=(map beak (unit dome:clay))] :: :: Domestic desk state. @@ -308,7 +311,7 @@ $: %c :: to %clay $> $? %info :: internal edit %merg :: merge desks - %fuse :: mega merge + %fuse :: merge many %pork :: %warp :: %werp :: @@ -1987,7 +1990,7 @@ =/ wir=wire /fuse/[syd]/(scot %p p.bec)/[q.bec]/(scot r.bec) [hen %pass wir %c %warp p.bec q.bec `[%sing %v r.bec /]] :: - :: we also want to clear the state (fiz) associated with this + :: We also want to clear the state (fiz) associated with this :: merge and print a warning if it's non trivial i.e. we're :: starting a new fuse before the previous one terminated. :: @@ -2011,7 +2014,7 @@ ^+ ..take-fuse ?~ riot :: - :: by setting fiz to *melt the merge is aborted - any further + :: By setting fiz to *melt the merge is aborted - any further :: responses we get for the merge will cause take-fuse to crash :: =. fiz *melt @@ -2060,12 +2063,12 @@ =/ merge-result=(unit merge-result) +.result ?~ merge-result :: - :: this merge was a no-op, just continue + :: This merge was a no-op, just continue :: $(merges t.merges) ?^ conflicts.u.merge-result :: - :: if there are merge conflicts send the error and abort the merge + :: If there are merge conflicts send the error and abort the merge :: =. fiz *melt =. hut.ran clean-hut-ran @@ -2077,7 +2080,7 @@ :: %& :: - :: convert the yuki to yaki + :: Convert the yuki to yaki :: =/ yuk=yuki +.new.u.merge-result =/ lobes=(map path lobe) @@ -2130,9 +2133,6 @@ |= [=ali=ship =ali=desk =germ ali-dome=dome:clay continuation-yaki=(unit yaki)] ^- (each (unit merge-result) [term tang]) |^ - :: term tang is the error, merge result being ~ means the merge was - :: a no-op - :: (each a b) is [| b] or [& a] ^- (each (unit merge-result) [term tang]) =/ ali-yaki=yaki (~(got by hut.ran) (~(got by hit.ali-dome) let.ali-dome)) =/ bob-yaki=(unit yaki) From ef228a0490fdf32df562da5299edf6dac45e84d0 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sun, 2 May 2021 23:57:40 +0300 Subject: [PATCH 087/136] compat: mingw: optionally cache dependency artifacts in cachix --- .github/workflows/build.yml | 8 +- pkg/urbit/compat/mingw/poor-mans-nix-shell.sh | 118 ++++++++++++++++++ pkg/urbit/configure | 30 +---- 3 files changed, 124 insertions(+), 32 deletions(-) create mode 100644 pkg/urbit/compat/mingw/poor-mans-nix-shell.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 51b8650730..e6922ab8a0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -122,6 +122,7 @@ jobs: defaults: run: shell: C:\msys64\msys2_shell.cmd -mingw64 -defterm -no-start -here -c ". <(cygpath '{0}')" + working-directory: ./pkg/urbit steps: - uses: actions/checkout@v2 @@ -130,10 +131,9 @@ jobs: # echo suppresses pacman prompt - run: echo|./configure - working-directory: ./pkg/urbit + env: + CACHIX_CACHE: ares + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - run: make build/urbit build/urbit-worker - working-directory: ./pkg/urbit - - run: build/urbit -d -B ../../bin/brass.pill -F zod && curl -f --data '{"source":{"dojo":"+hood/exit"},"sink":{"app":"hood"}}' http://localhost:12321 - working-directory: ./pkg/urbit diff --git a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh new file mode 100644 index 0000000000..fa6762cc2a --- /dev/null +++ b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh @@ -0,0 +1,118 @@ +# ensure required mingw packages are installed +mpkgs=(cmake curl gcc jq libsigsegv libuv make wslay) +pacman -S --needed autoconf automake-wrapper libtool patch ${mpkgs[@]/#/mingw-w64-x86_64-} + +declare -a cdirs +declare -a ldirs +sources=(../../nix/sources.json ../../nix/sources-mingw.json) + +hex2nixbase32 () { + local digits='0123456789abcdfghijklmnpqrsvwxyz' + local bits=0 + local left=0 # number of bits left in $bits + local i=0 + while ((1)) + do + while ((left>=5)) + do + echo -n ${digits:$((bits&31)):1} + bits=$((bits>>5)) + left=$((left-5)) + done + if ((i == ${#1})) + then + break + fi + char=0x${1:i:2} + i=$((i+2)) + bits=$((bits|(char<<(left)))) + left=$((left+8)) + done + echo -n ${digits:$bits:1} +} + +buildnixdep () { + local cache=https://app.cachix.org/api/v1/cache/${CACHIX_CACHE-} + local deriver=urbit-mingw-build + local hash= + local dir + if [ -n "$url" ] + then + local patch=compat/mingw/$key.patch + # create 'store hash' from sources.json data and patch + read hash _ < <(( + jq -Sscj --arg key "$key" --arg deriver "$deriver" 'add|to_entries|.[]|select(.key==$key)|{($deriver):.value}' ${sources[@]} + [ -e $patch ] && cat $patch)|sha256sum) + hash=$(hex2nixbase32 $hash) + dir=../$hash-$key + if [ -e $dir/.mingw~ ] + then + # dependency present, don't reupload + hash= + else + # dependency absent, check the binary cache if configured + if [ -n "${CACHIX_CACHE-}" ] + then + echo Checking binary cache for $hash-$key... + narinfo="$cache/${hash}.narinfo" + if curl -fLI "$narinfo" + then + url="$cache/$(curl -fL -H "Accept: application/json" "$narinfo"|jq -r '.url')" + echo Found $url + strip=0 + hash= + fi + fi + mkdir -p $dir + pushd $dir + curl -fL "$url"|(tar --strip $strip -xzf - || true) + popd + fi + else + # local dependency + dir=../$key + fi + # patch and build the dependency if necessary + # and append dependency paths to -I and -L arrays + . <(jq -sr --arg key "$key" --arg dir "$dir" 'add|to_entries|.[]|select(.key==$key)|" +pushd \($dir) +if [ ! -e .mingw~ ] +then" + ("../urbit/compat/mingw/\($key).patch"|" + [ -e \(.) ] && patch -p 1 <\(.)") + " + \(.value.mingw.prepare//"") + make \(.value.mingw.make//"") + touch .mingw~ +fi +popd +\(.value.mingw.include//"."|if type == "array" then . else [.] end|map("cdirs+=(-I\($dir)/\(.))")|join("\n")) +\(.value.mingw.lib//"."|if type == "array" then . else [.] end|map("ldirs+=(-L\($dir)/\(.))")|join("\n"))"' ${sources[@]}) + + # if configured, upload freshly built dependency to binary cache + if [ -n "$hash" -a -n "${CACHIX_AUTH_TOKEN-}" ] + then + ( + echo Uploading freshly built $hash-$key to binary cache... + tar -C $dir -czf $hash.tar . + local size=$(stat -c '%s' $hash.tar) + read filehash _ < <(sha256sum $hash.tar) + curl -fL -H "Content-Type: application/gzip" -H "Authorization: Bearer $CACHIX_AUTH_TOKEN" --data-binary @"$hash.tar" "$cache/nar" + curl -fL -H "Content-Type: application/json" -H "Authorization: Bearer $CACHIX_AUTH_TOKEN" --data-binary @- "$cache/${hash}.narinfo" < 512 )) || curl -L https://github.com/urbit/urbit/raw/urbit-v$URBIT_VERSION/bin/ivory.pill > ../../bin/ivory.pill - # poor man's nix-shell - declare -a cdirs - declare -a ldirs - . <(jq -sr ' -add|to_entries|.[]|select(.value.mingw)|.key as $key|"../\(.key)" as $dir|(if .value.url then " -if [ ! -d \($dir) ] -then - mkdir -p \($dir) - pushd \($dir) - curl -L \(.value.url)|(tar --strip \(.value.mingw.strip+1) -xzf - || true) - popd -fi" else "" end) + " -pushd \($dir) -if [ ! -e .mingw~ ] -then" + ("../urbit/compat/mingw/\($key).patch"|" - [ -e \(.) ] && patch -p 1 <\(.)") + " - \(.value.mingw.prepare//"") - touch .mingw~ -fi -make \(.value.mingw.make//"") -popd -\(.value.mingw.include//"."|if type == "array" then .[] else . end|"cdirs+=(-I../\($key)/\(.)) -")\(.value.mingw.lib//"."|if type == "array" then .[] else . end|"ldirs+=(-L../\($key)/\(.)) -")"' ../../nix/sources.json ../../nix/sources-mingw.json) - [ -e include/ca-bundle.h ] || xxd /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt include_ca_bundle_crt >include/ca-bundle.h [ -e include/ivory.h ] || xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h From 8dd35b7d184605a7efb02784e194bc21d50e617c Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 3 May 2021 01:23:00 +0300 Subject: [PATCH 088/136] compat: mingw: remap dependency directories --- pkg/urbit/compat/mingw/poor-mans-nix-shell.sh | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh index fa6762cc2a..c4dad9ec67 100644 --- a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh +++ b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh @@ -4,7 +4,10 @@ pacman -S --needed autoconf automake-wrapper libtool patch ${mpkgs[@]/#/mingw-w6 declare -a cdirs declare -a ldirs +declare -A hdeps sources=(../../nix/sources.json ../../nix/sources-mingw.json) +deriver=urbit-mingw-build +depdirs= hex2nixbase32 () { local digits='0123456789abcdfghijklmnpqrsvwxyz' @@ -31,19 +34,23 @@ hex2nixbase32 () { echo -n ${digits:$bits:1} } +hashnixdep () { + local patch=compat/mingw/$1.patch + local hash + read hash _ < <(( + # create 'store hash' from sources.json data and patch + jq -Sscj --arg key "$1" --arg deriver "$deriver" 'add|to_entries|.[]|select(.key==$key)|{($deriver):.value}' ${sources[@]} + [ -e $patch ] && cat $patch)|sha256sum) + hex2nixbase32 $hash +} + buildnixdep () { local cache=https://app.cachix.org/api/v1/cache/${CACHIX_CACHE-} - local deriver=urbit-mingw-build local hash= local dir if [ -n "$url" ] then - local patch=compat/mingw/$key.patch - # create 'store hash' from sources.json data and patch - read hash _ < <(( - jq -Sscj --arg key "$key" --arg deriver "$deriver" 'add|to_entries|.[]|select(.key==$key)|{($deriver):.value}' ${sources[@]} - [ -e $patch ] && cat $patch)|sha256sum) - hash=$(hex2nixbase32 $hash) + hash=${hdeps[$key]} dir=../$hash-$key if [ -e $dir/.mingw~ ] then @@ -79,8 +86,8 @@ pushd \($dir) if [ ! -e .mingw~ ] then" + ("../urbit/compat/mingw/\($key).patch"|" [ -e \(.) ] && patch -p 1 <\(.)") + " - \(.value.mingw.prepare//"") - make \(.value.mingw.make//"") + \(.value.mingw.prepare//""'"$depdirs"') + make \(.value.mingw.make//""'"$depdirs"') touch .mingw~ fi popd @@ -113,6 +120,13 @@ EOF fi } +# list dependencies, create store hashes and a directory replacement regex +. <(jq -sr 'add|to_entries|.[]|select(.value.mingw)|select(.value.url)|" +_hash=$(hashnixdep \(.key|@sh)) +hdeps[\(.key|@sh)]=$_hash +depdirs=\"$depdirs|gsub(\\\"\\\\\\.\\\\\\./\(.key)\\\";\\\"../$_hash-\(.key)\\\")\" +unset _hash"' ${sources[@]}) + # I have to go over the sources files several times # because jq does not have a way to invoke external programs . <(jq -sr 'add|to_entries|.[]|select(.value.mingw)|"key=\(.key|@sh) url=\(.value.url//""|@sh) strip=\(.value.mingw.strip+1) buildnixdep"' ${sources[@]}) From 47695e17d3043daa0aaed6ffb617833c0b2f3572 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 3 May 2021 11:38:53 +0300 Subject: [PATCH 089/136] compat: mingw: simplify poor-mans-nix-shell.sh --- pkg/urbit/compat/mingw/poor-mans-nix-shell.sh | 70 ++++++++++--------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh index c4dad9ec67..796929253d 100644 --- a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh +++ b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh @@ -34,20 +34,10 @@ hex2nixbase32 () { echo -n ${digits:$bits:1} } -hashnixdep () { - local patch=compat/mingw/$1.patch - local hash - read hash _ < <(( - # create 'store hash' from sources.json data and patch - jq -Sscj --arg key "$1" --arg deriver "$deriver" 'add|to_entries|.[]|select(.key==$key)|{($deriver):.value}' ${sources[@]} - [ -e $patch ] && cat $patch)|sha256sum) - hex2nixbase32 $hash -} - buildnixdep () { + echo Building dependency $key... local cache=https://app.cachix.org/api/v1/cache/${CACHIX_CACHE-} local hash= - local dir if [ -n "$url" ] then hash=${hdeps[$key]} @@ -79,20 +69,18 @@ buildnixdep () { # local dependency dir=../$key fi + # patch and build the dependency if necessary - # and append dependency paths to -I and -L arrays - . <(jq -sr --arg key "$key" --arg dir "$dir" 'add|to_entries|.[]|select(.key==$key)|" -pushd \($dir) -if [ ! -e .mingw~ ] -then" + ("../urbit/compat/mingw/\($key).patch"|" - [ -e \(.) ] && patch -p 1 <\(.)") + " - \(.value.mingw.prepare//""'"$depdirs"') - make \(.value.mingw.make//""'"$depdirs"') - touch .mingw~ -fi -popd -\(.value.mingw.include//"."|if type == "array" then . else [.] end|map("cdirs+=(-I\($dir)/\(.))")|join("\n")) -\(.value.mingw.lib//"."|if type == "array" then . else [.] end|map("ldirs+=(-L\($dir)/\(.))")|join("\n"))"' ${sources[@]}) + pushd $dir + if [ ! -e .mingw~ ] + then + local patch=../urbit/compat/mingw/$key.patch + [ -e $patch ] && patch -p 1 <$patch + $cmdprep + make $cmdmake + touch .mingw~ + fi + popd # if configured, upload freshly built dependency to binary cache if [ -n "$hash" -a -n "${CACHIX_AUTH_TOKEN-}" ] @@ -120,13 +108,31 @@ EOF fi } -# list dependencies, create store hashes and a directory replacement regex -. <(jq -sr 'add|to_entries|.[]|select(.value.mingw)|select(.value.url)|" -_hash=$(hashnixdep \(.key|@sh)) -hdeps[\(.key|@sh)]=$_hash -depdirs=\"$depdirs|gsub(\\\"\\\\\\.\\\\\\./\(.key)\\\";\\\"../$_hash-\(.key)\\\")\" -unset _hash"' ${sources[@]}) - # I have to go over the sources files several times # because jq does not have a way to invoke external programs -. <(jq -sr 'add|to_entries|.[]|select(.value.mingw)|"key=\(.key|@sh) url=\(.value.url//""|@sh) strip=\(.value.mingw.strip+1) buildnixdep"' ${sources[@]}) + +# list external dependencies, create hash map and directory replacement regex +# use -j and \u0000 to work around https://github.com/stedolan/jq/issues/1870 +while read -rd "" key json +do + # create 'store hash' from sources.json data and patch + patch=compat/mingw/$key.patch + read hash _ < <(( + echo -n $json + [ -e $patch ] && cat $patch)|sha256sum) + hash=$(hex2nixbase32 $hash) + hdeps[$key]=$hash + depdirs="$depdirs|gsub(\"\\\\.\\\\./$key\";\"../$hash-$key\")" +done < <(jq --arg deriver "$deriver" -Sscrj 'add|to_entries|.[]|select(.value.mingw)|select(.value.url)|.key," ",{($deriver):(.value)},"\u0000"' ${sources[@]}) + +# build dependencies, create include and library directory arrays +. <(jq -sr 'add|to_entries|.[]|select(.value.mingw)|" +unset dir +key=\(.key|@sh) \\ +url=\(.value.url//""|@sh) \\ +strip=\(.value.mingw.strip+1) \\ +cmdprep=\(.value.mingw.prepare//""'"$depdirs"'|@sh) \\ +cmdmake=\(.value.mingw.make//""'"$depdirs"'|@sh) \\ +buildnixdep # sets dir +\(.value.mingw.include//"."|if type == "array" then . else [.] end|map("cdirs+=(-I$dir/\(.))")|join("\n")) +\(.value.mingw.lib//"."|if type == "array" then . else [.] end|map("ldirs+=(-L$dir/\(.))")|join("\n"))"' ${sources[@]}) From aa2fd310fa89812704b7b4c9f3c8938fd57131cd Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 3 May 2021 12:30:15 +0300 Subject: [PATCH 090/136] compat: mingw: fix cmdmake splitting --- pkg/urbit/compat/mingw/poor-mans-nix-shell.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh index 796929253d..ca54e63723 100644 --- a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh +++ b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh @@ -76,8 +76,8 @@ buildnixdep () { then local patch=../urbit/compat/mingw/$key.patch [ -e $patch ] && patch -p 1 <$patch - $cmdprep - make $cmdmake + eval "$cmdprep" + eval make "$cmdmake" touch .mingw~ fi popd From 3865de241449e06a9c38e69977ebd95bf917093c Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Mon, 3 May 2021 15:18:40 +0300 Subject: [PATCH 091/136] compat: mingw: make smoke test use the same parameters as urbit-tests --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e6922ab8a0..02ce639019 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -136,4 +136,4 @@ jobs: CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - run: make build/urbit build/urbit-worker - - run: build/urbit -d -B ../../bin/brass.pill -F zod && curl -f --data '{"source":{"dojo":"+hood/exit"},"sink":{"app":"hood"}}' http://localhost:12321 + - run: build/urbit -l -d -B ../../bin/solid.pill -F bus && curl -f --data '{"source":{"dojo":"+hood/exit"},"sink":{"app":"hood"}}' http://localhost:12321 From 4c94c7edd4bb311195e47454638f678992b759a8 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Mon, 3 May 2021 17:33:56 -0400 Subject: [PATCH 092/136] ames: expire direct lanes faster after inactivity --- pkg/arvo/sys/vane/ames.hoon | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index ccf31d062f..b591024b50 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1967,20 +1967,12 @@ ++ on-memo |= [=bone payload=* valence=?(%plea %boon)] ^+ peer-core - :: if we haven't been trying to talk to %live, reset timer - :: - =? last-contact.qos.peer-state - ?& ?=(%live -.qos.peer-state) - %- ~(all by snd.peer-state) - |= =message-pump-state - =(~ live.packet-pump-state.message-pump-state) - == - now - :: =/ =message-blob (dedup-message (jim payload)) =. peer-core (run-message-pump bone %memo message-blob) :: - ?: &(=(%boon valence) ?=(?(%dead %unborn) -.qos.peer-state)) + ?: ?& =(%boon valence) + (gte now (add ~s30 last-contact.qos.peer-state)) + == check-clog peer-core :: +dedup-message: replace with any existing copy of this message From 2fd8e729e19982b82a1e4e11ecf1f3ec474f5b04 Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Mon, 3 May 2021 19:29:28 -0400 Subject: [PATCH 093/136] fuse: fix indents --- pkg/arvo/sys/vane/clay.hoon | 56 ++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 75205eeb35..2556df9225 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -1983,29 +1983,29 @@ |= [bas=beak con=(list [beak germ])] ^+ ..start-fuse =/ moves=(list move) - %+ turn - [[bas *germ] con] - |= [bec=beak germ] - ^- move - =/ wir=wire /fuse/[syd]/(scot %p p.bec)/[q.bec]/(scot r.bec) - [hen %pass wir %c %warp p.bec q.bec `[%sing %v r.bec /]] + %+ turn + [[bas *germ] con] + |= [bec=beak germ] + ^- move + =/ wir=wire /fuse/[syd]/(scot %p p.bec)/[q.bec]/(scot r.bec) + [hen %pass wir %c %warp p.bec q.bec `[%sing %v r.bec /]] :: :: We also want to clear the state (fiz) associated with this :: merge and print a warning if it's non trivial i.e. we're :: starting a new fuse before the previous one terminated. :: =/ err=tang - ?~ con.fiz - ~ - =/ discarded=tang - %+ turn - ~(tap in sto.fiz) - |= [k=beak v=(unit dome:clay)] - ^- tank - =/ received=tape ?~(v "missing" "received") - leaf+"{} {received}" - :- leaf+"fusing into {} from {} {} - overwriting prior fuse" - discarded + ?~ con.fiz + ~ + =/ discarded=tang + %+ turn + ~(tap in sto.fiz) + |= [k=beak v=(unit dome:clay)] + ^- tank + =/ received=tape ?~(v "missing" "received") + leaf+"{} {received}" + :_ discarded + leaf+"fusing into {} from {} {} - overwriting prior fuse" =. fiz (make-melt bas con) ((slog err) (emil moves)) :: @@ -2024,10 +2024,10 @@ :+ bas.fiz con.fiz (~(put by sto.fiz) bec `!<(dome:clay q.r.u.riot)) =/ all-done=flag - %- ~(all by sto.fiz) - |= res=(unit dome:clay) - ^- flag - !=(res ~) + %- ~(all by sto.fiz) + |= res=(unit dome:clay) + ^- flag + !=(res ~) ?. all-done ..take-fuse =| rag=rang @@ -2084,13 +2084,13 @@ :: =/ yuk=yuki +.new.u.merge-result =/ lobes=(map path lobe) - %- ~(run by q.yuk) - |= val=(each page lobe) - ^- lobe - ?- -.val - %& (page-to-lobe +.val) - %| +.val - == + %- ~(run by q.yuk) + |= val=(each page lobe) + ^- lobe + ?- -.val + %& (page-to-lobe +.val) + %| +.val + == (make-yaki p.yuk lobes now) == %= $ From 424d167905ddba5db33091290f1a85319b411d19 Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Mon, 3 May 2021 22:26:14 -0400 Subject: [PATCH 094/136] fuse: reset pills --- bin/brass.pill | 4 ++-- bin/ivory.pill | 4 ++-- bin/solid.pill | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bin/brass.pill b/bin/brass.pill index 4f5af448ab..f549dac587 100644 --- a/bin/brass.pill +++ b/bin/brass.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2d45514e6ebb8345911137a03980e709c8ea0c28b14b770a24a51190db7c8ed2 -size 8215365 +oid sha256:61e583dd7db795dac4a7c31bfd3ee8b240e679bb882e35d4e7d1acb5f9f2f3d6 +size 8270131 diff --git a/bin/ivory.pill b/bin/ivory.pill index ae03a26b96..7c2e8e4746 100644 --- a/bin/ivory.pill +++ b/bin/ivory.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b9807f664c9b4f6d6ce92c10f2959f28ad6e48a24a880556f828d3cf0cad2ee9 -size 4537425 +oid sha256:185ea5e76dc48695e55efc543377e0682e485f81b16e3b443f9be881d026d4f2 +size 2616564 diff --git a/bin/solid.pill b/bin/solid.pill index aaeb0a4d27..92802fcddc 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1ddeef9bf39394ff9c19a0d66c9fc45d218e4891dbd152f8f7696f8c0fa9747e -size 9877817 +oid sha256:bb221ce7316346bfaf1ddd953927c0f2afe7eaf5b160bed545f67ec7e5ccb005 +size 9410487 From 9db61a1566d32b1862a06ccf01ec136f8ee8a6ef Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Mon, 3 May 2021 22:26:49 -0400 Subject: [PATCH 095/136] fuse: simplify state restoration logic --- pkg/arvo/sys/vane/clay.hoon | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 2556df9225..3d504a1395 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -2010,6 +2010,8 @@ ((slog err) (emil moves)) :: ++ take-fuse + |^ + :: |= [bec=beak =riot] ^+ ..take-fuse ?~ riot @@ -2031,33 +2033,24 @@ ?. all-done ..take-fuse =| rag=rang - =/ clean-hut-ran hut.ran + =/ clean-state ..take-fuse =/ initial-dome=dome:clay (need (~(got by sto.fiz) bas.fiz)) =/ continuation-yaki=yaki - (~(got by hut.ran) (~(got by hit.initial-dome) let.initial-dome)) + (~(got by hut.ran) (~(got by hit.initial-dome) let.initial-dome)) =/ parents=(list tako) ~[(~(got by hit.initial-dome) let.initial-dome)] =/ merges con.fiz - :: - :: We must ensure that we don't leave garbage around. - :: In all cases we want to clear fiz and restore hut.ran - :: to clean-hut-ran. - :: |- ^+ ..take-fuse ?~ merges =/ t=tang [leaf+"{} fused from {} {}" ~] - =. fiz *melt - =. hut.ran clean-hut-ran - =. ..take-fuse (park | [%| continuation-yaki(p (flop parents))] rag) - (done %| %fuse-succeded t) + =. ..take-fuse (done-fuse clean-state %| %fuse-succeded t) + (park | [%| continuation-yaki(p (flop parents))] rag) =/ [bec=beak g=germ] i.merges =/ ali-dom=dome:clay (need (~(got by sto.fiz) bec)) =/ result (merge-helper p.bec q.bec g ali-dom `continuation-yaki) ?- -.result %| - =. fiz *melt - =. hut.ran clean-hut-ran - (done %| %fuse-merge-failed p.result) + (done-fuse clean-state %| %fuse-merge-failed p.result) :: %& =/ merge-result=(unit merge-result) +.result @@ -2070,9 +2063,7 @@ :: :: If there are merge conflicts send the error and abort the merge :: - =. fiz *melt - =. hut.ran clean-hut-ran - (done %& conflicts.u.merge-result) + (done-fuse clean-state %& conflicts.u.merge-result) =/ merged-yaki=yaki ?- -.new.u.merge-result %| @@ -2101,6 +2092,15 @@ parents [(~(got by hit.ali-dom) let.ali-dom) parents] == == + :: +done-fuse: restore state after a fuse is attempted, whether it + :: succeeds or fails. + :: + ++ done-fuse + |= [to-restore=_..take-fuse result=(each (set path) (pair term tang))] + ^+ ..take-fuse + =. fiz.to-restore *melt + (done:to-restore result) + -- :: ++ done |= result=(each (set path) (pair term tang)) From c86559071fe387dfb34a5b6aad9fe1d14ad05c55 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 4 May 2021 14:55:41 -0500 Subject: [PATCH 096/136] zuse: add +tab arm to +on, which allows one to take a subset excluding the start element with a max count --- pkg/arvo/sys/zuse.hoon | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index a843d4ed04..308df0b0da 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5398,6 +5398,44 @@ |- ?~ a a [n=[key.n.a (b val.n.a)] l=$(a l.a) r=$(a r.a)] + :: +tab: tabulate a subset excluding start element with a max count + :: + ++ tab + ~/ %tab + |= [a=(tree item) b=(unit key) c=@] + ^- (list item) + |^ + -:(tabulate (del-span a b) b c) + :: + ++ tabulate + |= [a=(tree item) b=(unit key) c=@] + ^- [(list item) @] + ?: ?&(?=(~ b) =(c 0)) + [~ 0] + =| f=[d=(list item) e=@] + |- ^- [(list item) @] + ?: ?|(?=(~ a) =(e.f c)) f + =. f $(a l.a) + ?: =(e.f c) f + =. f [[n.a d.f] +(e.f)] + ?: =(e.f c) f + $(a r.a) + :: + ++ del-span + |= [a=(tree item) b=(unit key)] + ^- (tree item) + ?~ a a + ?~ b a + :: found key + ?: =(u.b key.n.a) + (nip a(r ~)) + :: traverse to find key + ?: (compare key.n.a u.b) + :: found key to the left of end + a(r $(a r.a)) + :: found key to the right of end + $(a (nip a(r ~))) + -- :: +tap: convert to list, smallest to largest :: ++ tap From 5b8bebdafe11556c3bf9dff480e0a5880d548633 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Tue, 4 May 2021 23:23:36 +0300 Subject: [PATCH 097/136] compat: mingw: comment kill(2) stub implementation --- pkg/urbit/compat/mingw/compat.c | 44 ++++++++++++++++----------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/pkg/urbit/compat/mingw/compat.c b/pkg/urbit/compat/mingw/compat.c index f9b229cf7e..a0bd268411 100644 --- a/pkg/urbit/compat/mingw/compat.c +++ b/pkg/urbit/compat/mingw/compat.c @@ -134,33 +134,11 @@ int err_win_to_posix(DWORD winerr) return error; } -int kill(pid_t pid, int sig) -{ - if (pid > 0 && sig == SIGKILL) { - HANDLE h = OpenProcess(PROCESS_TERMINATE, FALSE, pid); - - if (TerminateProcess(h, -1)) { - CloseHandle(h); - return 0; - } - - errno = err_win_to_posix(GetLastError()); - CloseHandle(h); - return -1; - } - - // TODO: handle signals for self - // TODO: send SIGTERM as ctrl-c: https://stackoverflow.com/questions/813086/can-i-send-a-ctrl-c-sigint-to-an-application-on-windows - errno = EINVAL; - return -1; -} - static HANDLE timer_event; static HANDLE timer_thread; static int timer_signal; static int timer_interval; static int one_shot; -static __p_sig_fn_t timer_fn = SIG_DFL, sigint_fn = SIG_DFL; /* The timer works like this: * The thread, ticktack(), is a trivial routine that most of the time @@ -381,6 +359,28 @@ int msync(void *addr, size_t len, int flags) // ----------------------------------------------------------------------- +// vere uses kill() only to kill lockfile owner with SIGTERM or SIGKILL +// Windows does not have signals, so I handle SIGKILL as TerminateProcess() +// and return an error in all other cases +int kill(pid_t pid, int sig) +{ + if (pid > 0 && sig == SIGKILL) { + HANDLE h = OpenProcess(PROCESS_TERMINATE, FALSE, pid); + + if (TerminateProcess(h, -1)) { + CloseHandle(h); + return 0; + } + + errno = err_win_to_posix(GetLastError()); + CloseHandle(h); + return -1; + } + + errno = EINVAL; + return -1; +} + // libgcc built for mingw has included an implementation of mprotect // via VirtualProtect since olden days, but it takes int rather than size_t // and therefore fails or does unexpected things for >2GB blocks on 64-bit From 678773f65674c30e877d5681ed00d12a70f4bdd3 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Tue, 4 May 2021 23:25:20 +0300 Subject: [PATCH 098/136] vere: mingw: remove signal(SIGPIPE) calls: Win32 has no SIGPIPE --- pkg/urbit/vere/king.c | 2 -- pkg/urbit/worker/main.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c index bc65fc7e85..03658cb844 100644 --- a/pkg/urbit/vere/king.c +++ b/pkg/urbit/vere/king.c @@ -756,8 +756,6 @@ u3_king_commence() sig_s.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sig_s, 0); } - #else - signal(SIGPIPE, SIG_IGN); #endif // boot the ivory pill diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index 461aa9d5cf..c62ccd6e93 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -207,8 +207,6 @@ _cw_serf_commence(c3_i argc, c3_c* argv[]) sig_s.sa_handler = SIG_IGN; sigaction(SIGPIPE, &sig_s, 0); } - #else - signal(SIGPIPE, SIG_IGN); #endif // configure pipe to daemon process From 014a94a8d4133bb42d71a9534e1885141bcc8da1 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Tue, 4 May 2021 23:26:53 +0300 Subject: [PATCH 099/136] u3: remove unused #include --- pkg/urbit/noun/manage.c | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 8dee6bdbb7..0cedf6ca49 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -8,7 +8,6 @@ #include #include #include -#include #include // XX stack-overflow recovery should be gated by -a From 59227f86968350abfb5b4d9a3d6cd36bc6f1d1ee Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 4 May 2021 15:58:47 -0500 Subject: [PATCH 100/136] tests: added tests for +tab --- pkg/arvo/tests/sys/zuse/ordered-map.hoon | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pkg/arvo/tests/sys/zuse/ordered-map.hoon b/pkg/arvo/tests/sys/zuse/ordered-map.hoon index 4191d3e7ee..d0c7a81553 100644 --- a/pkg/arvo/tests/sys/zuse/ordered-map.hoon +++ b/pkg/arvo/tests/sys/zuse/ordered-map.hoon @@ -9,6 +9,7 @@ (items-from-keys (gulf 0 6)) :: =/ atom-map ((ordered-map @ud @tas) lte) +=/ gte-atom-map ((ordered-map @ud @tas) gte) :: |% ++ test-ordered-map-gas ^- tang @@ -27,6 +28,34 @@ !> test-items !> (tap:atom-map a) :: +++ test-ordered-map-tab ^- tang + :: + =/ a=(tree [@ud @tas]) (gas:gte-atom-map ~ test-items) + :: + %+ expect-eq + !> test-items + !> (tab:gte-atom-map a ~ 7) +:: +++ test-ordered-map-tab-starting-from ^- tang + :: + =/ a=(tree [@ud @tas]) (gas:gte-atom-map ~ test-items) + =/ small-test-items=(list [@ud @tas]) + (items-from-keys (gulf 3 6)) + :: + %+ expect-eq + !> small-test-items + !> (tab:gte-atom-map a [~ 2] 4) +:: +++ test-ordered-map-tab-starting-from-and-count ^- tang + :: + =/ a=(tree [@ud @tas]) (gas:gte-atom-map ~ test-items) + =/ small-test-items=(list [@ud @tas]) + (items-from-keys (gulf 4 6)) + :: + %+ expect-eq + !> small-test-items + !> (tab:gte-atom-map a [~ 2] 3) +:: ++ test-ordered-map-pop ^- tang :: =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) From b7db4d59df3bb88ae534e43df40fc2d56b17598c Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Wed, 5 May 2021 15:03:32 +0300 Subject: [PATCH 101/136] compat: always add compat dirs to -I --- pkg/urbit/configure | 14 +++++++++----- pkg/urbit/daemon/main.c | 2 ++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 3bec9bc85b..bc0e6727ea 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -42,8 +42,6 @@ do case $1 in --enable-debug) CFLAGS="${CFLAGS-} -O0 -g" - MEMORY_DEBUG=1 - CPU_DEBUG=1 ;; --disable-debug) CFLAGS="${CFLAGS-} -O3 -g" @@ -110,8 +108,8 @@ case $(tr A-Z a-z <<< $os) in ldextra="$ldextra -lzstd" # libuv implibs ldextra="$ldextra -luserenv -liphlpapi -lpsapi" - vcompat=mingw - CFLAGS="${CFLAGS-} ${cdirs[@]} -DCURL_STATICLIB -DH2O_NO_UNIX_SOCKETS -Icompat/mingw" + compat="${compat-} mingw" + CFLAGS="${CFLAGS-} ${cdirs[@]} -DCURL_STATICLIB -DH2O_NO_UNIX_SOCKETS" LDFLAGS="${LDFLAGS-} ${ldirs[@]} -static" PKG_CONFIG=">/dev/null echo" ;; @@ -149,11 +147,17 @@ for header in $headers do LDFLAGS="${LDFLAGS-} -I$header" done +compat="${compat-posix}" +for citem in $compat +do + CFLAGS="${CFLAGS-} -Icompat/$citem" +done + cat >config.mk <&2 diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 372bd28bc1..290b56befa 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -649,11 +649,13 @@ main(c3_i argc, } #endif + #if !defined(U3_OS_mingw) // Handle SIGTSTP as if it was SIGTERM. // // Configured here using signal() so as to be immediately available. // signal(SIGTSTP, _stop_exit); + #endif printf("~\n"); // printf("welcome.\n"); From c708820958586d2c5d015c59c2e2bbe9afcb90d4 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Wed, 5 May 2021 15:04:14 +0300 Subject: [PATCH 102/136] build-action: temporarily use PR-specific cache --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 02ce639019..c41b1de015 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -132,7 +132,7 @@ jobs: # echo suppresses pacman prompt - run: echo|./configure env: - CACHIX_CACHE: ares + CACHIX_CACHE: locpyl-tidnyd-test1 CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - run: make build/urbit build/urbit-worker From 6ba7f77b23df44a056977f0d5559dc0f5a138dc3 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Wed, 5 May 2021 15:06:38 +0300 Subject: [PATCH 103/136] compat: mingw: catch Ctrl-C in main process --- pkg/urbit/compat/mingw/ctrlc.c | 7 +++++++ pkg/urbit/compat/mingw/uv.patch | 13 +++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 pkg/urbit/compat/mingw/ctrlc.c diff --git a/pkg/urbit/compat/mingw/ctrlc.c b/pkg/urbit/compat/mingw/ctrlc.c new file mode 100644 index 0000000000..9be57981b9 --- /dev/null +++ b/pkg/urbit/compat/mingw/ctrlc.c @@ -0,0 +1,7 @@ +#include + +// initialize msvcrt signals early, otherwise Ctrl-C does nothing +static void __attribute__ ((constructor)) _init_crt_signals() +{ + signal(SIGINT, SIG_DFL); +} diff --git a/pkg/urbit/compat/mingw/uv.patch b/pkg/urbit/compat/mingw/uv.patch index b64cfd4739..1338b78355 100644 --- a/pkg/urbit/compat/mingw/uv.patch +++ b/pkg/urbit/compat/mingw/uv.patch @@ -15,3 +15,16 @@ index 0f2bb869b..f81245ec6 100644 if (err == ERROR_ACCESS_DENIED) { /* * SetNamedPipeHandleState can fail if the handle doesn't have either +diff --git a/src/win/tty.c b/src/win/tty.c +index c359d5601..1b9d4f853 100644 +--- a/src/win/tty.c ++++ b/src/win/tty.c +@@ -367,7 +367,7 @@ int uv_tty_set_mode(uv_tty_t* tty, uv_tty_mode_t mode) { + flags = ENABLE_ECHO_INPUT | ENABLE_LINE_INPUT | ENABLE_PROCESSED_INPUT; + break; + case UV_TTY_MODE_RAW: +- flags = ENABLE_WINDOW_INPUT; ++ flags = ENABLE_WINDOW_INPUT | ENABLE_PROCESSED_INPUT; + break; + case UV_TTY_MODE_IO: + return UV_ENOTSUP; From 947b9ec75cb7239ac329a17072f73808a8c8ea77 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Wed, 5 May 2021 11:56:04 -0500 Subject: [PATCH 104/136] zuse: +tab should paginate into the ordered-map starting from the highest element, not from the lowest --- pkg/arvo/sys/zuse.hoon | 14 ++++---- pkg/arvo/tests/sys/zuse/ordered-map.hoon | 44 +++++++++++++++++++----- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 308df0b0da..4244119398 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5405,7 +5405,7 @@ |= [a=(tree item) b=(unit key) c=@] ^- (list item) |^ - -:(tabulate (del-span a b) b c) + (flop -:(tabulate (del-span a b) b c)) :: ++ tabulate |= [a=(tree item) b=(unit key) c=@] @@ -5427,14 +5427,14 @@ ?~ a a ?~ b a :: found key - ?: =(u.b key.n.a) - (nip a(r ~)) + ?: =(key.n.a u.b) + (nip a(l ~)) :: traverse to find key ?: (compare key.n.a u.b) - :: found key to the left of end - a(r $(a r.a)) - :: found key to the right of end - $(a (nip a(r ~))) + :: found key to the left of start + $(a (nip a(l ~))) + :: found key to the right of start + a(l $(a l.a)) -- :: +tap: convert to list, smallest to largest :: diff --git a/pkg/arvo/tests/sys/zuse/ordered-map.hoon b/pkg/arvo/tests/sys/zuse/ordered-map.hoon index d0c7a81553..90987664f6 100644 --- a/pkg/arvo/tests/sys/zuse/ordered-map.hoon +++ b/pkg/arvo/tests/sys/zuse/ordered-map.hoon @@ -28,33 +28,61 @@ !> test-items !> (tap:atom-map a) :: -++ test-ordered-map-tab ^- tang +++ test-ordered-map-tab-gte ^- tang :: =/ a=(tree [@ud @tas]) (gas:gte-atom-map ~ test-items) :: %+ expect-eq - !> test-items + !> (flop test-items) !> (tab:gte-atom-map a ~ 7) :: -++ test-ordered-map-tab-starting-from ^- tang +++ test-ordered-map-tab-gte-starting-from ^- tang :: =/ a=(tree [@ud @tas]) (gas:gte-atom-map ~ test-items) =/ small-test-items=(list [@ud @tas]) - (items-from-keys (gulf 3 6)) + (items-from-keys (gulf 2 5)) :: %+ expect-eq - !> small-test-items - !> (tab:gte-atom-map a [~ 2] 4) + !> (flop small-test-items) + !> (tab:gte-atom-map a [~ 6] 4) :: -++ test-ordered-map-tab-starting-from-and-count ^- tang +++ test-ordered-map-tab-gte-count ^- tang :: =/ a=(tree [@ud @tas]) (gas:gte-atom-map ~ test-items) =/ small-test-items=(list [@ud @tas]) (items-from-keys (gulf 4 6)) :: + %+ expect-eq + !> (flop small-test-items) + !> (tab:gte-atom-map a ~ 3) +:: +++ test-ordered-map-tab ^- tang + :: + =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) + :: + %+ expect-eq + !> test-items + !> (tab:atom-map a ~ 7) +:: +++ test-ordered-map-tab-starting-from ^- tang + :: + =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) + =/ small-test-items=(list [@ud @tas]) + (items-from-keys (gulf 1 4)) + :: %+ expect-eq !> small-test-items - !> (tab:gte-atom-map a [~ 2] 3) + !> (tab:atom-map a [~ 0] 4) +:: +++ test-ordered-map-tab-count ^- tang + :: + =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) + =/ small-test-items=(list [@ud @tas]) + (items-from-keys (gulf 0 2)) + :: + %+ expect-eq + !> small-test-items + !> (tab:atom-map a ~ 3) :: ++ test-ordered-map-pop ^- tang :: From 4073cf9d76e44f832c1380ce00a28bed62595df8 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Wed, 5 May 2021 12:59:40 -0500 Subject: [PATCH 105/136] zuse: +tab marginal code cleanup to reduce use of lark --- pkg/arvo/sys/zuse.hoon | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 4244119398..7df956b2ee 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5405,20 +5405,20 @@ |= [a=(tree item) b=(unit key) c=@] ^- (list item) |^ - (flop -:(tabulate (del-span a b) b c)) + (flop e:(tabulate (del-span a b) b c)) :: ++ tabulate |= [a=(tree item) b=(unit key) c=@] - ^- [(list item) @] + ^- [d=@ e=(list item)] ?: ?&(?=(~ b) =(c 0)) - [~ 0] - =| f=[d=(list item) e=@] - |- ^- [(list item) @] - ?: ?|(?=(~ a) =(e.f c)) f + [0 ~] + =| f=[d=@ e=(list item)] + |- ^- [d=@ e=(list item)] + ?: ?|(?=(~ a) =(d.f c)) f =. f $(a l.a) - ?: =(e.f c) f - =. f [[n.a d.f] +(e.f)] - ?: =(e.f c) f + ?: =(d.f c) f + =. f [+(d.f) [n.a e.f]] + ?: =(d.f c) f $(a r.a) :: ++ del-span @@ -5426,14 +5426,10 @@ ^- (tree item) ?~ a a ?~ b a - :: found key ?: =(key.n.a u.b) (nip a(l ~)) - :: traverse to find key ?: (compare key.n.a u.b) - :: found key to the left of start $(a (nip a(l ~))) - :: found key to the right of start a(l $(a l.a)) -- :: +tap: convert to list, smallest to largest From 36ae091cff1b37ce0deb5a76c23a0adc2418e203 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Wed, 5 May 2021 13:09:43 -0500 Subject: [PATCH 106/136] zuse: +tab in ordered-map is more succinct and no longer nips --- pkg/arvo/sys/zuse.hoon | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 7df956b2ee..b1fc36113a 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5418,8 +5418,7 @@ =. f $(a l.a) ?: =(d.f c) f =. f [+(d.f) [n.a e.f]] - ?: =(d.f c) f - $(a r.a) + ?:(=(d.f c) f $(a r.a)) :: ++ del-span |= [a=(tree item) b=(unit key)] @@ -5427,9 +5426,9 @@ ?~ a a ?~ b a ?: =(key.n.a u.b) - (nip a(l ~)) + a(l r.a) ?: (compare key.n.a u.b) - $(a (nip a(l ~))) + $(a a(l r.a)) a(l $(a l.a)) -- :: +tap: convert to list, smallest to largest From 915c548037a6395759c77efaf7e4fc04a2e85e4b Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Wed, 5 May 2021 14:41:13 -0500 Subject: [PATCH 107/136] zuse: fix +del-span in +tab --- pkg/arvo/sys/zuse.hoon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index b1fc36113a..ff4109b492 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5426,9 +5426,9 @@ ?~ a a ?~ b a ?: =(key.n.a u.b) - a(l r.a) + r.a ?: (compare key.n.a u.b) - $(a a(l r.a)) + $(a r.a) a(l $(a l.a)) -- :: +tap: convert to list, smallest to largest From a25f8028f4d57e37849dbd6d9430e026a6489d85 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Wed, 5 May 2021 16:07:08 -0500 Subject: [PATCH 108/136] tests: add test to make sure subtree is being modified properly in +del-span --- pkg/arvo/sys/zuse.hoon | 2 +- pkg/arvo/tests/sys/zuse/ordered-map.hoon | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index ff4109b492..1ed24031cc 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5413,7 +5413,7 @@ ?: ?&(?=(~ b) =(c 0)) [0 ~] =| f=[d=@ e=(list item)] - |- ^- [d=@ e=(list item)] + |- ^+ f ?: ?|(?=(~ a) =(d.f c)) f =. f $(a l.a) ?: =(d.f c) f diff --git a/pkg/arvo/tests/sys/zuse/ordered-map.hoon b/pkg/arvo/tests/sys/zuse/ordered-map.hoon index 90987664f6..961882ac60 100644 --- a/pkg/arvo/tests/sys/zuse/ordered-map.hoon +++ b/pkg/arvo/tests/sys/zuse/ordered-map.hoon @@ -84,6 +84,14 @@ !> small-test-items !> (tab:atom-map a ~ 3) :: +++ test-ordered-map-tab-more-than-exist ^- tang + :: + =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) + :: + %+ expect-eq + !> test-items + !> (tab:atom-map a [~ 0] 8) +:: ++ test-ordered-map-pop ^- tang :: =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) From 0ed09b7a290c961cf09e7dc21fe31d52e84ddef6 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Wed, 5 May 2021 16:53:24 -0500 Subject: [PATCH 109/136] tests: fix wrong expected result with ordered-map test --- pkg/arvo/tests/sys/zuse/ordered-map.hoon | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/tests/sys/zuse/ordered-map.hoon b/pkg/arvo/tests/sys/zuse/ordered-map.hoon index 961882ac60..247b85c91b 100644 --- a/pkg/arvo/tests/sys/zuse/ordered-map.hoon +++ b/pkg/arvo/tests/sys/zuse/ordered-map.hoon @@ -86,10 +86,12 @@ :: ++ test-ordered-map-tab-more-than-exist ^- tang :: - =/ a=(tree [@ud @tas]) (gas:atom-map ~ test-items) + =/ specific-test-items=(list [@ud @tas]) + (items-from-keys (gulf 1 6)) + =/ a=(tree [@ud @tas]) (gas:atom-map ~ specific-test-items) :: %+ expect-eq - !> test-items + !> specific-test-items !> (tab:atom-map a [~ 0] 8) :: ++ test-ordered-map-pop ^- tang From a9d4306026ca72e75277c85d293c812268f2250e Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Wed, 5 May 2021 19:52:42 +0300 Subject: [PATCH 110/136] u3: rsignal refactoring --- pkg/urbit/compat/mingw/compat.c | 103 ------------------------------- pkg/urbit/compat/mingw/compat.h | 33 ++-------- pkg/urbit/compat/mingw/rsignal.h | 16 +++++ pkg/urbit/compat/mingw/setjmp.h | 15 +++++ pkg/urbit/compat/posix/rsignal.h | 11 ++++ pkg/urbit/include/c/portable.h | 1 + pkg/urbit/noun/manage.c | 32 ++++++---- 7 files changed, 67 insertions(+), 144 deletions(-) create mode 100644 pkg/urbit/compat/mingw/rsignal.h create mode 100644 pkg/urbit/compat/mingw/setjmp.h create mode 100644 pkg/urbit/compat/posix/rsignal.h diff --git a/pkg/urbit/compat/mingw/compat.c b/pkg/urbit/compat/mingw/compat.c index a0bd268411..0a60b56661 100644 --- a/pkg/urbit/compat/mingw/compat.c +++ b/pkg/urbit/compat/mingw/compat.c @@ -134,109 +134,6 @@ int err_win_to_posix(DWORD winerr) return error; } -static HANDLE timer_event; -static HANDLE timer_thread; -static int timer_signal; -static int timer_interval; -static int one_shot; - -/* The timer works like this: - * The thread, ticktack(), is a trivial routine that most of the time - * only waits to receive the signal to terminate. The main thread tells - * the thread to terminate by setting the timer_event to the signalled - * state. - * But ticktack() interrupts the wait state after the timer's interval - * length to call the signal handler. - */ - -static unsigned __stdcall ticktack(void *dummy) -{ - while (WaitForSingleObject(timer_event, timer_interval) == WAIT_TIMEOUT) { - raise(timer_signal); - if (one_shot) - break; - } - return 0; -} - -static int error(const char* s, ...) -{ - // TODO - va_list ap; - va_start(ap, s); - vfprintf(stderr, s, ap); - va_end(ap); - return -1; -} - -static int start_timer_thread(void) -{ - timer_event = CreateEvent(NULL, FALSE, FALSE, NULL); - if (timer_event) { - timer_thread = (HANDLE) _beginthreadex(NULL, 0, ticktack, NULL, 0, NULL); - if (!timer_thread ) - return errno = ENOMEM, - error("cannot start timer thread"); - } else - return errno = ENOMEM, - error("cannot allocate resources for timer"); - return 0; -} - -static void stop_timer_thread(void) -{ - if (timer_event) - SetEvent(timer_event); /* tell thread to terminate */ - if (timer_thread) { - int rc = WaitForSingleObject(timer_thread, 10000); - if (rc == WAIT_TIMEOUT) - error("timer thread did not terminate timely"); - else if (rc != WAIT_OBJECT_0) - error("waiting for timer thread failed: %lu", - GetLastError()); - CloseHandle(timer_thread); - } - if (timer_event) - CloseHandle(timer_event); - timer_event = NULL; - timer_thread = NULL; -} - -static inline int is_timeval_eq(const struct timeval *i1, const struct timeval *i2) -{ - return i1->tv_sec == i2->tv_sec && i1->tv_usec == i2->tv_usec; -} - -int setitimer(int type, struct itimerval *in, struct itimerval *out) -{ - static const struct timeval zero; - static int atexit_done; - - if (out != NULL) - return errno = EINVAL, - error("setitimer param 3 != NULL not implemented"); - if (!is_timeval_eq(&in->it_interval, &zero) && - !is_timeval_eq(&in->it_interval, &in->it_value)) - return errno = EINVAL, - error("setitimer: it_interval must be zero or eq it_value"); - - if (timer_thread) - stop_timer_thread(); - - if (is_timeval_eq(&in->it_value, &zero) && - is_timeval_eq(&in->it_interval, &zero)) - return 0; - - timer_signal = type == ITIMER_VIRTUAL ? SIGVTALRM : SIGALRM; - timer_interval = in->it_value.tv_sec * 1000 + in->it_value.tv_usec / 1000; - one_shot = is_timeval_eq(&in->it_interval, &zero); - if (!atexit_done) { - atexit(stop_timer_thread); - atexit_done = 1; - } - return start_timer_thread(); -} - // from msys2 mingw-packages-dev patches // ----------------------------------------------------------------------- diff --git a/pkg/urbit/compat/mingw/compat.h b/pkg/urbit/compat/mingw/compat.h index b0f2737e23..d70cfcc7a5 100644 --- a/pkg/urbit/compat/mingw/compat.h +++ b/pkg/urbit/compat/mingw/compat.h @@ -1,22 +1,6 @@ #ifndef _MINGW_IO_H #define _MINGW_IO_H -// msvcrt setjmp/longjmp are broken on 64-bit systems, use gcc builtins -typedef struct jmp_buf { - intptr_t buffer[5]; - int retval; -} jmp_buf; - -#define _setjmp setjmp -#define _longjmp longjmp -#define longjmp(buf, val) {buf.retval = (val); __builtin_longjmp(buf.buffer, 1);} -#define setjmp(buf) (__builtin_setjmp(buf.buffer) ? (buf.retval) : 0) - -// no profiling on MingW means signal masks are not used -#define sigjmp_buf jmp_buf -#define siglongjmp longjmp -#define sigsetjmp(A, B) setjmp(A) - #define mkdir(A, B) mkdir(A) char *realpath(const char *path, char *resolved_path); @@ -25,17 +9,10 @@ int utimes(const char *path, const struct timeval times[2]); int kill(pid_t pid, int signum); -#define SIGALRM 1233 -#define SIGVTALRM 1234 -#define SIGINFO 1235 -#define SIGUSR1 1236 -#define SIGTSTP 1238 - -#define ITIMER_REAL 0 -#define ITIMER_VIRTUAL 1 -struct itimerval { - struct timeval it_value, it_interval; -}; -int setitimer(int type, struct itimerval *in, struct itimerval *out); +#define SIGALRM (NSIG+0) +#define SIGVTALRM (NSIG+1) +#define SIGINFO (NSIG+2) +#define SIGUSR1 (NSIG+3) +#define SIG_COUNT (NSIG+4) #endif//_MINGW_IO_H \ No newline at end of file diff --git a/pkg/urbit/compat/mingw/rsignal.h b/pkg/urbit/compat/mingw/rsignal.h new file mode 100644 index 0000000000..2029847d09 --- /dev/null +++ b/pkg/urbit/compat/mingw/rsignal.h @@ -0,0 +1,16 @@ +#ifndef _RSIGNAL_H +#define _RSIGNAL_H + +#define rsignal_jmpbuf jmp_buf +#define rsignal_setjmp setjmp +#define rsignal_longjmp longjmp +#define rsignal_install_handler signal +#define rsignal_deinstall_handler(sig) signal((sig), SIG_IGN) +#define rsignal_setitimer(sig,in,out) 0 + +#define ITIMER_VIRTUAL 1 +struct itimerval { + struct timeval it_value, it_interval; +}; + +#endif//_RSIGNAL_H diff --git a/pkg/urbit/compat/mingw/setjmp.h b/pkg/urbit/compat/mingw/setjmp.h new file mode 100644 index 0000000000..497636872d --- /dev/null +++ b/pkg/urbit/compat/mingw/setjmp.h @@ -0,0 +1,15 @@ +#ifndef _MINGW_SETJMP_H +#define _MINGW_SETJMP_H + +// msvcrt setjmp/longjmp are broken on 64-bit systems, use gcc builtins +typedef struct jmp_buf { + intptr_t buffer[5]; + int retval; +} jmp_buf; + +#define _setjmp setjmp +#define _longjmp longjmp +#define longjmp(buf, val) {buf.retval = (val); __builtin_longjmp(buf.buffer, 1);} +#define setjmp(buf) (__builtin_setjmp(buf.buffer) ? (buf.retval) : 0) + +#endif//_MINGW_SETJMP_H diff --git a/pkg/urbit/compat/posix/rsignal.h b/pkg/urbit/compat/posix/rsignal.h new file mode 100644 index 0000000000..276006a18b --- /dev/null +++ b/pkg/urbit/compat/posix/rsignal.h @@ -0,0 +1,11 @@ +#ifndef _RSIGNAL_H +#define _RSIGNAL_H + +#define rsignal_jmpbuf sigjmp_buf +#define rsignal_setjmp(buf) sigsetjmp((buf), 1) +#define rsignal_longjmp siglongjmp +#define rsignal_install_handler signal +#define rsignal_deinstall_handler(sig) signal((sig), SIG_IGN) +#define rsignal_setitimer setitimer + +#endif//_RSIGNAL_H diff --git a/pkg/urbit/include/c/portable.h b/pkg/urbit/include/c/portable.h index 71bb7e5dd9..0647a1a6d2 100644 --- a/pkg/urbit/include/c/portable.h +++ b/pkg/urbit/include/c/portable.h @@ -81,6 +81,7 @@ # include # include # include +# include # include # include # include diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 0cedf6ca49..298ef0a650 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -2,6 +2,7 @@ ** */ #include "all.h" +#include "rsignal.h" #include "vere/vere.h" #include #include @@ -73,7 +74,14 @@ u3_noun arg); -static sigjmp_buf u3_Signal; +// u3m_signal uses restricted functionality signals for compatibility reasons: +// some platforms may not provide true POSIX asynchronous signals and their +// compat layer will then implement this restricted functionality subset. +// u3m_signal never needs to interrupt I/O operations, its signal handlers +// do not manipulate signals, do not modify shared state, and always either +// return or longjmp. +// +static rsignal_jmpbuf u3_Signal; #ifndef SIGSTKSZ # define SIGSTKSZ 16384 @@ -330,8 +338,8 @@ _cm_signal_deep(c3_w mil_w) #ifndef NO_OVERFLOW stackoverflow_install_handler(_cm_signal_handle_over, Sigstk, SIGSTKSZ); #endif - signal(SIGINT, _cm_signal_handle_intr); - signal(SIGTERM, _cm_signal_handle_term); + rsignal_install_handler(SIGINT, _cm_signal_handle_intr); + rsignal_install_handler(SIGTERM, _cm_signal_handle_term); // Provide a little emergency memory, for use in case things // go utterly haywire. @@ -347,11 +355,11 @@ _cm_signal_deep(c3_w mil_w) itm_u.it_value.tv_sec = (mil_w / 1000); itm_u.it_value.tv_usec = 1000 * (mil_w % 1000); - if ( setitimer(ITIMER_VIRTUAL, &itm_u, 0) ) { + if ( rsignal_setitimer(ITIMER_VIRTUAL, &itm_u, 0) ) { u3l_log("loom: set timer failed %s\r\n", strerror(errno)); } else { - signal(SIGVTALRM, _cm_signal_handle_alrm); + rsignal_install_handler(SIGVTALRM, _cm_signal_handle_alrm); } } @@ -363,9 +371,9 @@ _cm_signal_deep(c3_w mil_w) static void _cm_signal_done() { - signal(SIGINT, SIG_IGN); - signal(SIGTERM, SIG_IGN); - signal(SIGVTALRM, SIG_IGN); + rsignal_deinstall_handler(SIGINT); + rsignal_deinstall_handler(SIGTERM); + rsignal_deinstall_handler(SIGVTALRM); #ifndef NO_OVERFLOW stackoverflow_deinstall_handler(); @@ -376,7 +384,7 @@ _cm_signal_done() timerclear(&itm_u.it_interval); timerclear(&itm_u.it_value); - if ( setitimer(ITIMER_VIRTUAL, &itm_u, 0) ) { + if ( rsignal_setitimer(ITIMER_VIRTUAL, &itm_u, 0) ) { u3l_log("loom: clear timer failed %s\r\n", strerror(errno)); } } @@ -395,7 +403,7 @@ _cm_signal_done() void u3m_signal(u3_noun sig_l) { - siglongjmp(u3_Signal, sig_l); + rsignal_longjmp(u3_Signal, sig_l); } /* u3m_file(): load file, as atom, or bail. @@ -957,7 +965,7 @@ u3m_soft_top(c3_w mil_w, // timer ms */ _cm_signal_deep(mil_w); - if ( 0 != (sig_l = sigsetjmp(u3_Signal, 1)) ) { + if ( 0 != (sig_l = rsignal_setjmp(u3_Signal)) ) { // reinitialize trace state // u3t_init(); @@ -1620,8 +1628,6 @@ _cm_signals(void) u3l_log("boot: sigsegv install failed\n"); exit(1); } - // signal(SIGINT, _loom_stop); - # if defined(U3_OS_PROF) // Block SIGPROF, so that if/when we reactivate it on the From 2adcdaab1f28d2833fd80ab07e1be7a7c71e5c2b Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 6 May 2021 12:34:23 +0300 Subject: [PATCH 111/136] compat: mingw: fix signal numbers --- pkg/urbit/compat/mingw/compat.h | 9 ++++----- pkg/urbit/vere/king.c | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/urbit/compat/mingw/compat.h b/pkg/urbit/compat/mingw/compat.h index d70cfcc7a5..667c1ccf46 100644 --- a/pkg/urbit/compat/mingw/compat.h +++ b/pkg/urbit/compat/mingw/compat.h @@ -9,10 +9,9 @@ int utimes(const char *path, const struct timeval times[2]); int kill(pid_t pid, int signum); -#define SIGALRM (NSIG+0) -#define SIGVTALRM (NSIG+1) -#define SIGINFO (NSIG+2) -#define SIGUSR1 (NSIG+3) -#define SIG_COUNT (NSIG+4) +#define SIGUSR1 10 +#define SIGALRM 14 +#define SIGVTALRM 26 +#define SIG_COUNT 32 #endif//_MINGW_IO_H \ No newline at end of file diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c index 03658cb844..99f40032b9 100644 --- a/pkg/urbit/vere/king.c +++ b/pkg/urbit/vere/king.c @@ -544,7 +544,7 @@ _king_sign_init(void) // handle SIGINFO (if available) // -#ifndef U3_OS_linux +#ifdef SIGINFO { u3_usig* sig_u; @@ -600,7 +600,7 @@ _king_sign_cb(uv_signal_t* sil_u, c3_i num_i) // fallthru if defined // -#ifndef U3_OS_linux +#ifdef SIGINFO case SIGINFO: #endif case SIGUSR1: { From 3a930d75a16621e4825ee7fe5dec2e722088946b Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 6 May 2021 12:35:48 +0300 Subject: [PATCH 112/136] compat: mingw: single-thread rsignal implementation --- pkg/urbit/compat/mingw/rsignal.c | 162 +++++++++++++++++++++++++++++++ pkg/urbit/compat/mingw/rsignal.h | 19 ++-- pkg/urbit/include/c/portable.h | 2 + 3 files changed, 177 insertions(+), 6 deletions(-) create mode 100644 pkg/urbit/compat/mingw/rsignal.c diff --git a/pkg/urbit/compat/mingw/rsignal.c b/pkg/urbit/compat/mingw/rsignal.c new file mode 100644 index 0000000000..f87f786bf1 --- /dev/null +++ b/pkg/urbit/compat/mingw/rsignal.c @@ -0,0 +1,162 @@ +#include "all.h" +#include "rsignal.h" +#include + +int err_win_to_posix(DWORD winerr); + +// The current implementation of rsignal_ is single-threaded, +// but it can be extended to multi-threaded by replacing these +// static variables with a thread id-based hash map. +// +static __p_sig_fn_t _fns[SIG_COUNT]; +static volatile DWORD _tid; +static HANDLE _hvt; + +void rsignal_install_handler(int sig, __p_sig_fn_t fn) +{ + if (sig < 0 || sig >= SIG_COUNT) + return; + + DWORD newtid = GetCurrentThreadId(); + DWORD oldtid = InterlockedExchange(&_tid, newtid); + if (oldtid != 0 && oldtid != newtid) { + fprintf(stderr, "\r\nrsignal_install_handler: %u -> %u\r\n", oldtid, newtid); + return; + } + + __p_sig_fn_t oldfn = InterlockedExchangePointer((PVOID*)&_fns[sig], fn); + if (fn != 0 && oldfn != 0 && oldfn != fn) { + fprintf(stderr, "\r\nrsignal_install_handler: %p -> %p\r\n", oldfn, fn); + } +} + +void rsignal_deinstall_handler(int sig) +{ + rsignal_install_handler(sig, 0); +} + +void rsignal_raise(int sig) +{ + if (sig < 0 || sig >= SIG_COUNT) + return; + + __p_sig_fn_t oldfn = InterlockedExchangePointer((PVOID*)&_fns[sig], 0); + if (oldfn == 0) + return; + + HANDLE hthread = OpenThread(THREAD_ALL_ACCESS, FALSE, _tid); + if (!hthread) { + fprintf(stderr, "\r\nrsignal_raise: OpenThread(%u): %d\r\n", _tid, GetLastError()); + return; + } + + if (SuspendThread(hthread) < 0) { + fprintf(stderr, "\r\nrsignal_raise: SuspendThread(%u): %d\r\n", _tid, GetLastError()); + goto cleanup; + } + + oldfn(sig); + + if (!ResumeThread(hthread)) { + fprintf(stderr, "\r\nrsignal_raise: ResumeThread(%u): %d\r\n", _tid, GetLastError()); + + // abort because the main thread is stuck + abort(); + } + +cleanup: + CloseHandle(hthread); +} + +static void _rsignal_vt_cb(PVOID param, BOOLEAN timedOut) +{ + rsignal_raise(SIGVTALRM); +} + +int rsignal_setitimer(int type, struct itimerval *in, struct itimerval *out) +{ + if (in == 0) { + errno = EFAULT; + return -1; + } + + if (type != ITIMER_VIRTUAL || out != 0) { + errno = ENOTSUP; + return -1; + } + + if (_hvt != NULL) { + DeleteTimerQueueTimer(NULL, _hvt, NULL); + _hvt = NULL; + } + + if (timerisset(&in->it_value) && !CreateTimerQueueTimer(&_hvt, NULL, _rsignal_vt_cb, NULL, + in->it_value.tv_sec * 1000 + in->it_value.tv_usec / 1000, + in->it_interval.tv_sec * 1000 + in->it_interval.tv_usec / 1000, 0)) + { + errno = err_win_to_posix(GetLastError()); + return -1; + } else { + return 0; + } +} + +// direct import from ntdll.dll +extern DWORD64 __imp_KiUserExceptionDispatcher; + +static void _rsignal_longjmp(intptr_t* builtin_jb) +{ + __builtin_longjmp(builtin_jb, 1); +} + +void rsignal_post_longjmp(DWORD tid, intptr_t* builtin_jb) +{ + HANDLE hthread = OpenThread(THREAD_ALL_ACCESS, FALSE, tid); + if (!hthread) { + fprintf(stderr, "\r\nrsignal: OpenThread(%u): %d\r\n", tid, GetLastError()); + return; + } + + CONTEXT context; + context.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER; + if (!GetThreadContext(hthread, &context)) { + fprintf(stderr, "\r\nrsignal: GetThreadContext(%u): %d\r\n", tid, GetLastError()); + goto cleanup; + } + + // see if the thread is currently handling a structured exception + // if so, let the handler (usually the libsigsegv handler) finish + // and set up the the signal to run at the exception resume point + // otherwise, passing a parameter to fn is completely unreliable + // + DWORD64 kibase; + PRUNTIME_FUNCTION ki = RtlLookupFunctionEntry(__imp_KiUserExceptionDispatcher, &kibase, NULL); + CONTEXT c = context; + while (1) + { + DWORD64 base, frame; + PRUNTIME_FUNCTION f = RtlLookupFunctionEntry(c.Rip, &base, NULL); + if (!f) break; + if (f == ki) + { + // KiUserExceptionDispatcher has a "bare" frame + // with $rsp pointing to the CONTEXT structure + // + ((PCONTEXT)c.Rsp)->Rip = (DWORD64)_rsignal_longjmp; + ((PCONTEXT)c.Rsp)->Rcx = (DWORD64)builtin_jb; + goto cleanup; + } + PVOID handler_data; + RtlVirtualUnwind(0, base, c.Rip, f, &c, &handler_data, &frame, NULL); + } + + context.Rip = (DWORD64)_rsignal_longjmp; + context.Rcx = (DWORD64)builtin_jb; + if (!SetThreadContext(hthread, &context)) { + fprintf(stderr, "\r\nrsignal: SetThreadContext(%u): %d\r\n", tid, GetLastError()); + goto cleanup; + } + +cleanup: + CloseHandle(hthread); +} diff --git a/pkg/urbit/compat/mingw/rsignal.h b/pkg/urbit/compat/mingw/rsignal.h index 2029847d09..8c8c3d0f11 100644 --- a/pkg/urbit/compat/mingw/rsignal.h +++ b/pkg/urbit/compat/mingw/rsignal.h @@ -1,16 +1,23 @@ #ifndef _RSIGNAL_H #define _RSIGNAL_H -#define rsignal_jmpbuf jmp_buf -#define rsignal_setjmp setjmp -#define rsignal_longjmp longjmp -#define rsignal_install_handler signal -#define rsignal_deinstall_handler(sig) signal((sig), SIG_IGN) -#define rsignal_setitimer(sig,in,out) 0 +typedef struct { + jmp_buf jb; + unsigned long tid; +} rsignal_jmpbuf; + +#define rsignal_setjmp(buf) (buf.tid = GetCurrentThreadId(), setjmp(buf.jb)) +#define rsignal_longjmp(buf, val) if (buf.tid != GetCurrentThreadId()) {buf.jb.retval = (val); rsignal_post_longjmp(buf.tid, buf.jb.buffer);} else longjmp(buf.jb, val) + +void rsignal_install_handler(int sig, __p_sig_fn_t fn); +void rsignal_deinstall_handler(int sig); +void rsignal_post_longjmp(unsigned long tid, intptr_t* builtin_jb); #define ITIMER_VIRTUAL 1 struct itimerval { struct timeval it_value, it_interval; }; +int rsignal_setitimer(int type, struct itimerval *in, struct itimerval *out); + #endif//_RSIGNAL_H diff --git a/pkg/urbit/include/c/portable.h b/pkg/urbit/include/c/portable.h index 0647a1a6d2..16921ec681 100644 --- a/pkg/urbit/include/c/portable.h +++ b/pkg/urbit/include/c/portable.h @@ -73,6 +73,8 @@ # include # elif defined(U3_OS_mingw) +# define signal mingw_has_no_usable_signal +# define raise mingw_has_no_usable_raise # define _POSIX # include # include From 6702dad5f058cd017d278aae6da9aec997304588 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 6 May 2021 12:36:58 +0300 Subject: [PATCH 113/136] vere: mingw: pass Ctrl-C to serf --- pkg/urbit/daemon/main.c | 12 ++++++++++++ pkg/urbit/include/vere/vere.h | 3 +++ pkg/urbit/vere/king.c | 4 ++++ pkg/urbit/vere/lord.c | 7 +++++++ pkg/urbit/worker/main.c | 34 +++++++++++++++++++++++++++++++++- 5 files changed, 59 insertions(+), 1 deletion(-) diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index 290b56befa..ee91948225 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -753,6 +753,18 @@ main(c3_i argc, } } + #if defined(U3_OS_mingw) + // Initialize event used to transmit Ctrl-C to worker process + // + { + SECURITY_ATTRIBUTES sa = {sizeof(sa), NULL, TRUE}; + if ( NULL == (u3_Host.cev_u = CreateEvent(&sa, FALSE, FALSE, NULL)) ) { + u3l_log("boot: failed to create Ctrl-C event: %d\r\n", GetLastError()); + exit(1); + } + } + #endif + // Initialize OpenSSL for client and server // { diff --git a/pkg/urbit/include/vere/vere.h b/pkg/urbit/include/vere/vere.h index 56edece23a..20f2b98778 100644 --- a/pkg/urbit/include/vere/vere.h +++ b/pkg/urbit/include/vere/vere.h @@ -319,6 +319,9 @@ c3_d now_d; // event tick uv_loop_t* lup_u; // libuv event loop u3_usig* sig_u; // signal list + #if defined(U3_OS_mingw) + HANDLE cev_u; // Ctrl-C event handle + #endif u3_utty* uty_u; // linked terminal list u3_opts ops_u; // commandline options c3_i xit_i; // exit code for shutdown diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c index 99f40032b9..ca11a7bc36 100644 --- a/pkg/urbit/vere/king.c +++ b/pkg/urbit/vere/king.c @@ -590,6 +590,10 @@ _king_sign_cb(uv_signal_t* sil_u, c3_i num_i) case SIGINT: { u3l_log("\r\ninterrupt\r\n"); u3_term_ef_ctlc(); + + #if defined(U3_OS_mingw) + PulseEvent(u3_Host.cev_u); + #endif break; } diff --git a/pkg/urbit/vere/lord.c b/pkg/urbit/vere/lord.c index e8fb65d8a8..25ffe60962 100644 --- a/pkg/urbit/vere/lord.c +++ b/pkg/urbit/vere/lord.c @@ -1145,6 +1145,7 @@ u3_lord_init(c3_c* pax_c, c3_w wag_w, c3_d key_d[4], u3_lord_cb cb_u) c3_c key_c[256]; c3_c wag_c[11]; c3_c hap_c[11]; + c3_c cev_c[11]; c3_i err_i; sprintf(key_c, "%" PRIx64 ":%" PRIx64 ":%" PRIx64 ":%" PRIx64 "", @@ -1173,7 +1174,13 @@ u3_lord_init(c3_c* pax_c, c3_w wag_w, c3_d key_d[4], u3_lord_cb cb_u) arg_c[6] = "0"; } + #if defined(U3_OS_mingw) + sprintf(cev_c, "%u", u3_Host.cev_u); + arg_c[7] = cev_c; + arg_c[8] = 0; + #else arg_c[7] = 0; + #endif uv_pipe_init(u3L, &god_u->inn_u.pyp_u, 0); uv_timer_init(u3L, &god_u->out_u.tim_u); diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index c62ccd6e93..216d0c0470 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -155,6 +155,18 @@ _cw_serf_exit(void) u3t_trace_close(); } +#if defined(U3_OS_mingw) +extern void rsignal_raise(int); + +/* _mingw_ctrlc_cb(): invoked when the lord signals the Ctrl-C event +*/ +static void +_mingw_ctrlc_cb(PVOID param, BOOLEAN timedOut) +{ + rsignal_raise(SIGINT); +} +#endif + /* _cw_serf_commence(); initialize and run serf */ static void @@ -163,7 +175,23 @@ _cw_serf_commence(c3_i argc, c3_c* argv[]) c3_i inn_i, out_i; _cw_serf_stdio(&inn_i, &out_i); + #if defined(U3_OS_mingw) + c3_assert( 8 == argc ); + + // Initialize serf's end of Ctrl-C handling + // + { + HANDLE h; + if ( 1 != sscanf(argv[7], "%u", &h) ) { + fprintf(stderr, "serf: Ctrl-C event: bad handle %s: %s\r\n", argv[7], strerror(errno)); + } else + if ( !RegisterWaitForSingleObject(&h, h, _mingw_ctrlc_cb, NULL, INFINITE, 0) ) { + fprintf(stderr, "serf: Ctrl-C event: RegisterWaitForSingleObject(%u) failed (%d)\r\n", h, GetLastError()); + } + } + #else c3_assert( 7 == argc ); + #endif uv_loop_t* lup_u = uv_default_loop(); c3_c* dir_c = argv[2]; @@ -435,7 +463,11 @@ _cw_usage(c3_i argc, c3_c* argv[]) " cue persistent state:\n" " %s queu \n\n" " run as a 'serf':\n" - " %s serf \n", + " %s serf " + #if defined(U3_OS_mingw) + " " + #endif + "\n", argv[0], argv[0], argv[0], argv[0], argv[0], argv[0], argv[0]); } From 61a3efc0f059b88280eb80dbfd9ed6de54f95e3c Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 6 May 2021 12:42:25 +0300 Subject: [PATCH 114/136] compat: mingw: make sure .patch files check out without CRLF conversion --- pkg/urbit/.gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/urbit/.gitattributes b/pkg/urbit/.gitattributes index 97a98b07a2..def025cbfc 100644 --- a/pkg/urbit/.gitattributes +++ b/pkg/urbit/.gitattributes @@ -3,3 +3,4 @@ tests export-ignore hashtable_tests export-ignore shell.nix export-ignore +*.patch -text From 9590d23c87cc05a9d296a2fad8057cb9e2b7dac1 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 6 May 2021 13:24:40 +0300 Subject: [PATCH 115/136] compat: mingw: default to release build --- pkg/urbit/configure | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index bc0e6727ea..1f8ad364a2 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -37,14 +37,16 @@ xxd () { defmacro URBIT_VERSION "\"$URBIT_VERSION\"" +opt_debug= + while test $# != 0 do case $1 in --enable-debug) - CFLAGS="${CFLAGS-} -O0 -g" + opt_debug=1 ;; --disable-debug) - CFLAGS="${CFLAGS-} -O3 -g" + opt_debug=0 ;; *) echo "unrecognized option: $1" @@ -102,6 +104,13 @@ case $(tr A-Z a-z <<< $os) in defmacro U3_OS_mingw 1 + if ((opt_debug)) + then + CFLAGS="${CFLAGS-} -O0 -g" + else + CFLAGS="${CFLAGS-} -O3 -g" + fi + # increase default thread stack size and link Windows implibs ldextra="-Wl,--stack,67108864 -lbcrypt -lntdll -lws2_32" # libcurl implibs From 44722f3a092344c2afc7d8724b9bb5f50b638cde Mon Sep 17 00:00:00 2001 From: raghu <{ID}+{username}@users.noreply.github.com> Date: Tue, 11 May 2021 21:30:44 -0400 Subject: [PATCH 116/136] fuse: properly report success --- pkg/arvo/sys/vane/clay.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 3d504a1395..83b6c48021 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -2043,7 +2043,7 @@ ^+ ..take-fuse ?~ merges =/ t=tang [leaf+"{} fused from {} {}" ~] - =. ..take-fuse (done-fuse clean-state %| %fuse-succeded t) + =. ..take-fuse (done-fuse clean-state %& ~) (park | [%| continuation-yaki(p (flop parents))] rag) =/ [bec=beak g=germ] i.merges =/ ali-dom=dome:clay (need (~(got by sto.fiz) bec)) From 2c754ff08f381dde79693e75d2961ba191047d61 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Tue, 18 May 2021 12:29:16 -0500 Subject: [PATCH 117/136] zuse: added +ram method to ordered map and made comments on methods accurate --- pkg/arvo/sys/zuse.hoon | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pkg/arvo/sys/zuse.hoon b/pkg/arvo/sys/zuse.hoon index 9fda52f567..168c15863a 100644 --- a/pkg/arvo/sys/zuse.hoon +++ b/pkg/arvo/sys/zuse.hoon @@ -5077,10 +5077,6 @@ ++ ordered-map on :: +on: treap with user-specified horizontal order, ordered-map :: -:: Conceptually smaller items go on the left, so the item with the -:: smallest key can be popped off the head. If $key is `@` and -:: .compare is +lte, then the numerically smallest item is the head. -:: :: WARNING: ordered-map will not work properly if two keys can be :: unequal under noun equality but equal via the compare gate :: @@ -5144,7 +5140,7 @@ :: ?~(r.a %.y &((mor key.n.a key.n.r.a) $(a r.a, r `key.n.a))) == - :: +bap: convert to list, largest to smallest + :: +bap: convert to list, right to left :: ++ bap ~/ %bap @@ -5175,10 +5171,9 @@ :: +dip: stateful partial inorder traversal :: :: Mutates .state on each run of .f. Starts at .start key, or if - :: .start is ~, starts at the head (item with smallest key). Stops - :: when .f produces .stop=%.y. Traverses from smaller to larger - :: keys. Each run of .f can replace an item's value or delete the - :: item. + :: .start is ~, starts at the head. Stops when .f produces .stop=%.y. + :: Traverses from left to right keys. + :: Each run of .f can replace an item's value or delete the item. :: ++ dip ~/ %dip @@ -5339,7 +5334,7 @@ l.a(r $(l.a r.l.a)) r.a(l $(r.a l.r.a)) :: - :: +pop: produce .head (smallest item) and .rest or crash if empty + :: +pop: produce .head (leftmost item) and .rest or crash if empty :: ++ pop ~/ %pop @@ -5354,13 +5349,14 @@ ?: |(?=(~ rest.l) (mor key.n.a key.n.rest.l)) a(l rest.l) rest.l(r a(r r.rest.l)) - :: +pry: produce head (smallest item) or null + :: +pry: produce head (leftmost item) or null :: ++ pry ~/ %pry |= a=(tree item) ^- (unit item) ?~ a ~ + |- ?~ l.a `n.a $(a l.a) :: +put: ordered item insert @@ -5390,6 +5386,16 @@ ?: (mor key.n.a key.n.r) a(r r) r(l a(r l.r)) + :: +ram: produce tail (rightmost item) or null + :: + ++ ram + ~/ %ram + |= a=(tree item) + ^- (unit item) + ?~ a ~ + |- + ?~ r.a `n.a + $(a r.a) :: +run: apply gate to transform all values in place :: ++ run @@ -5431,7 +5437,7 @@ $(a r.a) a(l $(a l.a)) -- - :: +tap: convert to list, smallest to largest + :: +tap: convert to list, left to right :: ++ tap ~/ %tap From eb130136f0e42313eb2c6cc238a9fafc5853e3a1 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Tue, 18 May 2021 07:53:44 -1000 Subject: [PATCH 118/136] pill: update --- bin/solid.pill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 034ca34273..7bac0f38e7 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:d7b7cf24e56ab078cf1dcb82e4e7744f188c5221c08772d6cfb15f59ce81aaa5 -size 11198219 +oid sha256:88acd8aa1aae3d11579ada954b6e0b06c940de7856d22017cc1a1442de97fcf3 +size 13650762 From bbe3cbe8ee2904d5f9472763bb9d1689b14da3a8 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Tue, 18 May 2021 08:32:30 -1000 Subject: [PATCH 119/136] clay: formatting --- pkg/arvo/sys/vane/clay.hoon | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index edc25ba1e3..993ac2703a 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -2229,12 +2229,13 @@ ^- (each (unit merge-result) [term tang]) =/ ali-yaki=yaki (~(got by hut.ran) (~(got by hit.ali-dome) let.ali-dome)) =/ bob-yaki=(unit yaki) - ?~ continuation-yaki - ?~ let.dom - ~ - (~(get by hut.ran) (~(got by hit.dom) let.dom)) - continuation-yaki + ?~ continuation-yaki + ?~ let.dom + ~ + (~(get by hut.ran) (~(got by hit.dom) let.dom)) + continuation-yaki (merge-by-germ ali-yaki bob-yaki) + :: ++ merge-by-germ |= [=ali=yaki bob-yaki=(unit yaki)] ^- (each (unit merge-result) [term tang]) @@ -4498,7 +4499,6 @@ per=regs pew=regs == - +$ ford-cache-7 ford-cache +$ raft-6 $: rom=room-6 :: domestic From 804e2578ed9800568e28e225b604610d2653892a Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Tue, 18 May 2021 09:09:22 -1000 Subject: [PATCH 120/136] kiln: don't crash on fuse --- pkg/arvo/lib/hood/kiln.hoon | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/arvo/lib/hood/kiln.hoon b/pkg/arvo/lib/hood/kiln.hoon index 9f541bacdb..79b07a668a 100644 --- a/pkg/arvo/lib/hood/kiln.hoon +++ b/pkg/arvo/lib/hood/kiln.hoon @@ -501,6 +501,8 @@ ++ take |=(way=wire ?>(?=([@ ~] way) (work i.way))) :: general handler ++ take-mere :: |= [way=wire are=(each (set path) (pair term tang))] + ?. ?=([@ ~] way) + abet abet:abet:(mere:(take way) are) :: ++ take-coup-fancy :: From 0e8ff36857700d0711475007eabb17e446c60f05 Mon Sep 17 00:00:00 2001 From: Logan Allen Date: Wed, 19 May 2021 18:47:11 +0000 Subject: [PATCH 121/136] graph-validators: hotfix broken vale:dais caused by removal of +bunt from clay --- pkg/arvo/app/graph-store.hoon | 1 - pkg/arvo/mar/graph/validator/chat.hoon | 2 +- pkg/arvo/mar/graph/validator/link.hoon | 2 +- pkg/arvo/mar/graph/validator/post.hoon | 2 +- pkg/arvo/mar/graph/validator/publish.hoon | 2 +- 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/arvo/app/graph-store.hoon b/pkg/arvo/app/graph-store.hoon index 6763317fc5..6c7f778f01 100644 --- a/pkg/arvo/app/graph-store.hoon +++ b/pkg/arvo/app/graph-store.hoon @@ -1,6 +1,5 @@ :: graph-store [landscape] :: -:: /+ store=graph-store, sigs=signatures, res=resource, default-agent, dbug, verb ~% %graph-store-top ..part ~ |% diff --git a/pkg/arvo/mar/graph/validator/chat.hoon b/pkg/arvo/mar/graph/validator/chat.hoon index 53c10c9f0e..f1a0a16843 100644 --- a/pkg/arvo/mar/graph/validator/chat.hoon +++ b/pkg/arvo/mar/graph/validator/chat.hoon @@ -33,7 +33,7 @@ ++ grab |% ++ noun - |= p=* + |: p=`*`%*(. *indexed-post index.p [0 ~]) =/ ip ;;(indexed-post p) ?> ?=([@ ~] index.p.ip) ip diff --git a/pkg/arvo/mar/graph/validator/link.hoon b/pkg/arvo/mar/graph/validator/link.hoon index d877018648..3f0164c363 100644 --- a/pkg/arvo/mar/graph/validator/link.hoon +++ b/pkg/arvo/mar/graph/validator/link.hoon @@ -49,7 +49,7 @@ ++ grab |% ++ noun - |= p=* + |: p=`*`%*(. *indexed-post index.p [0 0 ~]) =/ ip ;;(indexed-post p) ?+ index.p.ip ~|(index+index.p.ip !!) :: top-level link post; title and url diff --git a/pkg/arvo/mar/graph/validator/post.hoon b/pkg/arvo/mar/graph/validator/post.hoon index a12e62e721..15c9a3f010 100644 --- a/pkg/arvo/mar/graph/validator/post.hoon +++ b/pkg/arvo/mar/graph/validator/post.hoon @@ -43,7 +43,7 @@ :: +noun: validate post :: ++ noun - |= p=* + |: p=`*`%*(. *indexed-post contents.p [%text '']~) =/ ip ;;(indexed-post p) ?> ?=(^ contents.p.ip) ip diff --git a/pkg/arvo/mar/graph/validator/publish.hoon b/pkg/arvo/mar/graph/validator/publish.hoon index c8da4405ca..d19812b3e1 100644 --- a/pkg/arvo/mar/graph/validator/publish.hoon +++ b/pkg/arvo/mar/graph/validator/publish.hoon @@ -58,7 +58,7 @@ :: +noun: validate publish note :: ++ noun - |= p=* + |: p=`*`%*(. *indexed-post index.p [0 ~]) =/ ip ;;(indexed-post p) ?+ index.p.ip !! :: top level post must have no content From 08c7f7f0956c05b9c9ab2f77506b6e70515fb3b4 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 20 May 2021 14:29:53 +0300 Subject: [PATCH 122/136] mingw: rerun build action From 934a88002edc7ba1ca525221b2353c5b040d9e6b Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 3 Jun 2021 00:59:08 +0300 Subject: [PATCH 123/136] u3: Makefile: include compat/*/*.mk --- pkg/urbit/Makefile | 3 ++- pkg/urbit/compat/mingw/compat.mk | 9 +++++++++ pkg/urbit/configure | 10 ++-------- 3 files changed, 13 insertions(+), 9 deletions(-) create mode 100644 pkg/urbit/compat/mingw/compat.mk diff --git a/pkg/urbit/Makefile b/pkg/urbit/Makefile index b82e936b55..2565bb8a5b 100644 --- a/pkg/urbit/Makefile +++ b/pkg/urbit/Makefile @@ -1,4 +1,5 @@ include config.mk +include $(foreach dir,$(compat),$(wildcard compat/$(dir)/*.mk)) jets = jets/tree.c $(wildcard jets/*/*.c) noun = $(wildcard noun/*.c) @@ -31,7 +32,7 @@ CFLAGS := $(CFLAGS) ################################################################################ -.PHONY: all test clean mkproper +.PHONY: all test clean mrproper ################################################################################ diff --git a/pkg/urbit/compat/mingw/compat.mk b/pkg/urbit/compat/mingw/compat.mk new file mode 100644 index 0000000000..3aece861b4 --- /dev/null +++ b/pkg/urbit/compat/mingw/compat.mk @@ -0,0 +1,9 @@ +# increase default thread stack size and link Windows implibs +LDFLAGS := $(LDFLAGS) -Wl,--stack,67108864 -lbcrypt -lntdll -lws2_32 +# libcurl +CFLAGS := $(CFLAGS) -DCURL_STATICLIB +LDFLAGS := $(LDFLAGS) -lzstd +# libh2o +CFLAGS := $(CFLAGS) -DH2O_NO_UNIX_SOCKETS +# libuv +LDFLAGS := $(LDFLAGS) -luserenv -liphlpapi -lpsapi diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 1f8ad364a2..e9d1fa74cb 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -111,14 +111,8 @@ case $(tr A-Z a-z <<< $os) in CFLAGS="${CFLAGS-} -O3 -g" fi - # increase default thread stack size and link Windows implibs - ldextra="-Wl,--stack,67108864 -lbcrypt -lntdll -lws2_32" - # libcurl implibs - ldextra="$ldextra -lzstd" - # libuv implibs - ldextra="$ldextra -luserenv -liphlpapi -lpsapi" compat="${compat-} mingw" - CFLAGS="${CFLAGS-} ${cdirs[@]} -DCURL_STATICLIB -DH2O_NO_UNIX_SOCKETS" + CFLAGS="${CFLAGS-} ${cdirs[@]}" LDFLAGS="${LDFLAGS-} ${ldirs[@]} -static" PKG_CONFIG=">/dev/null echo" ;; @@ -164,7 +158,7 @@ done cat >config.mk < Date: Thu, 3 Jun 2021 01:00:41 +0300 Subject: [PATCH 124/136] mingw: add rsignal_raise to rsignal.h; raise on same thread --- pkg/urbit/compat/mingw/rsignal.c | 5 +++++ pkg/urbit/compat/mingw/rsignal.h | 1 + pkg/urbit/worker/main.c | 3 +-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/compat/mingw/rsignal.c b/pkg/urbit/compat/mingw/rsignal.c index f87f786bf1..b49dff5153 100644 --- a/pkg/urbit/compat/mingw/rsignal.c +++ b/pkg/urbit/compat/mingw/rsignal.c @@ -44,6 +44,11 @@ void rsignal_raise(int sig) if (oldfn == 0) return; + if (_tid == GetCurrentThreadId()) { + oldfn(sig); + return; + } + HANDLE hthread = OpenThread(THREAD_ALL_ACCESS, FALSE, _tid); if (!hthread) { fprintf(stderr, "\r\nrsignal_raise: OpenThread(%u): %d\r\n", _tid, GetLastError()); diff --git a/pkg/urbit/compat/mingw/rsignal.h b/pkg/urbit/compat/mingw/rsignal.h index 8c8c3d0f11..5af8e2dcd7 100644 --- a/pkg/urbit/compat/mingw/rsignal.h +++ b/pkg/urbit/compat/mingw/rsignal.h @@ -9,6 +9,7 @@ typedef struct { #define rsignal_setjmp(buf) (buf.tid = GetCurrentThreadId(), setjmp(buf.jb)) #define rsignal_longjmp(buf, val) if (buf.tid != GetCurrentThreadId()) {buf.jb.retval = (val); rsignal_post_longjmp(buf.tid, buf.jb.buffer);} else longjmp(buf.jb, val) +void rsignal_raise(int sig); void rsignal_install_handler(int sig, __p_sig_fn_t fn); void rsignal_deinstall_handler(int sig); void rsignal_post_longjmp(unsigned long tid, intptr_t* builtin_jb); diff --git a/pkg/urbit/worker/main.c b/pkg/urbit/worker/main.c index 216d0c0470..b44add6170 100644 --- a/pkg/urbit/worker/main.c +++ b/pkg/urbit/worker/main.c @@ -3,6 +3,7 @@ ** the main loop of a serf process. */ #include "all.h" +#include "rsignal.h" #include #include @@ -156,8 +157,6 @@ _cw_serf_exit(void) } #if defined(U3_OS_mingw) -extern void rsignal_raise(int); - /* _mingw_ctrlc_cb(): invoked when the lord signals the Ctrl-C event */ static void From 08e45350847e73f15a59f23d557c2401d9ab7ea9 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 3 Jun 2021 01:41:41 +0300 Subject: [PATCH 125/136] vere: get rid of curl typecheck warnings for CURLOPT_WRITEFUNCTION --- pkg/urbit/vere/dawn.c | 4 +++- pkg/urbit/vere/king.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/vere/dawn.c b/pkg/urbit/vere/dawn.c index 349999102a..bc306a4611 100644 --- a/pkg/urbit/vere/dawn.c +++ b/pkg/urbit/vere/dawn.c @@ -44,8 +44,10 @@ _dawn_buf_to_oct(uv_buf_t buf_u) /* _dawn_curl_alloc(): allocate a response buffer for curl */ static size_t -_dawn_curl_alloc(void* dat_v, size_t uni_t, size_t mem_t, uv_buf_t* buf_u) +_dawn_curl_alloc(void* dat_v, size_t uni_t, size_t mem_t, void* buf_v) { + uv_buf_t* buf_u = buf_v; + size_t siz_t = uni_t * mem_t; buf_u->base = c3_realloc(buf_u->base, 1 + siz_t + buf_u->len); diff --git a/pkg/urbit/vere/king.c b/pkg/urbit/vere/king.c index ca11a7bc36..e61f6ee33c 100644 --- a/pkg/urbit/vere/king.c +++ b/pkg/urbit/vere/king.c @@ -215,8 +215,10 @@ _king_pier(u3_noun pier) ** XX deduplicate with dawn.c */ static size_t -_king_curl_alloc(void* dat_v, size_t uni_t, size_t mem_t, uv_buf_t* buf_u) +_king_curl_alloc(void* dat_v, size_t uni_t, size_t mem_t, void* buf_v) { + uv_buf_t* buf_u = buf_v; + size_t siz_t = uni_t * mem_t; buf_u->base = c3_realloc(buf_u->base, 1 + siz_t + buf_u->len); From 0ccbcd50389dde87c97c2087899c949eddd9afcd Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 3 Jun 2021 01:43:07 +0300 Subject: [PATCH 126/136] vere: mingw: bump curl version and get rid of CURLOPT_SSL_CTX_FUNCTION typecheck warning --- nix/sources-mingw.json | 6 +++--- pkg/urbit/compat/mingw/compat.mk | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nix/sources-mingw.json b/nix/sources-mingw.json index ae3a2d59f0..25f9c32d7f 100644 --- a/nix/sources-mingw.json +++ b/nix/sources-mingw.json @@ -6,14 +6,14 @@ "mingw": { "include": "include", "lib": "lib/.libs", - "prepare": "autoreconf -vfi && ./configure --disable-shared --disable-ldap --disable-rtsp --without-brotli --without-libidn2 --without-libpsl --without-nghttp2", + "prepare": "autoreconf -vfi && ./configure --disable-shared --disable-ldap --disable-rtsp --without-brotli --without-libidn2 --without-libpsl --without-nghttp2 --with-openssl", "make": "-C lib libcurl.la" }, "owner": "curl", "repo": "curl", - "rev": "curl-7_75_0", + "rev": "curl-7_77_0", "type": "tarball", - "url": "https://github.com/curl/curl/archive/curl-7_75_0.tar.gz", + "url": "https://github.com/curl/curl/archive/curl-7_77_0.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "lmdb": { diff --git a/pkg/urbit/compat/mingw/compat.mk b/pkg/urbit/compat/mingw/compat.mk index 3aece861b4..0bc9eaa87f 100644 --- a/pkg/urbit/compat/mingw/compat.mk +++ b/pkg/urbit/compat/mingw/compat.mk @@ -2,7 +2,7 @@ LDFLAGS := $(LDFLAGS) -Wl,--stack,67108864 -lbcrypt -lntdll -lws2_32 # libcurl CFLAGS := $(CFLAGS) -DCURL_STATICLIB -LDFLAGS := $(LDFLAGS) -lzstd +LDFLAGS := $(LDFLAGS) -lzstd -lcrypt32 # libh2o CFLAGS := $(CFLAGS) -DH2O_NO_UNIX_SOCKETS # libuv From b130a9dde939b7eca85000b049a32af991cce254 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 3 Jun 2021 01:45:50 +0300 Subject: [PATCH 127/136] vere: mingw: get rid of libsigsegv --- pkg/urbit/Makefile | 4 +- pkg/urbit/compat/mingw/compat.h | 1 + pkg/urbit/compat/mingw/poor-mans-nix-shell.sh | 2 +- .../compat/mingw/seh_handler_decorator.cc | 65 +++++++++++++++++++ .../compat/mingw/seh_handler_decorator.mk | 6 ++ pkg/urbit/configure | 1 + pkg/urbit/daemon/main.c | 4 ++ pkg/urbit/noun/manage.c | 49 +++++++++++++- 8 files changed, 126 insertions(+), 6 deletions(-) create mode 100644 pkg/urbit/compat/mingw/seh_handler_decorator.cc create mode 100644 pkg/urbit/compat/mingw/seh_handler_decorator.mk diff --git a/pkg/urbit/Makefile b/pkg/urbit/Makefile index 2565bb8a5b..82db53003d 100644 --- a/pkg/urbit/Makefile +++ b/pkg/urbit/Makefile @@ -77,9 +77,9 @@ build/urbit-worker: $(common_objs) $(worker_objs) @mkdir -p ./build @$(CC) $^ $(LDFLAGS) -o $@ -%.o: %.c $(headers) +%.o: %.c $(headers) $(CCDEPS) @echo CC $< - @$(CC) -I./include $(CFLAGS) -c $< -o $@ + @$(CC) -I./include $(CFLAGS) $< $(CCEXTRA) -c -o $@ tags: $(all_srcs) $(headers) ctags $^ diff --git a/pkg/urbit/compat/mingw/compat.h b/pkg/urbit/compat/mingw/compat.h index 667c1ccf46..42f159baeb 100644 --- a/pkg/urbit/compat/mingw/compat.h +++ b/pkg/urbit/compat/mingw/compat.h @@ -12,6 +12,7 @@ int kill(pid_t pid, int signum); #define SIGUSR1 10 #define SIGALRM 14 #define SIGVTALRM 26 +#define SIGSTK 31 #define SIG_COUNT 32 #endif//_MINGW_IO_H \ No newline at end of file diff --git a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh index ca54e63723..9cb37cb536 100644 --- a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh +++ b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh @@ -1,5 +1,5 @@ # ensure required mingw packages are installed -mpkgs=(cmake curl gcc jq libsigsegv libuv make wslay) +mpkgs=(cmake curl gcc jq libuv make wslay) pacman -S --needed autoconf automake-wrapper libtool patch ${mpkgs[@]/#/mingw-w64-x86_64-} declare -a cdirs diff --git a/pkg/urbit/compat/mingw/seh_handler_decorator.cc b/pkg/urbit/compat/mingw/seh_handler_decorator.cc new file mode 100644 index 0000000000..1ef350c7c2 --- /dev/null +++ b/pkg/urbit/compat/mingw/seh_handler_decorator.cc @@ -0,0 +1,65 @@ +#include +#include + +enum { INIT, CPAR, DQ, DQS, SQ, SQS }; +char line[1 << 16]; + +/* seh_handler_decorator: registers u3_exception_handler for all non-inline functions +*/ +int main(int argc, const char* argv[]) +{ + if (argc != 2) + return 1; + + int c, state = INIT, curly = 0, emit = 0; + + while (fgets(line, sizeof(line), stdin)) + { + if (line[0] == '#') + { + emit = !!strstr(line, argv[1]); + fputs(line, stdout); + } + else + for (int i = 0; line[i]; i++) + { + switch (state) { + case INIT: + case CPAR: + switch (line[i]) { + case ' ': + case '\t': + case '\n': + case '\r': + case '\f': break; + case '{': curly++; if (emit && curly == 1 && state == CPAR) goto emit_handler; goto reset; + case '}': curly--; goto reset; + case '"': state = DQ; break; + case '\'': state = SQ; break; + case ')': state = CPAR; break; + reset: + default: state = INIT; break; + } break; + case DQ: + switch (line[i]) { + case '\\': state = DQS; break; + case '"': state = INIT; break; + } break; + case DQS: state = DQ; break; + case SQ: + switch (line[i]) { + case '\\': state = SQS; break; + case '\'': state = INIT; break; + } break; + case SQS: state = SQ; break; + } + fputc(line[i], stdout); + continue; + emit_handler: + fputs("{__asm__(\".seh_handler u3_exception_handler,@except\\n\");", stdout); + state = INIT; + } + } + + return 0; +} diff --git a/pkg/urbit/compat/mingw/seh_handler_decorator.mk b/pkg/urbit/compat/mingw/seh_handler_decorator.mk new file mode 100644 index 0000000000..e02a569189 --- /dev/null +++ b/pkg/urbit/compat/mingw/seh_handler_decorator.mk @@ -0,0 +1,6 @@ +sehdexe := build/seh_handler_decorator.exe +CCDEPS := $(CCDEPS) $(sehdexe) +CCEXTRA = -E -o -|$(sehdexe) $<|$(CC) $(CFLAGS) -x cpp-output - + +$(sehdexe): compat/mingw/seh_handler_decorator.cc + @$(CC) $< -o $@ diff --git a/pkg/urbit/configure b/pkg/urbit/configure index e9d1fa74cb..3789b397dc 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -111,6 +111,7 @@ case $(tr A-Z a-z <<< $os) in CFLAGS="${CFLAGS-} -O3 -g" fi + deps="${deps/sigsegv}" compat="${compat-} mingw" CFLAGS="${CFLAGS-} ${cdirs[@]}" LDFLAGS="${LDFLAGS-} ${ldirs[@]} -static" diff --git a/pkg/urbit/daemon/main.c b/pkg/urbit/daemon/main.c index ee91948225..7912560a86 100644 --- a/pkg/urbit/daemon/main.c +++ b/pkg/urbit/daemon/main.c @@ -5,7 +5,9 @@ #define C3_GLOBAL #include "all.h" #include "vere/vere.h" +#if !defined(U3_OS_mingw) #include +#endif #include #include #include @@ -512,9 +514,11 @@ report(void) { printf("urbit %s\n", URBIT_VERSION); printf("gmp: %s\n", gmp_version); +#if !defined(U3_OS_mingw) printf("sigsegv: %d.%d\n", (libsigsegv_version >> 8) & 0xff, libsigsegv_version & 0xff); +#endif printf("openssl: %s\n", SSLeay_version(SSLEAY_VERSION)); printf("libuv: %s\n", uv_version_string()); printf("libh2o: %d.%d.%d\n", diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 298ef0a650..4c2abe90ac 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -8,7 +8,6 @@ #include #include #include -#include #include // XX stack-overflow recovery should be gated by -a @@ -83,12 +82,38 @@ // static rsignal_jmpbuf u3_Signal; +#if defined(U3_OS_mingw) +/* u3_exception_handler: replaces libsigsegv on MingW +*/ +EXCEPTION_DISPOSITION u3_exception_handler( + IN PEXCEPTION_RECORD ExceptionRecord, + IN ULONG64 EstablisherFrame, + IN OUT PCONTEXT ContextRecord, + IN OUT PDISPATCHER_CONTEXT DispatcherContext) +{ + if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && + ExceptionRecord->ExceptionInformation[0] == 1 && + u3e_fault((void*)ExceptionRecord->ExceptionInformation[1], 1)) + { + return ExceptionContinueExecution; + } + + if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { + rsignal_raise(SIGSTK); + } + + return ExceptionContinueSearch; +} +#else +#include + #ifndef SIGSTKSZ # define SIGSTKSZ 16384 #endif #ifndef NO_OVERFLOW static uint8_t Sigstk[SIGSTKSZ]; #endif +#endif #if 0 /* _cm_punt(): crudely print trace. @@ -135,17 +160,25 @@ static void _cm_overflow(void *arg1, void *arg2, void *arg3) static void _cm_signal_handle(c3_l sig_l) { +#ifndef U3_OS_mingw if ( c3__over == sig_l ) { +#ifndef NO_OVERFLOW sigsegv_leave_handler(_cm_overflow, NULL, NULL, NULL); - } - else { +#endif + } else +#endif + { u3m_signal(sig_l); } } #ifndef NO_OVERFLOW static void +#ifndef U3_OS_mingw _cm_signal_handle_over(int emergency, stackoverflow_context_t scp) +#else +_cm_signal_handle_over(int x) +#endif { _cm_signal_handle(c3__over); } @@ -336,7 +369,11 @@ _cm_signal_deep(c3_w mil_w) } #ifndef NO_OVERFLOW +#ifndef U3_OS_mingw stackoverflow_install_handler(_cm_signal_handle_over, Sigstk, SIGSTKSZ); +#else + rsignal_install_handler(SIGSTK, _cm_signal_handle_over); +#endif #endif rsignal_install_handler(SIGINT, _cm_signal_handle_intr); rsignal_install_handler(SIGTERM, _cm_signal_handle_term); @@ -376,7 +413,11 @@ _cm_signal_done() rsignal_deinstall_handler(SIGVTALRM); #ifndef NO_OVERFLOW +#ifndef U3_OS_mingw stackoverflow_deinstall_handler(); +#else + rsignal_deinstall_handler(SIGSTK); +#endif #endif { struct itimerval itm_u; @@ -1624,10 +1665,12 @@ _cm_limits(void) static void _cm_signals(void) { +# if !defined(U3_OS_mingw) if ( 0 != sigsegv_install_handler(u3e_fault) ) { u3l_log("boot: sigsegv install failed\n"); exit(1); } +# endif # if defined(U3_OS_PROF) // Block SIGPROF, so that if/when we reactivate it on the From 39032679ff124cb23719cc3a9911dbeb82654f49 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Thu, 3 Jun 2021 02:35:13 +0300 Subject: [PATCH 128/136] vere: mingw: fix seh_handler_decorator.mk --- pkg/urbit/compat/mingw/seh_handler_decorator.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/urbit/compat/mingw/seh_handler_decorator.mk b/pkg/urbit/compat/mingw/seh_handler_decorator.mk index e02a569189..8a06826ce0 100644 --- a/pkg/urbit/compat/mingw/seh_handler_decorator.mk +++ b/pkg/urbit/compat/mingw/seh_handler_decorator.mk @@ -1,6 +1,7 @@ -sehdexe := build/seh_handler_decorator.exe +sehdexe := build/seh_handler_decorator CCDEPS := $(CCDEPS) $(sehdexe) CCEXTRA = -E -o -|$(sehdexe) $<|$(CC) $(CFLAGS) -x cpp-output - $(sehdexe): compat/mingw/seh_handler_decorator.cc + @mkdir -p ./build @$(CC) $< -o $@ From 839a68e46e6056083c8f4b3c7d597f7a44c7037f Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Fri, 4 Jun 2021 12:33:18 +0300 Subject: [PATCH 129/136] vere: ignore more intermediate build files --- .gitignore | 1 + pkg/urbit/.gitignore | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index e79759dc67..e6c572d9b3 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,7 @@ release/ dist/ out/ work/ +pkg/*/*.a *.o # Landscape Dev diff --git a/pkg/urbit/.gitignore b/pkg/urbit/.gitignore index 6c94be2f8c..f78b9fecdf 100644 --- a/pkg/urbit/.gitignore +++ b/pkg/urbit/.gitignore @@ -1,3 +1,5 @@ # Configuration Result /config.mk /include/config.h +/include/ca-bundle.h +/include/ivory.h From fb2ee65d59e432df585fc18f0b8af6f30b6fc147 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Fri, 4 Jun 2021 12:34:20 +0300 Subject: [PATCH 130/136] compat: fix missing newlines at end of file --- nix/sources-mingw.json | 2 +- pkg/urbit/compat/mingw/compat.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/sources-mingw.json b/nix/sources-mingw.json index 25f9c32d7f..e3923bfdff 100644 --- a/nix/sources-mingw.json +++ b/nix/sources-mingw.json @@ -79,4 +79,4 @@ "mingw": { } } -} \ No newline at end of file +} diff --git a/pkg/urbit/compat/mingw/compat.h b/pkg/urbit/compat/mingw/compat.h index 42f159baeb..21290d7b55 100644 --- a/pkg/urbit/compat/mingw/compat.h +++ b/pkg/urbit/compat/mingw/compat.h @@ -15,4 +15,4 @@ int kill(pid_t pid, int signum); #define SIGSTK 31 #define SIG_COUNT 32 -#endif//_MINGW_IO_H \ No newline at end of file +#endif//_MINGW_IO_H From 35593949c39fe1024091ebf3daefc897ba2e8fc4 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Fri, 4 Jun 2021 12:36:24 +0300 Subject: [PATCH 131/136] vere: update argon2u dependency to urbit/argon2@a4c1e3f7 --- nix/sources.json | 6 +++--- pkg/urbit/compat/mingw/argon2u.patch | 22 ---------------------- 2 files changed, 3 insertions(+), 25 deletions(-) delete mode 100644 pkg/urbit/compat/mingw/argon2u.patch diff --git a/nix/sources.json b/nix/sources.json index 6bb8aa172b..4a4b8470d2 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -9,10 +9,10 @@ }, "owner": "urbit", "repo": "argon2", - "rev": "4da94a611ee62bad87ab2b131ffda3bcc0723d9c", - "sha256": "0bqq1hg367l4jkb6cqhxlblpvdbwz3l586qsfakwzfd9wdvnm3yc", + "rev": "a4c1e3f7138c2e577376beb99f964cf71e1c8b1b", + "sha256": "1j8a15fx2kn5aa3scmb5qnsfk627kfvsq5ppz9j0pv2d1xck527x", "type": "tarball", - "url": "https://github.com/urbit/argon2/archive/4da94a611ee62bad87ab2b131ffda3bcc0723d9c.tar.gz", + "url": "https://github.com/urbit/argon2/archive/a4c1e3f7138c2e577376beb99f964cf71e1c8b1b.tar.gz", "url_template": "https://github.com///archive/.tar.gz" }, "ed25519": { diff --git a/pkg/urbit/compat/mingw/argon2u.patch b/pkg/urbit/compat/mingw/argon2u.patch deleted file mode 100644 index fdabbe630e..0000000000 --- a/pkg/urbit/compat/mingw/argon2u.patch +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/src/encoding.c b/src/encoding.c -index 73c36f5..753a1e1 100644 ---- a/src/encoding.c -+++ b/src/encoding.c -@@ -370,7 +370,7 @@ int decode_string(argon2_context *ctx, const char *str, argon2_type type) { - #undef BIN - } - --void itoa(int i, char b[]){ -+static void encode_decimal(int i, char b[]){ - #ifdef ARGON2_JS - - // because this generates WASM error: -@@ -416,7 +416,7 @@ int encode_string(char *dst, size_t dst_len, argon2_context *ctx, - #define SX(x) \ - do { \ - char tmp[30]; \ -- itoa(x, tmp); \ -+ encode_decimal(x, tmp); \ - SS(tmp); \ - } while ((void)0, 0) - From a2a993cfa06d827807ca12915af29da153515b81 Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 5 Jun 2021 07:22:07 +0000 Subject: [PATCH 132/136] compat: mingw: add seh_handler_decorator comments, move SEH handler to compat/mingw/seh_handler.c --- pkg/urbit/Makefile | 2 ++ pkg/urbit/compat/mingw/seh_handler.c | 25 +++++++++++++ .../compat/mingw/seh_handler_decorator.cc | 4 +-- .../compat/mingw/seh_handler_decorator.mk | 19 +++++++++- pkg/urbit/noun/manage.c | 35 ++++++------------- 5 files changed, 58 insertions(+), 27 deletions(-) create mode 100644 pkg/urbit/compat/mingw/seh_handler.c diff --git a/pkg/urbit/Makefile b/pkg/urbit/Makefile index 82db53003d..c079c2af3b 100644 --- a/pkg/urbit/Makefile +++ b/pkg/urbit/Makefile @@ -77,6 +77,8 @@ build/urbit-worker: $(common_objs) $(worker_objs) @mkdir -p ./build @$(CC) $^ $(LDFLAGS) -o $@ +# CCDEPS and CCEXTRA are empty except in MingW build, +# which uses them to inject a C source transform step %.o: %.c $(headers) $(CCDEPS) @echo CC $< @$(CC) -I./include $(CFLAGS) $< $(CCEXTRA) -c -o $@ diff --git a/pkg/urbit/compat/mingw/seh_handler.c b/pkg/urbit/compat/mingw/seh_handler.c new file mode 100644 index 0000000000..ecf59f0a40 --- /dev/null +++ b/pkg/urbit/compat/mingw/seh_handler.c @@ -0,0 +1,25 @@ +#include "all.h" +#include "rsignal.h" +#include "vere/vere.h" + +/* _mingw_exception_filter: replaces libsigsegv on MingW +*/ +EXCEPTION_DISPOSITION _mingw_exception_filter( + IN PEXCEPTION_RECORD ExceptionRecord, + IN ULONG64 EstablisherFrame, + IN OUT PCONTEXT ContextRecord, + IN OUT PDISPATCHER_CONTEXT DispatcherContext) +{ + if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && + ExceptionRecord->ExceptionInformation[0] == 1 && + u3e_fault((void*)ExceptionRecord->ExceptionInformation[1], 1)) + { + return ExceptionContinueExecution; + } + + if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { + rsignal_raise(SIGSTK); + } + + return ExceptionContinueSearch; +} diff --git a/pkg/urbit/compat/mingw/seh_handler_decorator.cc b/pkg/urbit/compat/mingw/seh_handler_decorator.cc index 1ef350c7c2..acc97efcc3 100644 --- a/pkg/urbit/compat/mingw/seh_handler_decorator.cc +++ b/pkg/urbit/compat/mingw/seh_handler_decorator.cc @@ -4,7 +4,7 @@ enum { INIT, CPAR, DQ, DQS, SQ, SQS }; char line[1 << 16]; -/* seh_handler_decorator: registers u3_exception_handler for all non-inline functions +/* seh_handler_decorator: registers u3_exception_handler for all functions in given source file */ int main(int argc, const char* argv[]) { @@ -56,7 +56,7 @@ int main(int argc, const char* argv[]) fputc(line[i], stdout); continue; emit_handler: - fputs("{__asm__(\".seh_handler u3_exception_handler,@except\\n\");", stdout); + fputs("{__asm__(\".seh_handler _mingw_exception_filter,@except\\n\");", stdout); state = INIT; } } diff --git a/pkg/urbit/compat/mingw/seh_handler_decorator.mk b/pkg/urbit/compat/mingw/seh_handler_decorator.mk index 8a06826ce0..aa2a4c37de 100644 --- a/pkg/urbit/compat/mingw/seh_handler_decorator.mk +++ b/pkg/urbit/compat/mingw/seh_handler_decorator.mk @@ -1,4 +1,21 @@ -sehdexe := build/seh_handler_decorator +# This include file injects a step that transforms vere C source to register +# a SEH exception handler for each function that is declared in a .c file. +# It inserts a .seh_handler directive into each function body with __asm__. +# This directive affects the x64 unwind tables (.pdata and .xdata sections) +# emitted by the compiler. +# +# See gas/config/obj-coff-seh.h in binutils source for .seh_handler, and +# https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64 +# for a description of how stack unwinding and SEH work in x64 Windows. +# +# When this file sets CCEXTRA, the first invocation of $(CC) in Makefile +# writes preprocessor output to stdout (-E -o -), which is piped to a simple +# parser that inserts __asm__ statements to each function in the .c file +# being compiled. The second invocation of $(CC) reads transformed source +# from stdin (-x cpp-output -). $< argument to $(sehdexe) tells the parser +# which .c file is being compiled. + +sehdexe := build/seh_handler_decorator.exe CCDEPS := $(CCDEPS) $(sehdexe) CCEXTRA = -E -o -|$(sehdexe) $<|$(CC) $(CFLAGS) -x cpp-output - diff --git a/pkg/urbit/noun/manage.c b/pkg/urbit/noun/manage.c index 4c2abe90ac..e1f9c52da7 100644 --- a/pkg/urbit/noun/manage.c +++ b/pkg/urbit/noun/manage.c @@ -82,29 +82,7 @@ // static rsignal_jmpbuf u3_Signal; -#if defined(U3_OS_mingw) -/* u3_exception_handler: replaces libsigsegv on MingW -*/ -EXCEPTION_DISPOSITION u3_exception_handler( - IN PEXCEPTION_RECORD ExceptionRecord, - IN ULONG64 EstablisherFrame, - IN OUT PCONTEXT ContextRecord, - IN OUT PDISPATCHER_CONTEXT DispatcherContext) -{ - if (ExceptionRecord->ExceptionCode == EXCEPTION_ACCESS_VIOLATION && - ExceptionRecord->ExceptionInformation[0] == 1 && - u3e_fault((void*)ExceptionRecord->ExceptionInformation[1], 1)) - { - return ExceptionContinueExecution; - } - - if (ExceptionRecord->ExceptionCode == EXCEPTION_STACK_OVERFLOW) { - rsignal_raise(SIGSTK); - } - - return ExceptionContinueSearch; -} -#else +#if !defined(U3_OS_mingw) #include #ifndef SIGSTKSZ @@ -1665,7 +1643,16 @@ _cm_limits(void) static void _cm_signals(void) { -# if !defined(U3_OS_mingw) +# if defined(U3_OS_mingw) + // vere using libsigsegv on MingW is very slow, because libsigsegv + // works by installing a top-level SEH unhandled exception filter. + // The top-level filter runs only after Windows walks the whole stack, + // looking up registered exception filters for every stack frame, and + // finds no filter to handle the exception. + // Instead of libsigsegv, all vere functions register a SEH exception + // filter (see compat/mingw/seh_handler.c) that handles both memory + // access and stack overflow exceptions. It calls u3e_fault directly. +# else if ( 0 != sigsegv_install_handler(u3e_fault) ) { u3l_log("boot: sigsegv install failed\n"); exit(1); From c6121f3c010a11a49718d2df707a8a6ae8173ffe Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Sat, 5 Jun 2021 07:32:50 +0000 Subject: [PATCH 133/136] compat: mingw: put downloaded dependencies under $NIX_STORE (default to pkg/build so that git ignores them) --- pkg/urbit/compat/mingw/poor-mans-nix-shell.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh index 9cb37cb536..906cc15b1e 100644 --- a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh +++ b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh @@ -8,6 +8,14 @@ declare -A hdeps sources=(../../nix/sources.json ../../nix/sources-mingw.json) deriver=urbit-mingw-build depdirs= +nixpath=${NIX_STORE-../build} + +# LDFLAGS doesn't like absolute paths +if [ "${nixpath:0:1}" == "/" ] +then + mkdir -p $nixpath + nixpath=$(realpath --relative-to=. $nixpath) +fi hex2nixbase32 () { local digits='0123456789abcdfghijklmnpqrsvwxyz' @@ -41,7 +49,7 @@ buildnixdep () { if [ -n "$url" ] then hash=${hdeps[$key]} - dir=../$hash-$key + dir=$nixpath/$hash-$key if [ -e $dir/.mingw~ ] then # dependency present, don't reupload @@ -122,7 +130,8 @@ do [ -e $patch ] && cat $patch)|sha256sum) hash=$(hex2nixbase32 $hash) hdeps[$key]=$hash - depdirs="$depdirs|gsub(\"\\\\.\\\\./$key\";\"../$hash-$key\")" + # NB: this path substitution works only in local dependencies + depdirs="$depdirs|gsub(\"\\\\.\\\\./$key\";\"$nixpath/$hash-$key\")" done < <(jq --arg deriver "$deriver" -Sscrj 'add|to_entries|.[]|select(.value.mingw)|select(.value.url)|.key," ",{($deriver):(.value)},"\u0000"' ${sources[@]}) # build dependencies, create include and library directory arrays From e7ac8c0c2a187e442a4db4706a56ef03b42b94fa Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Wed, 16 Jun 2021 10:58:08 +0000 Subject: [PATCH 134/136] compat: mingw: fix dependency patch path --- pkg/urbit/compat/mingw/poor-mans-nix-shell.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh index 906cc15b1e..16e40d1fc8 100644 --- a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh +++ b/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh @@ -79,16 +79,16 @@ buildnixdep () { fi # patch and build the dependency if necessary - pushd $dir - if [ ! -e .mingw~ ] + if [ ! -e $dir/.mingw~ ] then - local patch=../urbit/compat/mingw/$key.patch - [ -e $patch ] && patch -p 1 <$patch + local patch=compat/mingw/$key.patch + [ -e $patch ] && patch -d $dir -p 1 <$patch + pushd $dir eval "$cmdprep" eval make "$cmdmake" touch .mingw~ + popd fi - popd # if configured, upload freshly built dependency to binary cache if [ -n "$hash" -a -n "${CACHIX_AUTH_TOKEN-}" ] From 23634f66f36962ba9aa85134fc9793ddc71d190a Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Fri, 2 Jul 2021 17:27:56 +0300 Subject: [PATCH 135/136] compat: add m1brew --- ...{sources-mingw.json => sources-pmnsh.json} | 24 ++++-- nix/sources.json | 33 ++++--- pkg/urbit/compat/m1brew/compat.mk | 7 ++ pkg/urbit/compat/m1brew/ed25519.patch | 13 +++ pkg/urbit/compat/m1brew/ent.patch | 12 +++ pkg/urbit/compat/m1brew/libscrypt.patch | 17 ++++ pkg/urbit/compat/m1brew/murmur3.patch | 13 +++ pkg/urbit/compat/m1brew/softfloat3.patch | 85 +++++++++++++++++++ pkg/urbit/compat/m1brew/use-static-libs.sh | 23 +++++ .../compat/{mingw => }/poor-mans-nix-shell.sh | 38 ++++----- pkg/urbit/configure | 57 +++++++++++-- pkg/urbit/include/c/portable.h | 6 +- 12 files changed, 280 insertions(+), 48 deletions(-) rename nix/{sources-mingw.json => sources-pmnsh.json} (87%) create mode 100644 pkg/urbit/compat/m1brew/compat.mk create mode 100644 pkg/urbit/compat/m1brew/ed25519.patch create mode 100644 pkg/urbit/compat/m1brew/ent.patch create mode 100644 pkg/urbit/compat/m1brew/libscrypt.patch create mode 100644 pkg/urbit/compat/m1brew/murmur3.patch create mode 100644 pkg/urbit/compat/m1brew/softfloat3.patch create mode 100755 pkg/urbit/compat/m1brew/use-static-libs.sh rename pkg/urbit/compat/{mingw => }/poor-mans-nix-shell.sh (74%) diff --git a/nix/sources-mingw.json b/nix/sources-pmnsh.json similarity index 87% rename from nix/sources-mingw.json rename to nix/sources-pmnsh.json index e3923bfdff..cd3e8a30fc 100644 --- a/nix/sources-mingw.json +++ b/nix/sources-pmnsh.json @@ -3,7 +3,7 @@ "branch": "master", "description": "A command line tool and library for transferring data with URL syntax", "homepage": "http://curl.se/", - "mingw": { + "pmnsh": { "include": "include", "lib": "lib/.libs", "prepare": "autoreconf -vfi && ./configure --disable-shared --disable-ldap --disable-rtsp --without-brotli --without-libidn2 --without-libpsl --without-nghttp2 --with-openssl", @@ -20,7 +20,7 @@ "branch": "mdb.master", "description": "LMDB library", "homepage": "http://www.lmdb.tech/", - "mingw": { + "pmnsh": { "strip": 2, "make": "liblmdb.a" }, @@ -35,7 +35,7 @@ "branch": "master", "description": "Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1.", "homepage": null, - "mingw": { + "pmnsh": { "include": "include", "lib": ".libs", "prepare": "./autogen.sh && ./configure --disable-shared --enable-module-recovery CFLAGS=-DSECP256K1_API=", @@ -52,11 +52,14 @@ "branch": "v1.x", "description": "Cross-platform asynchronous I/O", "homepage": "http://libuv.org/", - "mingw": { + "pmnsh": { "include": "include", "lib": ".libs", "prepare": "./autogen.sh && ./configure --disable-shared", - "make": "libuv.la" + "make": "libuv.la", + "compat": { + "m1brew": false + } }, "owner": "libuv", "repo": "libuv", @@ -66,17 +69,22 @@ "url_template": "https://github.com///archive/.tar.gz" }, "ent": { - "mingw": { + "pmnsh": { "prepare": "./configure" } }, "ge-additions": { - "mingw": { + "pmnsh": { "make": "CFLAGS=-I../ed25519" } }, "libaes_siv": { - "mingw": { + "pmnsh": { + "compat": { + "m1brew": { + "make": "CFLAGS=$(pkg-config --cflags openssl)" + } + } } } } diff --git a/nix/sources.json b/nix/sources.json index 4a4b8470d2..adb772dc2f 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -3,7 +3,7 @@ "branch": "master", "description": "With argon2u. Based off https://github.com/P-H-C/phc-winner-argon2", "homepage": "", - "mingw": { + "pmnsh": { "include": ["include", "src/blake2"], "make": "libargon2.a" }, @@ -19,7 +19,7 @@ "branch": "master", "description": "Submodule included by Urbit", "homepage": null, - "mingw": { + "pmnsh": { "strip": 1, "make": "all" }, @@ -35,10 +35,15 @@ "branch": "master", "description": "H2O - the optimized HTTP/1, HTTP/2, HTTP/3 server", "homepage": "https://h2o.examp1e.net", - "mingw": { + "pmnsh": { "include": "include", - "prepare": "cmake -G\"MSYS Makefiles\" .", - "make": "libh2o" + "prepare": "cmake .", + "make": "libh2o", + "compat": { + "mingw": { + "prepare": "cmake -G\"MSYS Makefiles\" ." + } + } }, "owner": "h2o", "repo": "h2o", @@ -76,7 +81,7 @@ "branch": "master", "description": null, "homepage": null, - "mingw": { + "pmnsh": { "make": "libscrypt.a CFLAGS_EXTRA=-ffast-math" }, "owner": "urbit", @@ -91,7 +96,7 @@ "branch": "master", "description": null, "homepage": null, - "mingw": { + "pmnsh": { "make": "static" }, "owner": "urbit", @@ -130,10 +135,18 @@ "branch": "master", "description": null, "homepage": null, - "mingw": { + "pmnsh": { "include": "source/include", - "lib": "build/Win64-MinGW-w64", - "make": "-C build/Win64-MinGW-w64 libsoftfloat3.a" + "compat": { + "m1brew": { + "lib": "build/template-FAST_INT64", + "make": "-C build/template-FAST_INT64 libsoftfloat3.a" + }, + "mingw": { + "lib": "build/Win64-MinGW-w64", + "make": "-C build/Win64-MinGW-w64 libsoftfloat3.a" + } + } }, "owner": "urbit", "repo": "berkeley-softfloat-3", diff --git a/pkg/urbit/compat/m1brew/compat.mk b/pkg/urbit/compat/m1brew/compat.mk new file mode 100644 index 0000000000..1c9733a036 --- /dev/null +++ b/pkg/urbit/compat/m1brew/compat.mk @@ -0,0 +1,7 @@ +# paths to brew packages +CFLAGS := $(CFLAGS) -I/opt/homebrew/include +LDFLAGS := $(LDFLAGS) -L/opt/homebrew/lib +# force linker to use static libraries +LDFLAGS := $(shell compat/m1brew/use-static-libs.sh $(LDFLAGS)) +# add extra osx libraries +LDFLAGS := $(LDFLAGS) -framework SystemConfiguration diff --git a/pkg/urbit/compat/m1brew/ed25519.patch b/pkg/urbit/compat/m1brew/ed25519.patch new file mode 100644 index 0000000000..1dfa957f1d --- /dev/null +++ b/pkg/urbit/compat/m1brew/ed25519.patch @@ -0,0 +1,13 @@ +diff --git a/Makefile b/Makefile +new file mode 100644 +--- /dev/null ++++ b/Makefile +@@ -0,0 +1,8 @@ ++.PHONY: all clean ++ ++all: *.c *.h ++ $(CC) -c -O3 -Wall -Werror *.c ++ $(AR) rcs libed25519.a *.o ++ ++clean: ++ rm -f *.o *.a diff --git a/pkg/urbit/compat/m1brew/ent.patch b/pkg/urbit/compat/m1brew/ent.patch new file mode 100644 index 0000000000..f26edf7a83 --- /dev/null +++ b/pkg/urbit/compat/m1brew/ent.patch @@ -0,0 +1,12 @@ +diff --git a/configure b/configure +--- a/configure ++++ b/configure +@@ -15,7 +15,7 @@ do + log "Trying IMPL=$IMPL" + + if IMPL=$impl make >/dev/null 2>/dev/null +- then sed -i 's|$(error IMPL must be set)|IMPL='"$impl"'|' Makefile ++ then sed -i "" 's|$(error IMPL must be set)|IMPL='"$impl"'|' Makefile + log "IMPL=$IMPL works" + exit 0 + else log "IMPL=$IMPL failed" diff --git a/pkg/urbit/compat/m1brew/libscrypt.patch b/pkg/urbit/compat/m1brew/libscrypt.patch new file mode 100644 index 0000000000..03e88629e8 --- /dev/null +++ b/pkg/urbit/compat/m1brew/libscrypt.patch @@ -0,0 +1,17 @@ +diff --git a/Makefile b/Makefile +index 783c537..3156ee2 100644 +--- a/Makefile ++++ b/Makefile +@@ -12,10 +12,9 @@ LDFLAGS_EXTRA?=-Wl,-z,relro + + all: reference + +-OBJS= crypto_scrypt-nosse.o sha256.o crypto-mcf.o b64.o crypto-scrypt-saltgen.o crypto_scrypt-check.o crypto_scrypt-hash.o slowequals.o ++OBJS= crypto_scrypt-nosse.o sha256.o crypto-mcf.o b64.o slowequals.o + +-libscrypt.so.0: $(OBJS) +- $(CC) $(LDFLAGS) -shared -o libscrypt.so.0 $(OBJS) -lm -lc ++libscrypt.a: $(OBJS) + ar rcs libscrypt.a $(OBJS) + + reference: libscrypt.so.0 main.o crypto_scrypt-hexconvert.o diff --git a/pkg/urbit/compat/m1brew/murmur3.patch b/pkg/urbit/compat/m1brew/murmur3.patch new file mode 100644 index 0000000000..cd83e9fdda --- /dev/null +++ b/pkg/urbit/compat/m1brew/murmur3.patch @@ -0,0 +1,13 @@ +diff --git a/makefile b/makefile +--- a/makefile ++++ b/makefile +@@ -12,5 +12,9 @@ shared: murmur3.c murmur3.h + $(CC) -fPIC -O3 -c murmur3.c + $(CC) -shared -Wl,--export-dynamic murmur3.o -o libmurmur3.so + ++static: murmur3.c murmur3.h ++ $(CC) -fPIC -O3 -c murmur3.c ++ $(AR) rcs libmurmur3.a murmur3.o ++ + clean: + rm -rf example *.o *.so diff --git a/pkg/urbit/compat/m1brew/softfloat3.patch b/pkg/urbit/compat/m1brew/softfloat3.patch new file mode 100644 index 0000000000..888d566f49 --- /dev/null +++ b/pkg/urbit/compat/m1brew/softfloat3.patch @@ -0,0 +1,85 @@ +diff --git a/build/template-FAST_INT64/Makefile b/build/template-FAST_INT64/Makefile +--- a/build/template-FAST_INT64/Makefile ++++ b/build/template-FAST_INT64/Makefile +@@ -34,28 +34,27 @@ + # + #============================================================================= + +-# Edit lines marked with `==>'. See "SoftFloat-source.html". ++SOURCE_DIR ?= ../../source ++SPECIALIZE_TYPE ?= 8086-SSE + +-==> SOURCE_DIR ?= ../../source +-==> SPECIALIZE_TYPE ?= 8086 ++SOFTFLOAT_OPTS ?= \ ++ -DSOFTFLOAT_ROUND_ODD -DINLINE_LEVEL=5 -DSOFTFLOAT_FAST_DIV32TO16 \ ++ -DSOFTFLOAT_FAST_DIV64TO32 + +-==> SOFTFLOAT_OPTS ?= \ +-==> -DSOFTFLOAT_ROUND_ODD -DINLINE_LEVEL=5 -DSOFTFLOAT_FAST_DIV32TO16 \ +-==> -DSOFTFLOAT_FAST_DIV64TO32 ++DELETE = rm -f ++C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include ++COMPILE_C = \ ++ cc -c -DSOFTFLOAT_FAST_INT64 $(SOFTFLOAT_OPTS) $(C_INCLUDES) -O2 -o $@ ++MAKELIB = ar crs $@ ++LIBNAME = libsoftfloat3 + +-==> DELETE = rm -f +-==> C_INCLUDES = -I. -I$(SOURCE_DIR)/$(SPECIALIZE_TYPE) -I$(SOURCE_DIR)/include +-==> COMPILE_C = \ +-==> cc -c -DSOFTFLOAT_FAST_INT64 $(SOFTFLOAT_OPTS) $(C_INCLUDES) -O2 -o $@ +-==> MAKELIB = ar crs $@ ++OBJ = .o ++LIB = .a + +-==> OBJ = .o +-==> LIB = .a +- +-==> OTHER_HEADERS = ++OTHER_HEADERS = + + .PHONY: all +-all: softfloat$(LIB) ++all: $(LIBNAME)$(LIB) + + OBJS_PRIMITIVES = \ + s_eq128$(OBJ) \ +@@ -381,11 +380,11 @@ $(OBJS_PRIMITIVES) $(OBJS_OTHERS): %$(OBJ): $(SOURCE_DIR)/%.c + $(OBJS_SPECIALIZE): %$(OBJ): $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/%.c + $(COMPILE_C) $(SOURCE_DIR)/$(SPECIALIZE_TYPE)/$*.c + +-softfloat$(LIB): $(OBJS_ALL) ++$(LIBNAME)$(LIB): $(OBJS_ALL) + $(DELETE) $@ + $(MAKELIB) $^ + + .PHONY: clean + clean: +- $(DELETE) $(OBJS_ALL) softfloat$(LIB) ++ $(DELETE) $(OBJS_ALL) $(LIBNAME)$(LIB) + +diff --git a/build/template-FAST_INT64/platform.h b/build/template-FAST_INT64/platform.h +--- a/build/template-FAST_INT64/platform.h ++++ b/build/template-FAST_INT64/platform.h +@@ -34,17 +34,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + =============================================================================*/ + +-// Edit lines marked with `==>'. See "SoftFloat-source.html". +- + /*---------------------------------------------------------------------------- + *----------------------------------------------------------------------------*/ +-==> #define LITTLEENDIAN 1 ++#define LITTLEENDIAN 1 + + /*---------------------------------------------------------------------------- + *----------------------------------------------------------------------------*/ +-==> #define INLINE inline ++#define INLINE inline + + /*---------------------------------------------------------------------------- + *----------------------------------------------------------------------------*/ +-==> #define THREAD_LOCAL _Thread_local ++#define THREAD_LOCAL _Thread_local + diff --git a/pkg/urbit/compat/m1brew/use-static-libs.sh b/pkg/urbit/compat/m1brew/use-static-libs.sh new file mode 100755 index 0000000000..7f7cf83a1c --- /dev/null +++ b/pkg/urbit/compat/m1brew/use-static-libs.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail +declare -a ldirs +for i in $@ +do + case $i in + -L*) ldirs+=(${i:2});; + esac +done +for i in $@ +do + case $i in + -l*) + lib=$(find ${ldirs[@]} -name lib${i:2}.a) + if [ "$lib" != "" ] + then + echo $lib + else + echo $i + fi;; + *) echo $i;; + esac +done diff --git a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh b/pkg/urbit/compat/poor-mans-nix-shell.sh similarity index 74% rename from pkg/urbit/compat/mingw/poor-mans-nix-shell.sh rename to pkg/urbit/compat/poor-mans-nix-shell.sh index 16e40d1fc8..ba8c5903bc 100644 --- a/pkg/urbit/compat/mingw/poor-mans-nix-shell.sh +++ b/pkg/urbit/compat/poor-mans-nix-shell.sh @@ -1,12 +1,10 @@ -# ensure required mingw packages are installed -mpkgs=(cmake curl gcc jq libuv make wslay) -pacman -S --needed autoconf automake-wrapper libtool patch ${mpkgs[@]/#/mingw-w64-x86_64-} - declare -a cdirs declare -a ldirs declare -A hdeps -sources=(../../nix/sources.json ../../nix/sources-mingw.json) -deriver=urbit-mingw-build +sources=(../../nix/sources.json ../../nix/sources-pmnsh.json) +patches=compat/$1 +deriver=urbit-$1-build +markfil=.$1~ depdirs= nixpath=${NIX_STORE-../build} @@ -50,7 +48,7 @@ buildnixdep () { then hash=${hdeps[$key]} dir=$nixpath/$hash-$key - if [ -e $dir/.mingw~ ] + if [ -e $dir/$markfil ] then # dependency present, don't reupload hash= @@ -79,14 +77,14 @@ buildnixdep () { fi # patch and build the dependency if necessary - if [ ! -e $dir/.mingw~ ] + if [ ! -e $dir/$markfil ] then - local patch=compat/mingw/$key.patch + local patch=$patches/$key.patch [ -e $patch ] && patch -d $dir -p 1 <$patch pushd $dir eval "$cmdprep" eval make "$cmdmake" - touch .mingw~ + touch $markfil popd fi @@ -118,30 +116,30 @@ EOF # I have to go over the sources files several times # because jq does not have a way to invoke external programs +jqprep='add|to_entries|.[]|.value.pmnsh as $p|select($p and $p.compat.'$1' != false)|(($p + $p.compat.'$1')|del(.compat)) as $o|' # list external dependencies, create hash map and directory replacement regex # use -j and \u0000 to work around https://github.com/stedolan/jq/issues/1870 while read -rd "" key json do # create 'store hash' from sources.json data and patch - patch=compat/mingw/$key.patch + patch=$patches/$key.patch read hash _ < <(( echo -n $json [ -e $patch ] && cat $patch)|sha256sum) hash=$(hex2nixbase32 $hash) hdeps[$key]=$hash - # NB: this path substitution works only in local dependencies - depdirs="$depdirs|gsub(\"\\\\.\\\\./$key\";\"$nixpath/$hash-$key\")" -done < <(jq --arg deriver "$deriver" -Sscrj 'add|to_entries|.[]|select(.value.mingw)|select(.value.url)|.key," ",{($deriver):(.value)},"\u0000"' ${sources[@]}) + depdirs="$depdirs|gsub(\"\\\\.\\\\./$key\";\"\\(\$d)/$hash-$key\")" +done < <(jq --arg deriver "$deriver" -Sscrj "$jqprep"'select(.value.url)|.key," ",{($deriver):(.value|del(.pmnsh) + ({'$1':$o}))},"\u0000"' ${sources[@]}) # build dependencies, create include and library directory arrays -. <(jq -sr 'add|to_entries|.[]|select(.value.mingw)|" +. <(jq --arg nixpath "$nixpath" -sr "$jqprep"'(if .value.url then ".." else $nixpath end) as $d|" unset dir key=\(.key|@sh) \\ url=\(.value.url//""|@sh) \\ -strip=\(.value.mingw.strip+1) \\ -cmdprep=\(.value.mingw.prepare//""'"$depdirs"'|@sh) \\ -cmdmake=\(.value.mingw.make//""'"$depdirs"'|@sh) \\ +strip=\($o.strip+1) \\ +cmdprep=\($o.prepare//""'"$depdirs"'|@sh) \\ +cmdmake=\($o.make//""'"$depdirs"'|@sh) \\ buildnixdep # sets dir -\(.value.mingw.include//"."|if type == "array" then . else [.] end|map("cdirs+=(-I$dir/\(.))")|join("\n")) -\(.value.mingw.lib//"."|if type == "array" then . else [.] end|map("ldirs+=(-L$dir/\(.))")|join("\n"))"' ${sources[@]}) +\($o.include//"."|if type == "array" then . else [.] end|map("cdirs+=(-I$dir/\(.))")|join("\n")) +\($o.lib//"."|if type == "array" then . else [.] end|map("ldirs+=(-L$dir/\(.))")|join("\n"))"' ${sources[@]}) diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 3789b397dc..5646eb4379 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -19,7 +19,7 @@ defmacro () { echo "#define $1 $2" >>include/config.h } -xxd () { +poor_mans_xxd () { cch=0 echo "unsigned char $2[] = {" while IFS='' read line @@ -93,14 +93,18 @@ esac # TODO Determine if the target cpu is little or big endian. case $(tr A-Z a-z <<< $os) in *mingw*) + # ensure required mingw packages are installed + mpkgs=(cmake curl gcc jq make) + pacman -S --needed autoconf automake-wrapper libtool patch ${mpkgs[@]/#/mingw-w64-x86_64-} + # set up dependencies and create cdirs and ldirs arrays - . compat/mingw/poor-mans-nix-shell.sh + . compat/poor-mans-nix-shell.sh mingw # support running off a tarball that doesn't contain binary pills (( $(stat -c %s ../../bin/ivory.pill) > 512 )) || curl -L https://github.com/urbit/urbit/raw/urbit-v$URBIT_VERSION/bin/ivory.pill > ../../bin/ivory.pill - [ -e include/ca-bundle.h ] || xxd /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt include_ca_bundle_crt >include/ca-bundle.h - [ -e include/ivory.h ] || xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h + [ -e include/ca-bundle.h ] || poor_mans_xxd /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt include_ca_bundle_crt >include/ca-bundle.h + [ -e include/ivory.h ] || poor_mans_xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h defmacro U3_OS_mingw 1 @@ -115,7 +119,45 @@ case $(tr A-Z a-z <<< $os) in compat="${compat-} mingw" CFLAGS="${CFLAGS-} ${cdirs[@]}" LDFLAGS="${LDFLAGS-} ${ldirs[@]} -static" - PKG_CONFIG=">/dev/null echo" + PKG_CONFIG=false + ;; + m1brew) + # ensure required packages are installed + brew install -q autoconf automake bash cmake coreutils gmp jq libsigsegv libtool libuv openssl pkgconfig + + if (( ${BASH_VERSION%%.*} < 5 )) + then + echo Running bash version $BASH_VERSION is too low, please restart bash to use freshly installed one + exit 1 + fi + + # for some reason pkg-config does not pick up openssl + export PKG_CONFIG_PATH="$(brew --prefix openssl)/lib/pkgconfig:${PKG_CONFIG_PATH-}" + + # set up dependencies and create cdirs and ldirs arrays + . compat/poor-mans-nix-shell.sh m1brew + + # support running off a tarball that doesn't contain binary pills + (( $(stat -f %z ../../bin/ivory.pill) > 512 )) || curl -L https://github.com/urbit/urbit/raw/urbit-v$URBIT_VERSION/bin/ivory.pill > ../../bin/ivory.pill + + [ -e include/ca-bundle.h ] || poor_mans_xxd /etc/ssl/cert.pem include_ca_bundle_crt >include/ca-bundle.h + [ -e include/ivory.h ] || poor_mans_xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h + + defmacro U3_OS_osx 1 + + deps="${deps/ssl/openssl}" + deps="${deps/uv/libuv}" + if ((opt_debug)) + then + CFLAGS="${CFLAGS-} -O0 -g" + else + # clang hangs on noun/allocate.c if -g is specified + CFLAGS="${CFLAGS-} -O3" + fi + + compat="${compat-} posix m1brew" + CFLAGS="${CFLAGS-} ${cdirs[@]}" + LDFLAGS="${LDFLAGS-} ${ldirs[@]}" ;; *linux*) defmacro U3_OS_linux 1 @@ -143,8 +185,9 @@ case $(tr A-Z a-z <<< $os) in esac for dep in ${osdeps-} $deps -do LDFLAGS="${LDFLAGS-} -l$dep" - ${PKG_CONFIG-pkg-config} --cflags --libs $dep 2>/dev/null || true +do + LDFLAGS="${LDFLAGS-} $(${PKG_CONFIG-pkg-config} --libs $dep 2>/dev/null || echo -l$dep)" + CFLAGS="${CFLAGS-} $(${PKG_CONFIG-pkg-config} --cflags $dep 2>/dev/null || true)" done for header in $headers diff --git a/pkg/urbit/include/c/portable.h b/pkg/urbit/include/c/portable.h index 16921ec681..4204769bf5 100644 --- a/pkg/urbit/include/c/portable.h +++ b/pkg/urbit/include/c/portable.h @@ -116,6 +116,9 @@ # define U3_OS_LoomBase 0x36000000 # endif # define U3_OS_LoomBits 29 // ie, 2^29 words == 2GB +# elif defined(U3_OS_osx) && defined(U3_CPU_aarch64) || defined(U3_OS_mingw) +# define U3_OS_LoomBase 0x28000000000 +# define U3_OS_LoomBits 29 // ie, 2^29 words == 2GB # elif defined(U3_OS_osx) || defined(U3_OS_bsd) # ifdef __LP64__ # define U3_OS_LoomBase 0x200000000 @@ -123,9 +126,6 @@ # define U3_OS_LoomBase 0x4000000 # endif # define U3_OS_LoomBits 29 // ie, 2^29 words == 2GB -# elif defined(U3_OS_mingw) -# define U3_OS_LoomBase 0x28000000000 -# define U3_OS_LoomBits 29 // ie, 2^29 words == 2GB # else # error "port: LoomBase" # endif From 5b4b6c6f1418f761844e20ad086d51e57e44f30f Mon Sep 17 00:00:00 2001 From: ~locpyl-tidnyd Date: Tue, 6 Jul 2021 16:25:23 +0300 Subject: [PATCH 136/136] compat: minor refactoring --- pkg/urbit/compat/create-include-files.sh | 22 ++++++++++ pkg/urbit/compat/m1brew/compat.mk | 7 +++ pkg/urbit/compat/mingw/compat.mk | 8 +++- pkg/urbit/compat/poor-mans-nix-shell.sh | 3 ++ pkg/urbit/configure | 56 +++--------------------- 5 files changed, 44 insertions(+), 52 deletions(-) create mode 100755 pkg/urbit/compat/create-include-files.sh diff --git a/pkg/urbit/compat/create-include-files.sh b/pkg/urbit/compat/create-include-files.sh new file mode 100755 index 0000000000..ff45ef7483 --- /dev/null +++ b/pkg/urbit/compat/create-include-files.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# support running off a tarball that doesn't contain binary pills +(( $($1 ../../bin/ivory.pill) > 512 )) || curl -L https://github.com/urbit/urbit/raw/urbit-v$URBIT_VERSION/bin/ivory.pill > ../../bin/ivory.pill + +poor_mans_xxd () { + cch=0 + echo "unsigned char $2[] = {" + while IFS='' read line + do + for i in $line + do + echo -n " 0x$i," + cch=$((cch+1)) + done + echo + done < <(od -An -v -tx1 $1) + echo "};" + echo "unsigned int $2_len = $cch;" +} + +[ -e include/ca-bundle.h ] || poor_mans_xxd $2 include_ca_bundle_crt >include/ca-bundle.h +[ -e include/ivory.h ] || poor_mans_xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h diff --git a/pkg/urbit/compat/m1brew/compat.mk b/pkg/urbit/compat/m1brew/compat.mk index 1c9733a036..5d612ba841 100644 --- a/pkg/urbit/compat/m1brew/compat.mk +++ b/pkg/urbit/compat/m1brew/compat.mk @@ -5,3 +5,10 @@ LDFLAGS := $(LDFLAGS) -L/opt/homebrew/lib LDFLAGS := $(shell compat/m1brew/use-static-libs.sh $(LDFLAGS)) # add extra osx libraries LDFLAGS := $(LDFLAGS) -framework SystemConfiguration + +ifdef debug +CFLAGS := $(CFLAGS) -O0 -g +else +# clang hangs on noun/allocate.c if -g is specified with -O3 +CFLAGS := $(CFLAGS) -O3 +endif diff --git a/pkg/urbit/compat/mingw/compat.mk b/pkg/urbit/compat/mingw/compat.mk index 0bc9eaa87f..f4254c188d 100644 --- a/pkg/urbit/compat/mingw/compat.mk +++ b/pkg/urbit/compat/mingw/compat.mk @@ -1,5 +1,5 @@ # increase default thread stack size and link Windows implibs -LDFLAGS := $(LDFLAGS) -Wl,--stack,67108864 -lbcrypt -lntdll -lws2_32 +LDFLAGS := $(LDFLAGS) -static -Wl,--stack,67108864 -lbcrypt -lntdll -lws2_32 # libcurl CFLAGS := $(CFLAGS) -DCURL_STATICLIB LDFLAGS := $(LDFLAGS) -lzstd -lcrypt32 @@ -7,3 +7,9 @@ LDFLAGS := $(LDFLAGS) -lzstd -lcrypt32 CFLAGS := $(CFLAGS) -DH2O_NO_UNIX_SOCKETS # libuv LDFLAGS := $(LDFLAGS) -luserenv -liphlpapi -lpsapi + +ifdef debug +CFLAGS := $(CFLAGS) -O0 -g +else +CFLAGS := $(CFLAGS) -O3 -g +endif diff --git a/pkg/urbit/compat/poor-mans-nix-shell.sh b/pkg/urbit/compat/poor-mans-nix-shell.sh index ba8c5903bc..45427e0285 100644 --- a/pkg/urbit/compat/poor-mans-nix-shell.sh +++ b/pkg/urbit/compat/poor-mans-nix-shell.sh @@ -143,3 +143,6 @@ cmdmake=\($o.make//""'"$depdirs"'|@sh) \\ buildnixdep # sets dir \($o.include//"."|if type == "array" then . else [.] end|map("cdirs+=(-I$dir/\(.))")|join("\n")) \($o.lib//"."|if type == "array" then . else [.] end|map("ldirs+=(-L$dir/\(.))")|join("\n"))"' ${sources[@]}) + +CFLAGS="${CFLAGS-} ${cdirs[@]}" +LDFLAGS="${LDFLAGS-} ${ldirs[@]}" diff --git a/pkg/urbit/configure b/pkg/urbit/configure index 5646eb4379..949b2847bd 100755 --- a/pkg/urbit/configure +++ b/pkg/urbit/configure @@ -19,22 +19,6 @@ defmacro () { echo "#define $1 $2" >>include/config.h } -poor_mans_xxd () { - cch=0 - echo "unsigned char $2[] = {" - while IFS='' read line - do - for i in $line - do - echo -n " 0x$i," - cch=$((cch+1)) - done - echo - done < <(od -An -v -tx1 $1) - echo "};" - echo "unsigned int $2_len = $cch;" -} - defmacro URBIT_VERSION "\"$URBIT_VERSION\"" opt_debug= @@ -46,7 +30,7 @@ do opt_debug=1 ;; --disable-debug) - opt_debug=0 + opt_debug= ;; *) echo "unrecognized option: $1" @@ -97,28 +81,13 @@ case $(tr A-Z a-z <<< $os) in mpkgs=(cmake curl gcc jq make) pacman -S --needed autoconf automake-wrapper libtool patch ${mpkgs[@]/#/mingw-w64-x86_64-} - # set up dependencies and create cdirs and ldirs arrays . compat/poor-mans-nix-shell.sh mingw - - # support running off a tarball that doesn't contain binary pills - (( $(stat -c %s ../../bin/ivory.pill) > 512 )) || curl -L https://github.com/urbit/urbit/raw/urbit-v$URBIT_VERSION/bin/ivory.pill > ../../bin/ivory.pill - - [ -e include/ca-bundle.h ] || poor_mans_xxd /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt include_ca_bundle_crt >include/ca-bundle.h - [ -e include/ivory.h ] || poor_mans_xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h + compat/create-include-files.sh 'stat -c %s' /etc/pki/ca-trust/extracted/openssl/ca-bundle.trust.crt defmacro U3_OS_mingw 1 - if ((opt_debug)) - then - CFLAGS="${CFLAGS-} -O0 -g" - else - CFLAGS="${CFLAGS-} -O3 -g" - fi - deps="${deps/sigsegv}" compat="${compat-} mingw" - CFLAGS="${CFLAGS-} ${cdirs[@]}" - LDFLAGS="${LDFLAGS-} ${ldirs[@]} -static" PKG_CONFIG=false ;; m1brew) @@ -134,30 +103,14 @@ case $(tr A-Z a-z <<< $os) in # for some reason pkg-config does not pick up openssl export PKG_CONFIG_PATH="$(brew --prefix openssl)/lib/pkgconfig:${PKG_CONFIG_PATH-}" - # set up dependencies and create cdirs and ldirs arrays . compat/poor-mans-nix-shell.sh m1brew - - # support running off a tarball that doesn't contain binary pills - (( $(stat -f %z ../../bin/ivory.pill) > 512 )) || curl -L https://github.com/urbit/urbit/raw/urbit-v$URBIT_VERSION/bin/ivory.pill > ../../bin/ivory.pill - - [ -e include/ca-bundle.h ] || poor_mans_xxd /etc/ssl/cert.pem include_ca_bundle_crt >include/ca-bundle.h - [ -e include/ivory.h ] || poor_mans_xxd ../../bin/ivory.pill u3_Ivory_pill >include/ivory.h + compat/create-include-files.sh 'stat -f %z' /etc/ssl/cert.pem defmacro U3_OS_osx 1 deps="${deps/ssl/openssl}" deps="${deps/uv/libuv}" - if ((opt_debug)) - then - CFLAGS="${CFLAGS-} -O0 -g" - else - # clang hangs on noun/allocate.c if -g is specified - CFLAGS="${CFLAGS-} -O3" - fi - compat="${compat-} posix m1brew" - CFLAGS="${CFLAGS-} ${cdirs[@]}" - LDFLAGS="${LDFLAGS-} ${ldirs[@]}" ;; *linux*) defmacro U3_OS_linux 1 @@ -201,10 +154,11 @@ do done cat >config.mk <&2