diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 4832a4d7d7..c40cfa27fe 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -9,7 +9,7 @@ on: - '.github/workflows/develop.yml' - '.github/workflows/release.yml' - '.github/workflows/master.yml' - - '.github/workflows/vere.yml' + - '.github/workflows/shared.yml' - 'pkg/arvo/**' - 'pkg/docker-image/**' - 'pkg/ent/**' @@ -24,7 +24,7 @@ on: jobs: call-vere: - uses: ./.github/workflows/vere.yml + uses: ./.github/workflows/shared.yml with: pace: 'edge' upload: >- diff --git a/.github/workflows/feature.yml b/.github/workflows/feature.yml index 01ca5af921..cdbc023dfd 100644 --- a/.github/workflows/feature.yml +++ b/.github/workflows/feature.yml @@ -7,7 +7,7 @@ on: - '.github/workflows/develop.yml' - '.github/workflows/release.yml' - '.github/workflows/master.yml' - - '.github/workflows/vere.yml' + - '.github/workflows/shared.yml' - 'pkg/arvo/**' - 'pkg/docker-image/**' - 'pkg/ent/**' @@ -22,7 +22,7 @@ on: jobs: call-vere: - uses: ./.github/workflows/vere.yml + uses: ./.github/workflows/shared.yml with: pace: 'edge' upload: >- diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index b04cd042e3..7b1022779f 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -9,7 +9,7 @@ on: - '.github/workflows/develop.yml' - '.github/workflows/release.yml' - '.github/workflows/master.yml' - - '.github/workflows/vere.yml' + - '.github/workflows/shared.yml' - 'pkg/arvo/**' - 'pkg/docker-image/**' - 'pkg/ent/**' @@ -24,7 +24,7 @@ on: jobs: call-vere: - uses: ./.github/workflows/vere.yml + uses: ./.github/workflows/shared.yml with: pace: 'live' upload: >- diff --git a/.github/workflows/next.yml b/.github/workflows/next.yml new file mode 100644 index 0000000000..0601faa0b3 --- /dev/null +++ b/.github/workflows/next.yml @@ -0,0 +1,31 @@ +name: Push to next/kelvin/* + +on: + push: + branches: + - 'next/kelvin/*' + paths: + - '.github/workflows/feature.yml' + - '.github/workflows/develop.yml' + - '.github/workflows/release.yml' + - '.github/workflows/master.yml' + - '.github/workflows/shared.yml' + - 'pkg/arvo/**' + - 'pkg/docker-image/**' + - 'pkg/ent/**' + - 'pkg/ge-additions/**' + - 'pkg/libaes_siv/**' + - 'pkg/urbit/**' + - 'pkg/urcrypt/**' + - 'tests/**' + - 'bin/**' + - 'nix/**' + - 'default.nix' + +jobs: + call-vere: + uses: ./.github/workflows/shared.yml + with: + upload: true + next: ${{ github.ref_name }} + secrets: inherit diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d2258a1f1..f863e79749 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ on: - '.github/workflows/develop.yml' - '.github/workflows/release.yml' - '.github/workflows/master.yml' - - '.github/workflows/vere.yml' + - '.github/workflows/shared.yml' - 'pkg/arvo/**' - 'pkg/docker-image/**' - 'pkg/ent/**' @@ -24,7 +24,7 @@ on: jobs: call-vere: - uses: ./.github/workflows/vere.yml + uses: ./.github/workflows/shared.yml with: pace: 'soon' upload: >- diff --git a/.github/workflows/vere.yml b/.github/workflows/shared.yml similarity index 79% rename from .github/workflows/vere.yml rename to .github/workflows/shared.yml index c24155c185..bb85467be7 100644 --- a/.github/workflows/vere.yml +++ b/.github/workflows/shared.yml @@ -14,6 +14,11 @@ on: type: string default: 'edge' required: false + next: + description: 'next kelvin version branch name' + type: string + default: null + required: false secrets: CACHIX_AUTH_TOKEN: required: false @@ -73,9 +78,15 @@ jobs: name: run urbit-tests run: | cp -RL tests pkg/arvo/tests - vere=$(curl https://bootstrap.urbit.org/vere/${{ inputs.pace }}/last) + if ${{ inputs.next != null }}; then + next=$(echo ${{ inputs.next }} | sed 's/[^0-9]//g') + base="https://bootstrap.urbit.org/vere/next/kelvin/${next}" + else + base="https://bootstrap.urbit.org/vere/${{ inputs.pace }}" + fi + vere=$(curl ${base}/last) + url="$(echo ${base}/v${vere}/vere-v${vere}-linux-x86_64)" echo $vere - url="$(echo https://bootstrap.urbit.org/vere/${{ inputs.pace }}/v${vere}/vere-v${vere}-linux-x86_64)" echo $url # put in .jam so it doesn't crash when it gets -A'd in curl -Lo pkg/arvo/vere.jam "$url" diff --git a/pkg/arvo/app/aqua.hoon b/pkg/arvo/app/aqua.hoon index a41f45ba33..de53ae0dc4 100644 --- a/pkg/arvo/app/aqua.hoon +++ b/pkg/arvo/app/aqua.hoon @@ -485,11 +485,10 @@ =/ =desk +.val =. userspace-ova.pil :: take all files from a userspace desk - =/ all-dirs=(list path) ~[/] :_ ~ %- unix-event:pill-lib - %+ %*(. file-ovum:pill-lib directories all-dirs) - desk /(scot %p our.hid)/[desk]/(scot %da now.hid) + %- file-ovum:pill-lib + [desk /(scot %p our.hid)/[desk]/(scot %da now.hid) ~] =^ ms state (poke-pill pil) (emit-cards ms) :: diff --git a/pkg/arvo/gen/brass.hoon b/pkg/arvo/gen/brass.hoon index a80cd5e903..099c9d5213 100644 --- a/pkg/arvo/gen/brass.hoon +++ b/pkg/arvo/gen/brass.hoon @@ -25,6 +25,7 @@ == :: prime=_| + exc=(list spur) == :- %pill ^- pill:pill @@ -45,4 +46,4 @@ |= =desk [desk /(scot %p p.bec)/[desk]/(scot %da now)] :: -(brass:pill sys dez prime) +(brass:pill sys dez prime exc) diff --git a/pkg/arvo/gen/hood/ames-snub.hoon b/pkg/arvo/gen/hood/ames-snub.hoon index 575e29c9d9..646f286d00 100644 --- a/pkg/arvo/gen/hood/ames-snub.hoon +++ b/pkg/arvo/gen/hood/ames-snub.hoon @@ -5,6 +5,6 @@ :::: :: :- %say -|= [^ ships=(list ship) ~] +|= [^ [form=?(%allow %deny) ships=(list ship)] ~] :- %helm-ames-snub -ships +[form ships] diff --git a/pkg/arvo/gen/solid.hoon b/pkg/arvo/gen/solid.hoon index f98d949480..1b3b6e0e27 100644 --- a/pkg/arvo/gen/solid.hoon +++ b/pkg/arvo/gen/solid.hoon @@ -29,6 +29,7 @@ :: dub=_| prime=_| + exc=(list spur) == :- %pill ^- pill:pill @@ -46,4 +47,4 @@ |= =desk [desk /(scot %p p.bec)/[desk]/(scot %da now)] :: -(solid:pill sys dez dub now prime) +(solid:pill sys dez dub now prime exc) diff --git a/pkg/arvo/lib/hood/helm.hoon b/pkg/arvo/lib/hood/helm.hoon index f40f18741c..399a17f7ff 100644 --- a/pkg/arvo/lib/hood/helm.hoon +++ b/pkg/arvo/lib/hood/helm.hoon @@ -202,8 +202,8 @@ abet:(emit %pass /helm/prod %arvo %a %prod ships) :: ++ poke-ames-snub - |= ships=(list ship) - abet:(emit %pass /helm/snub %arvo %a %snub ships) + |= snub=[form=?(%allow %deny) ships=(list ship)] + abet:(emit %pass /helm/snub %arvo %a %snub snub) :: ++ poke-atom |= ato=@ diff --git a/pkg/arvo/lib/hood/kiln.hoon b/pkg/arvo/lib/hood/kiln.hoon index 5f37c9b119..741245504e 100644 --- a/pkg/arvo/lib/hood/kiln.hoon +++ b/pkg/arvo/lib/hood/kiln.hoon @@ -583,15 +583,13 @@ ..abet =/ kel i.wic %- emil - =/ cards + =/ desks=(list [=desk =zest]) %+ murn ~(tap by rock) |= [=desk =zest wic=(set weft)] ?: |(=(%base desk) !?=(%live zest) (~(has in wic) kel)) ~ - `u=[%pass /kiln/bump/[desk] %arvo %c %zest desk %held] - ?~ cards - [%pass /kiln/bump/wick %arvo %c %wick ~]~ - cards + `u=[desk %held] + [%pass /kiln/bump/zeal %arvo %c %zeal desks]~ :: ++ poke-cancel |= a=@tas diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index cbb2fd4013..efa5efba3c 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -361,7 +361,7 @@ :: %init: vane boot :: %prod: re-send a packet per flow, to all peers if .ships is ~ :: %sift: limit verbosity to .ships - :: %snub: set packet blacklist to .ships + :: %snub: set packet blocklist to .ships :: %spew: set verbosity toggles :: %cong: adjust congestion control parameters :: %stir: recover from timer desync @@ -380,7 +380,7 @@ $>(%init vane-task) [%prod ships=(list ship)] [%sift ships=(list ship)] - [%snub ships=(list ship)] + [%snub form=?(%allow %deny) ships=(list ship)] [%spew veb=(list verb)] [%cong msg=@ud mem=@ud] [%stir arg=@t] @@ -522,7 +522,6 @@ :: heeds: listeners for %clog notifications :: closing: bones closed on the sender side :: corked: bones closed on both sender and receiver - :: krocs: bones that need to be sent again to the publisher :: +$ peer-state $: $: =symmetric-key @@ -540,7 +539,6 @@ heeds=(set duct) closing=(set bone) corked=(set bone) - krocs=(set bone) == :: $qos: quality of service; how is our connection to a peer doing? :: @@ -801,6 +799,7 @@ [%warp wer=ship rif=riff] :: internal file req [%werp who=ship wer=ship rif=riff-any] :: external file req [%wick ~] :: try upgrade + [%zeal lit=(list [=desk =zest])] :: batch zest [%zest des=desk liv=zest] :: live $>(%plea vane-task) :: ames request == :: @@ -1426,6 +1425,9 @@ :: start responding negatively to cors requests from origin :: [%reject-origin =origin] + :: %spew: set verbosity toggle + :: + [%spew veb=@] == :: +origin: request origin as specified in an Origin header :: diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index fcf539a446..5f09500fd3 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -612,20 +612,18 @@ :: snub: blocklist for incoming packets :: cong: parameters for marking a flow as clogged :: -:: Note: .corks is only still present for unreleased migration reasons -:: -:: +$ ames-state - $: peers=(map ship ship-state) :: TODO: remove krocs from peer-state + $: peers=(map ship ship-state) =unix=duct =life crypto-core=acru:ames =bug - corks=(set wire) :: TODO: remove next state update - snub=(set ship) + snub=[form=?(%allow %deny) ships=(set ship)] cong=[msg=@ud mem=@ud] == -+$ ames-state-4 ames-state-5 +:: ++$ azimuth-state [=symmetric-key =life =rift =public-key sponsor=ship] ++$ ames-state-4 ames-state-5 +$ ames-state-5 $: peers=(map ship ship-state-5) =unix=duct @@ -641,11 +639,7 @@ == :: +$ peer-state-5 - $: $: =symmetric-key - =life - =public-key - sponsor=ship - == + $: azimuth-state route=(unit [direct=? =lane]) =qos =ossuary @@ -655,6 +649,11 @@ heeds=(set duct) == :: ++$ bug-9 + $: veb=_[`?`%.n `?`%.n `?`%.n `?`%.n `?`%.n `?`%.n `?`%.n] + ships=(set ship) + == +:: +$ ames-state-6 $: peers=(map ship ship-state-6) =unix=duct @@ -669,12 +668,7 @@ == :: +$ peer-state-6 - $: $: =symmetric-key - =life - =rift - =public-key - sponsor=ship - == + $: azimuth-state route=(unit [direct=? =lane]) =qos =ossuary @@ -685,7 +679,7 @@ == :: +$ ames-state-7 - $: peers=(map ship ship-state) + $: peers=(map ship ship-state-7) =unix=duct =life crypto-core=acru:ames @@ -693,7 +687,7 @@ == :: +$ ames-state-8 - $: peers=(map ship ship-state) + $: peers=(map ship ship-state-7) =unix=duct =life crypto-core=acru:ames @@ -701,18 +695,8 @@ corks=(set wire) == :: -+$ bug-9 - $: veb=_[`?`%.n `?`%.n `?`%.n `?`%.n `?`%.n `?`%.n `?`%.n] - ships=(set ship) - == -:: -+$ bug-10 - $: veb=_[`?`%.n `?`%.n `?`%.n `?`%.n `?`%.n `?`%.n `?`%.n `?`%.n] - ships=(set ship) - == -:: +$ ames-state-9 - $: peers=(map ship ship-state) + $: peers=(map ship ship-state-7) =unix=duct =life crypto-core=acru:ames @@ -722,15 +706,54 @@ == :: +$ ames-state-10 - $: peers=(map ship ship-state) + $: peers=(map ship ship-state-7) =unix=duct =life crypto-core=acru:ames - bug=bug-10 + =bug corks=(set wire) snub=(set ship) == :: ++$ ship-state-7 + $% [%alien alien-agenda] + [%known peer-state-7] + == +:: ++$ peer-state-7 + $: azimuth-state + route=(unit [direct=? =lane]) + =qos + =ossuary + snd=(map bone message-pump-state) + rcv=(map bone message-sink-state) + nax=(set [=bone =message-num]) + heeds=(set duct) + closing=(set bone) + corked=(set bone) + krocs=(set bone) + == +:: ++$ ames-state-11 + $: peers=(map ship ship-state-7) + =unix=duct + =life + crypto-core=acru:ames + =bug + corks=(set wire) + snub=(set ship) + cong=[msg=@ud mem=@ud] + == +:: ++$ queued-event-11 + $% [%call =duct wrapped-task=(hobo task-11)] + [%take =wire =duct =sign] + == +:: ++$ task-11 + $% [%snub ships=(list ship)] + $<(%snub task) + == :: $bug: debug printing configuration :: :: veb: verbosity toggles @@ -857,7 +880,8 @@ [%8 ames-state-8] [%9 ames-state-9] [%10 ames-state-10] - [%11 ^ames-state] + [%11 ames-state-11] + [%12 ^ames-state] == :: |= [now=@da eny=@ rof=roof] @@ -980,7 +1004,7 @@ :: lifecycle arms; mostly pass-throughs to the contained adult ames :: ++ scry scry:adult-core - ++ stay [%11 %larva queued-events ames-state.adult-gate] + ++ stay [%12 %larva queued-events ames-state.adult-gate] ++ load |= $= old $% $: %4 @@ -1033,6 +1057,13 @@ [%adult state=ames-state-10] == == $: %11 + $% $: %larva + events=(qeu queued-event-11) + state=ames-state-11 + == + [%adult state=ames-state-11] + == == + $: %12 $% $: %larva events=(qeu queued-event) state=_ames-state.adult-gate @@ -1107,14 +1138,34 @@ =. queued-events events.old larval-gate :: - [%11 %adult *] (load:adult-core %11 state.old) + [%11 %adult *] + =. cached-state `[%11 state.old] + ~> %slog.0^leaf/"ames: larva reload" + larval-gate :: [%11 %larva *] ~> %slog.1^leaf/"ames: larva: load" - =. queued-events events.old - =. adult-gate (load:adult-core %11 state.old) + =. queued-events + :: "+rep:in on a +qeu looks strange, but works fine." + :: + %- ~(rep in events.old) + |= [e=queued-event-11 q=(qeu queued-event)] + %- ~(put to q) ^- queued-event + ?. ?=(%call -.e) e + =/ task=task-11 ((harden task-11) wrapped-task.e) + %= e + wrapped-task ?.(?=(%snub -.task) task [%snub %deny ships.task]) + == larval-gate - :: + :: + [%12 %adult *] (load:adult-core %12 state.old) + :: + [%12 %larva *] + ~> %slog.1^leaf/"ames: larva: load" + =. queued-events events.old + =. adult-gate (load:adult-core %12 state.old) + larval-gate + :: == :: +molt: re-evolve to adult-ames :: @@ -1137,7 +1188,9 @@ 10+(state-9-to-10:load:adult-core +.u.cached-state) =? u.cached-state ?=(%10 -.u.cached-state) 11+(state-10-to-11:load:adult-core +.u.cached-state) - ?> ?=(%11 -.u.cached-state) + =? u.cached-state ?=(%11 -.u.cached-state) + 12+(state-11-to-12:load:adult-core +.u.cached-state) + ?> ?=(%12 -.u.cached-state) =. ames-state.adult-gate +.u.cached-state [moz larval-core(cached-state ~)] -- @@ -1269,9 +1322,9 @@ :: +on-snub: handle request to change ship blacklist :: ++ on-snub - |= ships=(list ship) + |= [form=?(%allow %deny) ships=(list ship)] ^+ event-core - =. snub.ames-state (sy ships) + =. snub.ames-state [form (sy ships)] event-core :: +on-spew: handle request to set verbosity toggles on debug output :: @@ -1413,11 +1466,12 @@ ~/ %on-hear-packet |= [=lane =packet dud=(unit goof)] ^+ event-core - %- (ev-trace odd.veb sndr.packet |.("received packet")) + %- (ev-trace rcv.veb sndr.packet |.("received packet")) :: ?: =(our sndr.packet) event-core - ?: (~(has in snub.ames-state) sndr.packet) + ?: .= =(%deny form.snub.ames-state) + (~(has in ships.snub.ames-state) sndr.packet) %- (ev-trace rcv.veb sndr.packet |.("snubbed")) event-core :: @@ -3386,6 +3440,15 @@ ++ closing (~(has in closing.peer-state) bone) ++ corked (~(has in corked.peer-state) bone) ++ pump-core |=(=^bone (mu bone *message-pump-state)) + ++ received + |= =^bone + :: odd bone: %plea request message + :: even bone, 0 second bit: %boon response message + :: even bone, 1 second bit: nack-trace %boon message + :: + ?: =(1 (end 0 bone)) %plea + ?: =(0 (end 0 (rsh 0 bone))) %boon + %nack :: +| %entry-points :: +call: handle a $message-sink-task @@ -3393,18 +3456,24 @@ ++ call |= task=message-sink-task ^+ sink - :: if we get a plea request and have corked this flow, always ack - :: - ?: corked - =? peer-core &(?=(%hear -.task) =(1 (end 0 bone))) - %- (mi-trace odd.veb |.("hear plea on a corked bone={}")) - %+ send-shut-packet bone - [message-num.shut-packet.task %| %| ok=& lag=*@dr] - sink ?- -.task %drop sink(nax.state (~(del in nax.state) message-num.task)) %done (done ok.task) - %hear (hear [lane shut-packet ok]:task) + :: + %hear + ?. ?| corked + ?& %*(corked sink bone (mix 0b10 bone)) + =(%nack (received bone)) + == == + (hear [lane shut-packet ok]:task) + :: if we %hear a task on a corked bone, always ack + :: + =. peer-core + %+ send-shut-packet bone + [message-num.shut-packet.task %| %| ok=& lag=*@dr] + %. sink + %+ mi-trace odd.veb + |.("hear {<(received bone)>} on corked bone={}") == :: +| %tasks @@ -3574,18 +3643,12 @@ +| %implementation :: +handle-sink: dispatch message :: - :: odd bone: %plea request message - :: even bone, 0 second bit: %boon response message - :: even bone, 1 second bit: nack-trace %boon message - :: ++ handle-sink |= [=message-num message=* ok=?] - |^ ^+ sink - ?: =(1 (end 0 bone)) sink-plea - ?: =(0 (end 0 (rsh 0 bone))) sink-boon - sink-nack - :: XX FIXME: impure +abet pattern - ++ sink-plea + ^+ sink + |^ ?-((received bone) %plea ha-plea, %boon ha-boon, %nack ha-nack) + :: + ++ ha-plea ^+ sink ?: |(closing corked) sink %- %+ mi-trace msg.veb @@ -3619,7 +3682,7 @@ =. closing.peer-state (~(put in closing.peer-state) bone) (pe-emit duct %pass wire %a %plea her [%a /close ~]) :: - :: +sink-boon: handle response message, acking unconditionally + :: +ha-boon: handle response message, acking unconditionally :: :: .bone must be mapped in .ossuary.peer-state, or we crash. :: This means a malformed message will kill a flow. We @@ -3633,7 +3696,7 @@ :: TODO: This handles a previous crash in the client vane, but :: not in %ames itself. :: - ++ sink-boon + ++ ha-boon ^+ sink ?: |(closing corked) sink %- %+ mi-trace msg.veb |. @@ -3654,7 +3717,7 @@ :: (call %done ok=%.y) :: - ++ sink-nack + ++ ha-nack ^+ sink :: if we get a naxplanation for a %cork, the publisher hasn't :: received the OTA. The /recork timer will retry eventually. @@ -3712,7 +3775,7 @@ %jilt (on-jilt:event-core ship.task) %prod (on-prod:event-core ships.task) %sift (on-sift:event-core ships.task) - %snub (on-snub:event-core ships.task) + %snub (on-snub:event-core [form ships]:task) %spew (on-spew:event-core veb.task) %cong (on-cong:event-core [msg mem]:task) %stir (on-stir:event-core arg.task) @@ -3750,15 +3813,15 @@ [moves ames-gate] :: +stay: extract state before reload :: -++ stay [%11 %adult ames-state] +++ stay [%12 %adult ames-state] :: +load: load in old state after reload :: ++ load =< |= $= old-state - $% [%11 ^ames-state] + $% [%12 ^ames-state] == ^+ ames-gate - ?> ?=(%11 -.old-state) + ?> ?=(%12 -.old-state) ames-gate(ames-state +.old-state) :: all state transitions are called from larval ames :: @@ -3809,11 +3872,11 @@ :_ +.ames-state %- ~(run by peers.ames-state) |= ship-state=ship-state-6 - ^- ^ship-state + ^- ship-state-7 ?. ?=(%known -.ship-state) ship-state :- %known - ^- peer-state + ^- peer-state-7 :- +<.ship-state [route qos ossuary snd rcv nax heeds ~ ~ ~]:ship-state :: @@ -3846,13 +3909,26 @@ :: ++ state-10-to-11 |= ames-state=ames-state-10 - ^- ^^ames-state + ^- ames-state-11 =, ames-state :* peers unix-duct life crypto-core bug corks snub :: 5 messages and 100Kb of data outstanding :: [msg=5 mem=100.000] == + :: + ++ state-11-to-12 + |= ames-state=ames-state-11 + ^- ^^ames-state + :_ [unix-duct life crypto-core bug [%deny snub] cong]:ames-state + %- ~(run by peers.ames-state) + |= ship-state=ship-state-7 + ^- ^ship-state + ?. ?=(%known -.ship-state) + ship-state + %= ship-state + +> [route qos ossuary snd rcv nax heeds closing corked]:+>.ship-state + == -- :: +scry: dereference namespace :: @@ -3890,6 +3966,7 @@ :: /ax/peers/[ship]/forward-lane (list lane) :: /ax/bones/[ship] [snd=(set bone) rcv=(set bone)] :: /ax/snd-bones/[ship]/[bone] vase + :: /ax/snubbed (?(%allow %deny) (list ship)) :: ?. ?=(%x ren) ~ ?+ tyl ~ @@ -3969,5 +4046,8 @@ =/ res u.mps ``noun+!>(!>(res)) + :: + [%snubbed ~] + ``noun+!>([form.snub.ames-state ~(tap in ships.snub.ames-state)]) == -- diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index e46008de98..27491080c2 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -4970,9 +4970,22 @@ :: %wick =^ mos ruf - =/ den ((de now rof hen ruf) our %base) - abet:wick:den :: [wick] + abet:wick:((de now rof hen ruf) our %base) :: [wick] [mos ..^$] + :: + %zeal + =^ m1 ruf + =| mos=(list move) + |- ^+ [mos ruf] + ?~ lit.req + [mos ruf] + =/ den ((de now rof hen ruf) our desk.i.lit.req) + =^ mos-new ruf abet:(set-zest:den zest.i.lit.req) + $(mos (weld mos mos-new), lit.req t.lit.req) + =^ m2 ruf + abet:wick:((de now rof hen ruf) our %base) + =^ m3 ruf abet:goad:(lu now rof hen ruf) + [:(weld m1 m2 m3) ..^$] :: %zest =^ m1 ruf diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index e19f48d609..0972d75c9f 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -67,12 +67,8 @@ :: more structures :: |% -++ axle - $: :: date: date at which http-server's state was updated to this data structure - :: - date=%~2022.7.26 - :: server-state: state of inbound requests - :: ++$ axle + $: %~2023.2.17 =server-state == :: +server-state: state relating to open inbound HTTP connections @@ -112,6 +108,9 @@ :: outgoing-duct: to unix :: outgoing-duct=duct + :: verb: verbosity + :: + verb=@ == :: channel-request: an action requested on a channel :: @@ -584,7 +583,8 @@ [act [& secure address request] ~ 0] :: =. connections.state - (~(put by connections.state) duct connection) + %. (~(put by connections.state) duct connection) + (trace 2 |.("{} creating local")) :: :_ state (subscribe-to-app app.act inbound-request.connection) @@ -613,6 +613,7 @@ =/ connection=outstanding-connection [action [authenticated secure address request] ~ 0] =. connections.state + :: XX pretty sure this is superfluous - done in +handle-response (~(put by connections.state) duct connection) :: redirect to https if insecure, redirects enabled :: and secure port live @@ -864,6 +865,7 @@ %app :_ state :_ ~ + %- (trace 1 |.("leaving subscription to {}")) :* duct %pass /watch-response/[eyre-id] %g %deal [our our] app.action.u.connection %leave ~ @@ -1034,6 +1036,7 @@ =^ moz state (handle-response response) [[give-session-tokens (weld moves moz)] state] + %- (trace 1 |.("{(trip i.channels)} discarding channel due to logout")) =^ moz state (discard-channel:by-channel i.channels |) $(moves (weld moves moz), channels t.channels) @@ -1145,8 +1148,7 @@ :: POST methods are used solely for deleting channels (on-put-request channel-id request) :: - ~& %session-not-a-put - [~ state] + ((trace 0 |.("session not a put")) `state) :: +on-cancel-request: cancels an ongoing subscription :: :: One of our long lived sessions just got closed. We put the associated @@ -1156,15 +1158,15 @@ ^- [(list move) server-state] :: lookup the session id by duct :: - ?~ maybe-channel-id=(~(get by duct-to-key.channel-state.state) duct) - ~> %slog.[0 leaf+"eyre: no channel to cancel {}"] - [~ state] + %- (trace 1 |.("{} moving channel to waiting state")) :: - ~> %slog.[0 leaf+"eyre: canceling {}"] + ?~ maybe-channel-id=(~(get by duct-to-key.channel-state.state) duct) + ((trace 0 |.("{} no channel to move")) `state) :: =/ maybe-session (~(get by session.channel-state.state) u.maybe-channel-id) - ?~ maybe-session [~ state] + ?~ maybe-session + ((trace 1 |.("{} session doesn't exist")) `state) :: =/ heartbeat-cancel=(list move) ?~ heartbeat.u.maybe-session ~ @@ -1461,6 +1463,7 @@ ^- move :^ duct %pass (subscription-wire channel-id request-id ship app) + %- (trace 1 |.("subscribing to {} on {}")) :* %g %deal [our ship] app `task:agent:gall`[%watch path] == @@ -1487,8 +1490,9 @@ :: the client sent us a weird request referring to a subscription :: which isn't active. :: - ~& [%missing-subscription-in-unsubscribe channel-id subscription-id] - $(requests t.requests) + %. $(requests t.requests) + =* msg=tape "{(trip channel-id)} {}" + (trace 0 |.("missing subscription in unsubscribe {msg}")) :: =. gall-moves :_ gall-moves @@ -1496,6 +1500,7 @@ =, u.maybe-subscription :^ duc %pass (subscription-wire channel-id subscription-id.i.requests ship app) + %- (trace 1 |.("leaving subscription to {}")) :* %g %deal [our ship] app `task:agent:gall`[%leave ~] == @@ -1511,6 +1516,7 @@ $(requests t.requests) :: %delete + %- (trace 1 |.("{} discarding due to %delete PUT")) =^ moves state (discard-channel channel-id |) =. gall-moves @@ -1537,10 +1543,8 @@ (emit-event channel-id request-id sign) =/ =ship (slav %p i.extra) =* app=term i.t.extra - =/ =tape - %+ weld "eyre: removing watch for " - "non-existent channel {(trip channel-id)} on {(trip app)}" - %- (slog leaf+tape ~) + =* msg=tape "{(trip channel-id)} {(trip app)}" + %- (trace 0 |.("removing watch for non-existent channel {msg}")) :_ state :_ ~ ^- move @@ -1628,11 +1632,15 @@ ?& (gte num clog-threshold) (lth (add last-ack.u.channel clog-timeout) now) == - ~? clogged [%e %clogged channel-id request-id] :: if we're clogged, or we ran into an event we can't serialize, :: kill this gall subscription. :: - =* kicking |(clogged ?=(~ json)) + =* msg=tape "on {(trip channel-id)} for {(trip request-id)}" + =/ kicking=? + ?: clogged + ((trace 0 |.("clogged {msg}")) &) + ?. ?=(~ json) | + ((trace 0 |.("can't serialize event, kicking {msg}")) &) =? moves kicking :_ moves ::NOTE this shouldn't crash because we @@ -1640,6 +1648,7 @@ :: - only clog on %facts, which have a subscription associated, :: - and already checked whether we still have that subscription. =+ (~(got by subscriptions.u.channel) request-id) + %- (trace 1 |.("leaving subscription to {}")) :^ duct %pass (subscription-wire channel-id request-id ship app) [%g %deal [our ship] app %leave ~] @@ -1689,11 +1698,11 @@ ^- (unit desk) =/ sub (~(get by subscriptions.channel) request-id) ?~ sub - ((slog leaf+"eyre: no subscription for request-id {}" ~) ~) + ((trace 0 |.("no subscription for request-id {(trip request-id)}")) ~) =/ des=(unit (unit cage)) (rof ~ %gd [our app.u.sub da+now] ~) ?. ?=([~ ~ *] des) - ((slog leaf+"eyre: no desk for app {(trip app.u.sub)}" ~) ~) + ((trace 0 |.("no desk for app {}")) ~) `!<(=desk q.u.u.des) :: +channel-event-to-sign: attempt to recover a sign from a channel-event :: @@ -1710,11 +1719,11 @@ =/ val=(unit (unit cage)) (rof ~ %cb [our u.des da+now] /[have]) ?. ?=([~ ~ *] val) - ((slog leaf+"eyre: no mark {(trip have)}" ~) ~) + ((trace 0 |.("no mark {(trip have)}")) ~) =+ !<(=dais:clay q.u.u.val) =/ res (mule |.((vale:dais noun.event))) ?: ?=(%| -.res) - ((slog leaf+"eyre: stale fact of mark {(trip have)}" ~) ~) + ((trace 0 |.("stale fact of mark {(trip have)}")) ~) `[%fact have p.res] :: +sign-to-json: render sign from request-id as json channel event :: @@ -1733,15 +1742,14 @@ ?~ des [~ ~] :: =* have=mark p.cage.sign - =* desc=tape "from {(trip have)} to json" =/ convert=(unit vase) =/ cag=(unit (unit cage)) (rof ~ %cf [our u.des da+now] /[have]/json) ?. ?=([~ ~ *] cag) ~ `q.u.u.cag ?~ convert - ((slog leaf+"eyre: no convert {desc}" ~) [~ ~]) - ~| "conversion failed {desc}" + ((trace 0 |.("no convert from {(trip have)} to json")) [~ ~]) + ~| "conversion failed from {(trip have)} to json" [`[u.des have] `[%fact %json (slym u.convert q.q.cage.sign)]] ?~ jsyn ~ %- some @@ -1855,6 +1863,7 @@ %+ turn ~(tap by subscriptions.session) |= [request-id=@ud ship=@p app=term =path duc=^duct] ^- move + %- (trace 1 |.("{} leaving subscription to {}")) :^ duc %pass (subscription-wire channel-id request-id ship app) [%g %deal [our ship] app %leave ~] @@ -1870,6 +1879,7 @@ ?. ?=(%app -.action.connection) ~ :_ ~ + %- (trace 1 |.("leaving subscription to {}")) :* duct %pass /watch-response/[eyre-id] %g %deal [our our] app.action.connection %leave ~ @@ -1897,8 +1907,7 @@ :: verify that this is a valid response on the duct :: ?~ connection-state=(~(get by connections.state) duct) - ~& [%invalid-outstanding-connection duct] - [~ state] + ((trace 0 |.("{} invalid outstanding connection")) `state) :: |^ ^- [(list move) server-state] :: @@ -1906,8 +1915,7 @@ :: %start ?^ response-header.u.connection-state - ~& [%http-multiple-start duct] - error-connection + ((trace 0 |.("{} error multiple start")) error-connection) :: if request was authenticated, extend the session & cookie's life :: =^ response-header sessions.authentication-state.state @@ -1921,8 +1929,7 @@ ?~ session-id=(session-id-from-request request.inbound) :: cookies are the only auth method, so this is unexpected :: - ~& [%e %authenticated-without-cookie] - no-op + ((trace 0 |.("error authenticated without cookie")) no-op) ?. (~(has by sessions) u.session-id) :: if the session has expired since the request was opened, :: tough luck, we don't create/revive sessions here @@ -1954,6 +1961,7 @@ :: =. response-header.http-event response-header =. connections.state + %- (trace 2 |.("{} start")) %+ ~(put by connections.state) duct %_ connection response-header `response-header @@ -1967,10 +1975,11 @@ :: %continue ?~ response-header.u.connection-state - ~& [%http-continue-without-start duct] - error-connection + %. error-connection + (trace 0 |.("{} error continue without start")) :: =. connections.state + %- (trace 2 |.("{} continuing ")) %+ ~(jab by connections.state) duct |= connection=outstanding-connection =+ size=?~(data.http-event 0 p.u.data.http-event) @@ -1984,7 +1993,7 @@ %cancel :: todo: log this differently from an ise. :: - error-connection + ((trace 1 |.("cancel http event")) error-connection) == :: ++ pass-response @@ -1997,7 +2006,8 @@ :: remove all outstanding state for this connection :: =. connections.state - (~(del by connections.state) duct) + %. (~(del by connections.state) duct) + (trace 2 |.("{} completed")) state :: ++ error-connection @@ -2015,6 +2025,8 @@ ?. ?=(%app -.action.u.connection-state) ~ :_ ~ + %- %+ trace 1 + |.("leaving subscription to {}") :* duct %pass /watch-response/[eyre-id] %g %deal [our our] app.action.u.connection-state %leave ~ @@ -2124,6 +2136,11 @@ :- outgoing-duct.state =* ses sessions.authentication-state.state [%give %sessions (~(run in ~(key by ses)) (cury scot %uv))] + :: + ++ trace + |= [pri=@ print=(trap tape)] + ?: (lth verb.state pri) same + (slog leaf+"eyre: {(print)}" ~) -- :: ++ forwarded-params @@ -2426,6 +2443,10 @@ (~(del in approved) origin.task) (~(put in rejected) origin.task) [~ http-server-gate] + :: + %spew + =. verb.server-state.ax veb.task + `http-server-gate == :: ++ take @@ -2537,10 +2558,11 @@ ?> ?=([%behn %wake *] sign) ?^ error.sign [[duct %slip %d %flog %crud %wake u.error.sign]~ http-server-gate] - =/ discard-channel - discard-channel:by-channel:(per-server-event event-args) + =* id i.t.t.wire + %- %+ trace:(per-server-event event-args) 1 + |.("{(trip id)} cancelling channel due to timeout") =^ moves server-state.ax - (discard-channel i.t.t.wire &) + (discard-channel:by-channel:(per-server-event event-args) id &) [moves http-server-gate] :: %heartbeat @@ -2613,27 +2635,61 @@ :: ++ load => |% - ++ axle-old - %+ cork - axle - |= =axle - axle(date %~2020.10.18) - -- - |= old=$%(axle axle-old) + +$ axle-any + $% [%~2020.10.18 =server-state-0] + [%~2022.7.26 =server-state-0] + [%~2023.2.17 =server-state] + == + +$ server-state-0 + $: bindings=(list [=binding =duct =action]) + =cors-registry + connections=(map duct outstanding-connection) + =authentication-state + =channel-state + domains=(set turf) + =http-config + ports=[insecure=@ud secure=(unit @ud)] + outgoing-duct=duct + == + -- + |= old=axle-any ^+ ..^$ - :: ?- -.old %~2020.10.18 - %= $ - date.old %~2022.7.26 - :: - bindings.server-state.old - %+ insert-binding - [[~ /~/name] outgoing-duct.server-state.old [%name ~]] - bindings.server-state.old - == + =, server-state-0.old + %= ..^$ + ax ^- axle + :* %~2023.2.17 + (insert-binding [[~ /~/name] outgoing-duct [%name ~]] bindings) + cors-registry + connections + authentication-state + channel-state + domains + http-config + ports + outgoing-duct + 0 + == == :: %~2022.7.26 + =, server-state-0.old + %= ..^$ + ax ^- axle + :* %~2023.2.17 + bindings + cors-registry + connections + authentication-state + channel-state + domains + http-config + ports + outgoing-duct + 0 + == == + :: + %~2023.2.17 :: enable https redirects if certificate configured :: =. redirect.http-config.server-state.old diff --git a/pkg/base-dev/lib/pill.hoon b/pkg/base-dev/lib/pill.hoon index 4b07f57c63..0b4ae332fc 100644 --- a/pkg/base-dev/lib/pill.hoon +++ b/pkg/base-dev/lib/pill.hoon @@ -32,25 +32,23 @@ [/sys/hoon hoon/hoon] [/sys/arvo hoon/arvo] == +:: +dirs: filter for userspace directories to include +:: +++ dirs + |= [bas=path exc=(list spur)] + ^- (list spur) + ?~ exc ~(tap in dir:.^(arch %cy bas)) + %+ skim + ~(tap in dir:.^(arch %cy bas)) + |=(=spur =(~ (find ~[spur] exc))) :: +file-ovum: userspace filesystem load :: :: bas: full path to / directory :: ++ file-ovum - =/ directories=(list path) - :~ /app :: %gall applications - /gen :: :dojo generators - /lib :: libraries - /mar :: mark definitions - /sur :: structures - /sys :: system files - /ted :: :spider strands - /web :: %eyre web content - /desk :: desk manifest - == - |= [des=desk bas=path] + |= [des=desk bas=path exc=(list spur)] ^- unix-event - %. directories + %. (dirs bas exc) |= :: sal: all spurs to load from :: sal=(list spur) @@ -99,7 +97,7 @@ :: :: +file-ovum2: electric boogaloo :: -++ file-ovum2 |=(p=path `unix-event`[//arvo what/(user-files p)]) +++ file-ovum2 |=(z=[path (list spur)] `unix-event`[//arvo what/(user-files z)]) :: ++ prep-ovum |= dez=(list path) @@ -114,8 +112,8 @@ :: +user-files: all userspace hoon files :: ++ user-files - |= bas=path - %. directories:file-ovum + |= [bas=path exc=(list spur)] + %. (dirs bas exc) |= sal=(list spur) ^- (list (pair path (cask))) :: @@ -154,8 +152,9 @@ ++ solid :: sys: root path to boot system, `/~me/[desk]/now/sys` :: dez: secondary desks and their root paths + :: exc: list of desk folders to exclude :: - |= [sys=path dez=(list [desk path]) dub=? now=@da prime=?] + |= [sys=path dez=(list [desk path]) dub=? now=@da prime=? exc=(list spur)] ^- pill =/ bas=path (scag 3 sys) =/ compiler-path (weld sys /hoon) @@ -201,7 +200,8 @@ :- (boot-ovum:pill compiler-src arvo-src) %+ turn (snoc (turn dez tail) bas) - file-ovum2:pill + |= =path + (file-ovum2 [path exc]) .*(0 arvo-formula) |= [ovo=ovum ken=*] [~ (slum ken [now ovo])] @@ -230,15 +230,18 @@ :+ boot-ova ~ =. dez (snoc dez [%base bas]) %+ weld - (turn dez file-ovum) + %+ turn dez + |= [dek=desk bas=path] + (file-ovum [dek bas exc]) ?. prime ~ [(prep-ovum (turn dez tail))]~ :: ++ brass :: sys: root path to boot system, `/~me/[desk]/now/sys` :: dez: secondary desks and their root paths + :: exc: list of desk folders to exclude :: - |= [sys=path dez=(list [desk path]) prime=?] + |= [sys=path dez=(list [desk path]) prime=? exc=(list spur)] ^- pill =/ bas=path (scag 3 sys) :: compiler-source: hoon source file producing compiler, `sys/hoon` @@ -274,12 +277,14 @@ :: :+ %pill %brass :+ boot-ova - :~ (boot-ovum:pill compiler-source arvo-source) - (file-ovum2:pill bas) + :~ (boot-ovum compiler-source arvo-source) + (file-ovum2 [bas exc]) == =. dez (snoc dez [%base bas]) %+ weld - (turn dez file-ovum) + %+ turn dez + |= [dek=desk bas=path] + (file-ovum [dek bas exc]) ?. prime ~ [(prep-ovum (turn dez tail))]~ :: @@ -355,7 +360,7 @@ ::TODO will exclude non-:directories files, such as /changelog/txt =- (murn - same) ^- (list (unit ovum)) - :~ `(file-ovum as (en-beam beak /)) + :~ `(file-ovum as (en-beam beak /) ~) :: ?. pri ~ `(prep-ovum (en-beam beak /) ~) diff --git a/pkg/base-dev/lib/sss.hoon b/pkg/base-dev/lib/sss.hoon index a1a4a4b24c..5f02e3db02 100644 --- a/pkg/base-dev/lib/sss.hoon +++ b/pkg/base-dev/lib/sss.hoon @@ -201,7 +201,8 @@ ?^ dat=(get:wav wav.tide u.when.req) :_ pub :_ ~ (send scry/wave/u.dat src.bowl [dude u.when path]:req) - ?. (gth u.when.req key::(fall (ram:wav wav.tide) [key=+(u.when.req) **])) + ?: %+ lte u.when.req + key::(fall (ram:wav wav.tide) (pry:rok rok.tide) [=key val]:wav) :_ pub :_ ~ (send yore/~ src.bowl [dude u.when path]:req) :- ~[(send nigh/~ src.bowl [dude u.when path]:req)] diff --git a/pkg/landscape/sys.kelvin b/pkg/landscape/sys.kelvin index ea744fdf10..a36738916e 100644 --- a/pkg/landscape/sys.kelvin +++ b/pkg/landscape/sys.kelvin @@ -1,2 +1,3 @@ [%zuse 416] [%zuse 415] +[%zuse 414] \ No newline at end of file