From 953273004a12c2d143503a1da78cba5791c326b2 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Sat, 4 Jun 2022 17:18:21 -0500 Subject: [PATCH 01/67] ames: clog at 100kB instead of 5 messages --- pkg/arvo/sys/vane/ames.hoon | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index b2b4f8c9b..5fdc79af2 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2182,7 +2182,7 @@ ?: =(0 (end 0 bone)) ~ `u=message-pump-state - :: clogged: are five or more response messages unsent to this peer? + :: clogged: are outgoing messages to this peer using too much memory? :: =/ clogged=? =| acc=@ud @@ -2192,14 +2192,15 @@ =. acc %+ add acc %+ add - :: in-flight messages - :: - (sub [next current]:i.pumps) - :: queued messages - :: - ~(wyt in unsent-messages.i.pumps) + =- (roll - add) + (turn ~(tap to unsent-messages.i.pumps) |=(b=@ (met 3 b))) + ?~ unsent-fragments.i.pumps + 0 + (met 3 fragment.i.unsent-fragments.i.pumps) + :: 100.000 chosen so roughly 10.000 peers could be + :: clogged without killing the loom :: - ?: (gte acc 5) + ?: (gte acc 100.000) %.y $(pumps t.pumps) :: if clogged, notify client vanek From d2ea33ddfeb1a842d3bb4448dfdb87747b6d4dce Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Sun, 5 Jun 2022 08:49:47 -0500 Subject: [PATCH 02/67] ames: condense message size calculation Co-authored-by: fang --- pkg/arvo/sys/vane/ames.hoon | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 5fdc79af2..48cfd6b92 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2192,8 +2192,9 @@ =. acc %+ add acc %+ add - =- (roll - add) - (turn ~(tap to unsent-messages.i.pumps) |=(b=@ (met 3 b))) + %+ roll ~(tap to unsent-messages.i.pumps) + |= [a=@ b=@ud] + (add b (met 3 a)) ?~ unsent-fragments.i.pumps 0 (met 3 fragment.i.unsent-fragments.i.pumps) From 1380e83b55cb7a4e1cba09d0bb5df4d4f306bbd6 Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 15 Jun 2022 15:17:21 -0600 Subject: [PATCH 03/67] ames: clog on 5 messages and 100kB --- pkg/arvo/sys/vane/ames.hoon | 55 +++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 48cfd6b92..4a19c10f5 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2182,29 +2182,44 @@ ?: =(0 (end 0 bone)) ~ `u=message-pump-state - :: clogged: are outgoing messages to this peer using too much memory? :: =/ clogged=? - =| acc=@ud - |- ^- ? - ?~ pumps - %.n - =. acc - %+ add acc - %+ add - %+ roll ~(tap to unsent-messages.i.pumps) - |= [a=@ b=@ud] - (add b (met 3 a)) - ?~ unsent-fragments.i.pumps - 0 - (met 3 fragment.i.unsent-fragments.i.pumps) - :: 100.000 chosen so roughly 10.000 peers could be - :: clogged without killing the loom + |^ &(nuf-messages nuf-memory) :: - ?: (gte acc 100.000) - %.y - $(pumps t.pumps) - :: if clogged, notify client vanek + ++ nuf-messages + =| num=@ud + |- ^- ? + ?~ pumps | + =. num + ;: add num + (sub [next current]:i.pumps) + ~(wyt in unsent-messages.i.pumps) + == + ?: (gte num 5) + & + $(pumps t.pumps) + :: + ++ nuf-memory + =| mem=@ud + |- ^- ? + ?~ pumps | + =. mem + ;: add mem + %+ roll ~(tap to unsent-messages.i.pumps) + |=([a=@ b=@ud] (add b (met 3 a))) + :: + ?~ unsent-fragments.i.pumps + 0 + (met 3 fragment.i.unsent-fragments.i.pumps) + == + :: 100.000 chosen so roughly 10.000 peers could be + :: clogged without killing the loom + :: + ?: (gte mem 100.000) + & + $(pumps t.pumps) + -- + :: if clogged, notify client vane :: ?. clogged peer-core From 0190a477f51f9ce3f9f9b4392ff9d688076aec1b Mon Sep 17 00:00:00 2001 From: Ted Blackman Date: Wed, 15 Jun 2022 15:35:41 -0600 Subject: [PATCH 04/67] ames: adjust comments --- pkg/arvo/sys/vane/ames.hoon | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 4a19c10f5..3301e403a 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2185,6 +2185,7 @@ :: =/ clogged=? |^ &(nuf-messages nuf-memory) + :: +nuf-messages: are there enough messages to mark as clogged? :: ++ nuf-messages =| num=@ud @@ -2198,6 +2199,7 @@ ?: (gte num 5) & $(pumps t.pumps) + :: +nuf-memory: is enough memory used to mark as clogged? :: ++ nuf-memory =| mem=@ud @@ -2212,9 +2214,6 @@ 0 (met 3 fragment.i.unsent-fragments.i.pumps) == - :: 100.000 chosen so roughly 10.000 peers could be - :: clogged without killing the loom - :: ?: (gte mem 100.000) & $(pumps t.pumps) From ed96e6162c2f8b2fca9bcaf01889ca035fc37205 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Wed, 26 Oct 2022 13:01:32 +1000 Subject: [PATCH 05/67] arvo: retool for goerli instead of deprecated ropsten --- Makefile | 6 +++--- default.nix | 4 ++-- nix/pkgs/arvo/default.nix | 8 ++++---- nix/pkgs/marsSources/default.nix | 14 +++++++------- nix/pkgs/pill/brass.nix | 8 ++++---- nix/pkgs/pill/ivory.nix | 8 ++++---- pkg/arvo/app/claz.hoon | 2 +- pkg/arvo/app/roller.hoon | 4 ++-- pkg/arvo/gen/azimuth/watch.hoon | 2 +- pkg/arvo/gen/roller/endpoint.hoon | 2 +- pkg/arvo/gen/roller/goerli.hoon | 10 ++++++++++ pkg/arvo/gen/roller/ropsten.hoon | 10 ---------- pkg/arvo/lib/claz.hoon | 2 +- pkg/arvo/lib/dice.hoon | 4 ++-- pkg/arvo/sur/claz.hoon | 2 +- pkg/arvo/sur/dice.hoon | 2 +- pkg/arvo/ted/naive-csv.hoon | 2 +- pkg/base-dev/lib/azimuth.hoon | 18 +++++++++--------- sh/create-goerli-pills | 21 +++++++++++++++++++++ sh/create-ropsten-pills | 21 --------------------- 20 files changed, 75 insertions(+), 75 deletions(-) create mode 100644 pkg/arvo/gen/roller/goerli.hoon delete mode 100644 pkg/arvo/gen/roller/ropsten.hoon create mode 100755 sh/create-goerli-pills delete mode 100755 sh/create-ropsten-pills diff --git a/Makefile b/Makefile index dbadd5cd7..52aec5e1b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-all install cross release test pills ropsten-pills clean +.PHONY: build build-all install cross release test pills goerli-pills clean build: nix-build -A urbit --no-out-link @@ -17,8 +17,8 @@ pills: sh/update-brass-pill sh/update-ivory-pill -ropsten-pills: - sh/create-ropsten-pills +goerli-pills: + sh/create-goerli-pills interface: sh/build-interface diff --git a/default.nix b/default.nix index 3161c5363..f84a2e9d1 100644 --- a/default.nix +++ b/default.nix @@ -117,8 +117,8 @@ let pill = solid.lfs; }; - ivory-ropsten = ivory.override { arvo = arvo.ropsten; }; - brass-ropsten = brass.override { arvo = arvo.ropsten; }; + ivory-goerli = ivory.override { arvo = arvo.goerli; }; + brass-goerli = brass.override { arvo = arvo.goerli; }; # Create a .tgz of the primary binaries. tarball = let diff --git a/nix/pkgs/arvo/default.nix b/nix/pkgs/arvo/default.nix index 9d9d10806..502d2847a 100644 --- a/nix/pkgs/arvo/default.nix +++ b/nix/pkgs/arvo/default.nix @@ -5,16 +5,16 @@ stdenvNoCC.mkDerivation { src = marsSources; - outputs = [ "out" "ropsten" ]; + outputs = [ "out" "goerli" ]; - phases = [ "mainnetPhase" "ropstenPhase" ]; + phases = [ "mainnetPhase" "goerliPhase" ]; mainnetPhase = '' ln -s ${marsSources.out}/arvo $out ''; - ropstenPhase = '' - ln -s ${marsSources.ropsten}/arvo $ropsten + goerliPhase = '' + ln -s ${marsSources.goerli}/arvo $goerli ''; preferLocalBuild = true; diff --git a/nix/pkgs/marsSources/default.nix b/nix/pkgs/marsSources/default.nix index 80bbf303b..5811e19e5 100644 --- a/nix/pkgs/marsSources/default.nix +++ b/nix/pkgs/marsSources/default.nix @@ -6,16 +6,16 @@ stdenvNoCC.mkDerivation { buildInputs = [ bc ]; - outputs = [ "out" "ropsten" ]; + outputs = [ "out" "goerli" ]; - phases = [ "mainnetPhase" "ropstenPhase" ]; + phases = [ "mainnetPhase" "goerliPhase" ]; mainnetPhase = '' cp -r $src $out chmod -R u+w $out ''; - ropstenPhase = '' + goerliPhase = '' cp -r $src tmp chmod -R u+w tmp @@ -23,9 +23,9 @@ stdenvNoCC.mkDerivation { AMES=tmp/arvo/sys/vane/ames.hoon ACME=tmp/arvo/app/acme.hoon - # Replace the mainnet azimuth contract with the ropsten contract + # Replace the mainnet azimuth contract with the goerli contract sed --in-place \ - 's/\(\+\+ contracts \)mainnet\-contracts/\1ropsten-contracts/' \ + 's/\(\+\+ contracts \)mainnet\-contracts/\1goerli-contracts/' \ $ZUSE # Increment the %ames protocol version @@ -38,8 +38,8 @@ stdenvNoCC.mkDerivation { 's_https://acme-v02.api.letsencrypt.org/directory_https://acme-staging-v02.api.letsencrypt.org/directory_' \ $ACME - cp -r tmp $ropsten - chmod -R u+w $ropsten + cp -r tmp $goerli + chmod -R u+w $goerli ''; preferLocalBuild = true; diff --git a/nix/pkgs/pill/brass.nix b/nix/pkgs/pill/brass.nix index 36b18dd9a..6aa1c5dcf 100644 --- a/nix/pkgs/pill/brass.nix +++ b/nix/pkgs/pill/brass.nix @@ -1,5 +1,5 @@ { lib, stdenvNoCC, fetchGitHubLFS, bootFakeShip, solid, urbit, arvo, curl -, withRopsten ? false }: +, withGoerli ? false }: let @@ -9,9 +9,9 @@ in { build = import ./builder.nix { inherit stdenvNoCC urbit curl; - name = "brass" + lib.optionalString withRopsten "-ropsten"; + name = "brass" + lib.optionalString withGoerli "-goerli"; builder = ./brass.sh; - arvo = if withRopsten then arvo.ropsten else arvo; + arvo = if withGoerli then arvo.goerli else arvo; pier = bootFakeShip { inherit urbit; @@ -19,4 +19,4 @@ in { ship = "zod"; }; }; -} // lib.optionalAttrs (!withRopsten) { inherit lfs; } +} // lib.optionalAttrs (!withGoerli) { inherit lfs; } diff --git a/nix/pkgs/pill/ivory.nix b/nix/pkgs/pill/ivory.nix index 9eceef109..b4881909c 100644 --- a/nix/pkgs/pill/ivory.nix +++ b/nix/pkgs/pill/ivory.nix @@ -1,5 +1,5 @@ { lib, stdenvNoCC, fetchGitHubLFS, bootFakeShip, solid, urbit, arvo, curl, xxd -, withRopsten ? false }: +, withGoerli ? false }: let @@ -9,9 +9,9 @@ in { build = import ./builder.nix { inherit stdenvNoCC urbit curl; - name = "ivory" + lib.optionalString withRopsten "-ropsten"; + name = "ivory" + lib.optionalString withGoerli "-goerli"; builder = ./ivory.sh; - arvo = if withRopsten then arvo.ropsten else arvo; + arvo = if withGoerli then arvo.goerli else arvo; pier = bootFakeShip { inherit urbit; @@ -39,4 +39,4 @@ in { preferLocalBuild = true; }; -} // lib.optionalAttrs (!withRopsten) { inherit lfs; } +} // lib.optionalAttrs (!withGoerli) { inherit lfs; } diff --git a/pkg/arvo/app/claz.hoon b/pkg/arvo/app/claz.hoon index a8f1f42ce..64c20abcc 100644 --- a/pkg/arvo/app/claz.hoon +++ b/pkg/arvo/app/claz.hoon @@ -185,7 +185,7 @@ :: ?- network %mainnet 0x1 - %ropsten 0x3 + %goerli 0x5 %fakenet `@ux``@`1.337 [%other *] id.network == diff --git a/pkg/arvo/app/roller.hoon b/pkg/arvo/app/roller.hoon index a83d6d925..cb7ffb9fa 100644 --- a/pkg/arvo/app/roller.hoon +++ b/pkg/arvo/app/roller.hoon @@ -65,7 +65,7 @@ :: frequency: time to wait between sending batches (TODO fancier) :: endpoint: ethereum rpc endpoint to use :: contract: ethereum contract address - :: chain-id: mainnet, ropsten, local (https://chainid.network/) + :: chain-id: mainnet, goerli, local (https://chainid.network/) :: resend-time: time to resend a batch with higher gas prie :: update-rate: frequency to update the roller's predicted state :: fallback-gas-price: default batch gas price @@ -1019,7 +1019,7 @@ =, azimuth ?- net.config %mainnet mainnet-contracts - %ropsten ropsten-contracts + %goerli goerli-contracts %local local-contracts %default contracts == diff --git a/pkg/arvo/gen/azimuth/watch.hoon b/pkg/arvo/gen/azimuth/watch.hoon index 97c0d74b0..b46d32811 100644 --- a/pkg/arvo/gen/azimuth/watch.hoon +++ b/pkg/arvo/gen/azimuth/watch.hoon @@ -1,5 +1,5 @@ :: Change node url and network for azimuth :: :- %say -|= [* [url=@ta net=?(%mainnet %ropsten %local %default) ~] ~] +|= [* [url=@ta net=?(%mainnet %goerli %local %default) ~] ~] [%azimuth-poke %watch url net] diff --git a/pkg/arvo/gen/roller/endpoint.hoon b/pkg/arvo/gen/roller/endpoint.hoon index ce59f3073..26e271d6f 100644 --- a/pkg/arvo/gen/roller/endpoint.hoon +++ b/pkg/arvo/gen/roller/endpoint.hoon @@ -1,4 +1,4 @@ :: :- %say -|= [* [url=@t net=?(%mainnet %ropsten %local) ~] ~] +|= [* [url=@t net=?(%mainnet %goerli %local) ~] ~] [%roller-action %config %endpoint url net] diff --git a/pkg/arvo/gen/roller/goerli.hoon b/pkg/arvo/gen/roller/goerli.hoon new file mode 100644 index 000000000..4dacaf99a --- /dev/null +++ b/pkg/arvo/gen/roller/goerli.hoon @@ -0,0 +1,10 @@ +:: Configures /app/roller to listen to a Goerli Infura node +:: +:- %say +|= * +:* %roller-action + %config + %endpoint + 'https://goerli.infura.io/v3/2599df54929b47099bda360958d75aaf' + %goerli +== diff --git a/pkg/arvo/gen/roller/ropsten.hoon b/pkg/arvo/gen/roller/ropsten.hoon deleted file mode 100644 index af363f0dd..000000000 --- a/pkg/arvo/gen/roller/ropsten.hoon +++ /dev/null @@ -1,10 +0,0 @@ -:: Configures /app/roller to listen to a Ropsten Infura node -:: -:- %say -|= * -:* %roller-action - %config - %endpoint - 'https://ropsten.infura.io/v3/2599df54929b47099bda360958d75aaf' - %ropsten -== diff --git a/pkg/arvo/lib/claz.hoon b/pkg/arvo/lib/claz.hoon index 1a34ae9ff..3f7f40067 100644 --- a/pkg/arvo/lib/claz.hoon +++ b/pkg/arvo/lib/claz.hoon @@ -25,7 +25,7 @@ |= =network ?+ network ~&(%careful-fallback-contracts mainnet-contracts:azimuth) %mainnet mainnet-contracts:azimuth - %ropsten ropsten-contracts:azimuth + %goerli goerli-contracts:azimuth == :: ++ encode-claz-call diff --git a/pkg/arvo/lib/dice.hoon b/pkg/arvo/lib/dice.hoon index c59b7907b..4f5f44a3b 100644 --- a/pkg/arvo/lib/dice.hoon +++ b/pkg/arvo/lib/dice.hoon @@ -32,9 +32,9 @@ ^- [azimuth=@ux naive=@ux chain-id=@ launch=@] =< [azimuth naive chain-id launch] =, azimuth - ?- net + ?+ net !! %mainnet mainnet-contracts - %ropsten ropsten-contracts + %goerli goerli-contracts %local local-contracts %default contracts == diff --git a/pkg/arvo/sur/claz.hoon b/pkg/arvo/sur/claz.hoon index 2b3677ed8..88c3b246c 100644 --- a/pkg/arvo/sur/claz.hoon +++ b/pkg/arvo/sur/claz.hoon @@ -11,7 +11,7 @@ :: ++ network $? %mainnet - %ropsten + %goerli %fakenet [%other id=@] == diff --git a/pkg/arvo/sur/dice.hoon b/pkg/arvo/sur/dice.hoon index 239d0691b..5cde151aa 100644 --- a/pkg/arvo/sur/dice.hoon +++ b/pkg/arvo/sur/dice.hoon @@ -8,7 +8,7 @@ +$ sponsors (map ship [residents=(set ship) requests=(set ship)]) +$ history (map address:ethereum (tree hist-tx)) +$ events (list event-log:rpc:ethereum) -+$ net ?(%mainnet %ropsten %local %default) ++$ net ?(%mainnet %ropsten %goerli %local %default) +$ snap-state [%0 =id:block:jael nas=^state:naive =owners =sponsors] :: +$ config diff --git a/pkg/arvo/ted/naive-csv.hoon b/pkg/arvo/ted/naive-csv.hoon index 3ce3029eb..59745acbd 100644 --- a/pkg/arvo/ted/naive-csv.hoon +++ b/pkg/arvo/ted/naive-csv.hoon @@ -50,7 +50,7 @@ +$ address address:naive :: @ux +$ keccak @ux :: used for transaction and roll hashes +$ blocknum number:block :: @udblocknumber - +$ net net:dice :: ?(%mainnet %ropsten %local %default) + +$ net net:dice :: ?(%mainnet %goerli %local %default) +$ roll-dat :: all data required for each roll [[gas=@ud sender=address] =effects:naive] +$ block-dat :: all data required for each block diff --git a/pkg/base-dev/lib/azimuth.hoon b/pkg/base-dev/lib/azimuth.hoon index d9c6a1b2e..2bd907766 100644 --- a/pkg/base-dev/lib/azimuth.hoon +++ b/pkg/base-dev/lib/azimuth.hoon @@ -114,29 +114,29 @@ :: :: Testnet contract addresses :: - ++ ropsten-contracts + ++ goerli-contracts |% ++ azimuth - 0x308a.b6a6.024c.f198.b57e.008d.0ac9.ad02.1988.6579 + 0xbb61.fa68.3e4b.9104.18e2.7b00.a143.8a93.6234.df52 :: ++ ecliptic - 0x8b9f.86a2.8921.d9c7.05b3.113a.755f.b979.e1bd.1bce + 0xe129.0a32.9014.5e63.e6a8.ec1e.f661.6906.856d.0c8f :: ++ linear-star-release - 0x1f8e.dd03.1ee4.1474.0aed.b39b.84fb.8f2f.66ca.422f + 0x0 :: ++ conditional-star-release 0x0 :: ++ delegated-sending - 0x3e8c.a510.354b.c2fd.bbd6.1502.52d9.3105.c9c2.7bbe + 0x0 :: ++ naive - 0xe7cf.4b83.06d3.11ba.ca15.585f.e3f0.7cd0.441c.21d1 + 0x56e3.7137.cdaf.c026.a732.e8e8.40cd.621e.d50b.d210 :: - ++ launch 4.601.630 - ++ public launch - ++ chain-id 3 + ++ launch 7.834.742 + ++ public 7.834.800 + ++ chain-id 5 -- :: :: Local contract addresses diff --git a/sh/create-goerli-pills b/sh/create-goerli-pills new file mode 100755 index 000000000..f8e04add7 --- /dev/null +++ b/sh/create-goerli-pills @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# Usage: create-goerli-pills + +# Build the ivory and brass goerli pills, copying them to the local directory, +# named with the current give revision + +set -xeuo pipefail + +cd "${0%/*}/.." + +rev="$(git rev-parse HEAD)" +sha="${rev:0:5}" + +brass="brass-goerli-${sha}.pill" +ivory="ivory-goerli-${sha}.pill" + +cp -f "$(nix-build -A brass-goerli.build --no-out-link)" "$brass" +cp -f "$(nix-build -A ivory-goerli.build --no-out-link)" "$ivory" + +chmod 0644 "$brass" "$ivory" diff --git a/sh/create-ropsten-pills b/sh/create-ropsten-pills deleted file mode 100755 index d288795f3..000000000 --- a/sh/create-ropsten-pills +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# Usage: create-ropsten-pills - -# Build the ivory and brass ropsten pills, copying them to the local directory, -# named with the current give revision - -set -xeuo pipefail - -cd "${0%/*}/.." - -rev="$(git rev-parse HEAD)" -sha="${rev:0:5}" - -brass="brass-ropsten-${sha}.pill" -ivory="ivory-ropsten-${sha}.pill" - -cp -f "$(nix-build -A brass-ropsten.build --no-out-link)" "$brass" -cp -f "$(nix-build -A ivory-ropsten.build --no-out-link)" "$ivory" - -chmod 0644 "$brass" "$ivory" From 44fe459f6d1c180b8f5707d12efa8a7001838686 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Wed, 26 Oct 2022 13:58:36 +1000 Subject: [PATCH 06/67] roller: fix mint-lost --- pkg/arvo/app/roller.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/app/roller.hoon b/pkg/arvo/app/roller.hoon index cb7ffb9fa..b27264524 100644 --- a/pkg/arvo/app/roller.hoon +++ b/pkg/arvo/app/roller.hoon @@ -1017,7 +1017,7 @@ =/ [contract=@ux chain-id=@] =< [naive chain-id] =, azimuth - ?- net.config + ?+ net.config !! %mainnet mainnet-contracts %goerli goerli-contracts %local local-contracts From 50cfdb0f2a7e06533a36bd39433abc84b30b31b2 Mon Sep 17 00:00:00 2001 From: fang Date: Sun, 22 Jan 2023 21:42:57 +0100 Subject: [PATCH 07/67] clay: allow scrying out of pending requests Implements a /cx/[our]//[now]/cult/[desk] endpoint, for getting a set of pending requests for any given desk. We don't give the $cult for the desk as-is, but instead slim the $roves back down into $raves, remove clay protocol version metadata, and make sure to put our @p in place of empty "for" fields. --- pkg/arvo/sys/vane/clay.hoon | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index f9a6f5954..05e137d2b 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -5647,6 +5647,7 @@ %sweep ``[%sweep !>(sweep)] %rang ``[%rang !>(ran.ruf)] %tomb ``[%flag !>((tomb t.path))] + %cult ``[%cult !>((cult t.path))] %domes domes %tire ``[%tire !>(tore:(lu now rof *duct ruf))] %tyre ``[%tyre !>(tyr.ruf)] @@ -5672,6 +5673,22 @@ [[ship desk] [[let hit lab] tom nor liv ren]:dom.rede] ``[%domes !>(`cone`domes)] :: + ++ cult + |= =path + ^- (set [@p rave]) + %- %~ run in + %~ key by + ?~ path *^cult + qyx:(~(gut by dos.rom.ruf) i.path *dojo) + |= wove + :- ship:(fall for [ship=our @ud]) + ?- -.rove + %sing rove + %next [%next mood.rove] + %mult [%mult mool.rove] + %many [%many [track moat]:rove] + == + :: :: True if file is accessible :: ++ tomb From 6b80aec3c2f539c5ada95551a04a5f39d638c783 Mon Sep 17 00:00:00 2001 From: fang Date: Sun, 22 Jan 2023 21:46:16 +0100 Subject: [PATCH 08/67] gen: +desk-requests, counts pending clay requests Gives a summation of pending clay requests for a given desk, specifying the number of local and incoming-from-foreign requests, and for the latter, how many of those are awaiting the next revision of a desk (read: are currently |sync-ing from the desk). --- pkg/arvo/gen/desk-requests.hoon | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 pkg/arvo/gen/desk-requests.hoon diff --git a/pkg/arvo/gen/desk-requests.hoon b/pkg/arvo/gen/desk-requests.hoon new file mode 100644 index 000000000..6bc32102c --- /dev/null +++ b/pkg/arvo/gen/desk-requests.hoon @@ -0,0 +1,28 @@ +:: +desk-requests: count pending requests for a desk +:: +:- %say +|= $: [now=@da eny=@uvJ bec=beak] + [=desk ~] + ~ + == +:- %tang +^- tang +=/ cul=(list [@p rave:clay]) + %~ tap in + .^ (set [@p rave:clay]) + /cx/(scot %p p.bec)//(scot %da now)/cult/[desk] + == +:: +=/ [loc=_cul inc=_cul] + (skid cul |=([=@p rave:clay] =(p p.bec))) +=/ syc=_cul + =/ nex=@ud + +(ud:.^(cass:clay %cw /(scot %p p.bec)/[desk]/(scot %da now))) + (skim inc |=([@p =rave:clay] =([%sing %w ud+nex /] rave))) +:: +%- flop +:~ leaf+"total: {<(lent cul)>}" + leaf+"- local: {<(lent loc)>}" + leaf+"- incoming: {<(lent inc)>}" + leaf+" - for next: {<(lent syc)>}" +== From 3a7bd3a67f3b5446eba00f694f44b92c7536fc8b Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 5 Dec 2022 11:25:09 +0100 Subject: [PATCH 09/67] ames: add tool for removing failed %watch flows --- pkg/arvo/gen/hood/close-flows.hoon | 73 ++++++++++++++++++++++++++++++ pkg/arvo/lib/hood/helm.hoon | 12 +++++ pkg/arvo/sys/lull.hoon | 2 + pkg/arvo/sys/vane/ames.hoon | 26 +++++++++++ 4 files changed, 113 insertions(+) create mode 100644 pkg/arvo/gen/hood/close-flows.hoon diff --git a/pkg/arvo/gen/hood/close-flows.hoon b/pkg/arvo/gen/hood/close-flows.hoon new file mode 100644 index 000000000..1a32d2fef --- /dev/null +++ b/pkg/arvo/gen/hood/close-flows.hoon @@ -0,0 +1,73 @@ +:: |close-flows: sends a %cork to all ships that have nacked a %watch +:: +:: It runs in dry mode by default, printing the number of flows that +:: can be closed. To actually send the corks, run as: +:: +:: |close-flows, =dry-run | +:: +:- %say +|= $: [now=@da eny=@uvJ bec=beak] + [arg=~ dry-run=?] + == +:: +=/ peers-map + .^ (map ship ?(%alien %known)) + %ax /(scot %p p.bec)//(scot %da now)/peers + == +=/ peers=(list ship) + %+ murn ~(tap by peers-map) + |= [=ship val=?(%alien %known)] + ?: =(ship p.bec) + ~ :: this is weird, but we saw it + ?- val + %alien ~ + %known (some ship) + == +:: +=; krocs=(list [=ship =bone]) + helm-ames-kroc+[krocs dry-run] +:: +%- zing +%+ turn peers +|= =ship +=+ .^ =ship-state:ames + %ax /(scot %p p.bec)//(scot %da now)/peers/(scot %p ship) + == +=/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) +=/ forward-flows=(set @) + %- ~(gas in *(set @)) + %+ turn + %+ sort ~(tap by by-bone.ossuary.peer-state) + |= [[a=@ *] [b=@ *]] (lth a b) + head +:: +=/ nacks=(list bone) + %+ turn + (sort ~(tap by rcv.peer-state) |=([[a=@ *] [b=@ *]] (lth a b))) + head +:: +^- (list [@p @ud]) +%+ murn nacks +|= =bone +?. &(=(0 (end 0 bone)) =(1 (end 0 (rsh 0 bone)))) + :: not a naxplanation ack bone + :: + ~ +:: by only corking forward flows that have received +:: a nack we avoid corking the current subscription +:: +=+ target=(mix 0b10 bone) +:: make sure that the nack bone has a forward flow +:: +?. (~(has in forward-flows) target) ~ +?~ duct=(~(get by by-bone.ossuary.peer-state) target) + ~ +=/ =wire (snag 1 u.duct) +:: only wires with nonces are subscriptions +:: +?. ?=([%gall %use sub=@ @ %out @ @ nonce=@ pub=@ ~] wire) + ~ +:: extra security check so we know that the nonce is a number +:: +?~ (rush i.t.t.t.t.t.t.t.wire dem) ~ +`[ship target] diff --git a/pkg/arvo/lib/hood/helm.hoon b/pkg/arvo/lib/hood/helm.hoon index 0c812cbe8..1529c0b94 100644 --- a/pkg/arvo/lib/hood/helm.hoon +++ b/pkg/arvo/lib/hood/helm.hoon @@ -245,6 +245,17 @@ |= ~ =< abet (emit %pass /helm %arvo %a %stir '') :: +++ poke-ames-kroc + |= [krocs=(list [=ship =bone]) dry=?] =< abet + ?~ krocs (flog %text "No %ames flow ready to be close") + %- emil ^- (list card) + :- [%pass /di %arvo %d %flog text+"Closing {<(lent krocs)>} ames flows"] + ?: dry ~ + %+ turn krocs + |= [=ship =bone] + ^- card + [%pass /helm %arvo %a %kroc ship bone] +:: ++ poke-knob |= [error-tag=@tas level=?(%hush %soft %loud)] =< abet (emit %pass /helm %arvo %d %knob error-tag level) @@ -280,6 +291,7 @@ %helm-ames-sift =;(f (f !<(_+<.f vase)) poke-ames-sift) %helm-ames-verb =;(f (f !<(_+<.f vase)) poke-ames-verb) %helm-ames-wake =;(f (f !<(_+<.f vase)) poke-ames-wake) + %helm-ames-kroc =;(f (f !<(_+<.f vase)) poke-ames-kroc) %helm-atom =;(f (f !<(_+<.f vase)) poke-atom) %helm-automass =;(f (f !<(_+<.f vase)) poke-automass) %helm-cancel-automass =;(f (f !<(_+<.f vase)) poke-cancel-automass) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index a0159b021..e8e6bf5b2 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -352,6 +352,7 @@ :: %heed: track peer's responsiveness; gives %clog if slow :: %jilt: stop tracking peer's responsiveness :: %cork: request to delete message flow + :: %kroc: request to delete message flow on a specific bone :: %plea: request to send message :: :: System and Lifecycle Tasks @@ -370,6 +371,7 @@ [%heed =ship] [%jilt =ship] [%cork =ship] + [%kroc =ship =bone] $>(%plea vane-task) :: $>(%born vane-task) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 7404aaa18..71c422ddb 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1183,6 +1183,7 @@ %vega on-vega:event-core %plea (on-plea:event-core [ship plea]:task) %cork (on-cork:event-core ship.task) + %kroc (on-kroc:event-core ship.task bone.task) == :: [moves ames-gate] @@ -1908,6 +1909,31 @@ =/ rcvr [ship her-life.channel] "cork plea {}" abet:(on-memo:(make-peer-core peer-state channel) bone plea %plea) + :: +on-kroc: explicitly cork a flow on the provided bone + :: TODO: refactor + :: + ++ on-kroc + |= [=ship =bone] + ^+ event-core + =/ =plea [%$ /flow [%cork ~]] + =/ ship-state (~(get by peers.ames-state) ship) + ?. ?=([~ %known *] ship-state) + %+ enqueue-alien-todo ship + |= todos=alien-agenda + todos(messages [[duct plea] messages.todos]) + =/ =peer-state +.u.ship-state + ?. (~(has by by-bone.ossuary.peer-state) bone) + ~& >>> %dont-have-it^ship^bone + event-core + =/ =channel [[our ship] now channel-state -.peer-state] + :: + =. closing.peer-state (~(put in closing.peer-state) bone) + %- %^ trace msg.veb ship + |. ^- tape + =/ sndr [our our-life.channel] + =/ rcvr [ship her-life.channel] + "cork plea {}" + abet:(on-memo:(make-peer-core peer-state channel) bone plea %plea) :: +on-take-wake: receive wakeup or error notification from behn :: ++ on-take-wake From 05582d12c284caf44ca0626199953953e39dd5be Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 5 Dec 2022 11:47:48 +0100 Subject: [PATCH 10/67] ames: refactor +on-krock --- pkg/arvo/sys/vane/ames.hoon | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 71c422ddb..bcbf04eb5 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1449,6 +1449,7 @@ ~% %event-gate ..per-event ~ |= [[now=@da eny=@ rof=roof] =duct =ames-state] =* veb veb.bug.ames-state + =| cork-bone=(unit bone) :: modified by +on-kroc ~% %event-core ..$ ~ |% ++ event-core . @@ -1900,7 +1901,15 @@ =/ =peer-state +.u.ship-state =/ =channel [[our ship] now channel-state -.peer-state] :: - =^ =bone ossuary.peer-state (bind-duct ossuary.peer-state duct) + =/ [=bone ossuary=_ossuary.peer-state] + ?^ cork-bone [u.cork-bone ossuary.peer-state] + (bind-duct ossuary.peer-state duct) + =. ossuary.peer-state ossuary + :: + ?. (~(has by by-bone.ossuary.peer-state) bone) + %. event-core + %^ trace odd.veb ship + |.("trying to cork {}, not in the ossuary, ignoring") :: =. closing.peer-state (~(put in closing.peer-state) bone) %- %^ trace msg.veb ship @@ -1915,25 +1924,7 @@ ++ on-kroc |= [=ship =bone] ^+ event-core - =/ =plea [%$ /flow [%cork ~]] - =/ ship-state (~(get by peers.ames-state) ship) - ?. ?=([~ %known *] ship-state) - %+ enqueue-alien-todo ship - |= todos=alien-agenda - todos(messages [[duct plea] messages.todos]) - =/ =peer-state +.u.ship-state - ?. (~(has by by-bone.ossuary.peer-state) bone) - ~& >>> %dont-have-it^ship^bone - event-core - =/ =channel [[our ship] now channel-state -.peer-state] - :: - =. closing.peer-state (~(put in closing.peer-state) bone) - %- %^ trace msg.veb ship - |. ^- tape - =/ sndr [our our-life.channel] - =/ rcvr [ship her-life.channel] - "cork plea {}" - abet:(on-memo:(make-peer-core peer-state channel) bone plea %plea) + (%*(on-cork . cork-bone `bone) ship) :: +on-take-wake: receive wakeup or error notification from behn :: ++ on-take-wake From 8c17e5c32dd9b16b6f685580e34ebe0838c6543a Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 5 Dec 2022 12:36:37 +0100 Subject: [PATCH 11/67] ames: in |close-flows inspect wires in the duct we now don't assume where the wire is in the duct, and just pattern match until we find a subscription wire --- pkg/arvo/gen/hood/close-flows.hoon | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pkg/arvo/gen/hood/close-flows.hoon b/pkg/arvo/gen/hood/close-flows.hoon index 1a32d2fef..d1cf409f2 100644 --- a/pkg/arvo/gen/hood/close-flows.hoon +++ b/pkg/arvo/gen/hood/close-flows.hoon @@ -62,12 +62,18 @@ ?. (~(has in forward-flows) target) ~ ?~ duct=(~(get by by-bone.ossuary.peer-state) target) ~ -=/ =wire (snag 1 u.duct) -:: only wires with nonces are subscriptions +=; =wire + ?~(wire ~ `[ship target]) +:: TMI :: -?. ?=([%gall %use sub=@ @ %out @ @ nonce=@ pub=@ ~] wire) - ~ +=> .(duct `^duct`duct) +|- ^- wire +?~ duct ~ +:: inspect the wires in the duct, looking for subscriptions +:: +?. ?=([%gall %use sub=@ @ %out @ @ nonce=@ pub=@ ~] i.duct) + $(duct t.duct) :: extra security check so we know that the nonce is a number :: -?~ (rush i.t.t.t.t.t.t.t.wire dem) ~ -`[ship target] +?~ (rush i.t.t.t.t.t.t.t.i.duct dem) ~ +i.duct From 7c26716babe4de6e38cab66d2b3546ab7eb48960 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 5 Dec 2022 12:42:27 +0100 Subject: [PATCH 12/67] ames: remove stale comment --- pkg/arvo/sys/vane/ames.hoon | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index bcbf04eb5..76ecf528c 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1919,7 +1919,6 @@ "cork plea {}" abet:(on-memo:(make-peer-core peer-state channel) bone plea %plea) :: +on-kroc: explicitly cork a flow on the provided bone - :: TODO: refactor :: ++ on-kroc |= [=ship =bone] From 2c791daf0904a92854e43afbd420a94bc0ff195e Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 5 Dec 2022 13:09:12 +0100 Subject: [PATCH 13/67] ames: in |close-flows fix bug matching subs wire --- pkg/arvo/gen/hood/close-flows.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/gen/hood/close-flows.hoon b/pkg/arvo/gen/hood/close-flows.hoon index d1cf409f2..c47000a47 100644 --- a/pkg/arvo/gen/hood/close-flows.hoon +++ b/pkg/arvo/gen/hood/close-flows.hoon @@ -71,7 +71,7 @@ ?~ duct ~ :: inspect the wires in the duct, looking for subscriptions :: -?. ?=([%gall %use sub=@ @ %out @ @ nonce=@ pub=@ ~] i.duct) +?. ?=([%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] i.duct) $(duct t.duct) :: extra security check so we know that the nonce is a number :: From 2ebab4938ba158e98f65de46bcb66dd3b97e365e Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Tue, 6 Dec 2022 07:22:14 +0100 Subject: [PATCH 14/67] ames: address feedback in |close-flows --- pkg/arvo/gen/hood/close-flows.hoon | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/arvo/gen/hood/close-flows.hoon b/pkg/arvo/gen/hood/close-flows.hoon index c47000a47..5e089dbca 100644 --- a/pkg/arvo/gen/hood/close-flows.hoon +++ b/pkg/arvo/gen/hood/close-flows.hoon @@ -25,7 +25,7 @@ == :: =; krocs=(list [=ship =bone]) - helm-ames-kroc+[krocs dry-run] + helm-ames-kroc+[krocs dry-run] :: %- zing %+ turn peers @@ -38,12 +38,12 @@ %- ~(gas in *(set @)) %+ turn %+ sort ~(tap by by-bone.ossuary.peer-state) - |= [[a=@ *] [b=@ *]] (lth a b) + |= [[a=@ *] [b=@ *]] (lte a b) head :: =/ nacks=(list bone) %+ turn - (sort ~(tap by rcv.peer-state) |=([[a=@ *] [b=@ *]] (lth a b))) + (sort ~(tap by rcv.peer-state) |=([[a=@ *] [b=@ *]] (lte a b))) head :: ^- (list [@p @ud]) From 7d4959db2ee9298ff3fc255b5f219d363df53cba Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Tue, 6 Dec 2022 10:07:06 +0100 Subject: [PATCH 15/67] ames: don't send %kroc cards in |close-flows --- pkg/arvo/lib/hood/helm.hoon | 15 ++++++++++----- pkg/arvo/sys/lull.hoon | 1 - pkg/arvo/sys/vane/ames.hoon | 18 +----------------- 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/pkg/arvo/lib/hood/helm.hoon b/pkg/arvo/lib/hood/helm.hoon index 1529c0b94..2ef20fc01 100644 --- a/pkg/arvo/lib/hood/helm.hoon +++ b/pkg/arvo/lib/hood/helm.hoon @@ -249,12 +249,17 @@ |= [krocs=(list [=ship =bone]) dry=?] =< abet ?~ krocs (flog %text "No %ames flow ready to be close") %- emil ^- (list card) + :: TODO: extend closing flow overview + :: :- [%pass /di %arvo %d %flog text+"Closing {<(lent krocs)>} ames flows"] - ?: dry ~ - %+ turn krocs - |= [=ship =bone] - ^- card - [%pass /helm %arvo %a %kroc ship bone] + :: XX uncomment when updating %lull to include a %kroc task to ames + :: + ~ + :: ?: dry ~ + :: %+ turn krocs + :: |= [=ship =bone] + :: ^- card + :: [%pass /helm %arvo %a %kroc ship bone] :: ++ poke-knob |= [error-tag=@tas level=?(%hush %soft %loud)] =< abet diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index e8e6bf5b2..8ef1331a0 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -371,7 +371,6 @@ [%heed =ship] [%jilt =ship] [%cork =ship] - [%kroc =ship =bone] $>(%plea vane-task) :: $>(%born vane-task) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 76ecf528c..b6cabd5a3 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1183,7 +1183,6 @@ %vega on-vega:event-core %plea (on-plea:event-core [ship plea]:task) %cork (on-cork:event-core ship.task) - %kroc (on-kroc:event-core ship.task bone.task) == :: [moves ames-gate] @@ -1449,7 +1448,6 @@ ~% %event-gate ..per-event ~ |= [[now=@da eny=@ rof=roof] =duct =ames-state] =* veb veb.bug.ames-state - =| cork-bone=(unit bone) :: modified by +on-kroc ~% %event-core ..$ ~ |% ++ event-core . @@ -1901,15 +1899,7 @@ =/ =peer-state +.u.ship-state =/ =channel [[our ship] now channel-state -.peer-state] :: - =/ [=bone ossuary=_ossuary.peer-state] - ?^ cork-bone [u.cork-bone ossuary.peer-state] - (bind-duct ossuary.peer-state duct) - =. ossuary.peer-state ossuary - :: - ?. (~(has by by-bone.ossuary.peer-state) bone) - %. event-core - %^ trace odd.veb ship - |.("trying to cork {}, not in the ossuary, ignoring") + =^ bone ossuary.peer-state (bind-duct ossuary.peer-state duct) :: =. closing.peer-state (~(put in closing.peer-state) bone) %- %^ trace msg.veb ship @@ -1918,12 +1908,6 @@ =/ rcvr [ship her-life.channel] "cork plea {}" abet:(on-memo:(make-peer-core peer-state channel) bone plea %plea) - :: +on-kroc: explicitly cork a flow on the provided bone - :: - ++ on-kroc - |= [=ship =bone] - ^+ event-core - (%*(on-cork . cork-bone `bone) ship) :: +on-take-wake: receive wakeup or error notification from behn :: ++ on-take-wake From 81f01c456bc572faf2c8e1989c684cc3ef138101 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Thu, 8 Dec 2022 15:01:09 +0100 Subject: [PATCH 16/67] ames: clean up unused logic in |close-flows --- pkg/arvo/gen/hood/close-flows.hoon | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/pkg/arvo/gen/hood/close-flows.hoon b/pkg/arvo/gen/hood/close-flows.hoon index 5e089dbca..4c0cd45c8 100644 --- a/pkg/arvo/gen/hood/close-flows.hoon +++ b/pkg/arvo/gen/hood/close-flows.hoon @@ -34,21 +34,10 @@ %ax /(scot %p p.bec)//(scot %da now)/peers/(scot %p ship) == =/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) -=/ forward-flows=(set @) - %- ~(gas in *(set @)) - %+ turn - %+ sort ~(tap by by-bone.ossuary.peer-state) - |= [[a=@ *] [b=@ *]] (lte a b) - head -:: -=/ nacks=(list bone) - %+ turn - (sort ~(tap by rcv.peer-state) |=([[a=@ *] [b=@ *]] (lte a b))) - head :: ^- (list [@p @ud]) -%+ murn nacks -|= =bone +%+ murn ~(tap by rcv.peer-state) +|= [=bone *] ?. &(=(0 (end 0 bone)) =(1 (end 0 (rsh 0 bone)))) :: not a naxplanation ack bone :: @@ -59,7 +48,6 @@ =+ target=(mix 0b10 bone) :: make sure that the nack bone has a forward flow :: -?. (~(has in forward-flows) target) ~ ?~ duct=(~(get by by-bone.ossuary.peer-state) target) ~ =; =wire From 872db0efcc09f1f4efbebbfb06b002cced5ffccd Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Fri, 9 Dec 2022 16:13:41 +0100 Subject: [PATCH 17/67] ames, +close-flows: combine stale flows This combines two types of flows that can be closed: - nacking %watch pleas - resubscriptions that were not properly corked --- pkg/arvo/gen/hood/close-flows.hoon | 121 +++++++++++++++++++---------- pkg/arvo/lib/hood/helm.hoon | 2 +- pkg/arvo/sys/vane/gall.hoon | 13 ++++ 3 files changed, 93 insertions(+), 43 deletions(-) diff --git a/pkg/arvo/gen/hood/close-flows.hoon b/pkg/arvo/gen/hood/close-flows.hoon index 4c0cd45c8..012680916 100644 --- a/pkg/arvo/gen/hood/close-flows.hoon +++ b/pkg/arvo/gen/hood/close-flows.hoon @@ -1,4 +1,11 @@ -:: |close-flows: sends a %cork to all ships that have nacked a %watch +:: (TODO: rename) |close-flows: shows how many stale ames flows can be closed +:: (each ames flow is represented by its first bone) +:: +:: Run in verbose mode my default, to turn of: |close-flows, =veb | +:: +:: TODO: +:: corks all ames flows that are in a stale state due to some +:: previous unhandled errors :: :: It runs in dry mode by default, printing the number of flows that :: can be closed. To actually send the corks, run as: @@ -7,13 +14,12 @@ :: :- %say |= $: [now=@da eny=@uvJ bec=beak] - [arg=~ dry-run=?] + [arg=~ dry=? veb=?] == :: =/ peers-map - .^ (map ship ?(%alien %known)) - %ax /(scot %p p.bec)//(scot %da now)/peers - == + .^((map ship ?(%alien %known)) %ax /(scot %p p.bec)//(scot %da now)/peers) +:: =/ peers=(list ship) %+ murn ~(tap by peers-map) |= [=ship val=?(%alien %known)] @@ -24,44 +30,75 @@ %known (some ship) == :: -=; krocs=(list [=ship =bone]) - helm-ames-kroc+[krocs dry-run] +=/ agents-map + .^ (list [app=term sub-nonce=@ud run-nonce=@t =boat:gall =boar:gall]) + %gr (scot %p p.bec) %base (scot %da now) ~ + == :: -%- zing -%+ turn peers -|= =ship -=+ .^ =ship-state:ames - %ax /(scot %p p.bec)//(scot %da now)/peers/(scot %p ship) - == -=/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) +|^ ~? veb "{<~(wyt in nacks)>} flows from %nacking %watches" + ~? veb "{<~(wyt in resubs)>} flows from stale resubscriptions" + :+ %helm-ames-kroc dry + ~(tap in (~(uni in nacks) resubs)) :: -^- (list [@p @ud]) -%+ murn ~(tap by rcv.peer-state) -|= [=bone *] -?. &(=(0 (end 0 bone)) =(1 (end 0 (rsh 0 bone)))) - :: not a naxplanation ack bone +++ resubs + =+ peers=(~(gas in *(set ship)) peers) + %+ roll agents-map + |= $: [app=term next-nonce=@ run-nonce=@t =boat:gall =boar:gall] + resubs=(set [ship bone]) + == + =/ subs ~(tap in ~(key by boat)) + ?~ subs resubs + =/ [=wire =dock] i.subs + =/ sub=@ud (~(got by boar) wire dock) + :: ignores subscription with nonce=0, this is handled by +ap-rake + :: XX TODO handle also here? :: - ~ -:: by only corking forward flows that have received -:: a nack we avoid corking the current subscription + ?: =(sub 0) resubs + :: skip current subscription + :: + ?. (lth sub (dec next-nonce)) resubs + ?: !(~(has in peers) p.dock) resubs + =+ .^ =ship-state:ames + %ax /(scot %p p.bec)//(scot %da now)/peers/(scot %p p.dock) + == + =/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) + %- ~(rep by by-duct.ossuary.peer-state) + |= [[=duct =bone] resubs=_resubs] + =+ sub-nonce=(scot %ud sub) + =/ mod=^wire + :* %gall %use app run-nonce + %out (scot %p p.dock) q.dock + sub-nonce wire + == + ?. ?& ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] duct) + =(mod i.t.duct(i.t.t.t.t.t.t.t sub-nonce)) + == + resubs + (~(put in resubs) [p.dock bone]) :: -=+ target=(mix 0b10 bone) -:: make sure that the nack bone has a forward flow -:: -?~ duct=(~(get by by-bone.ossuary.peer-state) target) - ~ -=; =wire - ?~(wire ~ `[ship target]) -:: TMI -:: -=> .(duct `^duct`duct) -|- ^- wire -?~ duct ~ -:: inspect the wires in the duct, looking for subscriptions -:: -?. ?=([%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] i.duct) - $(duct t.duct) -:: extra security check so we know that the nonce is a number -:: -?~ (rush i.t.t.t.t.t.t.t.i.duct dem) ~ -i.duct +++ nacks + %+ roll peers + |= [=ship nacks=(set [ship bone])] + =+ .^ =ship-state:ames + %ax /(scot %p p.bec)//(scot %da now)/peers/(scot %p ship) + == + =/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) + :: + %+ roll ~(tap by rcv.peer-state) + |= [[=bone *] nacks=_nacks] + ?. &(=(0 (end 0 bone)) =(1 (end 0 (rsh 0 bone)))) + :: not a naxplanation ack bone + :: + nacks + :: by only corking forward flows that have received + :: a nack we avoid corking the current subscription + :: + =+ target=(mix 0b10 bone) + :: make sure that the nack bone has a forward flow + :: + ?~ duct=(~(get by by-bone.ossuary.peer-state) target) + nacks + ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] duct) + nacks + (~(put in nacks) [ship target]) +-- diff --git a/pkg/arvo/lib/hood/helm.hoon b/pkg/arvo/lib/hood/helm.hoon index 2ef20fc01..a00844fb2 100644 --- a/pkg/arvo/lib/hood/helm.hoon +++ b/pkg/arvo/lib/hood/helm.hoon @@ -246,7 +246,7 @@ (emit %pass /helm %arvo %a %stir '') :: ++ poke-ames-kroc - |= [krocs=(list [=ship =bone]) dry=?] =< abet + |= [dry=? krocs=(list [=ship =bone])] =< abet ?~ krocs (flog %text "No %ames flow ready to be close") %- emil ^- (list card) :: TODO: extend closing flow overview diff --git a/pkg/arvo/sys/vane/gall.hoon b/pkg/arvo/sys/vane/gall.hoon index 8ce10c655..00b1bff58 100644 --- a/pkg/arvo/sys/vane/gall.hoon +++ b/pkg/arvo/sys/vane/gall.hoon @@ -1989,6 +1989,19 @@ [~ ~] [~ ~ atom+!>(u.nonce)] :: + ?: ?& =(%r care) + ?=(~ path) + =([%$ %da now] coin) + =(our ship) + == + =| agents=(list [app=term sub-nonce=@ud run-nonce=@t =boat =boar]) + :+ ~ ~ + :- %noun !> ^+ agents + %+ roll ~(tap by yokes.state) + |= [[agent=term =yoke] agents=_agents] + :_ agents + [agent [sub-nonce run-nonce boat boar]:yoke] + :: ?. =(our ship) ~ ?. =([%$ %da now] coin) From ad712caccf55f778858c8c8583e3d34d2832c02c Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 12 Dec 2022 13:45:54 +0100 Subject: [PATCH 18/67] ames, gall: move |close-flows logic to vanes This modifies the %rake task in %gall, to select what kind of subscriptions we try to close: =mode %o: kill old pre-nonce subscriptions =mode %z: kill old pre-nonce subscriptions, including sub-nonce = 0 =mode %r: kills all stale resubscription flows It also adds a dry-run option to both tasks (%kroc in ames, %rake in gall) --- pkg/arvo/gen/hood/close-flows.hoon | 102 ++--------------------------- pkg/arvo/lib/hood/helm.hoon | 21 ++---- pkg/arvo/sys/lull.hoon | 5 +- pkg/arvo/sys/vane/ames.hoon | 42 +++++++++++- pkg/arvo/sys/vane/gall.hoon | 31 ++++++--- 5 files changed, 77 insertions(+), 124 deletions(-) diff --git a/pkg/arvo/gen/hood/close-flows.hoon b/pkg/arvo/gen/hood/close-flows.hoon index 012680916..521fc1272 100644 --- a/pkg/arvo/gen/hood/close-flows.hoon +++ b/pkg/arvo/gen/hood/close-flows.hoon @@ -1,104 +1,12 @@ -:: (TODO: rename) |close-flows: shows how many stale ames flows can be closed -:: (each ames flow is represented by its first bone) +:: |close-flows: corks all stale ames flows :: -:: Run in verbose mode my default, to turn of: |close-flows, =veb | +:: It runs in dry mode by default, printing the flows that can be closed: :: -:: TODO: -:: corks all ames flows that are in a stale state due to some -:: previous unhandled errors -:: -:: It runs in dry mode by default, printing the number of flows that -:: can be closed. To actually send the corks, run as: -:: -:: |close-flows, =dry-run | +:: |close-flows, =dry | :: :- %say |= $: [now=@da eny=@uvJ bec=beak] - [arg=~ dry=? veb=?] + [arg=~ dry=?] == :: -=/ peers-map - .^((map ship ?(%alien %known)) %ax /(scot %p p.bec)//(scot %da now)/peers) -:: -=/ peers=(list ship) - %+ murn ~(tap by peers-map) - |= [=ship val=?(%alien %known)] - ?: =(ship p.bec) - ~ :: this is weird, but we saw it - ?- val - %alien ~ - %known (some ship) - == -:: -=/ agents-map - .^ (list [app=term sub-nonce=@ud run-nonce=@t =boat:gall =boar:gall]) - %gr (scot %p p.bec) %base (scot %da now) ~ - == -:: -|^ ~? veb "{<~(wyt in nacks)>} flows from %nacking %watches" - ~? veb "{<~(wyt in resubs)>} flows from stale resubscriptions" - :+ %helm-ames-kroc dry - ~(tap in (~(uni in nacks) resubs)) -:: -++ resubs - =+ peers=(~(gas in *(set ship)) peers) - %+ roll agents-map - |= $: [app=term next-nonce=@ run-nonce=@t =boat:gall =boar:gall] - resubs=(set [ship bone]) - == - =/ subs ~(tap in ~(key by boat)) - ?~ subs resubs - =/ [=wire =dock] i.subs - =/ sub=@ud (~(got by boar) wire dock) - :: ignores subscription with nonce=0, this is handled by +ap-rake - :: XX TODO handle also here? - :: - ?: =(sub 0) resubs - :: skip current subscription - :: - ?. (lth sub (dec next-nonce)) resubs - ?: !(~(has in peers) p.dock) resubs - =+ .^ =ship-state:ames - %ax /(scot %p p.bec)//(scot %da now)/peers/(scot %p p.dock) - == - =/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) - %- ~(rep by by-duct.ossuary.peer-state) - |= [[=duct =bone] resubs=_resubs] - =+ sub-nonce=(scot %ud sub) - =/ mod=^wire - :* %gall %use app run-nonce - %out (scot %p p.dock) q.dock - sub-nonce wire - == - ?. ?& ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] duct) - =(mod i.t.duct(i.t.t.t.t.t.t.t sub-nonce)) - == - resubs - (~(put in resubs) [p.dock bone]) -:: -++ nacks - %+ roll peers - |= [=ship nacks=(set [ship bone])] - =+ .^ =ship-state:ames - %ax /(scot %p p.bec)//(scot %da now)/peers/(scot %p ship) - == - =/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) - :: - %+ roll ~(tap by rcv.peer-state) - |= [[=bone *] nacks=_nacks] - ?. &(=(0 (end 0 bone)) =(1 (end 0 (rsh 0 bone)))) - :: not a naxplanation ack bone - :: - nacks - :: by only corking forward flows that have received - :: a nack we avoid corking the current subscription - :: - =+ target=(mix 0b10 bone) - :: make sure that the nack bone has a forward flow - :: - ?~ duct=(~(get by by-bone.ossuary.peer-state) target) - nacks - ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] duct) - nacks - (~(put in nacks) [ship target]) --- +[%helm-kroc dry] diff --git a/pkg/arvo/lib/hood/helm.hoon b/pkg/arvo/lib/hood/helm.hoon index a00844fb2..8144921f8 100644 --- a/pkg/arvo/lib/hood/helm.hoon +++ b/pkg/arvo/lib/hood/helm.hoon @@ -245,21 +245,12 @@ |= ~ =< abet (emit %pass /helm %arvo %a %stir '') :: -++ poke-ames-kroc - |= [dry=? krocs=(list [=ship =bone])] =< abet - ?~ krocs (flog %text "No %ames flow ready to be close") +++ poke-kroc + |= dry=? =< abet %- emil ^- (list card) - :: TODO: extend closing flow overview - :: - :- [%pass /di %arvo %d %flog text+"Closing {<(lent krocs)>} ames flows"] - :: XX uncomment when updating %lull to include a %kroc task to ames - :: - ~ - :: ?: dry ~ - :: %+ turn krocs - :: |= [=ship =bone] - :: ^- card - :: [%pass /helm %arvo %a %kroc ship bone] + :~ [%pass /helm/kroc %arvo %a %kroc dry] + [%pass /helm/rake %arvo %g %rake ~ %r dry] + == :: ++ poke-knob |= [error-tag=@tas level=?(%hush %soft %loud)] =< abet @@ -296,7 +287,7 @@ %helm-ames-sift =;(f (f !<(_+<.f vase)) poke-ames-sift) %helm-ames-verb =;(f (f !<(_+<.f vase)) poke-ames-verb) %helm-ames-wake =;(f (f !<(_+<.f vase)) poke-ames-wake) - %helm-ames-kroc =;(f (f !<(_+<.f vase)) poke-ames-kroc) + %helm-kroc =;(f (f !<(_+<.f vase)) poke-kroc) %helm-atom =;(f (f !<(_+<.f vase)) poke-atom) %helm-automass =;(f (f !<(_+<.f vase)) poke-automass) %helm-cancel-automass =;(f (f !<(_+<.f vase)) poke-cancel-automass) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 8ef1331a0..78e2e697a 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -352,7 +352,7 @@ :: %heed: track peer's responsiveness; gives %clog if slow :: %jilt: stop tracking peer's responsiveness :: %cork: request to delete message flow - :: %kroc: request to delete message flow on a specific bone + :: %kroc: request to delete stale message flows :: %plea: request to send message :: :: System and Lifecycle Tasks @@ -371,6 +371,7 @@ [%heed =ship] [%jilt =ship] [%cork =ship] + [%kroc dry=?] $>(%plea vane-task) :: $>(%born vane-task) @@ -1809,7 +1810,7 @@ [%load =load] :: load agent [%nuke =dude] :: delete agent [%doff dude=(unit dude) ship=(unit ship)] :: kill subscriptions - [%rake dude=(unit dude) all=?] :: reclaim old subs + [%rake dude=(unit dude) mode=?(%o %z %r) dry=?] :: reclaim old subs $>(%init vane-task) :: set owner $>(%trim vane-task) :: trim state $>(%vega vane-task) :: report upgrade diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index b6cabd5a3..bd1a61436 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1183,6 +1183,7 @@ %vega on-vega:event-core %plea (on-plea:event-core [ship plea]:task) %cork (on-cork:event-core ship.task) + %kroc (on-kroc:event-core dry.task) == :: [moves ames-gate] @@ -1448,6 +1449,7 @@ ~% %event-gate ..per-event ~ |= [[now=@da eny=@ rof=roof] =duct =ames-state] =* veb veb.bug.ames-state + =| cork-bone=(unit bone) :: modified by +on-kroc ~% %event-core ..$ ~ |% ++ event-core . @@ -1899,7 +1901,15 @@ =/ =peer-state +.u.ship-state =/ =channel [[our ship] now channel-state -.peer-state] :: - =^ bone ossuary.peer-state (bind-duct ossuary.peer-state duct) + =/ [=bone ossuary=_ossuary.peer-state] + ?^ cork-bone [u.cork-bone ossuary.peer-state] + (bind-duct ossuary.peer-state duct) + =. ossuary.peer-state ossuary + :: + ?. (~(has by by-bone.ossuary.peer-state) bone) + %. event-core + %^ trace odd.veb ship + |.("trying to cork {}, not in the ossuary, ignoring") :: =. closing.peer-state (~(put in closing.peer-state) bone) %- %^ trace msg.veb ship @@ -1908,6 +1918,36 @@ =/ rcvr [ship her-life.channel] "cork plea {}" abet:(on-memo:(make-peer-core peer-state channel) bone plea %plea) + :: +on-kroc: cork all stale flows from failed subscriptions + :: + ++ on-kroc + |= dry=? + ^+ event-core + %+ roll ~(tap by peers.ames-state) + |= [[=ship =ship-state] core=_event-core] + ?. ?=(%known -.ship-state) core + =/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) + :: + %+ roll ~(tap by rcv.peer-state) + |= [[=bone *] core=_core] + ?. &(=(0 (end 0 bone)) =(1 (end 0 (rsh 0 bone)))) + :: not a naxplanation ack bone + :: + core + :: by only corking forward flows that have received + :: a nack we avoid corking the current subscription + :: + =+ target=(mix 0b10 bone) + :: make sure that the nack bone has a forward flow + :: + ?~ duct=(~(get by by-bone.ossuary.peer-state) target) + core + ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) + core + %- %^ trace |(dry odd.veb) ship + |.("kroc failed %watch plea {}") + ?: dry core + (%*(on-cork core cork-bone `target) ship) :: +on-take-wake: receive wakeup or error notification from behn :: ++ on-take-wake diff --git a/pkg/arvo/sys/vane/gall.hoon b/pkg/arvo/sys/vane/gall.hoon index 00b1bff58..f79abf45c 100644 --- a/pkg/arvo/sys/vane/gall.hoon +++ b/pkg/arvo/sys/vane/gall.hoon @@ -244,7 +244,7 @@ :: +mo-rake: send %cork's for old subscriptions if needed :: ++ mo-rake - |= [dude=(unit dude) all=?] + |= [dude=(unit dude) mode=?(%o %z %r) dry=?] ^+ mo-core =/ apps=(list (pair term yoke)) ?~ dude ~(tap by yokes.state) @@ -252,7 +252,7 @@ |- ^+ mo-core ?~ apps mo-core =/ ap-core (ap-yoke:ap p.i.apps [~ our] q.i.apps) - $(apps t.apps, mo-core ap-abet:(ap-rake:ap-core all)) + $(apps t.apps, mo-core ap-abet:(ap-rake:ap-core mode dry)) :: +mo-receive-core: receives an app core built by %ford. :: :: Presuming we receive a good core, we first check to see if the agent @@ -1512,36 +1512,49 @@ =? ap-core (gte 1 (~(got by boar.yoke) wyr dok)) (ap-pass wyr %agent dok %leave ~) $(subs t.subs) - :: +ap-rake: clean up the dead %leave's + :: +ap-rake: clean up the dead subscriptions + :: + :: =mode %o: kill old pre-nonce subscriptions + :: =mode %z: kill old pre-nonce subscriptions, including sub-nonce = 0 + :: =mode %r: kills all stale resubscription flows :: ++ ap-rake - |= all=? + |= [mode=?(%o %z %r) dry=?] =/ subs ~(tap in ~(key by boat.yoke)) |^ ^+ ap-core ?~ subs ap-core =/ [=wire =dock] i.subs - =/ non (~(got by boar.yoke) wire dock) - ?: &(!all =(0 non)) + =/ non=@ud (~(got by boar.yoke) wire dock) + ~& [mode sub-nonce=sub-nonce.yoke nonce=non] + ?. &(=(%zer mode) =(0 non)) $(subs t.subs) ?~ per=(scry-peer-state p.dock) $(subs t.subs) + :: skip current subscription in %res mode :: + ?. ?& =(%res mode) + !=(sub-nonce.yoke 0) + (lth non (dec sub-nonce.yoke)) + == + $(subs t.subs) + :: + =/ sub-nonce=@t ?.(=(%res mode) '0' (scot %ud non)) =/ dud=(set duct) =/ mod=^wire :* %gall %use agent-name run-nonce.yoke %out (scot %p p.dock) q.dock - '0' wire + sub-nonce wire == %- ~(rep by by-duct.ossuary.u.per) |= [[=duct =bone] out=(set duct)] ^+ out ?. ?& ?=([* [%gall %use @ @ %out @ @ @ *] *] duct) - =(mod i.t.duct(i.t.t.t.t.t.t.t '0')) + =(mod i.t.duct(i.t.t.t.t.t.t.t sub-nonce)) == out ?: (~(has in closing.u.per) bone) out ~> %slog.0^leaf+"gall: rake {}" - (~(put in out) duct) + ?:(dry out (~(put in out) duct)) :: %- ap-move (turn ~(tap in dud) |=(d=duct [+.d %pass -.d %a %cork p.dock])) From 1e04e9498e07f2edffbfd07b07fcef3d0ce51bab Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Sat, 7 Jan 2023 10:14:01 +0100 Subject: [PATCH 19/67] ames: clean all stale %watches in on-kroc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This removes the logic from cleaning up stale subscriptions in %gall, leaving +ap-rake as it was, and moves it to the +on-kroc arm in %ames. Failed subscriptions from nacking a %watch plea that were not properly corked (fixed in https://github.com/urbit/urbit/pull/6102) are a subset of the more general "stale re-subscription" issue, so we take care of all stale flows at the same time, by focusing on the current subscription—leaving all others to be corked automatically—and checking if it received a nack, to subsequently cork it. --- pkg/arvo/sys/lull.hoon | 2 +- pkg/arvo/sys/vane/ames.hoon | 55 +++++++++++++++++++++++++------------ pkg/arvo/sys/vane/gall.hoon | 44 ++++++----------------------- 3 files changed, 48 insertions(+), 53 deletions(-) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 78e2e697a..e05500f74 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -1810,7 +1810,7 @@ [%load =load] :: load agent [%nuke =dude] :: delete agent [%doff dude=(unit dude) ship=(unit ship)] :: kill subscriptions - [%rake dude=(unit dude) mode=?(%o %z %r) dry=?] :: reclaim old subs + [%rake dude=(unit dude) all=?] :: reclaim old subs $>(%init vane-task) :: set owner $>(%trim vane-task) :: trim state $>(%vega vane-task) :: report upgrade diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index bd1a61436..2175d02b7 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1918,7 +1918,7 @@ =/ rcvr [ship her-life.channel] "cork plea {}" abet:(on-memo:(make-peer-core peer-state channel) bone plea %plea) - :: +on-kroc: cork all stale flows from failed subscriptions + :: +on-kroc: cork all flows from failed subscriptions :: ++ on-kroc |= dry=? @@ -1927,27 +1927,48 @@ |= [[=ship =ship-state] core=_event-core] ?. ?=(%known -.ship-state) core =/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) + =/ dudes ;; (map dude:gall nonce=@) + =< q.q %- need %- need + (rof ~ %gf `beam`[[our %$ da+now] /]) :: - %+ roll ~(tap by rcv.peer-state) - |= [[=bone *] core=_core] - ?. &(=(0 (end 0 bone)) =(1 (end 0 (rsh 0 bone)))) - :: not a naxplanation ack bone + =/ subs=(jar path [bone nonce=@]) + %+ roll ~(tap by snd.peer-state) + |= [[=bone *] subs=(jar path [bone nonce=@])] + ?~ duct=(~(get by by-bone.ossuary.peer-state) bone) subs + ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) + subs + =/ nonce=@ (rash i.t.t.t.t.t.t.t.i.t.u.duct dem) + =/ =wire i.t.u.duct + :: don't include the sub-nonce in the key :: - core - :: by only corking forward flows that have received - :: a nack we avoid corking the current subscription + (~(add ja subs) (weld (scag 7 wire) (slag 8 wire)) [bone nonce]) + %+ roll ~(tap by subs) + |= [[=wire flows=(list [bone @])] core=_core] + :: + %+ roll flows + |= [[=bone sub-nonce=@] core=_core] + ?: (~(has in closing.peer-state) bone) core + =/ app=term ?>(?=([%gall %use sub=@ *] wire) i.t.t.wire) + =/ nonce=@ud (dec (~(got by dudes) app)) + =/ =path (slag 7 wire) + =/ log=tape "[bone={} nonce={} agent={}] {}" + =; corkable=? + ?.(corkable core (%*(on-cork core cork-bone `bone) ship)) + :: check if the current subscription also failed + :: + ?. =(sub-nonce nonce) + %. !dry + (trace |(dry odd.veb) ship |.((weld "stale %watch plea " log))) + :: we can safely cork the current subscription + :: if it received a nack on backward bone :: =+ target=(mix 0b10 bone) - :: make sure that the nack bone has a forward flow + :: not a naxplanation ack bone :: - ?~ duct=(~(get by by-bone.ossuary.peer-state) target) - core - ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) - core - %- %^ trace |(dry odd.veb) ship - |.("kroc failed %watch plea {}") - ?: dry core - (%*(on-cork core cork-bone `target) ship) + ?. &(=(0 (end 0 target)) =(1 (end 0 (rsh 0 target)))) + | + %. !dry + (trace |(dry odd.veb) ship |.((weld "failed %watch plea " log))) :: +on-take-wake: receive wakeup or error notification from behn :: ++ on-take-wake diff --git a/pkg/arvo/sys/vane/gall.hoon b/pkg/arvo/sys/vane/gall.hoon index f79abf45c..8ce10c655 100644 --- a/pkg/arvo/sys/vane/gall.hoon +++ b/pkg/arvo/sys/vane/gall.hoon @@ -244,7 +244,7 @@ :: +mo-rake: send %cork's for old subscriptions if needed :: ++ mo-rake - |= [dude=(unit dude) mode=?(%o %z %r) dry=?] + |= [dude=(unit dude) all=?] ^+ mo-core =/ apps=(list (pair term yoke)) ?~ dude ~(tap by yokes.state) @@ -252,7 +252,7 @@ |- ^+ mo-core ?~ apps mo-core =/ ap-core (ap-yoke:ap p.i.apps [~ our] q.i.apps) - $(apps t.apps, mo-core ap-abet:(ap-rake:ap-core mode dry)) + $(apps t.apps, mo-core ap-abet:(ap-rake:ap-core all)) :: +mo-receive-core: receives an app core built by %ford. :: :: Presuming we receive a good core, we first check to see if the agent @@ -1512,49 +1512,36 @@ =? ap-core (gte 1 (~(got by boar.yoke) wyr dok)) (ap-pass wyr %agent dok %leave ~) $(subs t.subs) - :: +ap-rake: clean up the dead subscriptions - :: - :: =mode %o: kill old pre-nonce subscriptions - :: =mode %z: kill old pre-nonce subscriptions, including sub-nonce = 0 - :: =mode %r: kills all stale resubscription flows + :: +ap-rake: clean up the dead %leave's :: ++ ap-rake - |= [mode=?(%o %z %r) dry=?] + |= all=? =/ subs ~(tap in ~(key by boat.yoke)) |^ ^+ ap-core ?~ subs ap-core =/ [=wire =dock] i.subs - =/ non=@ud (~(got by boar.yoke) wire dock) - ~& [mode sub-nonce=sub-nonce.yoke nonce=non] - ?. &(=(%zer mode) =(0 non)) + =/ non (~(got by boar.yoke) wire dock) + ?: &(!all =(0 non)) $(subs t.subs) ?~ per=(scry-peer-state p.dock) $(subs t.subs) - :: skip current subscription in %res mode :: - ?. ?& =(%res mode) - !=(sub-nonce.yoke 0) - (lth non (dec sub-nonce.yoke)) - == - $(subs t.subs) - :: - =/ sub-nonce=@t ?.(=(%res mode) '0' (scot %ud non)) =/ dud=(set duct) =/ mod=^wire :* %gall %use agent-name run-nonce.yoke %out (scot %p p.dock) q.dock - sub-nonce wire + '0' wire == %- ~(rep by by-duct.ossuary.u.per) |= [[=duct =bone] out=(set duct)] ^+ out ?. ?& ?=([* [%gall %use @ @ %out @ @ @ *] *] duct) - =(mod i.t.duct(i.t.t.t.t.t.t.t sub-nonce)) + =(mod i.t.duct(i.t.t.t.t.t.t.t '0')) == out ?: (~(has in closing.u.per) bone) out ~> %slog.0^leaf+"gall: rake {}" - ?:(dry out (~(put in out) duct)) + (~(put in out) duct) :: %- ap-move (turn ~(tap in dud) |=(d=duct [+.d %pass -.d %a %cork p.dock])) @@ -2002,19 +1989,6 @@ [~ ~] [~ ~ atom+!>(u.nonce)] :: - ?: ?& =(%r care) - ?=(~ path) - =([%$ %da now] coin) - =(our ship) - == - =| agents=(list [app=term sub-nonce=@ud run-nonce=@t =boat =boar]) - :+ ~ ~ - :- %noun !> ^+ agents - %+ roll ~(tap by yokes.state) - |= [[agent=term =yoke] agents=_agents] - :_ agents - [agent [sub-nonce run-nonce boat boar]:yoke] - :: ?. =(our ship) ~ ?. =([%$ %da now] coin) From 1d8214530a7edb709dbe4d12df59d89c9c21e238 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Sat, 7 Jan 2023 17:42:59 +0100 Subject: [PATCH 20/67] ames: fix |close-flows generator --- bin/solid.pill | 4 ++-- pkg/arvo/gen/hood/close-flows.hoon | 9 +++------ pkg/arvo/lib/hood/helm.hoon | 5 +---- pkg/arvo/sys/vane/ames.hoon | 21 +++++++++++---------- 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index a029847bf..7353108f5 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:55f1bcccf861b3a2247aabb6ec3349829272ef8c36cfc8a16ff0930aac87a24d -size 5748196 +oid sha256:511564074c56329460e238ebb4881ff4dd9d642ac6d0804bbd9aaa17adc2f8ee +size 5732943 diff --git a/pkg/arvo/gen/hood/close-flows.hoon b/pkg/arvo/gen/hood/close-flows.hoon index 521fc1272..24fc605a3 100644 --- a/pkg/arvo/gen/hood/close-flows.hoon +++ b/pkg/arvo/gen/hood/close-flows.hoon @@ -1,12 +1,9 @@ :: |close-flows: corks all stale ames flows :: -:: It runs in dry mode by default, printing the flows that can be closed: -:: -:: |close-flows, =dry | +:: It runs in dry mode by default, printing the flows that can be closed. +:: To actually close the flows, run with |close-flows, =dry | :: :- %say -|= $: [now=@da eny=@uvJ bec=beak] - [arg=~ dry=?] - == +|= [^ [dry=?]] :: [%helm-kroc dry] diff --git a/pkg/arvo/lib/hood/helm.hoon b/pkg/arvo/lib/hood/helm.hoon index 8144921f8..34b68bad8 100644 --- a/pkg/arvo/lib/hood/helm.hoon +++ b/pkg/arvo/lib/hood/helm.hoon @@ -247,10 +247,7 @@ :: ++ poke-kroc |= dry=? =< abet - %- emil ^- (list card) - :~ [%pass /helm/kroc %arvo %a %kroc dry] - [%pass /helm/rake %arvo %g %rake ~ %r dry] - == + (emit [%pass /helm/kroc %arvo %a %kroc dry]) :: ++ poke-knob |= [error-tag=@tas level=?(%hush %soft %loud)] =< abet diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 2175d02b7..6b553cc14 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1931,19 +1931,22 @@ =< q.q %- need %- need (rof ~ %gf `beam`[[our %$ da+now] /]) :: - =/ subs=(jar path [bone nonce=@]) + =/ subs=(jar path [bone sub-nonce=@]) %+ roll ~(tap by snd.peer-state) - |= [[=bone *] subs=(jar path [bone nonce=@])] + |= [[=bone *] subs=(jar path [bone sub-nonce=@])] ?~ duct=(~(get by by-bone.ossuary.peer-state) bone) subs ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) subs - =/ nonce=@ (rash i.t.t.t.t.t.t.t.i.t.u.duct dem) - =/ =wire i.t.u.duct + =/ =wire i.t.u.duct + =/ nonce=(unit @) (rush i.t.t.t.t.t.t.t.i.t.u.duct dem) + %- ~(add ja subs) + :_ [bone ?~(nonce 0 u.nonce)] :: 0 for old pre-nonce subscriptions + ?~ nonce wire :: don't include the sub-nonce in the key :: - (~(add ja subs) (weld (scag 7 wire) (slag 8 wire)) [bone nonce]) + (weld (scag 7 wire) (slag 8 wire)) %+ roll ~(tap by subs) - |= [[=wire flows=(list [bone @])] core=_core] + |= [[=wire flows=(list [bone sub-nonce=@])] core=_core] :: %+ roll flows |= [[=bone sub-nonce=@] core=_core] @@ -1954,13 +1957,11 @@ =/ log=tape "[bone={} nonce={} agent={}] {}" =; corkable=? ?.(corkable core (%*(on-cork core cork-bone `bone) ship)) - :: check if the current subscription also failed - :: - ?. =(sub-nonce nonce) + ?: &((gth sub-nonce 0) (lth sub-nonce nonce)) %. !dry (trace |(dry odd.veb) ship |.((weld "stale %watch plea " log))) :: we can safely cork the current subscription - :: if it received a nack on backward bone + :: if it received a nack on a backward bone :: =+ target=(mix 0b10 bone) :: not a naxplanation ack bone From e32c18b6776cc81d9c27449f47663da8aff1766c Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Fri, 3 Feb 2023 09:59:56 +0100 Subject: [PATCH 21/67] ames: style fixes and comments --- pkg/arvo/sys/vane/ames.hoon | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 6b553cc14..1107e19da 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1934,11 +1934,14 @@ =/ subs=(jar path [bone sub-nonce=@]) %+ roll ~(tap by snd.peer-state) |= [[=bone *] subs=(jar path [bone sub-nonce=@])] - ?~ duct=(~(get by by-bone.ossuary.peer-state) bone) subs + ?: (~(has in closing.peer-state) bone) + subs + ?~ duct=(~(get by by-bone.ossuary.peer-state) bone) + subs ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) subs =/ =wire i.t.u.duct - =/ nonce=(unit @) (rush i.t.t.t.t.t.t.t.i.t.u.duct dem) + =/ nonce=(unit @) (rush (snag 7 wire) dem) %- ~(add ja subs) :_ [bone ?~(nonce 0 u.nonce)] :: 0 for old pre-nonce subscriptions ?~ nonce wire @@ -1950,14 +1953,15 @@ :: %+ roll flows |= [[=bone sub-nonce=@] core=_core] - ?: (~(has in closing.peer-state) bone) core - =/ app=term ?>(?=([%gall %use sub=@ *] wire) i.t.t.wire) - =/ nonce=@ud (dec (~(got by dudes) app)) - =/ =path (slag 7 wire) - =/ log=tape "[bone={} nonce={} agent={}] {}" + =/ app=term ?>(?=([%gall %use sub=@ *] wire) i.t.t.wire) + =/ last-nonce=@ (dec (~(got by dudes) app)) + =/ =path (slag 7 wire) + =/ log=tape "[bone={} nonce={} agent={}] {}" =; corkable=? ?.(corkable core (%*(on-cork core cork-bone `bone) ship)) - ?: &((gth sub-nonce 0) (lth sub-nonce nonce)) + :: checks if this a stale re-subscriptions + :: + ?: &((gth sub-nonce 0) (lth sub-nonce last-nonce)) %. !dry (trace |(dry odd.veb) ship |.((weld "stale %watch plea " log))) :: we can safely cork the current subscription From ee76b97f491670c11084a3091f509af64c4eb458 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Fri, 3 Feb 2023 10:21:00 +0100 Subject: [PATCH 22/67] ames: fix |close-flow generator --- pkg/arvo/gen/hood/close-flows.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/gen/hood/close-flows.hoon b/pkg/arvo/gen/hood/close-flows.hoon index 24fc605a3..53bfbb7cb 100644 --- a/pkg/arvo/gen/hood/close-flows.hoon +++ b/pkg/arvo/gen/hood/close-flows.hoon @@ -4,6 +4,6 @@ :: To actually close the flows, run with |close-flows, =dry | :: :- %say -|= [^ [dry=?]] +|= [^ arg=~ dry=?] :: [%helm-kroc dry] From 24140d5f128cdd63b5b8e17c646d67fe91fa5cf0 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Fri, 3 Feb 2023 12:18:19 +0100 Subject: [PATCH 23/67] ames: clarify forward/backward bones --- pkg/arvo/sys/vane/ames.hoon | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 1107e19da..19d5bba5c 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1933,17 +1933,19 @@ :: =/ subs=(jar path [bone sub-nonce=@]) %+ roll ~(tap by snd.peer-state) - |= [[=bone *] subs=(jar path [bone sub-nonce=@])] - ?: (~(has in closing.peer-state) bone) + |= [[=forward=bone *] subs=(jar path [bone sub-nonce=@])] + ?: (~(has in closing.peer-state) forward-bone) subs - ?~ duct=(~(get by by-bone.ossuary.peer-state) bone) + ?~ duct=(~(get by by-bone.ossuary.peer-state) forward-bone) subs ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) subs =/ =wire i.t.u.duct =/ nonce=(unit @) (rush (snag 7 wire) dem) %- ~(add ja subs) - :_ [bone ?~(nonce 0 u.nonce)] :: 0 for old pre-nonce subscriptions + :: 0 for old pre-nonce subscriptions + :: + :_ [forward-bone ?~(nonce 0 u.nonce)] ?~ nonce wire :: don't include the sub-nonce in the key :: @@ -1967,10 +1969,10 @@ :: we can safely cork the current subscription :: if it received a nack on a backward bone :: - =+ target=(mix 0b10 bone) + =+ backward-bone=(mix 0b10 bone) :: not a naxplanation ack bone :: - ?. &(=(0 (end 0 target)) =(1 (end 0 (rsh 0 target)))) + ?. =(2 (mod backward-bone 4)) | %. !dry (trace |(dry odd.veb) ship |.((weld "failed %watch plea " log))) From c68a92f8b4916b84f891a4b813075e7d80a72259 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Fri, 3 Feb 2023 13:57:09 +0100 Subject: [PATCH 24/67] ames: add +state-flows Some of this functionality already exists in ames (see +on-kroc) so this generator will probably be removed once there's been enough testing done for removing stale flows on the live network . --- pkg/arvo/gen/stale-flows.hoon | 145 ++++++++++++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 pkg/arvo/gen/stale-flows.hoon diff --git a/pkg/arvo/gen/stale-flows.hoon b/pkg/arvo/gen/stale-flows.hoon new file mode 100644 index 000000000..bb0e9e269 --- /dev/null +++ b/pkg/arvo/gen/stale-flows.hoon @@ -0,0 +1,145 @@ +:: +stale-flows: prints number of ames flows that can be closed +:: +:: |stale-flows, =veb %0 :: TODO +:: |stale-flows, =veb %1 :: flows from nacking initial subscriptions +:: |stale-flows, =veb %2 :: stale flows that keep (re)trying to connect +:: |stale-flows, =veb %21 :: ... per app (only forward) +:: |stale-flows, =veb %3 :: stale resubscriptions +:: |stale-flows, =veb %31 :: (TODO) ... per app +:: +=> |% + +$ subs (jar path [ship bone @]) + +$ pags (jar app=term [dst=term =ship =path]) :: per-agent + +$ naks (set [ship bone]) + :: verbosity: + :: + + :: + +$ veb ?(%0 %1 %2 %21 %3 %31) + :: + ++ resubs + |= [=subs =veb] + ^- @ + ::=/ sorted + :: %+ sort ~(tap by subs) + :: |=([a=(list *) b=(list *)] (lte (lent a) (lent b))) + %+ roll ~(tap by subs)::sorted + |= [[k=path v=(list [ship bone @])] num=@] + ~? &(=(%3 veb) (gth (lent v) 1)) + "#{<(dec (lent v))>} stale resubs on {}" + ?. (gth (lent v) 1) num + (add (lent v) num) + -- +:: +:- %say +|= $: [now=@da eny=@uvJ bec=beak] + [arg=~ dry=? =veb] + == +:: +=/ peers-map + .^((map ship ?(%alien %known)) %ax /(scot %p p.bec)//(scot %da now)/peers) +:: +=/ peers=(list ship) + %+ murn ~(tap by peers-map) + |= [=ship val=?(%alien %known)] + ?: =(ship p.bec) + ~ :: this is weird, but we saw it + ?- val + %alien ~ + %known (some ship) + == +:: +=; [[=subs =pags backward=@ forward=@] =naks] + :- %tang %- flop + %+ weld + :~ leaf+"#{<~(wyt in naks)>} flows from %nacking %watches" + leaf+"#{} backward flows with >10 retries" + leaf+"#{} forward flows with >10 retries" + leaf+"#{<(resubs subs veb)>} stale resubscriptions" + == + ?. =(%21 veb) ~ + :- leaf+"----------------------------------" + %+ turn ~(tap by pags) + |= [app=term v=(list [dst=term =ship =path])] + :- %leaf + %+ weld "#{<(lent v)>} flows for {} with >10 retries" + ?. =(1 (lent v)) ~ + ?> ?=(^ v) + " on {} to {} at {}" +:: +%+ roll peers +|= [=ship [=subs p=pags b=@ f=@] n=naks] +=+ .^ =ship-state:ames + %ax /(scot %p p.bec)//(scot %da now)/peers/(scot %p ship) + == +=/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) +:: +|^ [stale nacks] +:: +++ stale + %+ roll ~(tap by snd.peer-state) + |= [[=bone message-pump-state:ames] subs=_subs pags=_p backward=_b forward=_f] + =, packet-pump-state + :- ?~ duct=(~(get by by-bone.ossuary.peer-state) bone) subs + ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) + subs + =/ =wire i.t.u.duct + =/ nonce=(unit @) (rush i.t.t.t.t.t.t.t.i.t.u.duct dem) + %- ~(add ja subs) + :_ [ship bone ?~(nonce 0 u.nonce)] :: 0, 1? + ?~ nonce wire + :: don't include the sub-nonce in the key + :: + (weld (scag 7 wire) (slag 8 wire)) + %+ roll ~(tap in live) + |= [[* [packet-state:ames *]] pags=_pags out=[b=_backward f=_forward]] + :: + :: only forward flows + :: + =? pags &(=(0 (end 0 bone)) (gth retries 10)) + ?~ duct=(~(get by by-bone.ossuary.peer-state) bone) + pags + ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) + pags + =/ =wire i.t.u.duct + (~(add ja pags) (snag 2 wire) (snag 8 wire) ship (slag 9 wire)) + :: + ~? &(=(%2 veb) (gth retries 10)) + =+ arrow=?:(=(0 (end 0 bone)) "<-" "->") + "{arrow} ({(cite:title ship)}) bone #{}, retries: #{}" + :- pags + =? out (gth retries 10) + ?: =(0 (end 0 bone)) + [b.out +(f.out)] + [+(b.out) f.out] + out +:: +++ nacks + %+ roll ~(tap by rcv.peer-state) + |= [[=bone *] nacks=_n] + ?. &(=(0 (end 0 bone)) =(1 (end 0 (rsh 0 bone)))) + :: not a naxplanation ack bone + :: + nacks + :: by only corking forward flows that have received + :: a nack we avoid corking the current subscription + :: + =+ target=(mix 0b10 bone) + :: make sure that the nack bone has a forward flow + :: + ?~ duct=(~(get by by-bone.ossuary.peer-state) target) + nacks + ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) + nacks + =/ =wire i.t.u.duct + ?> ?=([%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] wire) + =/ app=term i.t.t.wire + =/ nonce=@ + =- ?~(- 0 u.-) + (rush i.t.t.t.t.t.t.t.wire dem) + =/ =path t.t.t.t.t.t.t.t.wire + ~? =(%1 veb) "[bone={} nonce={} agent={}] {}" + (~(put in nacks) [ship target]) +-- + + From a024b68b26f88ef630afdadccd6172537fe4ef7f Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Fri, 3 Feb 2023 14:07:57 +0100 Subject: [PATCH 25/67] ames: sort by number of re-trying flows per app --- pkg/arvo/gen/stale-flows.hoon | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/gen/stale-flows.hoon b/pkg/arvo/gen/stale-flows.hoon index bb0e9e269..4cedb1076 100644 --- a/pkg/arvo/gen/stale-flows.hoon +++ b/pkg/arvo/gen/stale-flows.hoon @@ -59,7 +59,9 @@ == ?. =(%21 veb) ~ :- leaf+"----------------------------------" - %+ turn ~(tap by pags) + %+ turn %+ sort ~(tap by pags) + |= [[* v=(list)] [* w=(list)]] + (gth (lent v) (lent w)) |= [app=term v=(list [dst=term =ship =path])] :- %leaf %+ weld "#{<(lent v)>} flows for {} with >10 retries" From 5f9d6b23d10a58c4db0e838c13be60ab5924353b Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Sun, 5 Feb 2023 12:25:55 +0100 Subject: [PATCH 26/67] ames: improve logging in dry mode for on-kroc --- pkg/arvo/sys/vane/ames.hoon | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 19d5bba5c..c31bab31a 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1923,9 +1923,15 @@ ++ on-kroc |= dry=? ^+ event-core + :: + =; [corks=@ core=_event-core] + ?. dry core + %.(core (slog leaf/"ames: #{} flows can be corked" ~)) + :: %+ roll ~(tap by peers.ames-state) - |= [[=ship =ship-state] core=_event-core] - ?. ?=(%known -.ship-state) core + |= [[=ship =ship-state] corks=@ core=_event-core] + ?. ?=(%known -.ship-state) + corks^core =/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) =/ dudes ;; (map dude:gall nonce=@) =< q.q %- need %- need @@ -1951,21 +1957,23 @@ :: (weld (scag 7 wire) (slag 8 wire)) %+ roll ~(tap by subs) - |= [[=wire flows=(list [bone sub-nonce=@])] core=_core] + |= [[=wire flows=(list [bone sub-nonce=@])] corks=_corks core=_core] :: %+ roll flows - |= [[=bone sub-nonce=@] core=_core] + |= [[=bone sub-nonce=@] corks=_corks core=_core] =/ app=term ?>(?=([%gall %use sub=@ *] wire) i.t.t.wire) =/ last-nonce=@ (dec (~(got by dudes) app)) =/ =path (slag 7 wire) - =/ log=tape "[bone={} nonce={} agent={}] {}" + =/ log=tape "[bone={} agent={}] {}" =; corkable=? - ?.(corkable core (%*(on-cork core cork-bone `bone) ship)) + =? corks corkable +(corks) + =? core &(corkable !dry) (%*(on-cork core cork-bone `bone) ship) + corks^core :: checks if this a stale re-subscriptions :: ?: &((gth sub-nonce 0) (lth sub-nonce last-nonce)) - %. !dry - (trace |(dry odd.veb) ship |.((weld "stale %watch plea " log))) + %. & + (trace &(dry odd.veb) ship |.((weld "stale %watch plea " log))) :: we can safely cork the current subscription :: if it received a nack on a backward bone :: @@ -1974,8 +1982,8 @@ :: ?. =(2 (mod backward-bone 4)) | - %. !dry - (trace |(dry odd.veb) ship |.((weld "failed %watch plea " log))) + %. & + (trace &(dry odd.veb) ship |.((weld "failed %watch plea " log))) :: +on-take-wake: receive wakeup or error notification from behn :: ++ on-take-wake From 108e0cd0231d9d1a214380f3c3873434affaf192 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 6 Feb 2023 12:56:00 +0100 Subject: [PATCH 27/67] ames: fix syntax error --- pkg/arvo/sys/vane/ames.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index c31bab31a..851cf53c2 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1967,7 +1967,7 @@ =/ log=tape "[bone={} agent={}] {}" =; corkable=? =? corks corkable +(corks) - =? core &(corkable !dry) (%*(on-cork core cork-bone `bone) ship) + =? core &(corkable !dry) (%*(on-cork core cork-bone `bone) ship) corks^core :: checks if this a stale re-subscriptions :: From 1522e4ab96966c215d3756e77a687b3c400a300f Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 6 Feb 2023 15:11:30 +0100 Subject: [PATCH 28/67] ames: don't scry dudes for every peer --- pkg/arvo/sys/vane/ames.hoon | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 851cf53c2..dc938e87f 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1928,15 +1928,15 @@ ?. dry core %.(core (slog leaf/"ames: #{} flows can be corked" ~)) :: + =/ dudes ;; (map dude:gall nonce=@) + =< q.q %- need %- need + (rof ~ %gf `beam`[[our %$ da+now] /]) + :: %+ roll ~(tap by peers.ames-state) |= [[=ship =ship-state] corks=@ core=_event-core] ?. ?=(%known -.ship-state) corks^core =/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) - =/ dudes ;; (map dude:gall nonce=@) - =< q.q %- need %- need - (rof ~ %gf `beam`[[our %$ da+now] /]) - :: =/ subs=(jar path [bone sub-nonce=@]) %+ roll ~(tap by snd.peer-state) |= [[=forward=bone *] subs=(jar path [bone sub-nonce=@])] From fbb95b0d9fa042332d93e3736fd785970db5dc04 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 6 Feb 2023 15:16:59 +0100 Subject: [PATCH 29/67] ames: check if we have a flow for a nax ack bone --- pkg/arvo/sys/vane/ames.hoon | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index dc938e87f..36303ac9f 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1978,10 +1978,12 @@ :: if it received a nack on a backward bone :: =+ backward-bone=(mix 0b10 bone) - :: not a naxplanation ack bone + :: the backward bone is a naxplanation ack bone and we have a flow for it :: ?. =(2 (mod backward-bone 4)) | + ?~ (~(get by rcv.peer-state) backward-bone) + | %. & (trace &(dry odd.veb) ship |.((weld "failed %watch plea " log))) :: +on-take-wake: receive wakeup or error notification from behn From 46c27d33cd8fe9ae3d3c62c426b86062660cca98 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 6 Feb 2023 16:48:32 +0100 Subject: [PATCH 30/67] ames: fix check for stale resubscriptions --- bin/solid.pill | 4 ++-- pkg/arvo/sys/vane/ames.hoon | 28 +++++++++++----------------- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 7353108f5..5650539dc 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:511564074c56329460e238ebb4881ff4dd9d642ac6d0804bbd9aaa17adc2f8ee -size 5732943 +oid sha256:2035ef65290065edbd99a86f9f5a36978617bc1983131fa474a9a5c0e91dc15d +size 5998440 diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 36303ac9f..02be753f5 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1928,10 +1928,6 @@ ?. dry core %.(core (slog leaf/"ames: #{} flows can be corked" ~)) :: - =/ dudes ;; (map dude:gall nonce=@) - =< q.q %- need %- need - (rof ~ %gf `beam`[[our %$ da+now] /]) - :: %+ roll ~(tap by peers.ames-state) |= [[=ship =ship-state] corks=@ core=_event-core] ?. ?=(%known -.ship-state) @@ -1959,27 +1955,25 @@ %+ roll ~(tap by subs) |= [[=wire flows=(list [bone sub-nonce=@])] corks=_corks core=_core] :: - %+ roll flows - |= [[=bone sub-nonce=@] corks=_corks core=_core] - =/ app=term ?>(?=([%gall %use sub=@ *] wire) i.t.t.wire) - =/ last-nonce=@ (dec (~(got by dudes) app)) - =/ =path (slag 7 wire) - =/ log=tape "[bone={} agent={}] {}" + %- tail + %+ roll (sort flows |=([[@ n=@] [@ m=@]] (lte n m))) + |= [[=bone nonce=@] resubs=_(lent flows) corks=_corks core=_core] + =/ app=term ?>(?=([%gall %use sub=@ *] wire) i.t.t.wire) + =/ =path (slag 7 wire) + =/ log=tape "[bone={} agent={} nonce={}] {}" =; corkable=? =? corks corkable +(corks) =? core &(corkable !dry) (%*(on-cork core cork-bone `bone) ship) - corks^core - :: checks if this a stale re-subscriptions + (dec resubs)^corks^core + :: checks if this is a stale re-subscription :: - ?: &((gth sub-nonce 0) (lth sub-nonce last-nonce)) + ?. =(resubs 1) %. & (trace &(dry odd.veb) ship |.((weld "stale %watch plea " log))) - :: we can safely cork the current subscription - :: if it received a nack on a backward bone + :: the current subscription can be safely corked if there + :: is a flow with a naxplanation ack on a backward bone :: =+ backward-bone=(mix 0b10 bone) - :: the backward bone is a naxplanation ack bone and we have a flow for it - :: ?. =(2 (mod backward-bone 4)) | ?~ (~(get by rcv.peer-state) backward-bone) From e0a37c40341d2874dce20a98a1dc0efcea0a7fde Mon Sep 17 00:00:00 2001 From: fang Date: Mon, 6 Feb 2023 19:19:00 +0100 Subject: [PATCH 31/67] webterm: v1.1.1 For release with a new glob that includes #6258. --- pkg/webterm/desk.docket-0 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/webterm/desk.docket-0 b/pkg/webterm/desk.docket-0 index 531bd1d58..3f3840fb4 100644 --- a/pkg/webterm/desk.docket-0 +++ b/pkg/webterm/desk.docket-0 @@ -1,9 +1,9 @@ :~ title+'Terminal' info+'A web interface to your Urbit\'s command line.' color+0x2e.4347 - glob-http+['https://bootstrap.urbit.org/glob-0v5.hvjci.n7c4h.1onl6.34g14.fut7c.glob' 0v5.hvjci.n7c4h.1onl6.34g14.fut7c] + glob-http+['https://bootstrap.urbit.org/glob-0vep1vb.e09fv.h88mg.endg6.3mip6.glob' 0vep1vb.e09fv.h88mg.endg6.3mip6] base+'webterm' - version+[1 1 0] + version+[1 1 1] website+'https://tlon.io' license+'MIT' == From 575296fcac96dc19d5e8325176f4669861fddbc6 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Tue, 7 Feb 2023 10:24:10 -0800 Subject: [PATCH 32/67] Update MAINTAINERS.md add post-release deployment instructions --- MAINTAINERS.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index c58c227a9..8967439af 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -126,7 +126,8 @@ git push origin - [ ] ssh into `~zod` - [ ] Check to ensure that nobody else is ssh'd into `~zod`, by running `screen -ls` and verifying no sessions are attached. - [ ] Attach to the screen session using `screen -x` -- [ ] Install the contents of the `%kids` desk on `~marnec-dozzod-marzod` into our `%base`: `|merge %base ~marnec-dozzod-marzod %kids, =gem %only-that` +- [ ] Install the contents of the `%kids` desk on `~marnec-dozzod-marzod` into our `%base`: `|merge %base ~marnec-dozzod-marzod %kids, =gem % +` - [ ] Check that `~zod` has updated to the latest release. For a Kelvin release, you can run `zuse` in the Dojo. Each non-Kelvin release might its own way of checking whether the update has completed, possibly through checking the `%cz` hash of the `%base` desk matches the hash on `~marnec-dozzod-marzod` by comparing the outputs of `+vat %base` on both ships. - [ ] Merge `~zod`'s `%base` desk into its `%kids` desk to distribute the new code to the network: `|merge %kids our %base, =gem %only-that` - [ ] Before exiting the screen session on `~zod`, make sure the screen session is not left in copy mode for a long period of time, since that will disrupt `~zod`'s operation. @@ -143,6 +144,17 @@ git push origin - [ ] Merge `master` back into `develop`. - [ ] Cut a new release branch from `develop`. The branch should have the format `release/urbit-os-vX.XX` +### Post-Release Deployments + +We'll now need to deploy the new release candidate to the pre-release moons. + +- On `~marnec-dozzod-marzod`: + - [ ] `|merge %base ~binnec-dozzod-marzod %kids, =gem %only-that` to update `~marnec` with the contents of the GH release branch + - [ ] `|merge %kids our %base, =gem %only-that` to OTA the release candidate to subscribers +- When ready to deploy the release candidate to App Developers, on `~doznec-dozzod-marzod`: + - [ ] `|merge %base ~marnec-dozzod-marzod %kids, =gem %only-that` to update `~doznec` with the release candidate + - [ ] `|merge %kids our %base, =gem %only-that` to OTA the release candidate to subscribers + ### Post-Release Checks - [ ] Check that `~marzod` and other distribution stars are receiving the update by running `|ames-sift ~zod` and `|ames-verb %rcv %ges`. You should see lots of packets from `~zod`. Once you have confirmed packets are flowing, run `|ames-sift` and `|ames-verb` with no arguments to reset the verbosity state. - [ ] Check that planets are receiving the update. They should start updating within an hour or so. From 8b2f555d6e1e3bc173ad701db6a6206c35ca1bb0 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Tue, 7 Feb 2023 10:25:30 -0800 Subject: [PATCH 33/67] Update MAINTAINERS.md --- MAINTAINERS.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 8967439af..d5f28ad85 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -126,8 +126,7 @@ git push origin - [ ] ssh into `~zod` - [ ] Check to ensure that nobody else is ssh'd into `~zod`, by running `screen -ls` and verifying no sessions are attached. - [ ] Attach to the screen session using `screen -x` -- [ ] Install the contents of the `%kids` desk on `~marnec-dozzod-marzod` into our `%base`: `|merge %base ~marnec-dozzod-marzod %kids, =gem % -` +- [ ] Install the contents of the `%kids` desk on `~marnec-dozzod-marzod` into our `%base`: `|merge %base ~marnec-dozzod-marzod %kids, =gem %only-that` - [ ] Check that `~zod` has updated to the latest release. For a Kelvin release, you can run `zuse` in the Dojo. Each non-Kelvin release might its own way of checking whether the update has completed, possibly through checking the `%cz` hash of the `%base` desk matches the hash on `~marnec-dozzod-marzod` by comparing the outputs of `+vat %base` on both ships. - [ ] Merge `~zod`'s `%base` desk into its `%kids` desk to distribute the new code to the network: `|merge %kids our %base, =gem %only-that` - [ ] Before exiting the screen session on `~zod`, make sure the screen session is not left in copy mode for a long period of time, since that will disrupt `~zod`'s operation. From 032ee85b4fcf6723374e686acaaa32bfdb5e1317 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Tue, 7 Feb 2023 10:27:11 -0800 Subject: [PATCH 34/67] Update MAINTAINERS.md --- MAINTAINERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index d5f28ad85..9e04d4a29 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -143,7 +143,7 @@ git push origin - [ ] Merge `master` back into `develop`. - [ ] Cut a new release branch from `develop`. The branch should have the format `release/urbit-os-vX.XX` -### Post-Release Deployments +### Release Next Release Candidate We'll now need to deploy the new release candidate to the pre-release moons. From e44a661076fd248f600a3696dada00f8ba31a5fe Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Tue, 7 Feb 2023 10:35:43 -0800 Subject: [PATCH 35/67] Update MAINTAINERS.md --- MAINTAINERS.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 9e04d4a29..bedd9f666 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -126,7 +126,10 @@ git push origin - [ ] ssh into `~zod` - [ ] Check to ensure that nobody else is ssh'd into `~zod`, by running `screen -ls` and verifying no sessions are attached. - [ ] Attach to the screen session using `screen -x` -- [ ] Install the contents of the `%kids` desk on `~marnec-dozzod-marzod` into our `%base`: `|merge %base ~marnec-dozzod-marzod %kids, =gem %only-that` +- [ ] Ensure that the release candidate was correctly propagated through the prerelease moons + - Use `-read %z ~SHIP %DESK da+now /` to check desk hashes + - [ ] Run the above check on the following [SHIP DESK] pairs: [~marnec-dozzod-marzod %base], [~marnec-dozzod-marzod %kids], [~doznec-dozzod-marzod %base], [~doznec-dozzod-marzod %kids] — **they should all match** +- [ ] Install the contents of the `%kids` desk on `~doznec-dozzod-marzod` into our `%base`: `|merge %base ~doznec-dozzod-marzod %kids, =gem %only-that` - [ ] Check that `~zod` has updated to the latest release. For a Kelvin release, you can run `zuse` in the Dojo. Each non-Kelvin release might its own way of checking whether the update has completed, possibly through checking the `%cz` hash of the `%base` desk matches the hash on `~marnec-dozzod-marzod` by comparing the outputs of `+vat %base` on both ships. - [ ] Merge `~zod`'s `%base` desk into its `%kids` desk to distribute the new code to the network: `|merge %kids our %base, =gem %only-that` - [ ] Before exiting the screen session on `~zod`, make sure the screen session is not left in copy mode for a long period of time, since that will disrupt `~zod`'s operation. @@ -145,7 +148,7 @@ git push origin ### Release Next Release Candidate -We'll now need to deploy the new release candidate to the pre-release moons. +We'll now need to **IMMEDIATELY** deploy the new release candidate to the pre-release moons. - On `~marnec-dozzod-marzod`: - [ ] `|merge %base ~binnec-dozzod-marzod %kids, =gem %only-that` to update `~marnec` with the contents of the GH release branch From 1ecc010bacdbc980892d3c235f7ee1e97dab8652 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Tue, 7 Feb 2023 10:42:29 -0800 Subject: [PATCH 36/67] Update MAINTAINERS.md --- MAINTAINERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index bedd9f666..ebbe9e94e 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -148,7 +148,7 @@ git push origin ### Release Next Release Candidate -We'll now need to **IMMEDIATELY** deploy the new release candidate to the pre-release moons. +We'll now need to **IMMEDIATELY** deploy the new release candidate to the pre-release moon(s). - On `~marnec-dozzod-marzod`: - [ ] `|merge %base ~binnec-dozzod-marzod %kids, =gem %only-that` to update `~marnec` with the contents of the GH release branch From 62b6e8c8f8ab23dda0242d6e5a720d3d3158a3db Mon Sep 17 00:00:00 2001 From: ryjm Date: Mon, 6 Feb 2023 15:37:23 -0500 Subject: [PATCH 37/67] clay: desk name sanity check Ensures no desk with an illegal name can be created by sanity checking in `%fuse`, `%park`, and `%merge`. --- pkg/arvo/sys/vane/clay.hoon | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 6f31ae2bf..b69100016 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -4828,6 +4828,8 @@ %merg :: direct state up ?: =(%$ des.req) ~&(%merg-no-desk !!) + ?. ((sane %tas) des.req) + ~&(%merg-bad-desk-name !!) =^ mos ruf =/ den ((de now rof hen ruf) our des.req) abet:(start-merge:den her.req dem.req cas.req how.req) @@ -4836,6 +4838,8 @@ %fuse ?: =(%$ des.req) ~&(%fuse-no-desk !!) + ?. ((sane %tas) des.req) + ~&(%fuse-bad-desk-name !!) =^ mos ruf =/ den ((de now rof hen ruf) our des.req) abet:(start-fuse:den bas.req con.req) @@ -4881,6 +4885,8 @@ [(weld moves-1 moves-2) ..^^$] :: %park + ?. ((sane %tas) des.req) + ~&(%park-bad-desk !!) =^ mos ruf =/ den ((de now rof hen ruf) our des.req) abet:(park:den | & [yok ran]:req) From 972588e573274033fd2792052fb3789744300f0d Mon Sep 17 00:00:00 2001 From: Jake Miller Date: Wed, 8 Feb 2023 07:11:50 -0500 Subject: [PATCH 38/67] add bad desk name to stack trace Co-authored-by: fang --- pkg/arvo/sys/vane/clay.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index b69100016..c9db16e65 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -4829,7 +4829,7 @@ ?: =(%$ des.req) ~&(%merg-no-desk !!) ?. ((sane %tas) des.req) - ~&(%merg-bad-desk-name !!) + ~|([%merg-bad-desk-name des.req] !!) =^ mos ruf =/ den ((de now rof hen ruf) our des.req) abet:(start-merge:den her.req dem.req cas.req how.req) From a5512fe9bc95cd7db1fdf6ef28029512dc5f4289 Mon Sep 17 00:00:00 2001 From: ryjm Date: Wed, 8 Feb 2023 07:38:44 -0500 Subject: [PATCH 39/67] add bad desk name to all traces --- pkg/arvo/sys/vane/clay.hoon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index c9db16e65..276bb012c 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -4839,7 +4839,7 @@ ?: =(%$ des.req) ~&(%fuse-no-desk !!) ?. ((sane %tas) des.req) - ~&(%fuse-bad-desk-name !!) + ~&([%fuse-bad-desk-name des.req] !!) =^ mos ruf =/ den ((de now rof hen ruf) our des.req) abet:(start-fuse:den bas.req con.req) @@ -4886,7 +4886,7 @@ :: %park ?. ((sane %tas) des.req) - ~&(%park-bad-desk !!) + ~&([%park-bad-desk des.req] !!) =^ mos ruf =/ den ((de now rof hen ruf) our des.req) abet:(park:den | & [yok ran]:req) From bad133cf9b2e60241a189b7fb417c9441eb4edb4 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Wed, 8 Feb 2023 16:20:41 +0100 Subject: [PATCH 40/67] ames: style fixes --- pkg/arvo/gen/stale-flows.hoon | 42 ++++++++++++++++------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/pkg/arvo/gen/stale-flows.hoon b/pkg/arvo/gen/stale-flows.hoon index 4cedb1076..ec64dac16 100644 --- a/pkg/arvo/gen/stale-flows.hoon +++ b/pkg/arvo/gen/stale-flows.hoon @@ -1,19 +1,15 @@ :: +stale-flows: prints number of ames flows that can be closed :: -:: |stale-flows, =veb %0 :: TODO :: |stale-flows, =veb %1 :: flows from nacking initial subscriptions :: |stale-flows, =veb %2 :: stale flows that keep (re)trying to connect :: |stale-flows, =veb %21 :: ... per app (only forward) :: |stale-flows, =veb %3 :: stale resubscriptions -:: |stale-flows, =veb %31 :: (TODO) ... per app :: => |% +$ subs (jar path [ship bone @]) +$ pags (jar app=term [dst=term =ship =path]) :: per-agent +$ naks (set [ship bone]) - :: verbosity: - :: - + :: verbosity :: +$ veb ?(%0 %1 %2 %21 %3 %31) :: @@ -50,24 +46,24 @@ == :: =; [[=subs =pags backward=@ forward=@] =naks] - :- %tang %- flop - %+ weld - :~ leaf+"#{<~(wyt in naks)>} flows from %nacking %watches" - leaf+"#{} backward flows with >10 retries" - leaf+"#{} forward flows with >10 retries" - leaf+"#{<(resubs subs veb)>} stale resubscriptions" - == - ?. =(%21 veb) ~ - :- leaf+"----------------------------------" - %+ turn %+ sort ~(tap by pags) - |= [[* v=(list)] [* w=(list)]] - (gth (lent v) (lent w)) - |= [app=term v=(list [dst=term =ship =path])] - :- %leaf - %+ weld "#{<(lent v)>} flows for {} with >10 retries" - ?. =(1 (lent v)) ~ - ?> ?=(^ v) - " on {} to {} at {}" + :- %tang %- flop + %+ weld + :~ leaf+"#{<~(wyt in naks)>} flows from %nacking %watches" + leaf+"#{} backward flows with >10 retries" + leaf+"#{} forward flows with >10 retries" + leaf+"#{<(resubs subs veb)>} stale resubscriptions" + == + ?. =(%21 veb) ~ + :- leaf+"----------------------------------" + %+ turn %+ sort ~(tap by pags) + |= [[* v=(list)] [* w=(list)]] + (gth (lent v) (lent w)) + |= [app=term v=(list [dst=term =ship =path])] + :- %leaf + %+ weld "#{<(lent v)>} flows for {} with >10 retries" + ?. =(1 (lent v)) ~ + ?> ?=(^ v) + " on {} to {} at {}" :: %+ roll peers |= [=ship [=subs p=pags b=@ f=@] n=naks] From 90360f76d922a9bf4cf12a687269828caaa798e3 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Wed, 8 Feb 2023 16:21:56 +0100 Subject: [PATCH 41/67] ames: remove extra lines --- pkg/arvo/gen/stale-flows.hoon | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/arvo/gen/stale-flows.hoon b/pkg/arvo/gen/stale-flows.hoon index ec64dac16..d4d889313 100644 --- a/pkg/arvo/gen/stale-flows.hoon +++ b/pkg/arvo/gen/stale-flows.hoon @@ -139,5 +139,3 @@ ~? =(%1 veb) "[bone={} nonce={} agent={}] {}" (~(put in nacks) [ship target]) -- - - From dd812283ac6b303bb2f205e6b82c90a3dc88b155 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Thu, 9 Feb 2023 10:47:49 +0100 Subject: [PATCH 42/67] ames: optimize memory measurement Co-authored-by: Joe Bryan --- pkg/arvo/sys/vane/ames.hoon | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 6254ea8f3..81e41e248 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2595,14 +2595,11 @@ |- ^- ? ?~ pumps | =. mem - ;: add mem - %+ roll ~(tap to unsent-messages.i.pumps) - |=([a=@ b=@ud] (add b (met 3 a))) - :: - ?~ unsent-fragments.i.pumps - 0 - (met 3 fragment.i.unsent-fragments.i.pumps) - == + %+ add + %- ~(rep in unsent-messages.i.pumps) + |=([a=@ b=_mem] (add b (met 3 a))) + ?~ unsent-fragments.i.pumps. 0 + (met 3 fragment.i.unsent-fragments.i.pumps) ?: (gte mem 100.000) & $(pumps t.pumps) From abbd95f2ce636050a81e59be7bec2df18cb73fe2 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Thu, 9 Feb 2023 13:12:50 +0100 Subject: [PATCH 43/67] ames: fix typo --- pkg/arvo/sys/vane/ames.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 81e41e248..25cfd1e54 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -2598,7 +2598,7 @@ %+ add %- ~(rep in unsent-messages.i.pumps) |=([a=@ b=_mem] (add b (met 3 a))) - ?~ unsent-fragments.i.pumps. 0 + ?~ unsent-fragments.i.pumps 0 (met 3 fragment.i.unsent-fragments.i.pumps) ?: (gte mem 100.000) & From 0497790b86e21a20d62eefafe7b24660a52b4d54 Mon Sep 17 00:00:00 2001 From: ryjm Date: Thu, 9 Feb 2023 08:43:58 -0500 Subject: [PATCH 44/67] ~& -> ~| --- pkg/arvo/sys/vane/clay.hoon | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/arvo/sys/vane/clay.hoon b/pkg/arvo/sys/vane/clay.hoon index 276bb012c..00e27e007 100644 --- a/pkg/arvo/sys/vane/clay.hoon +++ b/pkg/arvo/sys/vane/clay.hoon @@ -4827,7 +4827,7 @@ :: %merg :: direct state up ?: =(%$ des.req) - ~&(%merg-no-desk !!) + ~|(%merg-no-desk !!) ?. ((sane %tas) des.req) ~|([%merg-bad-desk-name des.req] !!) =^ mos ruf @@ -4837,9 +4837,9 @@ :: %fuse ?: =(%$ des.req) - ~&(%fuse-no-desk !!) + ~|(%fuse-no-desk !!) ?. ((sane %tas) des.req) - ~&([%fuse-bad-desk-name des.req] !!) + ~|([%fuse-bad-desk-name des.req] !!) =^ mos ruf =/ den ((de now rof hen ruf) our des.req) abet:(start-fuse:den bas.req con.req) @@ -4886,7 +4886,7 @@ :: %park ?. ((sane %tas) des.req) - ~&([%park-bad-desk des.req] !!) + ~|([%park-bad-desk des.req] !!) =^ mos ruf =/ den ((de now rof hen ruf) our des.req) abet:(park:den | & [yok ran]:req) From 1c0bf965256aad27e7658f6633a38afe4d91f481 Mon Sep 17 00:00:00 2001 From: Philip Monk Date: Thu, 9 Feb 2023 13:21:43 -0700 Subject: [PATCH 45/67] clay: correctly restart syncs when source breaches Fixes #6313 --- pkg/arvo/lib/hood/kiln.hoon | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/pkg/arvo/lib/hood/kiln.hoon b/pkg/arvo/lib/hood/kiln.hoon index e82d8c26e..b1be5cf62 100644 --- a/pkg/arvo/lib/hood/kiln.hoon +++ b/pkg/arvo/lib/hood/kiln.hoon @@ -1095,6 +1095,7 @@ %+ lard /init =/ m (strand:rand ,vase) ;< =riot:clay bind:m (warp:strandio her sud ~ %sing %y ud+1 /) + ?> ?=(^ riot) ~> %slog.(fmt "activated install into {here}") ;< now=@da bind:m get-time:strandio ;< =riot:clay bind:m (warp:strandio her sud ~ %sing %w da+now /) @@ -1109,6 +1110,7 @@ %+ lard /next =/ m (strand:rand ,vase) ;< =riot:clay bind:m (warp:strandio her sud ~ %sing %w ud+let /) + ?> ?=(^ riot) ~> %slog.(fmt "downloading update for {here}") ;< =riot:clay bind:m (warp:strandio her sud ~ %sing %v ud+let /) ?> ?=(^ riot) @@ -1176,14 +1178,6 @@ :: %main ?> ?=(%mere +<.sign-arvo) - :: This case is maintained by superstition. If you remove it, - :: carefully test that if the source ship is breached, we - :: correctly reset let to 0 - :: - ?: ?=([%| %ali-unavailable *] p.sign-arvo) - =+ "kiln: merge into {here} failed, maybe because sunk; restarting" - %- (slog leaf/- p.p.sign-arvo) - init ?: ?=(%| -.p.sign-arvo) =+ "kiln: merge into {here} failed, waiting for next revision" %- (slog leaf/- p.p.sign-arvo) @@ -1200,12 +1194,6 @@ ?> ?=(%mere +<.sign-arvo) ?~ kid ..abet - :: See %main for this case - :: - ?: ?=([%| %ali-unavailable *] p.sign-arvo) - =+ "kids merge to {} failed, maybe peer sunk; restarting" - ~> %slog.(fmt -) - init :: Just notify; we've already started listening for the next :: version :: From 3506b1edcdef4220e90cb9469260fa2c6efb5f86 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Fri, 10 Feb 2023 14:37:05 +0100 Subject: [PATCH 46/67] ames: fix number of stale flows log --- pkg/arvo/gen/stale-flows.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/gen/stale-flows.hoon b/pkg/arvo/gen/stale-flows.hoon index d4d889313..c5e956fd6 100644 --- a/pkg/arvo/gen/stale-flows.hoon +++ b/pkg/arvo/gen/stale-flows.hoon @@ -24,7 +24,7 @@ ~? &(=(%3 veb) (gth (lent v) 1)) "#{<(dec (lent v))>} stale resubs on {}" ?. (gth (lent v) 1) num - (add (lent v) num) + (add (dec (lent v)) num) -- :: :- %say From 23879acf0f65b11ae8f6cce9effd2b362f02d761 Mon Sep 17 00:00:00 2001 From: Josh Lehman Date: Fri, 10 Feb 2023 06:55:20 -0800 Subject: [PATCH 47/67] Update MAINTAINERS.md --- MAINTAINERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MAINTAINERS.md b/MAINTAINERS.md index ebbe9e94e..40ecb6491 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -148,7 +148,7 @@ git push origin ### Release Next Release Candidate -We'll now need to **IMMEDIATELY** deploy the new release candidate to the pre-release moon(s). +We'll now need to **IMMEDIATELY** deploy the new release candidate to the pre-release moon(s). Otherwise PRs merged during this window will bypass the testing period on `~binnec` and go straight to the release candidate. - On `~marnec-dozzod-marzod`: - [ ] `|merge %base ~binnec-dozzod-marzod %kids, =gem %only-that` to update `~marnec` with the contents of the GH release branch From 93b9423bcc9ca75e4c241d555d4a47bba6e76ef1 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Fri, 10 Feb 2023 16:13:28 +0100 Subject: [PATCH 48/67] ames: improve logs for stale retries --- pkg/arvo/gen/stale-flows.hoon | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pkg/arvo/gen/stale-flows.hoon b/pkg/arvo/gen/stale-flows.hoon index c5e956fd6..f7b668b2f 100644 --- a/pkg/arvo/gen/stale-flows.hoon +++ b/pkg/arvo/gen/stale-flows.hoon @@ -49,8 +49,8 @@ :- %tang %- flop %+ weld :~ leaf+"#{<~(wyt in naks)>} flows from %nacking %watches" - leaf+"#{} backward flows with >10 retries" - leaf+"#{} forward flows with >10 retries" + leaf+"#{} live backward flows with (keep retrying)" + leaf+"#{} live forward flows with (keep retrying)" leaf+"#{<(resubs subs veb)>} stale resubscriptions" == ?. =(%21 veb) ~ @@ -76,7 +76,10 @@ :: ++ stale %+ roll ~(tap by snd.peer-state) - |= [[=bone message-pump-state:ames] subs=_subs pags=_p backward=_b forward=_f] + |= $: [=bone message-pump-state:ames] + subs=_subs pags=_p + backward=_b forward=_f + == =, packet-pump-state :- ?~ duct=(~(get by by-bone.ossuary.peer-state) bone) subs ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) @@ -90,7 +93,10 @@ :: (weld (scag 7 wire) (slag 8 wire)) %+ roll ~(tap in live) - |= [[* [packet-state:ames *]] pags=_pags out=[b=_backward f=_forward]] + |= $: [[msg=@ frag=@] [packet-state:ames *]] + pags=_pags + out=[b=_backward f=_forward] + == :: :: only forward flows :: @@ -104,7 +110,9 @@ :: ~? &(=(%2 veb) (gth retries 10)) =+ arrow=?:(=(0 (end 0 bone)) "<-" "->") - "{arrow} ({(cite:title ship)}) bone #{}, retries: #{}" + =+ closing=(~(has in closing.peer-state) bone) + %+ weld "{arrow} ({(cite:title ship)}) bone=#{} " + "closing={} msg=#{} frag=#{} #{}" :- pags =? out (gth retries 10) ?: =(0 (end 0 bone)) From c92c2c9279f612fd897738b7b51709dd3b6aac2e Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Fri, 10 Feb 2023 16:19:18 +0100 Subject: [PATCH 49/67] ames: log stale flows in closing state --- pkg/arvo/sys/vane/ames.hoon | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 02be753f5..345d33c89 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1935,9 +1935,15 @@ =/ =peer-state:ames ?>(?=(%known -.ship-state) +.ship-state) =/ subs=(jar path [bone sub-nonce=@]) %+ roll ~(tap by snd.peer-state) - |= [[=forward=bone *] subs=(jar path [bone sub-nonce=@])] + |= $: [=forward=bone message-pump-state:ames] + subs=(jar path [bone sub-nonce=@]) + == ?: (~(has in closing.peer-state) forward-bone) - subs + %. subs + %^ trace &(dry odd.veb) ship + |. + %+ weld "stale flow bone={} in closing, " + "#{<~(wyt in live:packet-pump-state)>} packets retrying" ?~ duct=(~(get by by-bone.ossuary.peer-state) forward-bone) subs ?. ?=([* [%gall %use sub=@ @ %out @ @ nonce=@ pub=@ *] *] u.duct) From b8b7e1951f3441cb7058f3bbb8c19e8d157595e1 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Mon, 13 Feb 2023 12:04:41 +0100 Subject: [PATCH 50/67] ames: add %cong task to adjust clog --- pkg/arvo/sys/lull.hoon | 3 + pkg/arvo/sys/vane/ames.hoon | 110 +++++++++++++++++++++++++----------- 2 files changed, 80 insertions(+), 33 deletions(-) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index e05500f74..f05466613 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -363,6 +363,8 @@ :: %sift: limit verbosity to .ships :: %snub: set packet blacklist to .ships :: %spew: set verbosity toggles + :: %cong: adjust congestion control parameters + :: %stir: recover from timer desync :: %trim: release memory :: %vega: kernel reload notification :: @@ -380,6 +382,7 @@ [%sift ships=(list ship)] [%snub ships=(list ship)] [%spew veb=(list verb)] + [%cong msg=@ud mem=@ud] [%stir arg=@t] $>(%trim vane-task) $>(%vega vane-task) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index 25cfd1e54..68e65834e 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -609,6 +609,7 @@ :: crypto-core: interface for encryption and signing :: bug: debug printing configuration :: snub: blocklist for incoming packets +:: cong: parameters for marking a flow as clogged :: :: Note: .corks is only still present for unreleased migration reasons :: @@ -621,6 +622,7 @@ =bug corks=(set wire) snub=(set ship) + cong=[msg=@ud mem=@ud] == +$ ames-state-4 ames-state-5 +$ ames-state-5 @@ -703,6 +705,11 @@ ships=(set ship) == :: ++$ bug-10 + $: veb=_[`?`%.n `?`%.n `?`%.n `?`%.n `?`%.n `?`%.n `?`%.n `?`%.n] + ships=(set ship) + == +:: +$ ames-state-9 $: peers=(map ship ship-state) =unix=duct @@ -713,6 +720,16 @@ snub=(set ship) == :: ++$ ames-state-10 + $: peers=(map ship ship-state) + =unix=duct + =life + crypto-core=acru:ames + bug=bug-10 + corks=(set wire) + snub=(set ship) + == +:: :: $bug: debug printing configuration :: :: veb: verbosity toggles @@ -876,7 +893,8 @@ [%7 ames-state-7] [%8 ames-state-8] [%9 ames-state-9] - [%10 ^ames-state] + [%10 ames-state-10] + [%11 ^ames-state] == :: |= [now=@da eny=@ rof=roof] @@ -999,7 +1017,7 @@ :: lifecycle arms; mostly pass-throughs to the contained adult ames :: ++ scry scry:adult-core - ++ stay [%10 %larva queued-events ames-state.adult-gate] + ++ stay [%11 %larva queued-events ames-state.adult-gate] ++ load |= $= old $% $: %4 @@ -1045,6 +1063,13 @@ [%adult state=ames-state-9] == == $: %10 + $% $: %larva + events=(qeu queued-event) + state=ames-state-10 + == + [%adult state=ames-state-10] + == == + $: %11 $% $: %larva events=(qeu queued-event) state=_ames-state.adult-gate @@ -1109,12 +1134,22 @@ =. queued-events events.old larval-gate :: - [%10 %adult *] (load:adult-core %10 state.old) + [%10 %adult *] + =. cached-state `[%10 state.old] + ~> %slog.0^leaf/"ames: larva reload" + larval-gate :: [%10 %larva *] ~> %slog.1^leaf/"ames: larva: load" =. queued-events events.old - =. adult-gate (load:adult-core %10 state.old) + larval-gate + :: + [%11 %adult *] (load:adult-core %11 state.old) + :: + [%11 %larva *] + ~> %slog.1^leaf/"ames: larva: load" + =. queued-events events.old + =. adult-gate (load:adult-core %11 state.old) larval-gate :: == @@ -1137,7 +1172,9 @@ 9+(state-8-to-9:load:adult-core +.u.cached-state) =? u.cached-state ?=(%9 -.u.cached-state) 10+(state-9-to-10:load:adult-core +.u.cached-state) - ?> ?=(%10 -.u.cached-state) + =? u.cached-state ?=(%10 -.u.cached-state) + 11+(state-10-to-11:load:adult-core +.u.cached-state) + ?> ?=(%11 -.u.cached-state) =. ames-state.adult-gate +.u.cached-state [moz larval-core(cached-state ~)] -- @@ -1178,6 +1215,7 @@ %sift (on-sift:event-core ships.task) %snub (on-snub:event-core ships.task) %spew (on-spew:event-core veb.task) + %cong (on-cong:event-core [msg mem]:task) %stir (on-stir:event-core arg.task) %trim on-trim:event-core %vega on-vega:event-core @@ -1214,19 +1252,19 @@ [moves ames-gate] :: +stay: extract state before reload :: -++ stay [%10 %adult ames-state] +++ stay [%11 %adult ames-state] :: +load: load in old state after reload :: ++ load =< |= $= old-state - $% [%10 ^ames-state] + $% [%11 ^ames-state] == ^+ ames-gate - ?> ?=(%10 -.old-state) + ?> ?=(%11 -.old-state) ames-gate(ames-state +.old-state) + :: all state transitions are called from larval ames :: |% - :: +state-4-to-5 called from larval-ames :: ++ state-4-to-5 |= ames-state=ames-state-4 @@ -1244,7 +1282,6 @@ message-pump-state ship-state ames-state - :: +state-5-to-6 called from larval-ames :: ++ state-5-to-6 |= ames-state=ames-state-5 @@ -1267,7 +1304,6 @@ :_ +.peer-state =, -.peer-state [symmetric-key life rift public-key sponsor] - :: +state-6-to-7 called from larval-ames :: ++ state-6-to-7 |= ames-state=ames-state-6 @@ -1282,41 +1318,42 @@ ^- peer-state :- +<.ship-state [route qos ossuary snd rcv nax heeds ~ ~ ~]:ship-state - :: +state-7-to-8 called from larval-ames :: ++ state-7-to-8 |= ames-state=ames-state-7 ^- ames-state-8 - :* peers.ames-state - unix-duct.ames-state - life.ames-state - crypto-core.ames-state - bug.ames-state + =, ames-state + :* peers unix-duct life crypto-core bug *(set wire) == + :: ++ state-8-to-9 |= ames-state=ames-state-8 ^- ames-state-9 - :* peers.ames-state - unix-duct.ames-state - life.ames-state - crypto-core.ames-state - bug.ames-state - corks.ames-state + =, ames-state + :* peers unix-duct life crypto-core bug corks *(set ship) == + :: ++ state-9-to-10 |= ames-state=ames-state-9 - ^- ^^ames-state - :* peers.ames-state - unix-duct.ames-state - life.ames-state - crypto-core.ames-state + ^- ames-state-10 + =, ames-state + :* peers unix-duct life crypto-core %= bug.ames-state - veb [&1 &2 &3 &4 &5 &6 |6 %.n]:veb.bug.ames-state + veb [&1 &2 &3 &4 &5 &6 |6 %.n]:veb.bug == - corks.ames-state - snub.ames-state + corks snub + == + :: + ++ state-10-to-11 + |= ames-state=ames-state-10 + ^- ^^ames-state + =, ames-state + :* peers unix-duct life crypto-core bug corks snub + :: 5 messages and 100Kb of data outstanding + :: + [msg=5 mem=100.000] == -- :: +scry: dereference namespace @@ -1583,6 +1620,13 @@ =. peer-core (run-message-pump:peer-core i.bones %prod ~) $(bones t.bones) -- + :: +on-cong: adjust congestion control parameters + :: + ++ on-cong + |= [msg=@ud mem=@ud] + ^+ event-core + =. cong.ames-state msg^mem + event-core :: +on-stir: recover from timer desync, setting new timers as needed :: :: .arg is unused, meant to ease future debug commands @@ -2585,7 +2629,7 @@ (sub [next current]:i.pumps) ~(wyt in unsent-messages.i.pumps) == - ?: (gte num 5) + ?: (gte num msg.cong.ames-state) & $(pumps t.pumps) :: +nuf-memory: is enough memory used to mark as clogged? @@ -2600,7 +2644,7 @@ |=([a=@ b=_mem] (add b (met 3 a))) ?~ unsent-fragments.i.pumps 0 (met 3 fragment.i.unsent-fragments.i.pumps) - ?: (gte mem 100.000) + ?: (gte mem mem.cong.ames-state) & $(pumps t.pumps) -- From d8ff95db8491f711a5dfaffb7964683fec281e3d Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 1 Aug 2022 01:50:36 +0300 Subject: [PATCH 51/67] eyre: add /~/name endpoint --- pkg/arvo/sys/lull.hoon | 3 ++ pkg/arvo/sys/vane/eyre.hoon | 87 ++++++++++++++++++++++++++++--------- tests/sys/vane/eyre.hoon | 2 +- 3 files changed, 70 insertions(+), 22 deletions(-) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index e05500f74..f565e237a 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -1551,6 +1551,9 @@ :: gall scry endpoint :: [%scry ~] + :: internal name endpoint, returns ship @p + :: + [%name ~] :: respond with the default file not found page :: [%four-oh-four ~] diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index f3ebe9583..88998ad69 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -70,7 +70,7 @@ ++ axle $: :: date: date at which http-server's state was updated to this data structure :: - date=%~2020.10.18 + date=%~2022.7.26 :: server-state: state of inbound requests :: =server-state @@ -744,44 +744,58 @@ :: %scry (handle-scry authenticated address request(url suburl)) + :: + %name + (handle-name authenticated request) :: %four-oh-four %^ return-static-data-on-duct 404 'text/html' (error-page 404 authenticated url.request ~) == + :: +handle-name: respond with our @p or 403 + :: + ++ handle-name + |= [authenticated=? =request:http] + ^- (quip move server-state) + ?. authenticated + (error-response authenticated url.request 403 ~) + ?. =(%'GET' method.request) + (error-response authenticated url.request 405 "may only GET name") + %^ return-static-data-on-duct 200 'text/plain' + (as-octs:mimes:html (scot %p our)) :: +handle-scry: respond with scry result, 404 or 500 :: ++ handle-scry |= [authenticated=? =address =request:http] |^ ^- (quip move server-state) ?. authenticated - (error-response 403 ~) + (error-response authenticated url.request 403 ~) ?. =(%'GET' method.request) - (error-response 405 "may only GET scries") + (error-response authenticated url.request 405 "may only GET scries") :: make sure the path contains an app to scry into :: =+ req=(parse-request-line url.request) ?. ?=(^ site.req) - (error-response 400 "scry path must start with app name") + (error-response authenticated url.request 400 "scry path must start with app name") :: attempt the scry that was asked for :: =/ res=(unit (unit cage)) (do-scry %gx i.site.req (snoc t.site.req (fall ext.req %mime))) - ?~ res (error-response 500 "failed scry") - ?~ u.res (error-response 404 "no scry result") + ?~ res (error-response authenticated url.request 500 "failed scry") + ?~ u.res (error-response authenticated url.request 404 "no scry result") =* mark p.u.u.res =* vase q.u.u.res :: attempt to find conversion gate to mime :: =/ tub=(unit tube:clay) (find-tube i.site.req mark %mime) - ?~ tub (error-response 500 "no tube from {(trip mark)} to mime") + ?~ tub (error-response authenticated url.request 500 "no tube from {(trip mark)} to mime") :: attempt conversion, then send results :: =/ mym=(each mime tang) (mule |.(!<(mime (u.tub vase)))) ?- -.mym - %| (error-response 500 "failed tube from {(trip mark)} to mime") + %| (error-response authenticated url.request 500 "failed tube from {(trip mark)} to mime") %& %+ return-static-data-on-duct 200 [(rsh 3 (spat p.p.mym)) q.p.mym] == @@ -804,11 +818,6 @@ ^- (unit (unit cage)) (rof ~ care [our desk da+now] path) :: - ++ error-response - |= [status=@ud =tape] - ^- (quip move server-state) - %^ return-static-data-on-duct status 'text/html' - (error-page status authenticated url.request tape) -- :: +subscribe-to-app: subscribe to app and poke it with request data :: @@ -848,7 +857,7 @@ %leave ~ == :: - ?(%authentication %logout) + ?(%authentication %logout %name) [~ state] :: %channel @@ -875,6 +884,13 @@ data=[~ data] complete=%.y == + :: +error-response: return error data all at once + :: + ++ error-response + |= [authenticated=? url=@t status=@ud =tape] + ^- (quip move server-state) + %^ return-static-data-on-duct status 'text/html' + (error-page status authenticated url tape) :: +authentication: per-event authentication as this Urbit's owner :: :: Right now this hard codes the authentication page using the old +code @@ -1071,7 +1087,7 @@ ^- @t %- crip =; max-age=tape - "urbauth-{(scow %p our)}={(scow %uv session)}; Path=/; Max-Age={max-age}" + "urbauth-{(scow %p our)}={(scow %uv session)}; Path=/; Max-Age={max-age}; HttpOnly" %- format-ud-as-integer ?. extend 0 (div (msec:milly session-timeout) 1.000) @@ -2207,6 +2223,7 @@ [[~ /~/logout] duct [%logout ~]] [[~ /~/channel] duct [%channel ~]] [[~ /~/scry] duct [%scry ~]] + [[~ /~/name] duct [%name ~]] == [~ http-server-gate] :: %trim: in response to memory pressure @@ -2577,15 +2594,43 @@ :: +load: migrate old state to new state (called on vane reload) :: ++ load - |= old=axle + => |% + ++ axle-old + %+ cork + axle + |= =axle + axle(date %~2020.10.18) + -- + |= old=$%(axle axle-old) ^+ ..^$ - :: enable https redirects if certificate configured :: - =. redirect.http-config.server-state.old - ?& ?=(^ secure.ports.server-state.old) - ?=(^ secure.http-config.server-state.old) + ~! %loading + ?- -.old + %~2020.10.18 + =/ [* d=duct *] + %+ snag 0 + %+ skim bindings.server-state.old + |= [=binding =duct =action] + =(%authentication -.action) + =/ new + %= old + date %~2022.7.26 + :: + bindings.server-state + ^- (list [binding duct action]) + :- [[~ /~/name] d [%name ~]] bindings.server-state.old == - ..^$(ax old) + $(old new) + :: + %~2022.7.26 + :: enable https redirects if certificate configured + :: + =. redirect.http-config.server-state.old + ?& ?=(^ secure.ports.server-state.old) + ?=(^ secure.http-config.server-state.old) + == + ..^$(ax old) + == :: +stay: produce current state :: ++ stay `axle`ax diff --git a/tests/sys/vane/eyre.hoon b/tests/sys/vane/eyre.hoon index f8fd9ba0f..c9c115a6e 100644 --- a/tests/sys/vane/eyre.hoon +++ b/tests/sys/vane/eyre.hoon @@ -2347,5 +2347,5 @@ :: ++ cookie-string %^ cat 3 cookie-value - '; Path=/; Max-Age=604800' + '; Path=/; Max-Age=604800; HttpOnly' -- From 70298fd8b3986c58a8ccc32304d8bd1f08b9572d Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 1 Aug 2022 01:51:30 +0300 Subject: [PATCH 52/67] http-api: get ship name from /~/name endpoint instead of cookie --- pkg/npm/http-api/src/Urbit.ts | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/pkg/npm/http-api/src/Urbit.ts b/pkg/npm/http-api/src/Urbit.ts index 870d4dae4..6b01ee37e 100644 --- a/pkg/npm/http-api/src/Urbit.ts +++ b/pkg/npm/http-api/src/Urbit.ts @@ -157,6 +157,23 @@ export class Urbit { return airlock; } + /** + * Gets the name of the ship accessible at this.url and stores it to this.ship + * + */ + async getShipName(): Promise { + if (this.ship) { + return Promise.resolve(); + } + + const nameResp = await fetch(`${this.url}/~/name`, { + method: 'get', + credentials: 'include', + }); + const name = await nameResp.text(); + this.ship = name.substring(1); + } + /** * Connects to the Urbit ship. Nothing can be done until this is called. * That's why we roll it into this.authenticate @@ -174,17 +191,22 @@ export class Urbit { method: 'post', body: `password=${this.code}`, credentials: 'include', - }).then((response) => { + }).then(async response => { if (this.verbose) { console.log('Received authentication response', response); } - const cookie = response.headers.get('set-cookie'); - if (!this.ship) { - this.ship = new RegExp(/urbauth-~([\w-]+)/).exec(cookie)[1]; + + if (response.status !== 204) { + throw new Error('Login failed with status ' + response.status); } + if (!isBrowser) { + const cookie = response.headers.get('set-cookie'); this.cookie = cookie; } + + this.getShipName(); + }); } From 9291fed27f787a029a4f93768c5d09fa729ee480 Mon Sep 17 00:00:00 2001 From: pkova Date: Thu, 11 Aug 2022 19:30:37 +0300 Subject: [PATCH 53/67] eyre: revert +error-response refactor --- pkg/arvo/sys/vane/eyre.hoon | 39 +++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index 88998ad69..bcd5e0f50 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -756,46 +756,53 @@ :: ++ handle-name |= [authenticated=? =request:http] - ^- (quip move server-state) + |^ ^- (quip move server-state) ?. authenticated - (error-response authenticated url.request 403 ~) + (error-response 403 ~) ?. =(%'GET' method.request) - (error-response authenticated url.request 405 "may only GET name") + (error-response 405 "may only GET name") %^ return-static-data-on-duct 200 'text/plain' (as-octs:mimes:html (scot %p our)) + :: + ++ error-response + |= [status=@ud =tape] + ^- (quip move server-state) + %^ return-static-data-on-duct status 'text/html' + (error-page status authenticated url.request tape) + -- :: +handle-scry: respond with scry result, 404 or 500 :: ++ handle-scry |= [authenticated=? =address =request:http] |^ ^- (quip move server-state) ?. authenticated - (error-response authenticated url.request 403 ~) + (error-response 403 ~) ?. =(%'GET' method.request) - (error-response authenticated url.request 405 "may only GET scries") + (error-response 405 "may only GET scries") :: make sure the path contains an app to scry into :: =+ req=(parse-request-line url.request) ?. ?=(^ site.req) - (error-response authenticated url.request 400 "scry path must start with app name") + (error-response 400 "scry path must start with app name") :: attempt the scry that was asked for :: =/ res=(unit (unit cage)) (do-scry %gx i.site.req (snoc t.site.req (fall ext.req %mime))) - ?~ res (error-response authenticated url.request 500 "failed scry") - ?~ u.res (error-response authenticated url.request 404 "no scry result") + ?~ res (error-response 500 "failed scry") + ?~ u.res (error-response 404 "no scry result") =* mark p.u.u.res =* vase q.u.u.res :: attempt to find conversion gate to mime :: =/ tub=(unit tube:clay) (find-tube i.site.req mark %mime) - ?~ tub (error-response authenticated url.request 500 "no tube from {(trip mark)} to mime") + ?~ tub (error-response 500 "no tube from {(trip mark)} to mime") :: attempt conversion, then send results :: =/ mym=(each mime tang) (mule |.(!<(mime (u.tub vase)))) ?- -.mym - %| (error-response authenticated url.request 500 "failed tube from {(trip mark)} to mime") + %| (error-response 500 "failed tube from {(trip mark)} to mime") %& %+ return-static-data-on-duct 200 [(rsh 3 (spat p.p.mym)) q.p.mym] == @@ -818,6 +825,11 @@ ^- (unit (unit cage)) (rof ~ care [our desk da+now] path) :: + ++ error-response + |= [status=@ud =tape] + ^- (quip move server-state) + %^ return-static-data-on-duct status 'text/html' + (error-page status authenticated url.request tape) -- :: +subscribe-to-app: subscribe to app and poke it with request data :: @@ -884,13 +896,6 @@ data=[~ data] complete=%.y == - :: +error-response: return error data all at once - :: - ++ error-response - |= [authenticated=? url=@t status=@ud =tape] - ^- (quip move server-state) - %^ return-static-data-on-duct status 'text/html' - (error-page status authenticated url tape) :: +authentication: per-event authentication as this Urbit's owner :: :: Right now this hard codes the authentication page using the old +code From e3d19f9e8dd56830777404cc5d09ffaed5b1d7c4 Mon Sep 17 00:00:00 2001 From: pkova Date: Thu, 11 Aug 2022 19:32:49 +0300 Subject: [PATCH 54/67] eyre: refactor +load to be prettier --- pkg/arvo/sys/vane/eyre.hoon | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index bcd5e0f50..dadf19290 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -2617,15 +2617,13 @@ %+ skim bindings.server-state.old |= [=binding =duct =action] =(%authentication -.action) - =/ new - %= old - date %~2022.7.26 - :: - bindings.server-state - ^- (list [binding duct action]) - :- [[~ /~/name] d [%name ~]] bindings.server-state.old + ~& [%duct d] + %= $ + date.old %~2022.7.26 + :: + bindings.server-state.old + (insert-binding [[~ /~/name] d [%name ~]] bindings.server-state.old) == - $(old new) :: %~2022.7.26 :: enable https redirects if certificate configured From 712bcba2b2411b1bdfa1f1610cfa45bba7506e2b Mon Sep 17 00:00:00 2001 From: pkova Date: Thu, 11 Aug 2022 19:45:11 +0300 Subject: [PATCH 55/67] eyre: hardcode %init task duct in +load --- pkg/arvo/sys/vane/eyre.hoon | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index dadf19290..1d25b5dbc 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -2612,17 +2612,13 @@ ~! %loading ?- -.old %~2020.10.18 - =/ [* d=duct *] - %+ snag 0 - %+ skim bindings.server-state.old - |= [=binding =duct =action] - =(%authentication -.action) - ~& [%duct d] %= $ date.old %~2022.7.26 :: bindings.server-state.old - (insert-binding [[~ /~/name] d [%name ~]] bindings.server-state.old) + %+ insert-binding + [[~ /~/name] ~[/dill //term/1] [%name ~]] + bindings.server-state.old == :: %~2022.7.26 From ea36b1d1d4b1a0bbeae9709382a8da07997a39c3 Mon Sep 17 00:00:00 2001 From: pkova Date: Thu, 11 Aug 2022 19:51:05 +0300 Subject: [PATCH 56/67] http-api: make status check in connect more permissive --- pkg/npm/http-api/src/Urbit.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/npm/http-api/src/Urbit.ts b/pkg/npm/http-api/src/Urbit.ts index 6b01ee37e..5db949dba 100644 --- a/pkg/npm/http-api/src/Urbit.ts +++ b/pkg/npm/http-api/src/Urbit.ts @@ -196,7 +196,7 @@ export class Urbit { console.log('Received authentication response', response); } - if (response.status !== 204) { + if (response.status >= 200 && response.status < 300) { throw new Error('Login failed with status ' + response.status); } From bb73dc9975cac259fa15decfba8f36b427733007 Mon Sep 17 00:00:00 2001 From: pkova Date: Thu, 11 Aug 2022 20:49:16 +0300 Subject: [PATCH 57/67] eyre: replace hardcoded duct in +load with equivalent outgoing-duct --- pkg/arvo/sys/vane/eyre.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index 1d25b5dbc..111d12fae 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -2617,7 +2617,7 @@ :: bindings.server-state.old %+ insert-binding - [[~ /~/name] ~[/dill //term/1] [%name ~]] + [[~ /~/name] outgoing-duct.server-state.old [%name ~]] bindings.server-state.old == :: From 6d273ff03566dc2988b116e357718a5cb777ba02 Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 29 Aug 2022 14:50:23 +0300 Subject: [PATCH 58/67] lull: describe %name endpoint better --- pkg/arvo/sys/lull.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index f565e237a..f16ffedde 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -1551,7 +1551,7 @@ :: gall scry endpoint :: [%scry ~] - :: internal name endpoint, returns ship @p + :: respond with the @p the requester is authenticated as :: [%name ~] :: respond with the default file not found page From 0202ff069c9b0d519c7905634eb9c1c3fdc6f502 Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 29 Aug 2022 14:50:52 +0300 Subject: [PATCH 59/67] eyre: fix indentation --- pkg/arvo/sys/vane/eyre.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index 111d12fae..b2a644e2d 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -2615,7 +2615,7 @@ %= $ date.old %~2022.7.26 :: - bindings.server-state.old + bindings.server-state.old %+ insert-binding [[~ /~/name] outgoing-duct.server-state.old [%name ~]] bindings.server-state.old From 5fab65cb7ed2887ad203a88531f9ac0f949f4a08 Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 29 Aug 2022 14:51:09 +0300 Subject: [PATCH 60/67] http-api: parse ship name from cookie if possible --- pkg/npm/http-api/src/Urbit.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/npm/http-api/src/Urbit.ts b/pkg/npm/http-api/src/Urbit.ts index 5db949dba..26798a9f4 100644 --- a/pkg/npm/http-api/src/Urbit.ts +++ b/pkg/npm/http-api/src/Urbit.ts @@ -200,8 +200,13 @@ export class Urbit { throw new Error('Login failed with status ' + response.status); } + const cookie = response.headers.get('set-cookie'); + + if (!this.ship && cookie) { + this.ship = new RegExp(/urbauth-~([\w-]+)/).exec(cookie)[1]; + } + if (!isBrowser) { - const cookie = response.headers.get('set-cookie'); this.cookie = cookie; } From b51a4ee1345e996af3449242813faf30faa16b61 Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 13 Feb 2023 14:34:46 +0200 Subject: [PATCH 61/67] pill: update solid.pill --- bin/solid.pill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/solid.pill b/bin/solid.pill index 5650539dc..846a327fa 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2035ef65290065edbd99a86f9f5a36978617bc1983131fa474a9a5c0e91dc15d -size 5998440 +oid sha256:544744b6d1360692b83be7f3e505d0710ea6ce541912509601e81fec2a2cea2e +size 5992253 From 8c08842e98edc9eeb9e813d2a378c2d4a0874e76 Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 13 Feb 2023 14:38:48 +0200 Subject: [PATCH 62/67] http-api: remove changes that went in in PR #6084 --- pkg/npm/http-api/src/Urbit.ts | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/pkg/npm/http-api/src/Urbit.ts b/pkg/npm/http-api/src/Urbit.ts index 26798a9f4..870d4dae4 100644 --- a/pkg/npm/http-api/src/Urbit.ts +++ b/pkg/npm/http-api/src/Urbit.ts @@ -157,23 +157,6 @@ export class Urbit { return airlock; } - /** - * Gets the name of the ship accessible at this.url and stores it to this.ship - * - */ - async getShipName(): Promise { - if (this.ship) { - return Promise.resolve(); - } - - const nameResp = await fetch(`${this.url}/~/name`, { - method: 'get', - credentials: 'include', - }); - const name = await nameResp.text(); - this.ship = name.substring(1); - } - /** * Connects to the Urbit ship. Nothing can be done until this is called. * That's why we roll it into this.authenticate @@ -191,27 +174,17 @@ export class Urbit { method: 'post', body: `password=${this.code}`, credentials: 'include', - }).then(async response => { + }).then((response) => { if (this.verbose) { console.log('Received authentication response', response); } - - if (response.status >= 200 && response.status < 300) { - throw new Error('Login failed with status ' + response.status); - } - const cookie = response.headers.get('set-cookie'); - - if (!this.ship && cookie) { + if (!this.ship) { this.ship = new RegExp(/urbauth-~([\w-]+)/).exec(cookie)[1]; } - if (!isBrowser) { this.cookie = cookie; } - - this.getShipName(); - }); } From 54680c33a81a4a0c16c5d1c6ddf57ffbc12e79b0 Mon Sep 17 00:00:00 2001 From: fang Date: Mon, 13 Feb 2023 15:12:24 +0100 Subject: [PATCH 63/67] eyre: clean up stray ~! --- pkg/arvo/sys/vane/eyre.hoon | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index b2a644e2d..3adc422a4 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -2609,7 +2609,6 @@ |= old=$%(axle axle-old) ^+ ..^$ :: - ~! %loading ?- -.old %~2020.10.18 %= $ From 49a24647da4cd07c03b31e42b17f19b66182c1f5 Mon Sep 17 00:00:00 2001 From: pkova Date: Mon, 13 Feb 2023 20:58:59 +0200 Subject: [PATCH 64/67] eyre: revert inclusion of HttpOnly header for now --- pkg/arvo/sys/vane/eyre.hoon | 2 +- tests/sys/vane/eyre.hoon | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index 3adc422a4..9aabcda8a 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -1092,7 +1092,7 @@ ^- @t %- crip =; max-age=tape - "urbauth-{(scow %p our)}={(scow %uv session)}; Path=/; Max-Age={max-age}; HttpOnly" + "urbauth-{(scow %p our)}={(scow %uv session)}; Path=/; Max-Age={max-age}" %- format-ud-as-integer ?. extend 0 (div (msec:milly session-timeout) 1.000) diff --git a/tests/sys/vane/eyre.hoon b/tests/sys/vane/eyre.hoon index c9c115a6e..f8fd9ba0f 100644 --- a/tests/sys/vane/eyre.hoon +++ b/tests/sys/vane/eyre.hoon @@ -2347,5 +2347,5 @@ :: ++ cookie-string %^ cat 3 cookie-value - '; Path=/; Max-Age=604800; HttpOnly' + '; Path=/; Max-Age=604800' -- From 21158d6d54dde80db34d08809d638ae377506555 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Tue, 14 Feb 2023 06:06:41 +0100 Subject: [PATCH 65/67] ames: style fixes and comments --- pkg/arvo/gen/hood/close-flows.hoon | 4 ++-- pkg/arvo/lib/hood/helm.hoon | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/arvo/gen/hood/close-flows.hoon b/pkg/arvo/gen/hood/close-flows.hoon index 53bfbb7cb..8804c9e34 100644 --- a/pkg/arvo/gen/hood/close-flows.hoon +++ b/pkg/arvo/gen/hood/close-flows.hoon @@ -1,4 +1,4 @@ -:: |close-flows: corks all stale ames flows +:: Deletes all stale ames flows from failed (re) subscriptions :: :: It runs in dry mode by default, printing the flows that can be closed. :: To actually close the flows, run with |close-flows, =dry | @@ -6,4 +6,4 @@ :- %say |= [^ arg=~ dry=?] :: -[%helm-kroc dry] +[%helm-ames-kroc dry] diff --git a/pkg/arvo/lib/hood/helm.hoon b/pkg/arvo/lib/hood/helm.hoon index 34b68bad8..b5a63287e 100644 --- a/pkg/arvo/lib/hood/helm.hoon +++ b/pkg/arvo/lib/hood/helm.hoon @@ -245,9 +245,9 @@ |= ~ =< abet (emit %pass /helm %arvo %a %stir '') :: -++ poke-kroc +++ poke-ames-kroc |= dry=? =< abet - (emit [%pass /helm/kroc %arvo %a %kroc dry]) + (emit %pass /helm %arvo %a %kroc dry) :: ++ poke-knob |= [error-tag=@tas level=?(%hush %soft %loud)] =< abet @@ -284,7 +284,7 @@ %helm-ames-sift =;(f (f !<(_+<.f vase)) poke-ames-sift) %helm-ames-verb =;(f (f !<(_+<.f vase)) poke-ames-verb) %helm-ames-wake =;(f (f !<(_+<.f vase)) poke-ames-wake) - %helm-kroc =;(f (f !<(_+<.f vase)) poke-kroc) + %helm-ames-kroc =;(f (f !<(_+<.f vase)) poke-ames-kroc) %helm-atom =;(f (f !<(_+<.f vase)) poke-atom) %helm-automass =;(f (f !<(_+<.f vase)) poke-automass) %helm-cancel-automass =;(f (f !<(_+<.f vase)) poke-cancel-automass) From c503647eb3286f077435a4c36d453329cb8a70a2 Mon Sep 17 00:00:00 2001 From: yosoyubik Date: Tue, 14 Feb 2023 06:07:59 +0100 Subject: [PATCH 66/67] ames: no-op on %kroc task --- pkg/arvo/sys/vane/ames.hoon | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/arvo/sys/vane/ames.hoon b/pkg/arvo/sys/vane/ames.hoon index f5d20a354..3a0692a4b 100644 --- a/pkg/arvo/sys/vane/ames.hoon +++ b/pkg/arvo/sys/vane/ames.hoon @@ -1962,11 +1962,14 @@ =/ rcvr [ship her-life.channel] "cork plea {}" abet:(on-memo:(make-peer-core peer-state channel) bone plea %plea) - :: +on-kroc: cork all flows from failed subscriptions + :: +on-kroc: cork all stale flows from failed subscriptions :: ++ on-kroc |= dry=? ^+ event-core + :: no-op + :: + ?: & %.(event-core (slog leaf/"ames: %kroc task not ready" ~)) :: =; [corks=@ core=_event-core] ?. dry core From 55afb9d0b5a1bc477de7df865ed94b3f102d1ad6 Mon Sep 17 00:00:00 2001 From: pkova Date: Tue, 21 Feb 2023 20:48:56 +0200 Subject: [PATCH 67/67] lull, eyre: add intermediate type for /~/name endpoint The previous changes implementing the /~/name endpoint were breaking, since we changed the type of `$action:eyre`. This commit keeps the /~/name endpoint functional, but adds adapters to eyre scries that returns the old `$action:eyre` type. These adapters and their associated intermediate types can be removed the next time we burn a kelvin. --- pkg/arvo/sys/lull.hoon | 3 +- pkg/arvo/sys/vane/eyre.hoon | 85 +++++++++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index 8527f3994..9856f9311 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -1555,8 +1555,9 @@ :: [%scry ~] :: respond with the @p the requester is authenticated as + :: TODO: put this back in when we burn the next kelvin :: - [%name ~] + :: [%name ~] :: respond with the default file not found page :: [%four-oh-four ~] diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index 9aabcda8a..e1716c624 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -75,6 +75,32 @@ :: =server-state == +:: +:: +outstanding-connection-new: intermediate type to enable +:: /~/name endpoint without +:: breaking type change in lull +:: ++$ outstanding-connection-new + $: :: action: the action that had matched + :: + action=action-new + :: inbound-request: the original request which caused this connection + :: + =inbound-request + :: response-header: set when we get our first %start + :: + response-header=(unit response-header:http) + :: bytes-sent: the total bytes sent in response + :: + bytes-sent=@ud + == +:: +action-new: intermediate type to enable /~/name endpoint +:: without breaking type change in lull +:: ++$ action-new + $% action + [%name ~] + == :: +server-state: state relating to open inbound HTTP connections :: +$ server-state @@ -87,13 +113,13 @@ :: TODO: It would be nice if we had a path trie. We could decompose :: the :binding into a (map (unit @t) (trie knot =action)). :: - bindings=(list [=binding =duct =action]) + bindings=(list [=binding =duct action=action-new]) :: cors-registry: state used and managed by the +cors core :: =cors-registry :: connections: open http connections not fully complete :: - connections=(map duct outstanding-connection) + connections=(map duct outstanding-connection-new) :: authentication-state: state managed by the +authentication core :: =authentication-state @@ -580,7 +606,7 @@ :: =/ act [%app app=%lens] :: - =/ connection=outstanding-connection + =/ connection=outstanding-connection-new [act [& secure address request] ~ 0] :: =. connections.state @@ -604,13 +630,13 @@ (fall (forwarded-for u.forwards) address) :: =/ host (get-header:http 'host' headers) - =/ [=action suburl=@t] + =/ [action=action-new suburl=@t] (get-action-for-binding host url.request) :: =/ authenticated (request-is-logged-in:authentication request) :: record that we started an asynchronous response :: - =/ connection=outstanding-connection + =/ connection=outstanding-connection-new [action [authenticated secure address request] ~ 0] =. connections.state (~(put by connections.state) duct connection) @@ -1935,7 +1961,7 @@ (session-cookie-string u.session-id &) headers.response-header.http-event :: - =/ connection=outstanding-connection + =/ connection=outstanding-connection-new (~(got by connections.state) duct) :: if the request was a simple cors request from an approved origin :: append the necessary cors headers to the response @@ -1971,7 +1997,7 @@ :: =. connections.state %+ ~(jab by connections.state) duct - |= connection=outstanding-connection + |= connection=outstanding-connection-new =+ size=?~(data.http-event 0 p.u.data.http-event) connection(bytes-sent (add bytes-sent.connection size)) :: @@ -2024,7 +2050,7 @@ :: Adds =binding =action if there is no conflicting bindings. :: ++ add-binding - |= [=binding =action] + |= [=binding action=action-new] ^- [(list move) server-state] =^ success bindings.state :: prevent binding in reserved namespaces @@ -2045,7 +2071,7 @@ %_ state bindings %+ skip bindings.state - |= [item-binding=^binding item-duct=^duct =action] + |= [item-binding=^binding item-duct=^duct action=action-new] ^- ? &(=(item-binding binding) =(item-duct duct)) == @@ -2053,7 +2079,7 @@ :: ++ get-action-for-binding |= [raw-host=(unit @t) url=@t] - ^- [=action suburl=@t] + ^- [action=action-new suburl=@t] :: process :raw-host :: :: If we are missing a 'Host:' header, if that header is a raw IP @@ -2159,8 +2185,8 @@ :: +insert-binding: add a new binding, replacing any existing at its path :: ++ insert-binding - |= $: new=[=binding =duct =action] - bindings=(list [=binding =duct =action]) + |= $: new=[=binding =duct action=action-new] + bindings=(list [=binding =duct action=action-new]) == ^+ bindings ?~ bindings [new]~ @@ -2223,7 +2249,7 @@ :: =. bindings.server-state.ax =- (roll - insert-binding) - ^- (list [binding ^duct action]) + ^- (list [binding ^duct action-new]) :~ [[~ /~/login] duct [%authentication ~]] [[~ /~/logout] duct [%logout ~]] [[~ /~/channel] duct [%channel ~]] @@ -2596,6 +2622,31 @@ -- :: ++ http-server-gate ..$ +:: +bindings-old: filter /~/name endpoint from bindings +:: +++ bindings-old + |= new=(list [b=binding d=duct a=action-new]) + =| old=(list [binding duct action]) + |- ^+ old + ?~ new old + =/ l + ?: ?=([%name ~] a.i.new) + old + (snoc old [b.i.new d.i.new a.i.new]) + $(new t.new, old l) +:: +connections-old: filter /~/name endpoint from connections +:: +++ connections-old + |= new=(map duct outstanding-connection-new) + =| old=(map duct outstanding-connection) + =/ l=(list [d=duct o=outstanding-connection-new]) ~(tap by new) + |- ^+ old + ?~ l old + =/ x + ?: ?=([%name ~] -.o.i.l) + old + (~(put by old) d.i.l o.i.l) + $(l t.l, old x) :: +load: migrate old state to new state (called on vane reload) :: ++ load @@ -2650,9 +2701,9 @@ =* who p.why ?: =(tyl /whey) =/ maz=(list mass) - :~ bindings+&+bindings.server-state.ax + :~ bindings+&+(bindings-old bindings.server-state.ax) auth+&+authentication-state.server-state.ax - connections+&+connections.server-state.ax + connections+&+(connections-old connections.server-state.ax) channels+&+channel-state.server-state.ax axle+&+ax == @@ -2692,8 +2743,8 @@ ?. ?=(%$ ren) [~ ~] ?+ syd [~ ~] - %bindings ``noun+!>(bindings.server-state.ax) - %connections ``noun+!>(connections.server-state.ax) + %bindings ``noun+!>((bindings-old bindings.server-state.ax)) + %connections ``noun+!>((connections-old connections.server-state.ax)) %authentication-state ``noun+!>(authentication-state.server-state.ax) %channel-state ``noun+!>(channel-state.server-state.ax) ::