From 02549975334bd001c4e232bce68052ea17288e76 Mon Sep 17 00:00:00 2001 From: Patrick O'Sullivan Date: Fri, 4 Nov 2022 16:04:22 -0500 Subject: [PATCH 01/10] Groups/Garden: Add S3 region to s3 settings --- pkg/landscape/app/s3-store.hoon | 33 +++++++++++++++++++--- pkg/landscape/gen/s3-store/set-region.hoon | 10 +++++++ pkg/landscape/lib/s3-json.hoon | 3 ++ pkg/landscape/sur/s3-0.hoon | 27 ++++++++++++++++++ pkg/landscape/sur/s3.hoon | 7 +++++ 5 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 pkg/landscape/gen/s3-store/set-region.hoon create mode 100644 pkg/landscape/sur/s3-0.hoon diff --git a/pkg/landscape/app/s3-store.hoon b/pkg/landscape/app/s3-store.hoon index 061a35cf1..cee795389 100644 --- a/pkg/landscape/app/s3-store.hoon +++ b/pkg/landscape/app/s3-store.hoon @@ -9,12 +9,14 @@ +$ card card:agent:gall +$ versioned-state $% state-zero + state-one == :: -+$ state-zero [%0 =credentials =configuration] ++$ state-zero [%0 =credentials:zero:past =configuration:zero:past] ++$ state-one [%1 =credentials =configuration] -- :: -=| state-zero +=| state-one =* state - :: %- agent:dbug @@ -28,8 +30,28 @@ ++ on-init on-init:def ++ on-save !>(state) ++ on-load - |= old-vase=vase - [~ this(state !<(state-zero old-vase))] + |= =vase + =/ old !<(versioned-state vase) + |^ + ?- -.old + %1 `this(state old) + %0 `this(state (state-0-to-1 old)) + == + ++ state-0-to-1 + |= zer=state-zero + ^- state-one + :* %1 + credentials.zer + (configuration-0-to-1 configuration.zer) + == + ++ configuration-0-to-1 + |= conf=configuration:zero:past + ^- ^configuration + :* buckets.conf + current-bucket.conf + '' + == + -- :: ++ on-poke ~/ %s3-poke @@ -56,6 +78,9 @@ :: %set-secret-access-key state(secret-access-key.credentials secret-access-key.act) + :: + %set-region + state(region.configuration region.act) :: %set-current-bucket %_ state diff --git a/pkg/landscape/gen/s3-store/set-region.hoon b/pkg/landscape/gen/s3-store/set-region.hoon new file mode 100644 index 000000000..d87b09f93 --- /dev/null +++ b/pkg/landscape/gen/s3-store/set-region.hoon @@ -0,0 +1,10 @@ +:: s3-store|set-current-bucket: set current bucket for S3 +:: +/- *s3 +:- %say +|= $: [now=@da eny=@uvJ =beak] + [[region=@t ~] ~] + == +:- %s3-action +^- action +[%set-region region] diff --git a/pkg/landscape/lib/s3-json.hoon b/pkg/landscape/lib/s3-json.hoon index e4a531daa..04cf848b9 100644 --- a/pkg/landscape/lib/s3-json.hoon +++ b/pkg/landscape/lib/s3-json.hoon @@ -10,6 +10,7 @@ :~ [%set-endpoint so:dejs] [%set-access-key-id so:dejs] [%set-secret-access-key so:dejs] + [%set-region so:dejs] [%add-bucket so:dejs] [%remove-bucket so:dejs] [%set-current-bucket so:dejs] @@ -25,6 +26,7 @@ :~ ?- -.upd %set-current-bucket [%'setCurrentBucket' s+bucket.upd] %add-bucket [%'addBucket' s+bucket.upd] + %set-region [%'setRegion' s+region.upd] %remove-bucket [%'removeBucket' s+bucket.upd] %set-endpoint [%'setEndpoint' s+endpoint.upd] %set-access-key-id [%'setAccessKeyId' s+access-key-id.upd] @@ -44,6 +46,7 @@ %- pairs:enjs :~ [%buckets a+(turn ~(tap in buckets.configuration.upd) |=(a=@t s+a))] [%'currentBucket' s+current-bucket.configuration.upd] + [%'region' s+region.configuration.upd] == == == diff --git a/pkg/landscape/sur/s3-0.hoon b/pkg/landscape/sur/s3-0.hoon new file mode 100644 index 000000000..4f0ca04cb --- /dev/null +++ b/pkg/landscape/sur/s3-0.hoon @@ -0,0 +1,27 @@ +|% ++$ credentials + $: endpoint=@t + access-key-id=@t + secret-access-key=@t + == +:: ++$ configuration + $: buckets=(set @t) + current-bucket=@t + == +:: ++$ action + $% [%set-endpoint endpoint=@t] + [%set-access-key-id access-key-id=@t] + [%set-secret-access-key secret-access-key=@t] + [%add-bucket bucket=@t] + [%remove-bucket bucket=@t] + [%set-current-bucket bucket=@t] + == +:: ++$ update + $% [%credentials =credentials] + [%configuration =configuration] + action + == +-- diff --git a/pkg/landscape/sur/s3.hoon b/pkg/landscape/sur/s3.hoon index 4f0ca04cb..a79ca68ce 100644 --- a/pkg/landscape/sur/s3.hoon +++ b/pkg/landscape/sur/s3.hoon @@ -1,4 +1,9 @@ +/- zer=s3-0 |% +++ past + |% + ++ zero zer + -- +$ credentials $: endpoint=@t access-key-id=@t @@ -8,6 +13,7 @@ +$ configuration $: buckets=(set @t) current-bucket=@t + region=@t == :: +$ action @@ -17,6 +23,7 @@ [%add-bucket bucket=@t] [%remove-bucket bucket=@t] [%set-current-bucket bucket=@t] + [%set-region region=@t] == :: +$ update From 48978bd940ae92c9ee43ede5e6dd176402284cf6 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Wed, 14 Dec 2022 12:46:36 -0600 Subject: [PATCH 02/10] group-store: allow rebuilding --- pkg/landscape/app/group-store.hoon | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index 7e26a80ff..b373024d6 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -140,6 +140,7 @@ ?+ q.vase !! %migrate poke-migrate:gc %export poke-export:gc + %rebuild poke-rebuild:gc == :: ?(%group-update-0 %group-action) @@ -235,6 +236,18 @@ :: |_ bol=bowl:gall +* io ~(. agentio bol) +++ poke-rebuild + ^- (quip card _state) + =/ wex ~(tap by wex.bol) + |- + ?~ wex + `state + =/ [[=wire =ship =term] [acked=? =(pole knot)]] + i.wex + ?. ?=([%gladio ship=@ ~] pole) + $(wex t.wex) + $(wex t.wex, wait (~(put in wait) (slav %p ship.pole))) +:: ++ poke-export ^- (quip card _state) :_ state @@ -256,6 +269,8 @@ ^- card [%pass /gladio/(scot %p ship) %agent [ship %groups] %watch /init] :: + +:: ++ backoff-migrate |= =ship ^- card From 4df4a16881085b8896db1400536731e9f2de5e65 Mon Sep 17 00:00:00 2001 From: Liam Fitzgerald Date: Wed, 14 Dec 2022 13:29:17 -0600 Subject: [PATCH 03/10] group-store: add %rebuild poke --- pkg/landscape/app/group-store.hoon | 41 ++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index b373024d6..c7c429a55 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -238,15 +238,36 @@ +* io ~(. agentio bol) ++ poke-rebuild ^- (quip card _state) - =/ wex ~(tap by wex.bol) - |- - ?~ wex - `state - =/ [[=wire =ship =term] [acked=? =(pole knot)]] - i.wex - ?. ?=([%gladio ship=@ ~] pole) - $(wex t.wex) - $(wex t.wex, wait (~(put in wait) (slav %p ship.pole))) + |^ + =. wait + put-missing + =^ cards state + watch-missing + [cards state] + :: + ++ watch-missing + =/ wait ~(tap in wait) + =| cards=(list card) + |- + ?~ wait + [cards state] + ?: (~(has by wex.bol) [/gladio/(scot %p i.wait) i.wait dap.bol]) + $(wait t.wait) + =. cards + :_(cards (watch-init-migrate i.wait)) + $(wait t.wait) + :: + ++ put-missing + =/ wex ~(tap by wex.bol) + |- + ?~ wex + wait + =/ [[=wire =ship =term] [acked=? =(pole knot)]] + i.wex + ?. ?=([%gladio ship=@ ~] pole) + $(wex t.wex) + $(wex t.wex, wait (~(put in wait) (slav %p ship.pole))) + -- :: ++ poke-export ^- (quip card _state) @@ -269,8 +290,6 @@ ^- card [%pass /gladio/(scot %p ship) %agent [ship %groups] %watch /init] :: - -:: ++ backoff-migrate |= =ship ^- card From 4e3698f327c6749e5be750d28f2a91ddec598487 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Tue, 3 Jan 2023 17:47:26 -0600 Subject: [PATCH 04/10] group-store: instead of only watching missing, restart any migration subscriptions that might be bad --- pkg/landscape/app/group-store.hoon | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index 16e66fe35..2eb5f3d9c 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -242,19 +242,22 @@ =. wait put-missing =^ cards state - watch-missing + rewatch [cards state] :: - ++ watch-missing + ++ rewatch =/ wait ~(tap in wait) =| cards=(list card) |- ?~ wait [cards state] - ?: (~(has by wex.bol) [/gladio/(scot %p i.wait) i.wait dap.bol]) - $(wait t.wait) + =/ wir /gladio/(scot %p i.wait) =. cards :_(cards (watch-init-migrate i.wait)) + :: if we have a subscription already, leave first to restart + =? cards + (~(has by wex.bol) [wir i.wait dap.bol]) + :_(cards [%pass wir %agent [our.bol dap.bol] %leave ~]) $(wait t.wait) :: ++ put-missing From 264678a908fa4e913aad28259ee5eff47e6f39a4 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Tue, 3 Jan 2023 18:17:02 -0600 Subject: [PATCH 05/10] group-store: correct target --- pkg/landscape/app/group-store.hoon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index 2eb5f3d9c..b41645948 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -257,7 +257,7 @@ :: if we have a subscription already, leave first to restart =? cards (~(has by wex.bol) [wir i.wait dap.bol]) - :_(cards [%pass wir %agent [our.bol dap.bol] %leave ~]) + :_(cards [%pass wir %agent [i.wait %groups] %leave ~]) $(wait t.wait) :: ++ put-missing From 936835cd0c2cba5f513c888456d4244b5eb6dc47 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Tue, 3 Jan 2023 18:50:05 -0600 Subject: [PATCH 06/10] groups-store: fixing wire check for leaves --- pkg/landscape/app/group-store.hoon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index b41645948..5efa484c6 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -256,8 +256,8 @@ :_(cards (watch-init-migrate i.wait)) :: if we have a subscription already, leave first to restart =? cards - (~(has by wex.bol) [wir i.wait dap.bol]) - :_(cards [%pass wir %agent [i.wait %groups] %leave ~]) + (~(has by wex.bol) [wir i.wait %groups]) + :_(cards [%pass wir %agent [i.wait %groups] %leave ~]) $(wait t.wait) :: ++ put-missing From 563aed6b7f0c8eba06174885b275d0054158b551 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Wed, 4 Jan 2023 11:17:44 -0600 Subject: [PATCH 07/10] group-store: automatically run rebuild on load --- pkg/landscape/app/group-store.hoon | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index 5efa484c6..6b3bfc9f6 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -84,7 +84,10 @@ =| cards=(list card) |^ ?- -.old - %3 [(flop cards) this(state old)] + %3 + :_ this(state old) + :_ cards + [%pass /pyre/rebuild %agent [our dap]:bowl %poke noun+!>(%rebuild)] :: %2 %_ $ From 9316aebec75558203d065f2c3576dbfafe1776d2 Mon Sep 17 00:00:00 2001 From: Hunter Miller Date: Wed, 4 Jan 2023 12:40:18 -0600 Subject: [PATCH 08/10] group-store: only run rebuild once --- pkg/landscape/app/group-store.hoon | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index 6b3bfc9f6..bd4dcf982 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -41,6 +41,7 @@ state-one state-two state-three + state-four == :: +$ state-zero @@ -61,9 +62,15 @@ =groups wait=(set ship) == +:: ++$ state-four + $: %4 + =groups + wait=(set ship) + == -- :: -=| state-three +=| state-four =* state - :: %- agent:dbug @@ -84,10 +91,15 @@ =| cards=(list card) |^ ?- -.old - %3 - :_ this(state old) - :_ cards - [%pass /pyre/rebuild %agent [our dap]:bowl %poke noun+!>(%rebuild)] + %4 [(flop cards) this(state old)] + :: + %3 + %_ $ + old [%4 +.old] + cards + :_ cards + [%pass /pyre/rebuild %agent [our dap]:bowl %poke noun+!>(%rebuild)] + == :: %2 %_ $ @@ -391,8 +403,8 @@ |= arc=* ^- (quip card _state) |^ - =/ sty=state-three - [%3 (remake-groups ;;((tree [resource tree-group]) +.arc)) ~] + =/ sty=state-four + [%4 (remake-groups ;;((tree [resource tree-group]) +.arc)) ~] :_ sty %+ roll ~(tap by groups.sty) |= [[rid=resource grp=group] out=(list card)] From 00f9908cb6fe616bbf6f510b00bc613d5e2926b5 Mon Sep 17 00:00:00 2001 From: fang Date: Thu, 5 Jan 2023 18:53:44 +0100 Subject: [PATCH 09/10] ci: remove the mingw build flow As discussed internally at the end of last year, we are discontinuing Windows support for vere. The tldr is that we presently don't have the resources to support non-*nix systems at the level of quality we want to deliver. Notably Windows is already very much a second-class citizen (#5822 and others). Windows users may want to run Urbit through WSL instead. As such, the Window build step should be removed from our CI. --- .github/workflows/vere.yml | 79 +------------------------------------- 1 file changed, 1 insertion(+), 78 deletions(-) diff --git a/.github/workflows/vere.yml b/.github/workflows/vere.yml index eb94e8ea9..9315d8bf4 100644 --- a/.github/workflows/vere.yml +++ b/.github/workflows/vere.yml @@ -162,86 +162,9 @@ jobs: - if: ${{ matrix.os == 'ubuntu-latest' }} run: nix-build -A docker-image - mingw: - runs-on: windows-latest - defaults: - run: - shell: > - C:\msys64\msys2_shell.cmd -mingw64 -defterm -no-start -here -c - ". <(cygpath '{0}')" - working-directory: ./pkg/urbit - - steps: - - uses: actions/checkout@v2 - with: - lfs: true - - # echo suppresses pacman prompt - - run: echo|./configure - env: - CACHIX_CACHE: ares - CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - - - run: mingw32-make build/urbit - - run: mingw32-make test - - run: > - build/urbit -l -d -B ../../bin/solid.pill -F bus && - curl -f --data '{"source":{"dojo":"+hood/exit"},"sink":{"app":"hood"}}' - http://localhost:12321 - - - name: confirm binary is mostly static - run: | - if [ -z "$(ldd build/urbit | grep -vi "windows/system32")"]; then - echo "it's mostly static" - exit 0 - else - echo "dynamic links found:" - ldd build/urbit - exit 1 - fi - - - uses: actions/setup-python@v2 - if: inputs.upload - with: - python-version: 3.7 - - - uses: google-github-actions/setup-gcloud@v0.6.0 - if: inputs.upload - env: - # see https://github.com/google-github-actions/setup-gcloud/issues/100 - CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe - with: - service_account_key: ${{ secrets.GCS_SERVICE_ACCOUNT_KEY }} - project_id: ${{ secrets.GCS_PROJECT }} - export_default_credentials: true - - - name: upload binary to bootstrap.urbit.org - if: inputs.upload - env: - CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe - shell: bash - run: | - if [ "real" == "$VERSION_TYPE" ]; then - version="$(cat ./version)" - else - version="${GITHUB_SHA:0:9}" - fi - - system="x86_64-windows" - target="gs://${UPLOAD_BASE}/${VERE_PACE}/${version}/vere-v${version}-${system}.exe" - - gsutil cp -n ./build/urbit.exe "$target" - exitcode=$? - - test $exitcode -eq 0 && - echo "upload to $target complete." || - echo "upload to $target failed."; - exit $exitcode - - after: runs-on: ubuntu-latest - needs: [urbit, mingw] + needs: [urbit] if: inputs.upload steps: - uses: google-github-actions/setup-gcloud@v0.2.0 From b1a5e109f56fbe6eb0cf55cc63bd96360f75d191 Mon Sep 17 00:00:00 2001 From: fang Date: Thu, 5 Jan 2023 18:56:46 +0100 Subject: [PATCH 10/10] ci: remove frontend-tests flow These are no longer being used as much as they were, most of the stuff it was hitting lives in separate repos now. Can be ripped out as such. --- .github/workflows/frontend-test.yml | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 .github/workflows/frontend-test.yml diff --git a/.github/workflows/frontend-test.yml b/.github/workflows/frontend-test.yml deleted file mode 100644 index 89e68662a..000000000 --- a/.github/workflows/frontend-test.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: frontend-test - -on: - pull_request: - paths: - - 'pkg/interface/**' - - 'pkg/btc-wallet/**' - - 'pkg/npm/**' - -jobs: - frontend-test: - runs-on: ubuntu-latest - name: "Test changed frontend packages" - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - run: git fetch --prune - - name: 'Setup root deps' - run: npm ci - - name: 'Setup dependencies' - run: npm run bootstrap - - name: 'Run tests' - run: npm run test -- --since origin/$GITHUB_BASE_REF --include-dependents