diff --git a/.github/workflows/frontend-test.yml b/.github/workflows/frontend-test.yml deleted file mode 100644 index 89e68662a..000000000 --- a/.github/workflows/frontend-test.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: frontend-test - -on: - pull_request: - paths: - - 'pkg/interface/**' - - 'pkg/btc-wallet/**' - - 'pkg/npm/**' - -jobs: - frontend-test: - runs-on: ubuntu-latest - name: "Test changed frontend packages" - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - run: git fetch --prune - - name: 'Setup root deps' - run: npm ci - - name: 'Setup dependencies' - run: npm run bootstrap - - name: 'Run tests' - run: npm run test -- --since origin/$GITHUB_BASE_REF --include-dependents diff --git a/.github/workflows/vere.yml b/.github/workflows/vere.yml index eb94e8ea9..9315d8bf4 100644 --- a/.github/workflows/vere.yml +++ b/.github/workflows/vere.yml @@ -162,86 +162,9 @@ jobs: - if: ${{ matrix.os == 'ubuntu-latest' }} run: nix-build -A docker-image - mingw: - runs-on: windows-latest - defaults: - run: - shell: > - C:\msys64\msys2_shell.cmd -mingw64 -defterm -no-start -here -c - ". <(cygpath '{0}')" - working-directory: ./pkg/urbit - - steps: - - uses: actions/checkout@v2 - with: - lfs: true - - # echo suppresses pacman prompt - - run: echo|./configure - env: - CACHIX_CACHE: ares - CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - - - run: mingw32-make build/urbit - - run: mingw32-make test - - run: > - build/urbit -l -d -B ../../bin/solid.pill -F bus && - curl -f --data '{"source":{"dojo":"+hood/exit"},"sink":{"app":"hood"}}' - http://localhost:12321 - - - name: confirm binary is mostly static - run: | - if [ -z "$(ldd build/urbit | grep -vi "windows/system32")"]; then - echo "it's mostly static" - exit 0 - else - echo "dynamic links found:" - ldd build/urbit - exit 1 - fi - - - uses: actions/setup-python@v2 - if: inputs.upload - with: - python-version: 3.7 - - - uses: google-github-actions/setup-gcloud@v0.6.0 - if: inputs.upload - env: - # see https://github.com/google-github-actions/setup-gcloud/issues/100 - CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe - with: - service_account_key: ${{ secrets.GCS_SERVICE_ACCOUNT_KEY }} - project_id: ${{ secrets.GCS_PROJECT }} - export_default_credentials: true - - - name: upload binary to bootstrap.urbit.org - if: inputs.upload - env: - CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe - shell: bash - run: | - if [ "real" == "$VERSION_TYPE" ]; then - version="$(cat ./version)" - else - version="${GITHUB_SHA:0:9}" - fi - - system="x86_64-windows" - target="gs://${UPLOAD_BASE}/${VERE_PACE}/${version}/vere-v${version}-${system}.exe" - - gsutil cp -n ./build/urbit.exe "$target" - exitcode=$? - - test $exitcode -eq 0 && - echo "upload to $target complete." || - echo "upload to $target failed."; - exit $exitcode - - after: runs-on: ubuntu-latest - needs: [urbit, mingw] + needs: [urbit] if: inputs.upload steps: - uses: google-github-actions/setup-gcloud@v0.2.0 diff --git a/bin/multi-brass.pill b/bin/multi-brass.pill index e1229bb75..d95c40bd6 100644 --- a/bin/multi-brass.pill +++ b/bin/multi-brass.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3306a5b916caa838c67cb079410c6e5bb158054520129ffb9f9bdb144ab1b691 -size 7418493 +oid sha256:ea8626444e4f0213e39c21ded20607145ee85a947afc592f182f46e7f598ef30 +size 7748671 diff --git a/pkg/garden/app/docket.hoon b/pkg/garden/app/docket.hoon index 671834ccc..54e595a69 100644 --- a/pkg/garden/app/docket.hoon +++ b/pkg/garden/app/docket.hoon @@ -394,7 +394,10 @@ :: !=(%kids desk) :: == :: [dap.bowl %no-docket-file-for desk] - `state + ?. (~(has by charges) desk) + `state + :- ~[del-fact:cha] + state(charges (~(del by charges) desk)) :: always update the docket in state to match clay's :: =/ =docket docket:cha diff --git a/pkg/interface/src/views/apps/publish/components/Note.tsx b/pkg/interface/src/views/apps/publish/components/Note.tsx index 988e81eea..b5f7f5f2c 100644 --- a/pkg/interface/src/views/apps/publish/components/Note.tsx +++ b/pkg/interface/src/views/apps/publish/components/Note.tsx @@ -26,6 +26,7 @@ interface NoteProps { } export function NoteContent({ post }) { + console.log(post.contents); return ( diff --git a/pkg/landscape/app/contact-store.hoon b/pkg/landscape/app/contact-store.hoon index e3f750715..bac7cd9ec 100644 --- a/pkg/landscape/app/contact-store.hoon +++ b/pkg/landscape/app/contact-store.hoon @@ -123,7 +123,15 @@ :: ensure difference =/ old=(unit contact:store) (~(get by rolodex) ship) ?. ?| ?=(~ old) - !=(contact(last-updated *@da) u.old(last-updated *@da)) + :: if new contact is before existing contact, no-op + :: + :: NB: last-updated.contact is often *@da, so this + :: effectively stops add from applying if we already have + :: a contact for them + :: + ?& (gth last-updated.contact last-updated.u.old) + !=(contact(last-updated *@da) u.old(last-updated *@da)) + == == [~ state] ~| "cannot add a data url to cover!" diff --git a/pkg/landscape/app/graph-pull-hook.hoon b/pkg/landscape/app/graph-pull-hook.hoon index 1a134e614..364e0d00d 100644 --- a/pkg/landscape/app/graph-pull-hook.hoon +++ b/pkg/landscape/app/graph-pull-hook.hoon @@ -54,12 +54,7 @@ ++ on-pull-nack |= [=resource =tang] ^- (quip card _this) - %- (slog leaf+"nacked {}" tang) - :_ this - ?. (~(has in get-keys:gra) resource) ~ - =- [%pass /pull-nack %agent [our.bowl %graph-store] %poke %graph-update-3 -]~ - !> ^- update:store - [now.bowl [%archive-graph resource]] + `this :: ++ on-pull-kick |= =resource diff --git a/pkg/landscape/app/graph-store.hoon b/pkg/landscape/app/graph-store.hoon index 2ba5d9a19..3170ead6e 100644 --- a/pkg/landscape/app/graph-store.hoon +++ b/pkg/landscape/app/graph-store.hoon @@ -11,16 +11,17 @@ [%3 network:one:store] [%4 network:store] [%5 network:store] - state-6 + [%6 network:store] + state-7 == ::- -+$ state-6 [%6 network:store] ++$ state-7 [%7 network:store] ++ orm orm:store ++ orm-log orm-log:store ++ mar %graph-update-3 -- :: -=| state-6 +=| state-7 =* state - :: %- agent:dbug @@ -96,7 +97,43 @@ (scag 2 (tap:orm-log update-log)) == :: - %6 [cards this(state old)] + %6 + =/ old-dms + %- ~(gas by *(map resource:store marked-graph:store)) + %+ skim ~(tap by graphs.old) + |=([r=resource:store *] (is-old-dm:upgrade:store r)) + =/ backup (backup:upgrade:store bowl) + %_ $ + -.old %7 + archive.old ~ + update-logs.old + %- ~(gas by *(map resource:store update-log:store)) + %+ murn ~(tap by update-logs.old) + |= [r=resource:store =update-log:store] + ?: (is-old-dm:upgrade:store r) + ~ + `[r (strip-sigs-log:upgrade:store update-log)] + :: + graphs.old + %- ~(gas by *(map resource:store marked-graph:store)) + %+ murn ~(tap by graphs.old) + |= [r=resource:store =graph:store mar=(unit mark)] + ?: (is-old-dm:upgrade:store r) + ~ + `[r (strip-sigs-graph:upgrade:store graph) mar] + :: + cards + ;: welp + cards + :: + (nuke-groups:upgrade:store bowl) + :: + (turn ~(tap by archive.old) backup) + (turn ~(tap by old-dms) backup) + == + == + :: + %7 [cards this(state old)] == :: ++ on-watch @@ -129,9 +166,21 @@ ?+ mark (on-poke:def mark vase) %graph-update-3 (graph-update !<(update:store vase)) %import (poke-import q.vase) + %migrated (poke-migrated !<(resource:store vase)) == [cards this] :: + ++ poke-migrated + |= r=resource:res + ^- (quip card _state) + =/ =path /(rap 3 'backup-' (scot %p entity.r) '-' name.r ~)/noun + =/ graph (~(got by graphs) r) + :- [%pass /migrate %agent [our.bowl %hood] %poke drum-put+!>([path (jam r graph)])]~ + %_ state + graphs (~(del by graphs) r) + update-logs (~(del by update-logs) r) + == + :: ++ graph-update |= =update:store ^- (quip card _state) diff --git a/pkg/landscape/app/group-pull-hook.hoon b/pkg/landscape/app/group-pull-hook.hoon index f97582e70..aad34e33b 100644 --- a/pkg/landscape/app/group-pull-hook.hoon +++ b/pkg/landscape/app/group-pull-hook.hoon @@ -45,10 +45,7 @@ ++ on-pull-nack |= [=resource =tang] ^- (quip card _this) - %- (slog tang) - :_ this - =- [%pass / %agent [our.bowl %group-store] %poke -]~ - group-update-0+!>([%remove-group resource ~]) + `this :: ++ on-pull-kick |= =resource diff --git a/pkg/landscape/app/group-store.hoon b/pkg/landscape/app/group-store.hoon index f869a4c0a..bd4dcf982 100644 --- a/pkg/landscape/app/group-store.hoon +++ b/pkg/landscape/app/group-store.hoon @@ -31,13 +31,17 @@ :: /- *group /+ store=group-store, default-agent, verb, dbug, resource, *migrate, agentio +/+ gladio |% +$ card card:agent:gall +++ ota-host ~sogryp-dister-dozzod-dozzod :: +$ versioned-state $% state-zero state-one state-two + state-three + state-four == :: +$ state-zero @@ -52,13 +56,25 @@ $: %2 =groups == +:: ++$ state-three + $: %3 + =groups + wait=(set ship) + == +:: ++$ state-four + $: %4 + =groups + wait=(set ship) + == -- :: -=| state-two +=| state-four =* state - :: %- agent:dbug -%+ verb | +%+ verb & ^- agent:gall =< |_ =bowl:gall @@ -72,10 +88,32 @@ ++ on-load |= =old=vase =/ old !<(versioned-state old-vase) + =| cards=(list card) |^ - ?- -.old - %2 `this(state old) - :: + ?- -.old + %4 [(flop cards) this(state old)] + :: + %3 + %_ $ + old [%4 +.old] + cards + :_ cards + [%pass /pyre/rebuild %agent [our dap]:bowl %poke noun+!>(%rebuild)] + == + :: + %2 + %_ $ + old [%3 groups.old ~] + cards + %- welp + :_ cards + :~ [%pass /pyre/export %agent [our dap]:bowl %poke noun+!>(%export)] + [%pass /pyre/migrate %agent [our dap]:bowl %poke noun+!>(%migrate)] + [%pass / %agent [our %hood]:bowl %poke %kiln-install !>([%groups ota-host %groups])] + [%pass / %agent [our %hood]:bowl %poke %kiln-install !>([%talk ota-host %talk])] + == + == + :: %1 %_ $ -.old %2 @@ -112,6 +150,13 @@ =^ cards state ?+ mark (on-poke:def mark vase) %sane (poke-sane:gc !<(?(%check %fix) vase)) + :: + %noun + ?+ q.vase !! + %migrate poke-migrate:gc + %export poke-export:gc + %rebuild poke-rebuild:gc + == :: ?(%group-update-0 %group-action) (poke-group-update:gc !<(update:store vase)) @@ -125,6 +170,8 @@ |= =path ^- (quip card _this) ?> (team:title our.bowl src.bowl) + ?: ?=([%wait ~] path) + `this ?> ?=([%groups ~] path) :_ this [%give %fact ~ %group-update-0 !>([%initial groups])]~ @@ -135,6 +182,8 @@ |= =path ^- (unit (unit cage)) ?+ path (on-peek:def path) + [%x %wait ~] + ``ships+!>(~(tap in wait)) [%y %groups ~] ``noun+!>(`(set resource)`~(key by groups)) :: @@ -159,28 +208,38 @@ ++ on-agent |= [=wire =sign:agent:gall] ^- (quip card _this) - ?. ?=([%try-rejoin @ *] wire) - (on-agent:def wire sign) - ?> ?=(%poke-ack -.sign) - =/ rid=resource (de-path:resource t.t.wire) - ?~ p.sign - =/ =cage - [%pull-hook-action !>([%add entity.rid rid])] - :_ this - [%pass / %agent [our.bowl %group-pull-hook] %poke cage]~ - =/ nack-count=@ud (slav %ud i.t.wire) - =/ wakeup=@da - (add now.bowl (mul ~s1 (bex (min 19 nack-count)))) - :_ this - [%pass wire %arvo %b %wait wakeup]~ + =^ cards state + ?+ wire [- state]:(on-agent:def wire sign) + [%pyre *] (take-pyre:gc t.wire sign) + [%gladio @ ~] (take-migrate:gc sign) + :: + [%try-rejoin @ *] + ?> ?=(%poke-ack -.sign) + =/ rid=resource (de-path:resource t.t.wire) + ?~ p.sign + =/ =cage + [%pull-hook-action !>([%add entity.rid rid])] + :_ state + [%pass / %agent [our.bowl %group-pull-hook] %poke cage]~ + =/ nack-count=@ud (slav %ud i.t.wire) + =/ wakeup=@da + (add now.bowl (mul ~s1 (bex (min 19 nack-count)))) + :_ state + [%pass wire %arvo %b %wait wakeup]~ + == + [cards this] :: ++ on-arvo - |= [=wire =sign-arvo] + |= [=(pole knot) =sign-arvo] ^- (quip card _this) - ?. ?=([%try-rejoin @ *] wire) - (on-arvo:def wire sign-arvo) - =/ =resource (de-path:resource t.t.wire) - =/ nack-count=@ud (slav %ud i.t.wire) + ?: ?=([%gladio %backoff ship=@ ~] pole) + =^ cards state + (take-backoff:gc (slav %p ship.pole) sign-arvo) + [cards this] + ?. ?=([%try-rejoin count=@ res=*] pole) + (on-arvo:def pole sign-arvo) + =/ =resource (de-path:resource res.pole) + =/ nack-count=@ud (slav %ud count.pole) ?> ?=([%behn %wake *] sign-arvo) ~? ?=(^ error.sign-arvo) "behn errored in backoff timers, continuing anyway" @@ -192,6 +251,107 @@ :: |_ bol=bowl:gall +* io ~(. agentio bol) +++ poke-rebuild + ^- (quip card _state) + |^ + =. wait + put-missing + =^ cards state + rewatch + [cards state] + :: + ++ rewatch + =/ wait ~(tap in wait) + =| cards=(list card) + |- + ?~ wait + [cards state] + =/ wir /gladio/(scot %p i.wait) + =. cards + :_(cards (watch-init-migrate i.wait)) + :: if we have a subscription already, leave first to restart + =? cards + (~(has by wex.bol) [wir i.wait %groups]) + :_(cards [%pass wir %agent [i.wait %groups] %leave ~]) + $(wait t.wait) + :: + ++ put-missing + =/ wex ~(tap by wex.bol) + |- + ?~ wex + wait + =/ [[=wire =ship =term] [acked=? =(pole knot)]] + i.wex + ?. ?=([%gladio ship=@ ~] pole) + $(wex t.wex) + $(wex t.wex, wait (~(put in wait) (slav %p ship.pole))) + -- +:: +++ poke-export + ^- (quip card _state) + :_ state + =; =cage + [%pass /export %agent [our.bol %hood] %poke cage]~ + drum-put+!>([/groups/jam ~(export gladio bol)]) +:: +++ poke-migrate + ^- (quip card _state) + =^ cards-1=(list card) wait + (~(migrate-start gladio bol) wait) + =/ cards-2=(list card) + (turn ~(tap in wait) watch-init-migrate) + =/ cards (welp cards-1 cards-2) + [cards state(wait wait)] +:: +++ watch-init-migrate + |= =ship + ^- card + [%pass /gladio/(scot %p ship) %agent [ship %groups] %watch /init] +:: +++ backoff-migrate + |= =ship + ^- card + [%pass /gladio/backoff/(scot %p ship) %arvo %b %wait (add ~h1 now.bol)] +:: +++ take-pyre + |= [=wire =sign:agent:gall] + ^- (quip card _state) + :_ state + ?> ?=(%poke-ack -.sign) + ?~ p.sign + ~ + [%pass / %pyre leaf/"{} failed" u.p.sign]~ +:: +++ take-backoff + |= [=ship sign=sign-arvo] + ^- (quip card _state) + ?> ?=([%behn %wake *] sign) + ?: ?=(^ error.sign) + `state + :_ state + ~[(watch-init-migrate ship)] +:: +++ take-migrate + |= =sign:agent:gall + ^- (quip card _state) + ?. (~(has in wait) src.bol) + :: already succeeded + `state + ?- -.sign + ?(%poke-ack %fact) `state + %kick :_(state (watch-init-migrate src.bol)^~) + %watch-ack + ?~ p.sign + :: they have public release + ~& migrating/src.bol + =. wait (~(del in wait) src.bol) + :_ state + :- [%give %fact ~[/wait] ships+!>(~(tap in wait))] + (~(migrate-ship gladio bol) src.bol) + :_ state + ~[(backoff-migrate src.bol)] + == +:: ++ peek-group |= rid=resource ^- (unit group) @@ -243,8 +403,8 @@ |= arc=* ^- (quip card _state) |^ - =/ sty=state-two - [%2 (remake-groups ;;((tree [resource tree-group]) +.arc))] + =/ sty=state-four + [%4 (remake-groups ;;((tree [resource tree-group]) +.arc)) ~] :_ sty %+ roll ~(tap by groups.sty) |= [[rid=resource grp=group] out=(list card)] diff --git a/pkg/landscape/app/metadata-pull-hook.hoon b/pkg/landscape/app/metadata-pull-hook.hoon index f2d08fd30..2877a098d 100644 --- a/pkg/landscape/app/metadata-pull-hook.hoon +++ b/pkg/landscape/app/metadata-pull-hook.hoon @@ -321,15 +321,7 @@ ++ on-pull-nack |= [=resource =tang] ^- (quip card _this) - =/ =associations:metadata - (metadata-for-group:met resource) - :_ this - %+ turn ~(tap by associations) - |= [=md-resource:metadata =association:metadata] - %+ poke-our:pass:io:hc %metadata-store - :- %metadata-update-2 - !> ^- update:metadata - [%remove resource md-resource] + `this :: ++ on-pull-kick |= =resource diff --git a/pkg/landscape/app/s3-store.hoon b/pkg/landscape/app/s3-store.hoon index 061a35cf1..cee795389 100644 --- a/pkg/landscape/app/s3-store.hoon +++ b/pkg/landscape/app/s3-store.hoon @@ -9,12 +9,14 @@ +$ card card:agent:gall +$ versioned-state $% state-zero + state-one == :: -+$ state-zero [%0 =credentials =configuration] ++$ state-zero [%0 =credentials:zero:past =configuration:zero:past] ++$ state-one [%1 =credentials =configuration] -- :: -=| state-zero +=| state-one =* state - :: %- agent:dbug @@ -28,8 +30,28 @@ ++ on-init on-init:def ++ on-save !>(state) ++ on-load - |= old-vase=vase - [~ this(state !<(state-zero old-vase))] + |= =vase + =/ old !<(versioned-state vase) + |^ + ?- -.old + %1 `this(state old) + %0 `this(state (state-0-to-1 old)) + == + ++ state-0-to-1 + |= zer=state-zero + ^- state-one + :* %1 + credentials.zer + (configuration-0-to-1 configuration.zer) + == + ++ configuration-0-to-1 + |= conf=configuration:zero:past + ^- ^configuration + :* buckets.conf + current-bucket.conf + '' + == + -- :: ++ on-poke ~/ %s3-poke @@ -56,6 +78,9 @@ :: %set-secret-access-key state(secret-access-key.credentials secret-access-key.act) + :: + %set-region + state(region.configuration region.act) :: %set-current-bucket %_ state diff --git a/pkg/landscape/desk.docket-0 b/pkg/landscape/desk.docket-0 deleted file mode 100644 index e6b65c6ac..000000000 --- a/pkg/landscape/desk.docket-0 +++ /dev/null @@ -1,10 +0,0 @@ -:~ title+'Groups' - info+'A suite of applications to communicate on Urbit' - color+0xee.5432 - glob-http+['https://bootstrap.urbit.org/glob-0v7.2rpmd.966js.dt2sj.ggv4a.n15nq.glob' 0v7.2rpmd.966js.dt2sj.ggv4a.n15nq] - - base+'landscape' - version+[1 1 0] - website+'https://tlon.io' - license+'MIT' -== diff --git a/pkg/landscape/gen/s3-store/set-region.hoon b/pkg/landscape/gen/s3-store/set-region.hoon new file mode 100644 index 000000000..d87b09f93 --- /dev/null +++ b/pkg/landscape/gen/s3-store/set-region.hoon @@ -0,0 +1,10 @@ +:: s3-store|set-current-bucket: set current bucket for S3 +:: +/- *s3 +:- %say +|= $: [now=@da eny=@uvJ =beak] + [[region=@t ~] ~] + == +:- %s3-action +^- action +[%set-region region] diff --git a/pkg/landscape/lib/gladio.hoon b/pkg/landscape/lib/gladio.hoon new file mode 100644 index 000000000..a46f3d968 --- /dev/null +++ b/pkg/landscape/lib/gladio.hoon @@ -0,0 +1,193 @@ +:: Migrate scripts +/- gra=graph-store +/- met=metadata-store +/- grp=group-store +/- i=migrate +/- *group +|_ =bowl:gall ++$ card card:agent:gall +:: if false, indicates that OTA should be done in one go, in order to +:: allow for testing on partial testnets +++ split-ota & +++ import-club + |= [=^groups =associations:met =network:gra] + %- ~(gas by *imports:club:i) + %+ murn ~(tap by graphs.network) + |= [=flag:i graph=graph:gra mar=(unit mark)] + ^- (unit [flag:i import:club:i]) + ?. =(mar `%graph-validator-chat) + ~ + ?~ assoc=(~(get by associations) [%graph flag]) + ~& missing-assoc-club/flag + ~ + ?~ group=(~(get by groups) group.u.assoc) + ~& missing-group/[flag group.u.assoc] + ~ + ?. hidden.u.group + ~ + `[flag members.u.group u.assoc graph] +:: +++ import-flags + |= [our=ship =^groups =associations:met =network:gra] + |= =mark + ^- (set flag:i) + %- ~(gas in *(set flag:i)) + %+ skim + ~(tap in ~(key by ((import-for-mark ~ groups associations network) mark))) + |= =flag:i + !=(our p.flag) +:: +++ import-for-mark + |= [her=(unit ship) =^groups =associations:met =network:gra] + |= =mark + ^- imports:graph:i + %- ~(gas by *imports:graph:i) + %+ murn ~(tap by graphs.network) + |= [=flag:i graph=graph:gra mar=(unit ^mark)] + ?. |(=(`p.flag her) =(her ~)) + ~ + ?. =(mar `mark) :: XX: correct detection? + ~ + ?~ assoc=(~(get by associations) [%graph flag]) + ~& missing-assoc/flag^mark + ~ + ?~ group=(~(get by groups) group.u.assoc) + ~& missing-group/[flag group.u.assoc] + ~ + ?: hidden.u.group + ~ + =/ writers=(set ship) + (~(get ju tags.u.group) %graph flag %writers) + ?~ log=(~(get by update-logs.network) flag) + ~& missing-log/flag :: XX: doesn't need to fail, but suspect case + ~ + `[flag writers u.assoc u.log graph] + +++ scry + |= [=dude:gall =path] + %- welp + :_ path + /gx/(scot %p our.bowl)/[dude]/(scot %da now.bowl) +++ groups + ~+ .^([@ =^groups *] (scry %group-store /export/noun)) +++ groups-raw + .^(* (scry %group-store /export/noun)) +++ network + ~+ .^([@ =network:gra] (scry %graph-store /export/noun)) +++ network-raw + .^(* (scry %graph-store /export/noun)) +++ associations + ~+ .^([@ =associations:met ~] (scry %metadata-store /export/noun)) +++ associations-raw + .^(* (scry %metadata-store /export/noun)) +++ export + %- jam + ^- * + :~ [%group-store groups-raw] + [%metadata-store associations-raw] + == +++ peers + |= =network:gra + =- (~(del in -) our.bowl) + %- ~(gas in *(set ship)) + (turn ~(tap in ~(key by graphs.network)) head) +++ poke-our + |= [=dude:gall =cage] + [%pass /gladio %agent [our.bowl dude] %poke cage] +++ migrate-start + |= wait=(set ship) + ^- (quip card (set ship)) + =+ network + =+ associations + =+ groups + =/ ships (peers network) + =/ dms (~(get by graphs:network) [our.bowl %dm-inbox]) + =/ import (import-for-mark `our.bowl groups associations network) + =/ clubs (import-club groups associations network) + =/ chats=imports:graph:i + (import %graph-validator-chat) + =/ diarys=imports:graph:i + (import %graph-validator-publish) + =/ links=imports:graph:i + (import %graph-validator-link) + =/ =imports:groups:i + %- ~(gas by *imports:groups:i) + %+ murn ~(tap by groups) + |= [=flag:i =group] + ^- (unit [_flag import:groups:i]) + ?: hidden.group + ~ + ?~ assoc=(~(get by associations) [%groups flag]) + ~& missing-group-assoc/flag + ~ + =/ chans=(map flag:i association:met) + %- ~(gas by *(map flag:i association:met)) + %+ murn ~(tap by associations) + |= [res=md-resource:met ass=association:met] + ^- (unit [flag:i association:met]) + ?. =(group.ass flag) ~ + `[resource.res ass] + =/ roles=(set flag:i) + %- ~(gas in *(set flag:i)) + %+ murn ~(tap by chans) + |= [=flag:i =association:met] + ^- (unit flag:i) + ?^ link=(~(get by links) flag) + ?: =(writers.u.link ~) ~ + `flag + ?^ diary=(~(get by diarys) flag) + ?: =(writers.u.diary ~) ~ + `flag + ?^ chat=(~(get by chats) flag) + ?: =(writers.u.chat ~) ~ + `flag + ~ + `[flag u.assoc chans roles group] + =/ dms (~(get by graphs:network) [our.bowl %dm-inbox]) + =/ flag-importer (import-flags our.bowl groups associations network) + =+ :* chat-flags=(flag-importer %graph-validator-chat) + heap-flags=(flag-importer %graph-validator-link) + diary-flags=(flag-importer %graph-validator-publish) + == + =/ setup=(list card) + %+ welp (migrate-ship our.bowl) + :* (poke-our %groups group-import+!>(imports)) + (poke-our %chat import-flags+!>(chat-flags)) + (poke-our %heap import-flags+!>(heap-flags)) + (poke-our %diary import-flags+!>(diary-flags)) + (poke-our %chat club-imports+!>(clubs)) + ?~ dms ~ + (poke-our %chat dm-imports+!>(p.u.dms))^~ + == + ?. split-ota + :_ ~ + (welp setup (zing (turn ~(tap in (~(del in ships) our.bowl)) migrate-ship))) + [setup (~(uni in ships) wait)] +:: +++ migrate-ship + |= her=ship + ^- (list card) + =+ groups + =+ network + =+ associations + =/ import (import-for-mark `her groups associations network) + =/ chats=imports:graph:i + (import %graph-validator-chat) + =/ diarys=imports:graph:i + (import %graph-validator-publish) + =/ links=imports:graph:i + (import %graph-validator-link) + =/ graph-flags + %. ~(key by links) + =- ~(uni in -) + (~(uni in ~(key by chats)) ~(key by diarys)) + %+ welp + %+ turn ~(tap in graph-flags) + |= =flag:i + ^- card + (poke-our %graph-store migrated+!>(flag)) + :~ (poke-our %chat graph-imports+!>(chats)) + (poke-our %diary graph-imports+!>(diarys)) + (poke-our %heap graph-imports+!>(links)) + == +-- diff --git a/pkg/landscape/lib/graph-store.hoon b/pkg/landscape/lib/graph-store.hoon index df11d0349..d81afb866 100644 --- a/pkg/landscape/lib/graph-store.hoon +++ b/pkg/landscape/lib/graph-store.hoon @@ -528,6 +528,81 @@ :: ++ upgrade |% + ++ is-old-dm |=(r=resource =('dm--' (end [3 4] name.r))) + ++ backup + |= =bowl:gall + |= [r=resource m=marked-graph] + ^- card:agent:gall + =/ pax /(rap 3 'archive-' (scot %p entity.r) '-' name.r ~)/noun + =/ =cage drum-put+!>([pax (jam r m)]) + [%pass /archive %agent [our.bowl %hood] %poke cage] + ++ strip-sigs-graph + |= g=graph + ^+ g + =* loop $ + %+ gas:orm *graph + %+ turn (tap:orm g) + |= [key=@ val=node] :: optional: also strip out deleted messages? + =? children.val ?=(%graph -.children.val) + [%graph loop(g p.children.val)] + :- key + ?. ?=(%& -.post.val) + val + val(signatures.p.post ~) + ++ strip-sigs-log + |= u=update-log + %+ gas:orm-log *update-log + %+ turn (tap:orm-log u) + |= [key=@ upd=logged-update] + :- key + :- p.upd + ?+ -.q.upd q.upd + %add-graph + q.upd(graph (strip-sigs-graph graph.q.upd)) + :: + %add-signatures + q.upd(signatures ~) + :: + %remove-signatures + q.upd(signatures ~) + :: + %add-nodes + %= q.upd + nodes + %- ~(run by nodes.q.upd) + |= =node + ^+ node + %= node + children + ?. ?=(%graph -.children.node) + children.node + [%graph (strip-sigs-graph p.children.node)] + :: + post + ?. ?=(%& -.post.node) + post.node + =. signatures.p.post.node ~ + post.node + == + == + == + :: + ++ nuke-groups + |= =bowl:gall + |^ ^- (list card:agent:gall) + ?. .^(? (gall-scry %u %groups)) + ~ + =+ .^(=desk (gall-scry %d %groups)) + :~ [%pass /nuke %agent [our.bowl %hood] %poke kiln-nuke+!>([desk &])] + [%pass /nuke %agent [our.bowl %docket] %poke docket-uninstall+!>(desk)] + [%pass /nuke %agent [our.bowl %docket] %poke docket-uninstall+!>(%talk)] + == + :: + ++ gall-scry + |= [=care:clay dap=dude:gall] + ^- path + /(cat 3 %g care)/(scot %p our.bowl)/[dap]/(scot %da now.bowl) + -- :: :: +two :: @@ -758,9 +833,9 @@ -- ++ import |= [arc=* our=ship] - ^- (quip card:agent:gall [%6 network]) + ^- (quip card:agent:gall [%7 network]) |^ - =/ sty [%6 (remake-network ;;(tree-network +.arc))] + =/ sty [%7 (remake-network ;;(tree-network +.arc))] :_ sty %+ turn ~(tap by graphs.sty) |= [rid=resource =marked-graph] diff --git a/pkg/landscape/lib/s3-json.hoon b/pkg/landscape/lib/s3-json.hoon index e4a531daa..04cf848b9 100644 --- a/pkg/landscape/lib/s3-json.hoon +++ b/pkg/landscape/lib/s3-json.hoon @@ -10,6 +10,7 @@ :~ [%set-endpoint so:dejs] [%set-access-key-id so:dejs] [%set-secret-access-key so:dejs] + [%set-region so:dejs] [%add-bucket so:dejs] [%remove-bucket so:dejs] [%set-current-bucket so:dejs] @@ -25,6 +26,7 @@ :~ ?- -.upd %set-current-bucket [%'setCurrentBucket' s+bucket.upd] %add-bucket [%'addBucket' s+bucket.upd] + %set-region [%'setRegion' s+region.upd] %remove-bucket [%'removeBucket' s+bucket.upd] %set-endpoint [%'setEndpoint' s+endpoint.upd] %set-access-key-id [%'setAccessKeyId' s+access-key-id.upd] @@ -44,6 +46,7 @@ %- pairs:enjs :~ [%buckets a+(turn ~(tap in buckets.configuration.upd) |=(a=@t s+a))] [%'currentBucket' s+current-bucket.configuration.upd] + [%'region' s+region.configuration.upd] == == == diff --git a/pkg/landscape/sur/migrate.hoon b/pkg/landscape/sur/migrate.hoon new file mode 100644 index 000000000..3bf567298 --- /dev/null +++ b/pkg/landscape/sur/migrate.hoon @@ -0,0 +1,23 @@ +/- met=metadata-store, gra=graph-store +/- *group +|% ++$ flag (pair ship term) +++ graph + |% + +$ import + [writers=(set ship) =association:met =update-log:gra =graph:gra] + +$ imports (map flag import) + -- +:: +++ groups + |% + +$ import [=association:met chans=(map flag =association:met) roles=(set flag) =group] + +$ imports (map flag import) + -- +++ club + |% + +$ import [ships=(set ship) =association:met =graph:gra] + +$ imports (map flag import) + -- +-- + diff --git a/pkg/landscape/sur/s3-0.hoon b/pkg/landscape/sur/s3-0.hoon new file mode 100644 index 000000000..4f0ca04cb --- /dev/null +++ b/pkg/landscape/sur/s3-0.hoon @@ -0,0 +1,27 @@ +|% ++$ credentials + $: endpoint=@t + access-key-id=@t + secret-access-key=@t + == +:: ++$ configuration + $: buckets=(set @t) + current-bucket=@t + == +:: ++$ action + $% [%set-endpoint endpoint=@t] + [%set-access-key-id access-key-id=@t] + [%set-secret-access-key secret-access-key=@t] + [%add-bucket bucket=@t] + [%remove-bucket bucket=@t] + [%set-current-bucket bucket=@t] + == +:: ++$ update + $% [%credentials =credentials] + [%configuration =configuration] + action + == +-- diff --git a/pkg/landscape/sur/s3.hoon b/pkg/landscape/sur/s3.hoon index 4f0ca04cb..a79ca68ce 100644 --- a/pkg/landscape/sur/s3.hoon +++ b/pkg/landscape/sur/s3.hoon @@ -1,4 +1,9 @@ +/- zer=s3-0 |% +++ past + |% + ++ zero zer + -- +$ credentials $: endpoint=@t access-key-id=@t @@ -8,6 +13,7 @@ +$ configuration $: buckets=(set @t) current-bucket=@t + region=@t == :: +$ action @@ -17,6 +23,7 @@ [%add-bucket bucket=@t] [%remove-bucket bucket=@t] [%set-current-bucket bucket=@t] + [%set-region region=@t] == :: +$ update diff --git a/pkg/landscape/ted/graph/recover-archive.hoon b/pkg/landscape/ted/graph/recover-archive.hoon new file mode 100644 index 000000000..455ee9feb --- /dev/null +++ b/pkg/landscape/ted/graph/recover-archive.hoon @@ -0,0 +1,19 @@ +/- spider, graph=graph-store, met=metadata-store, *group, group-store, push-hook +/+ strandio, resource, graph-view +=> +|% +++ strand strand:spider +++ poke poke:strandio +++ poke-our poke-our:strandio +-- +=, strand=strand:spider +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +=+ !<([~ pax=path] arg) +;< =bowl:spider bind:m get-bowl:strandio +=+ .^([rid=resource mar=marked-graph:graph] %cx pax) +;< ~ bind:m + (poke-our:strandio %graph-store %graph-update-3 !>([now.bowl %add-graph p.mar q.mar &])) +(pure:m *vase) diff --git a/pkg/landscape/ted/keep.hoon b/pkg/landscape/ted/keep.hoon new file mode 100644 index 000000000..3eb758cae --- /dev/null +++ b/pkg/landscape/ted/keep.hoon @@ -0,0 +1,14 @@ +/- spider, docket +/+ strandio +=, strand=strand:spider +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +=+ !<(jim=@ arg) +;< =bowl:spider bind:m get-bowl:strandio +=/ home=path /(scot %p our.bowl)/[q.byk.bowl]/(scot %da now.bowl) +=/ =path /(cat 3 'backup-' (scot %da now.bowl))/noun + :: XX remove prints +;< ~ bind:m (poke-our:strandio %hood drum-put+!>([path jim])) +(pure:m *vase) diff --git a/pkg/landscape/ted/migrate.hoon b/pkg/landscape/ted/migrate.hoon new file mode 100644 index 000000000..ab92d379f --- /dev/null +++ b/pkg/landscape/ted/migrate.hoon @@ -0,0 +1,91 @@ +:: Migrate scripts +/- spider +/- gra=graph-store +/- met=metadata-store +/- grp=group-store +/- i=migrate +/- *group +/+ strandio +=, strand=strand:spider +|% +++ import-for-mark + |= [=groups =associations:met =network:gra] + |= =mark + ^- imports:graph:i + %- ~(gas by *imports:graph:i) + %+ murn ~(tap by graphs.network) + |= [=flag:i graph=graph:gra mar=(unit ^mark)] + ?. =(mar `mark) :: XX: correct detection? + ~ + ?~ assoc=(~(get by associations) [%graph flag]) + ~& missing-assoc/flag^mark + ~ + ?~ group=(~(get by groups) group.u.assoc) + ~& missing-group/[flag group.u.assoc] + ~ + =/ writers=(set ship) + (~(get ju tags.u.group) %graph flag %writers) + ?~ log=(~(get by update-logs.network) flag) + ~& missing-log/flag :: XX: doesn't need to fail, but suspect case + ~ + `[flag writers u.assoc u.log graph] +:: +-- +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +;< =bowl:spider bind:m get-bowl:strandio +;< [%2 =groups] bind:m + (scry:strandio ,[%2 =groups] /gx/group-store/export/noun) +;< [%6 =network:gra] bind:m + (scry:strandio ,[%6 =network:gra] /gx/graph-store/export/noun) +;< =associations:met bind:m + (scry:strandio ,associations:met /gx/metadata-store/associations/noun) +=/ import (import-for-mark groups associations network) +=/ chats=imports:graph:i + (import %graph-validator-chat) +=/ diarys=imports:graph:i + (import %graph-validator-publish) +=/ links=imports:graph:i + (import %graph-validator-link) +=/ =imports:groups:i + %- ~(gas by *imports:groups:i) + %+ murn ~(tap by groups) + |= [=flag:i =group] + ^- (unit [_flag import:groups:i]) + ?~ assoc=(~(get by associations) [%groups flag]) + ~& missing-group-assoc/flag + ~ + =/ chans=(map flag:i association:met) + %- ~(gas by *(map flag:i association:met)) + %+ murn ~(tap by associations) + |= [res=md-resource:met ass=association:met] + ^- (unit [flag:i association:met]) + ?. =(group.ass flag) ~ + `[resource.res ass] + =/ roles=(set flag:i) + %- ~(gas in *(set flag:i)) + %+ murn ~(tap by chans) + |= [=flag:i =association:met] + ^- (unit flag:i) + ?^ link=(~(get by links) flag) + ?: =(writers.u.link ~) ~ + `flag + ?^ diary=(~(get by diarys) flag) + ?: =(writers.u.diary ~) ~ + `flag + ?^ chat=(~(get by chats) flag) + ?: =(writers.u.chat ~) ~ + `flag + ~ + `[flag u.assoc chans roles group] +;< ~ bind:m (poke-our:strandio %groups group-import+!>(imports)) +;< ~ bind:m (poke-our:strandio %chat graph-imports+!>(chats)) +;< ~ bind:m (poke-our:strandio %diary graph-imports+!>(diarys)) +;< ~ bind:m (poke-our:strandio %heap graph-imports+!>(links)) +;< ~ bind:m + ?~ dms=(~(get by graphs.network) [our.bowl %dm-inbox]) + (pure:(strand ,~) ~) + (poke-our:strandio %chat %dm-imports !>(p.u.dms)) +(pure:m *vase) diff --git a/pkg/urbit/version b/pkg/urbit/version index 9f76d37b7..07fe6f6c9 100644 --- a/pkg/urbit/version +++ b/pkg/urbit/version @@ -1 +1 @@ -1.13 \ No newline at end of file +1.15 \ No newline at end of file