Merge branch 'next/arvo' into m/ruin-louder

.github/actions/glob/entrypoint.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-
-cd "$GITHUB_WORKSPACE" || exit
-
-echo "$2" | base64 -d > service-account
-echo "$3" | base64 -d > id_ssh
-echo "$4" | base64 -d > id_ssh.pub
-
-chmod 600 service-account
-chmod 600 id_ssh
-chmod 600 id_ssh.pub
-
-janeway release glob-all --dev --no-pill \
-    --credentials service-account \
-    --ssh-key id_ssh \
-    --ci \
-  | bash
-
-SHORTHASH=$(git rev-parse --short HEAD)
-
-janeway release prepare-ota arvo-glob-"$SHORTHASH" "$1" \
-    --credentials service-account \
-    --ssh-key id_ssh \
-    --ci \
-  | bash
-
-janeway release perform-ota "$1" \
-    --credentials service-account \
-    --ssh-key id_ssh \
-    --ci \
-  | bash
-

.github/actions/ota/Dockerfile

@@ -1,4 +1,4 @@
-FROM tloncorp/janeway:v0.15.4
+FROM tloncorp/janeway:v0.17.0
 COPY entrypoint.sh /entrypoint.sh
 EXPOSE 22/tcp
 ENTRYPOINT ["/entrypoint.sh"]

.github/actions/ota/action.yml

@@ -1,18 +1,21 @@
-name: 'glob'
-description: 'Create a glob and deploy it to a moon'
+name: 'ota'
+description: 'perform an OTA update of arvo on a remote ship'
 inputs:
   ship:
-    description: "Ship to deploy to"
+    description: "target ship"
     required: true
   credentials:
     description: "base64-encoded GCP Service Account credentials"
     required: true
   ssh-sec-key:
-    description: "A base64-encoded SSH secret key for the container to use"
+    description: "base64-encoded SSH secret key for the container to use"
     required: true
   ssh-pub-key:
-    description: "The corresponding base64-encoded SSH public key"
+    description: "base64-encoded corresponding SSH public key"
     required: true
+  ref:
+    description: "git ref of arvo source to check out"
+    required: false
 
 runs:
   using: 'docker'
@@ -22,4 +25,4 @@ runs:
     - ${{ inputs.credentials }}
     - ${{ inputs.ssh-sec-key }}
     - ${{ inputs.ssh-pub-key }}
-
+    - ${{ inputs.ref }}

.github/actions/ota/entrypoint.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+echo "$2" | base64 -d > /service-account
+echo "$3" | base64 -d > /id_ssh
+echo "$4" | base64 -d > /id_ssh.pub
+
+chmod 600 /service-account
+chmod 600 /id_ssh
+chmod 600 /id_ssh.pub
+
+janeway \
+    --ci \
+    --verbose \
+    --credentials /service-account \
+    --ssh-key /id_ssh \
+    release ota \
+    arvo \
+    "$1" \
+    ${5:+"--ref"} ${5:+"$5"} \
+  | bash

.github/workflows/chromatic.yml

@@ -1,27 +0,0 @@
-name: Chromatic Deployment
-
-on:
-  pull_request:
-    paths:
-      - 'pkg/interface/**'
-  push:
-    paths:
-      - 'pkg/interface/**'
-    branches:
-      - 'release/next-userspace'
-
-jobs:
-  chromatic-deployment:
-    runs-on: ubuntu-latest
-    name: "Deploy Chromatic"
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - run: npm i && npm run bootstrap
-      - name: Publish to Chromatic
-        uses: chromaui/action@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
-          workingDir: pkg/interface

.github/workflows/frontend-test.yml

@@ -1,24 +0,0 @@
-name: frontend-test
-
-on:
-  pull_request:
-    paths:
-      - 'pkg/interface/**'
-      - 'pkg/btc-wallet/**'
-      - 'pkg/npm/**'
-
-jobs:
-  frontend-test:
-    runs-on: ubuntu-latest
-    name: "Test changed frontend packages"
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-      - run: git fetch --prune
-      - name: 'Setup root deps'
-        run: npm ci
-      - name: 'Setup dependencies'
-        run: npm run bootstrap
-      - name: 'Run tests'
-        run: npm run test -- --since origin/$GITHUB_BASE_REF  --include-dependents

.github/workflows/glob.yml

@@ -1,20 +0,0 @@
-name: glob
-on:
-  push:
-    branches:
-      - 'release/next-userspace'
-jobs:
-  glob:
-    runs-on: ubuntu-latest
-    name: "Create and deploy a glob to ~hanruc-nalfus-nidsut-tomdun"
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          lfs: true
-      - uses: ./.github/actions/glob
-        with:
-          ship: 'hanruc-nalfus-nidsut-tomdun'
-          credentials: ${{ secrets.JANEWAY_SERVICE_KEY }}
-          ssh-sec-key: ${{ secrets.JANEWAY_SSH_SEC_KEY }}
-          ssh-pub-key: ${{ secrets.JANEWAY_SSH_PUB_KEY }}
-

.github/workflows/merge-master.yml

@@ -1,27 +0,0 @@
-name: merge
-on:
-  push:
-    branches:
-      - 'master'
-jobs:
-  merge-to-next-js:
-    runs-on: ubuntu-latest
-    name: "Merge master to release/next-userspace"
-    steps:
-      - uses: actions/checkout@v2
-      - uses: devmasx/merge-branch@v1.3.1
-        with:
-          type: now
-          target_branch: release/next-userspace
-          github_token: ${{ secrets.JANEWAY_BOT_TOKEN }}
-
-  merge-to-group-timer:
-    runs-on: ubuntu-latest
-    name: "Merge master to ops/group-timer"
-    steps:
-      - uses: actions/checkout@v2
-      - uses: devmasx/merge-branch@v1.3.1
-        with:
-          type: now
-          target_branch: ops/group-timer
-          github_token: ${{ secrets.JANEWAY_BOT_TOKEN }}

.github/workflows/merge-release.yml

@@ -1,17 +0,0 @@
-name: ops-merge
-on:
-  push:
-    branches:
-      - 'release/*'
-jobs:
-  merge-release-to-ops:
-    runs-on: ubuntu-latest
-    name: "Merge to ops-tlon"
-    steps:
-      - uses: actions/checkout@v2
-      - uses: devmasx/merge-branch@v1.3.1
-        with:
-          type: now
-          target_branch: ops-tlon
-          github_token: ${{ secrets.JANEWAY_BOT_TOKEN }}
-

.github/workflows/ota.yml

@@ -0,0 +1,19 @@
+name: ota
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'next/arvo'
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    name: "make an OTA update to arvo on ~binnec-dozzod-marzod"
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/ota
+        with:
+          ship: 'canary'
+          credentials: ${{ secrets.JANEWAY_SERVICE_KEY }}
+          ssh-sec-key: ${{ secrets.JANEWAY_SSH_SEC_KEY }}
+          ssh-pub-key: ${{ secrets.JANEWAY_SSH_PUB_KEY }}
+          ref: 'next/arvo'

.github/workflows/vere.yml

@@ -162,86 +162,9 @@ jobs:
       - if: ${{ matrix.os == 'ubuntu-latest' }}
         run: nix-build -A docker-image
 
-  mingw:
-    runs-on: windows-latest
-    defaults:
-      run:
-        shell: >
-          C:\msys64\msys2_shell.cmd -mingw64 -defterm -no-start -here -c
-          ". <(cygpath '{0}')"
-        working-directory: ./pkg/urbit
-
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          lfs: true
-
-      # echo suppresses pacman prompt
-      - run: echo|./configure
-        env:
-          CACHIX_CACHE: ares
-          CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
-
-      - run: mingw32-make build/urbit
-      - run: mingw32-make test
-      - run: >
-          build/urbit -l -d -B ../../bin/solid.pill -F bus &&
-          curl -f --data '{"source":{"dojo":"+hood/exit"},"sink":{"app":"hood"}}'
-          http://localhost:12321
-
-      - name: confirm binary is mostly static
-        run: |
-          if [ -z "$(ldd build/urbit | grep -vi "windows/system32")"]; then
-            echo "it's mostly static"
-            exit 0
-          else
-            echo "dynamic links found:"
-            ldd build/urbit
-            exit 1
-          fi
-
-      - uses: actions/setup-python@v2
-        if: inputs.upload
-        with:
-          python-version: 3.7
-
-      - uses: google-github-actions/setup-gcloud@v0.6.0
-        if: inputs.upload
-        env:
-          # see https://github.com/google-github-actions/setup-gcloud/issues/100
-          CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe
-        with:
-          service_account_key: ${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}
-          project_id: ${{ secrets.GCS_PROJECT }}
-          export_default_credentials: true
-
-      - name: upload binary to bootstrap.urbit.org
-        if: inputs.upload
-        env:
-          CLOUDSDK_PYTHON: ${{env.pythonLocation}}\python.exe
-        shell: bash
-        run: |
-          if [ "real" == "$VERSION_TYPE" ]; then
-            version="$(cat ./version)"
-          else
-            version="${GITHUB_SHA:0:9}"
-          fi
-
-          system="x86_64-windows"
-          target="gs://${UPLOAD_BASE}/${VERE_PACE}/${version}/vere-v${version}-${system}.exe"
-
-          gsutil cp -n ./build/urbit.exe "$target"
-          exitcode=$?
-
-          test $exitcode -eq 0 &&
-            echo "upload to $target complete." ||
-            echo "upload to $target failed.";
-          exit $exitcode
-
-
   after:
     runs-on: ubuntu-latest
-    needs: [urbit, mingw]
+    needs: [urbit]
     if: inputs.upload
     steps:
       - uses: google-github-actions/setup-gcloud@v0.2.0

bin/multi-brass.pill

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3306a5b916caa838c67cb079410c6e5bb158054520129ffb9f9bdb144ab1b691
-size 7418493
+oid sha256:ea8626444e4f0213e39c21ded20607145ee85a947afc592f182f46e7f598ef30
+size 7748671

pkg/garden/app/docket.hoon

@@ -394,7 +394,10 @@
         ::          !=(%kids desk)
         ::      ==
         ::    [dap.bowl %no-docket-file-for desk]
-        `state
+        ?.  (~(has by charges) desk)
+          `state
+        :-  ~[del-fact:cha]
+        state(charges (~(del by charges) desk))
       ::  always update the docket in state to match clay's
       ::
       =/  =docket            docket:cha

pkg/interface/src/views/apps/publish/components/Note.tsx

@@ -26,6 +26,7 @@ interface NoteProps {
 }
 
 export function NoteContent({ post }) {
+  console.log(post.contents);
   return (
       <Box color="black" className="md" style={{ overflowWrap: 'break-word', overflow: 'hidden' }}>
         <GraphContent tall={true} contents={post.contents.slice(1)} showOurContact />

pkg/landscape/app/contact-store.hoon

@@ -123,7 +123,15 @@
       ::  ensure difference
       =/  old=(unit contact:store)  (~(get by rolodex) ship)
       ?.  ?|  ?=(~ old)
-              !=(contact(last-updated *@da) u.old(last-updated *@da))
+              ::  if new contact is before existing contact, no-op
+              ::
+              ::  NB: last-updated.contact is often *@da, so this
+              ::  effectively stops add from applying if we already have
+              ::  a contact for them
+              ::
+              ?&  (gth last-updated.contact last-updated.u.old)
+                  !=(contact(last-updated *@da) u.old(last-updated *@da))
+              ==
           ==
         [~ state]
       ~|  "cannot add a data url to cover!"

pkg/landscape/app/graph-pull-hook.hoon

@@ -54,12 +54,7 @@
 ++  on-pull-nack
   |=  [=resource =tang]
   ^-  (quip card _this)
-  %-  (slog leaf+"nacked {<resource>}" tang)
-  :_  this
-  ?.  (~(has in get-keys:gra) resource)  ~
-  =-  [%pass /pull-nack %agent [our.bowl %graph-store] %poke %graph-update-3 -]~
-  !>  ^-  update:store
-  [now.bowl [%archive-graph resource]]
+  `this
 ::
 ++  on-pull-kick
   |=  =resource

pkg/landscape/app/graph-store.hoon

@@ -11,16 +11,17 @@
       [%3 network:one:store]
       [%4 network:store]
       [%5 network:store]
-      state-6
+      [%6 network:store]
+      state-7
   ==
 ::-
-+$  state-6  [%6 network:store]
++$  state-7  [%7 network:store]
 ++  orm      orm:store
 ++  orm-log  orm-log:store
 ++  mar      %graph-update-3
 --
 ::
-=|  state-6
+=|  state-7
 =*  state  -
 ::
 %-  agent:dbug
@@ -96,7 +97,43 @@
       (scag 2 (tap:orm-log update-log))
     ==
   ::
-    %6  [cards this(state old)]
+      %6
+    =/  old-dms
+      %-  ~(gas by *(map resource:store marked-graph:store))
+      %+  skim  ~(tap by graphs.old)
+      |=([r=resource:store *] (is-old-dm:upgrade:store r))
+    =/  backup  (backup:upgrade:store bowl)
+    %_    $
+        -.old  %7
+        archive.old  ~
+        update-logs.old
+      %-  ~(gas by *(map resource:store update-log:store))
+      %+  murn  ~(tap by update-logs.old)
+      |=  [r=resource:store =update-log:store]
+      ?:  (is-old-dm:upgrade:store r)
+        ~
+      `[r (strip-sigs-log:upgrade:store update-log)]
+    ::
+        graphs.old
+      %-  ~(gas by *(map resource:store marked-graph:store))
+      %+  murn  ~(tap by graphs.old)
+      |=  [r=resource:store =graph:store mar=(unit mark)]
+      ?:  (is-old-dm:upgrade:store r)
+        ~
+      `[r (strip-sigs-graph:upgrade:store graph) mar]
+    ::
+        cards
+      ;:  welp
+        cards
+      ::
+        (nuke-groups:upgrade:store bowl)
+      ::
+        (turn ~(tap by archive.old) backup)
+        (turn ~(tap by old-dms) backup)
+      ==
+    ==
+  ::
+      %7  [cards this(state old)]
   ==
 ::
 ++  on-watch
@@ -129,9 +166,21 @@
     ?+  mark           (on-poke:def mark vase)
       %graph-update-3  (graph-update !<(update:store vase))
       %import          (poke-import q.vase)
+      %migrated        (poke-migrated !<(resource:store vase))
     ==
   [cards this]
   ::
+  ++  poke-migrated
+    |=  r=resource:res
+    ^-  (quip card _state)
+    =/  =path  /(rap 3 'backup-' (scot %p entity.r) '-' name.r ~)/noun
+    =/  graph  (~(got by graphs) r)
+    :-  [%pass /migrate %agent [our.bowl %hood] %poke drum-put+!>([path (jam r graph)])]~
+    %_  state
+      graphs       (~(del by graphs) r)
+      update-logs  (~(del by update-logs) r)
+    ==
+  ::
   ++  graph-update
     |=  =update:store
     ^-  (quip card _state)

pkg/landscape/app/group-pull-hook.hoon

@@ -45,10 +45,7 @@
 ++  on-pull-nack
   |=   [=resource =tang]
   ^-  (quip card _this)
-  %-  (slog tang)
-  :_  this
-  =-  [%pass / %agent [our.bowl %group-store] %poke -]~
-  group-update-0+!>([%remove-group resource ~])
+  `this
 ::
 ++  on-pull-kick
   |=  =resource

pkg/landscape/app/group-store.hoon

@@ -31,13 +31,17 @@
 ::
 /-  *group
 /+  store=group-store, default-agent, verb, dbug, resource, *migrate, agentio
+/+  gladio
 |%
 +$  card  card:agent:gall
+++  ota-host  ~sogryp-dister-dozzod-dozzod
 ::
 +$  versioned-state
   $%  state-zero
       state-one
       state-two
+      state-three
+      state-four
   ==
 ::
 +$  state-zero
@@ -52,13 +56,25 @@
   $:  %2
       =groups
   ==
+::
++$  state-three
+  $:  %3
+      =groups
+      wait=(set ship)
+  ==
+::
++$  state-four
+  $:  %4
+      =groups
+      wait=(set ship)
+  ==
 --
 ::
-=|  state-two
+=|  state-four
 =*  state  -
 ::
 %-  agent:dbug
-%+  verb  |
+%+  verb  &
 ^-  agent:gall
 =<
   |_  =bowl:gall
@@ -72,10 +88,32 @@
   ++  on-load
     |=  =old=vase
     =/  old  !<(versioned-state old-vase)
+    =|  cards=(list card)
     |^
-    ?-  -.old
-      %2  `this(state old)
-      ::
+    ?-    -.old
+        %4  [(flop cards) this(state old)]
+    ::
+        %3 
+      %_    $
+          old    [%4 +.old] 
+          cards
+        :_  cards
+        [%pass /pyre/rebuild %agent [our dap]:bowl %poke noun+!>(%rebuild)]
+      ==
+    ::
+        %2
+      %_    $
+          old    [%3 groups.old ~]
+          cards  
+        %-  welp 
+        :_  cards
+        :~  [%pass /pyre/export %agent [our dap]:bowl %poke noun+!>(%export)]
+            [%pass /pyre/migrate %agent [our dap]:bowl %poke noun+!>(%migrate)]
+            [%pass / %agent [our %hood]:bowl %poke %kiln-install !>([%groups ota-host %groups])]
+            [%pass / %agent [our %hood]:bowl %poke %kiln-install !>([%talk ota-host %talk])]
+        ==
+      ==
+    ::
         %1  
       %_    $
         -.old  %2
@@ -112,6 +150,13 @@
     =^  cards  state
       ?+    mark  (on-poke:def mark vase)
         %sane  (poke-sane:gc !<(?(%check %fix) vase))
+      ::
+          %noun 
+        ?+  q.vase  !!
+          %migrate  poke-migrate:gc
+          %export   poke-export:gc
+          %rebuild  poke-rebuild:gc
+        ==
       ::
           ?(%group-update-0 %group-action)
         (poke-group-update:gc !<(update:store vase))
@@ -125,6 +170,8 @@
     |=  =path
     ^-  (quip card _this)
     ?>  (team:title our.bowl src.bowl)
+    ?:  ?=([%wait ~] path)
+      `this
     ?>  ?=([%groups ~] path)
     :_  this
     [%give %fact ~ %group-update-0 !>([%initial groups])]~
@@ -135,6 +182,8 @@
     |=  =path
     ^-  (unit (unit cage))
     ?+  path  (on-peek:def path)
+        [%x %wait ~]
+      ``ships+!>(~(tap in wait))
         [%y %groups ~]
       ``noun+!>(`(set resource)`~(key by groups))
     ::
@@ -159,28 +208,38 @@
   ++  on-agent
     |=  [=wire =sign:agent:gall]
     ^-  (quip card _this)
-    ?.  ?=([%try-rejoin @ *] wire)
-      (on-agent:def wire sign)
-    ?>  ?=(%poke-ack -.sign)
-    =/  rid=resource  (de-path:resource t.t.wire)
-    ?~  p.sign
-      =/  =cage
-        [%pull-hook-action !>([%add entity.rid rid])]
-      :_  this
-      [%pass / %agent [our.bowl %group-pull-hook] %poke cage]~
-    =/  nack-count=@ud  (slav %ud i.t.wire)
-    =/  wakeup=@da
-      (add now.bowl (mul ~s1 (bex (min 19 nack-count))))
-    :_  this
-    [%pass wire %arvo %b %wait wakeup]~
+    =^  cards  state
+      ?+    wire  [- state]:(on-agent:def wire sign)
+          [%pyre *]      (take-pyre:gc t.wire sign)
+          [%gladio @ ~]  (take-migrate:gc sign)
+      ::
+          [%try-rejoin @ *]
+        ?>  ?=(%poke-ack -.sign)
+        =/  rid=resource  (de-path:resource t.t.wire)
+        ?~  p.sign
+          =/  =cage
+            [%pull-hook-action !>([%add entity.rid rid])]
+          :_  state
+          [%pass / %agent [our.bowl %group-pull-hook] %poke cage]~
+        =/  nack-count=@ud  (slav %ud i.t.wire)
+        =/  wakeup=@da
+          (add now.bowl (mul ~s1 (bex (min 19 nack-count))))
+        :_  state
+        [%pass wire %arvo %b %wait wakeup]~
+      ==
+    [cards this]
   ::
   ++  on-arvo
-    |=  [=wire =sign-arvo]
+    |=  [=(pole knot) =sign-arvo]
     ^-  (quip card _this)
-    ?.  ?=([%try-rejoin @ *] wire)
-      (on-arvo:def wire sign-arvo)
-    =/  =resource       (de-path:resource t.t.wire)
-    =/  nack-count=@ud  (slav %ud i.t.wire)
+    ?:  ?=([%gladio %backoff ship=@ ~] pole)
+      =^  cards  state
+        (take-backoff:gc (slav %p ship.pole) sign-arvo)
+      [cards this]
+    ?.  ?=([%try-rejoin count=@ res=*] pole)
+      (on-arvo:def pole sign-arvo)
+    =/  =resource       (de-path:resource res.pole)
+    =/  nack-count=@ud  (slav %ud count.pole)
     ?>  ?=([%behn %wake *] sign-arvo)
     ~?  ?=(^ error.sign-arvo)
       "behn errored in backoff timers, continuing anyway"
@@ -192,6 +251,107 @@
 ::
 |_  bol=bowl:gall
 +*  io  ~(. agentio bol)
+++  poke-rebuild
+  ^-  (quip card _state)
+  |^  
+  =.  wait
+    put-missing
+  =^  cards  state
+    rewatch
+  [cards state]
+  ::
+  ++  rewatch
+    =/  wait  ~(tap in wait)
+    =|  cards=(list card)
+    |-
+    ?~  wait
+      [cards state]
+    =/  wir  /gladio/(scot %p i.wait)
+    =.  cards
+      :_(cards (watch-init-migrate i.wait))
+    ::  if we have a subscription already, leave first to restart
+    =?  cards
+        (~(has by wex.bol) [wir i.wait %groups])
+      :_(cards [%pass wir %agent [i.wait %groups] %leave ~])
+    $(wait t.wait)
+  ::
+  ++  put-missing
+    =/  wex  ~(tap by wex.bol)
+    |-
+    ?~  wex
+      wait
+    =/  [[=wire =ship =term] [acked=? =(pole knot)]]
+      i.wex
+    ?.  ?=([%gladio ship=@ ~] pole)
+      $(wex t.wex)
+    $(wex t.wex, wait (~(put in wait) (slav %p ship.pole)))
+  --
+::
+++  poke-export
+  ^-  (quip card _state)
+  :_  state
+  =;  =cage
+    [%pass /export %agent [our.bol %hood] %poke cage]~
+  drum-put+!>([/groups/jam ~(export gladio bol)])
+::
+++  poke-migrate
+  ^-  (quip card _state)
+  =^  cards-1=(list card)  wait
+    (~(migrate-start gladio bol) wait)
+  =/  cards-2=(list card)
+    (turn ~(tap in wait) watch-init-migrate)
+  =/  cards  (welp cards-1 cards-2)
+  [cards state(wait wait)]
+::
+++  watch-init-migrate
+  |=  =ship
+  ^-  card
+  [%pass /gladio/(scot %p ship) %agent [ship %groups] %watch /init]
+::
+++  backoff-migrate
+  |=  =ship
+  ^-  card
+  [%pass /gladio/backoff/(scot %p ship) %arvo %b %wait (add ~h1 now.bol)]
+::
+++  take-pyre
+  |=  [=wire =sign:agent:gall]
+  ^-  (quip card _state)
+  :_  state
+  ?>  ?=(%poke-ack -.sign)
+  ?~  p.sign
+    ~
+  [%pass / %pyre leaf/"{<wire>} failed" u.p.sign]~
+::
+++  take-backoff
+  |=  [=ship sign=sign-arvo]
+  ^-  (quip card _state)
+  ?>  ?=([%behn %wake *] sign)
+  ?:  ?=(^ error.sign)
+    `state
+  :_  state
+  ~[(watch-init-migrate ship)]
+::
+++  take-migrate
+  |=  =sign:agent:gall
+  ^-  (quip card _state)
+  ?.  (~(has in wait) src.bol)
+    ::  already succeeded
+    `state
+  ?-    -.sign  
+      ?(%poke-ack %fact)  `state
+      %kick               :_(state (watch-init-migrate src.bol)^~)
+      %watch-ack
+    ?~  p.sign
+      ::  they have public release
+      ~&  migrating/src.bol
+      =.  wait  (~(del in wait) src.bol)
+      :_  state
+      :-  [%give %fact ~[/wait] ships+!>(~(tap in wait))]
+      (~(migrate-ship gladio bol) src.bol)
+    :_  state
+    ~[(backoff-migrate src.bol)]
+  ==
+::
 ++  peek-group
   |=  rid=resource
   ^-  (unit group)
@@ -243,8 +403,8 @@
   |=  arc=*
   ^-  (quip card _state)
   |^
-  =/  sty=state-two
-    [%2 (remake-groups ;;((tree [resource tree-group]) +.arc))]
+  =/  sty=state-four
+    [%4 (remake-groups ;;((tree [resource tree-group]) +.arc)) ~]
   :_  sty
   %+  roll  ~(tap by groups.sty)
   |=  [[rid=resource grp=group] out=(list card)]

pkg/landscape/app/metadata-pull-hook.hoon

@@ -321,15 +321,7 @@
 ++  on-pull-nack
   |=   [=resource =tang]
   ^-  (quip card _this)
-  =/  =associations:metadata
-    (metadata-for-group:met resource)
-  :_  this
-  %+  turn  ~(tap by associations)
-  |=  [=md-resource:metadata =association:metadata]
-  %+  poke-our:pass:io:hc  %metadata-store
-  :-  %metadata-update-2
-  !>  ^-   update:metadata
-  [%remove resource md-resource]
+  `this
 ::
 ++  on-pull-kick
   |=  =resource

pkg/landscape/app/s3-store.hoon

@@ -9,12 +9,14 @@
 +$  card  card:agent:gall
 +$  versioned-state
   $%  state-zero
+      state-one
   ==
 ::
-+$  state-zero  [%0 =credentials =configuration]
++$  state-zero  [%0 =credentials:zero:past =configuration:zero:past]
++$  state-one   [%1 =credentials =configuration]
 --
 ::
-=|  state-zero
+=|  state-one
 =*  state  -
 ::
 %-  agent:dbug
@@ -28,8 +30,28 @@
 ++  on-init   on-init:def
 ++  on-save   !>(state)
 ++  on-load
-  |=  old-vase=vase
-  [~ this(state !<(state-zero old-vase))]
+  |=  =vase
+  =/  old  !<(versioned-state vase)
+  |^
+  ?-  -.old
+    %1  `this(state old)
+    %0  `this(state (state-0-to-1 old))
+  ==
+  ++  state-0-to-1
+    |=  zer=state-zero
+    ^-  state-one
+    :*  %1
+        credentials.zer
+        (configuration-0-to-1 configuration.zer)
+    ==
+  ++  configuration-0-to-1
+    |=  conf=configuration:zero:past
+    ^-  ^configuration
+    :*  buckets.conf
+        current-bucket.conf
+        ''
+    ==
+  --
 ::
 ++  on-poke
   ~/  %s3-poke
@@ -56,6 +78,9 @@
     ::
         %set-secret-access-key
       state(secret-access-key.credentials secret-access-key.act)
+    ::
+        %set-region
+      state(region.configuration region.act)
     ::
         %set-current-bucket
       %_  state

pkg/landscape/desk.docket-0

@@ -1,10 +0,0 @@
-:~  title+'Groups'
-    info+'A suite of applications to communicate on Urbit'
-    color+0xee.5432
-    glob-http+['https://bootstrap.urbit.org/glob-0v7.2rpmd.966js.dt2sj.ggv4a.n15nq.glob' 0v7.2rpmd.966js.dt2sj.ggv4a.n15nq]
-
-    base+'landscape'
-    version+[1 1 0]
-    website+'https://tlon.io'
-    license+'MIT'
-==

pkg/landscape/gen/s3-store/set-region.hoon

@@ -0,0 +1,10 @@
+::  s3-store|set-current-bucket: set current bucket for S3
+::
+/-  *s3
+:-  %say
+|=  $:  [now=@da eny=@uvJ =beak]
+        [[region=@t ~] ~]
+    ==
+:-  %s3-action
+^-  action
+[%set-region region]

pkg/landscape/lib/gladio.hoon

@@ -0,0 +1,193 @@
+:: Migrate scripts
+/-  gra=graph-store
+/-  met=metadata-store
+/-  grp=group-store
+/-  i=migrate
+/-  *group
+|_  =bowl:gall
++$  card  card:agent:gall
+::  if false, indicates that OTA should be done in one go, in order to
+::  allow for testing on partial testnets
+++  split-ota   &
+++  import-club
+  |=  [=^groups =associations:met =network:gra]
+  %-  ~(gas by *imports:club:i)
+  %+  murn  ~(tap by graphs.network)
+  |=  [=flag:i graph=graph:gra mar=(unit mark)]
+  ^-  (unit [flag:i import:club:i])
+  ?.  =(mar `%graph-validator-chat)
+    ~
+  ?~  assoc=(~(get by associations) [%graph flag])
+    ~&  missing-assoc-club/flag
+    ~
+  ?~  group=(~(get by groups) group.u.assoc)
+    ~&  missing-group/[flag group.u.assoc]
+    ~
+  ?.  hidden.u.group
+    ~
+  `[flag members.u.group u.assoc graph]
+::
+++  import-flags
+  |=  [our=ship =^groups =associations:met =network:gra]
+  |=  =mark
+  ^-  (set flag:i)
+  %-  ~(gas in *(set flag:i))
+  %+  skim
+    ~(tap in ~(key by ((import-for-mark ~ groups associations network) mark)))
+  |=  =flag:i
+  !=(our p.flag)
+::
+++  import-for-mark
+  |=  [her=(unit ship) =^groups =associations:met =network:gra]
+  |=  =mark
+  ^-  imports:graph:i
+  %-  ~(gas by *imports:graph:i)
+  %+  murn  ~(tap by graphs.network)
+  |=  [=flag:i graph=graph:gra mar=(unit ^mark)]
+  ?.  |(=(`p.flag her) =(her ~))
+    ~
+  ?.  =(mar `mark)  ::  XX: correct detection?
+    ~
+  ?~  assoc=(~(get by associations) [%graph flag])
+    ~&  missing-assoc/flag^mark
+    ~
+  ?~  group=(~(get by groups) group.u.assoc)
+    ~&  missing-group/[flag group.u.assoc]
+    ~
+  ?:  hidden.u.group
+    ~
+  =/  writers=(set ship)  
+    (~(get ju tags.u.group) %graph flag %writers)
+  ?~  log=(~(get by update-logs.network) flag)
+    ~&  missing-log/flag  :: XX: doesn't need to fail, but suspect case
+    ~
+  `[flag writers u.assoc u.log graph]
+
+++  scry
+  |=  [=dude:gall =path]
+  %-  welp 
+  :_  path
+  /gx/(scot %p our.bowl)/[dude]/(scot %da now.bowl)
+++  groups
+  ~+  .^([@ =^groups *] (scry %group-store /export/noun))
+++  groups-raw
+  .^(* (scry %group-store /export/noun))
+++  network
+  ~+  .^([@ =network:gra] (scry %graph-store /export/noun))
+++  network-raw
+  .^(* (scry %graph-store /export/noun))
+++  associations
+  ~+  .^([@ =associations:met ~] (scry %metadata-store /export/noun))
+++  associations-raw
+  .^(* (scry %metadata-store /export/noun))
+++  export
+  %-  jam
+  ^-  *
+  :~  [%group-store groups-raw]
+      [%metadata-store associations-raw]
+  ==
+++  peers
+  |=  =network:gra
+  =-  (~(del in -) our.bowl)
+  %-  ~(gas in *(set ship))
+  (turn ~(tap in ~(key by graphs.network)) head)
+++  poke-our
+  |=  [=dude:gall =cage]
+  [%pass /gladio %agent [our.bowl dude] %poke cage]
+++  migrate-start
+  |=  wait=(set ship)
+  ^-  (quip card (set ship))
+  =+  network
+  =+  associations
+  =+  groups
+  =/  ships   (peers network)
+  =/  dms  (~(get by graphs:network) [our.bowl %dm-inbox])
+  =/  import  (import-for-mark `our.bowl groups associations network)
+  =/  clubs  (import-club groups associations network)
+  =/  chats=imports:graph:i
+    (import %graph-validator-chat)
+  =/  diarys=imports:graph:i
+    (import %graph-validator-publish)
+  =/  links=imports:graph:i
+    (import %graph-validator-link)
+  =/  =imports:groups:i
+    %-  ~(gas by *imports:groups:i)
+    %+  murn  ~(tap by groups)
+    |=  [=flag:i =group]
+    ^-  (unit [_flag import:groups:i])
+    ?:  hidden.group
+      ~
+    ?~  assoc=(~(get by associations) [%groups flag])
+      ~&  missing-group-assoc/flag
+      ~
+    =/  chans=(map flag:i association:met)
+      %-  ~(gas by *(map flag:i association:met))
+      %+  murn  ~(tap by associations)
+      |=  [res=md-resource:met ass=association:met]
+      ^-  (unit [flag:i association:met])
+      ?.  =(group.ass flag)  ~
+      `[resource.res ass]
+    =/  roles=(set flag:i)
+      %-  ~(gas in *(set flag:i))
+      %+  murn  ~(tap by chans)
+      |=  [=flag:i =association:met]
+      ^-  (unit flag:i)
+      ?^  link=(~(get by links) flag)
+        ?:  =(writers.u.link ~)  ~
+        `flag
+      ?^  diary=(~(get by diarys) flag)
+        ?:  =(writers.u.diary ~)  ~
+        `flag
+      ?^  chat=(~(get by chats) flag)
+        ?:  =(writers.u.chat ~)  ~
+        `flag
+      ~
+    `[flag u.assoc chans roles group]
+  =/  dms  (~(get by graphs:network) [our.bowl %dm-inbox])
+  =/  flag-importer  (import-flags our.bowl groups associations network)
+  =+  :*  chat-flags=(flag-importer %graph-validator-chat)
+          heap-flags=(flag-importer %graph-validator-link)
+          diary-flags=(flag-importer %graph-validator-publish)
+      ==
+  =/  setup=(list card)
+    %+  welp  (migrate-ship our.bowl)
+    :*  (poke-our %groups group-import+!>(imports))
+        (poke-our %chat import-flags+!>(chat-flags))
+        (poke-our %heap import-flags+!>(heap-flags))
+        (poke-our %diary import-flags+!>(diary-flags))
+        (poke-our %chat club-imports+!>(clubs))
+        ?~  dms  ~
+        (poke-our %chat dm-imports+!>(p.u.dms))^~
+    ==
+  ?.  split-ota
+    :_  ~
+    (welp setup (zing (turn ~(tap in (~(del in ships) our.bowl)) migrate-ship)))
+  [setup (~(uni in ships) wait)]
+::
+++  migrate-ship
+  |=  her=ship
+  ^-  (list card)
+  =+  groups
+  =+  network
+  =+  associations
+  =/  import  (import-for-mark `her groups associations network)
+  =/  chats=imports:graph:i
+    (import %graph-validator-chat)
+  =/  diarys=imports:graph:i
+    (import %graph-validator-publish)
+  =/  links=imports:graph:i
+    (import %graph-validator-link)
+  =/  graph-flags
+    %.  ~(key by links)
+    =-  ~(uni in -)
+    (~(uni in ~(key by chats)) ~(key by diarys))
+  %+  welp
+    %+  turn  ~(tap in graph-flags)
+    |=  =flag:i
+    ^-  card
+    (poke-our %graph-store migrated+!>(flag))
+  :~  (poke-our %chat graph-imports+!>(chats))
+      (poke-our %diary graph-imports+!>(diarys))
+      (poke-our %heap graph-imports+!>(links))
+        ==
+--

pkg/landscape/lib/graph-store.hoon

@@ -528,6 +528,81 @@
 ::
 ++  upgrade
   |%
+  ++  is-old-dm  |=(r=resource =('dm--' (end [3 4] name.r)))
+  ++  backup
+    |=  =bowl:gall
+    |=  [r=resource m=marked-graph]
+    ^-  card:agent:gall
+    =/  pax  /(rap 3 'archive-' (scot %p entity.r) '-' name.r ~)/noun
+    =/  =cage  drum-put+!>([pax (jam r m)])
+    [%pass /archive %agent [our.bowl %hood] %poke cage]
+  ++  strip-sigs-graph
+    |=  g=graph
+    ^+  g
+    =*  loop  $
+    %+  gas:orm  *graph
+    %+  turn  (tap:orm g)
+    |=  [key=@ val=node]  :: optional: also strip out deleted messages?
+    =?  children.val  ?=(%graph -.children.val)
+      [%graph loop(g p.children.val)]
+    :-  key
+    ?.  ?=(%& -.post.val)
+      val
+    val(signatures.p.post ~)
+  ++  strip-sigs-log
+    |=  u=update-log
+    %+  gas:orm-log  *update-log
+    %+  turn  (tap:orm-log u)
+    |=  [key=@ upd=logged-update]
+    :-  key
+    :-  p.upd
+    ?+    -.q.upd  q.upd
+        %add-graph
+      q.upd(graph (strip-sigs-graph graph.q.upd))
+    ::
+        %add-signatures
+      q.upd(signatures ~)
+    ::
+        %remove-signatures
+      q.upd(signatures ~)
+    ::
+        %add-nodes
+      %=    q.upd
+          nodes
+        %-  ~(run by nodes.q.upd)
+        |=  =node
+        ^+  node
+        %=    node
+            children
+          ?.  ?=(%graph -.children.node)
+            children.node
+          [%graph (strip-sigs-graph p.children.node)]
+        ::
+            post
+          ?.  ?=(%& -.post.node)
+            post.node
+          =.  signatures.p.post.node  ~
+          post.node
+        ==
+      ==
+    ==
+  ::
+  ++  nuke-groups
+    |=  =bowl:gall
+    |^  ^-  (list card:agent:gall)
+    ?.  .^(? (gall-scry %u %groups))
+      ~
+    =+  .^(=desk (gall-scry %d %groups))
+    :~  [%pass /nuke %agent [our.bowl %hood] %poke kiln-nuke+!>([desk &])]
+        [%pass /nuke %agent [our.bowl %docket] %poke docket-uninstall+!>(desk)]
+        [%pass /nuke %agent [our.bowl %docket] %poke docket-uninstall+!>(%talk)]
+    ==
+    ::
+    ++  gall-scry
+      |=  [=care:clay dap=dude:gall]
+      ^-  path
+      /(cat 3 %g care)/(scot %p our.bowl)/[dap]/(scot %da now.bowl)
+    --
   ::
   ::  +two
   ::
@@ -758,9 +833,9 @@
   --
 ++  import
   |=  [arc=* our=ship]
-  ^-  (quip card:agent:gall [%6 network])
+  ^-  (quip card:agent:gall [%7 network])
   |^
-  =/  sty  [%6 (remake-network ;;(tree-network +.arc))]
+  =/  sty  [%7 (remake-network ;;(tree-network +.arc))]
   :_  sty
   %+  turn  ~(tap by graphs.sty)
   |=  [rid=resource =marked-graph]

pkg/landscape/lib/s3-json.hoon

@@ -10,6 +10,7 @@
     :~  [%set-endpoint so:dejs]
         [%set-access-key-id so:dejs]
         [%set-secret-access-key so:dejs]
+        [%set-region so:dejs]
         [%add-bucket so:dejs]
         [%remove-bucket so:dejs]
         [%set-current-bucket so:dejs]
@@ -25,6 +26,7 @@
   :~  ?-  -.upd
           %set-current-bucket  [%'setCurrentBucket' s+bucket.upd]
           %add-bucket          [%'addBucket' s+bucket.upd]
+          %set-region          [%'setRegion' s+region.upd]
           %remove-bucket       [%'removeBucket' s+bucket.upd]
           %set-endpoint        [%'setEndpoint' s+endpoint.upd]
           %set-access-key-id   [%'setAccessKeyId' s+access-key-id.upd]
@@ -44,6 +46,7 @@
         %-  pairs:enjs
         :~  [%buckets a+(turn ~(tap in buckets.configuration.upd) |=(a=@t s+a))]
             [%'currentBucket' s+current-bucket.configuration.upd]
+            [%'region' s+region.configuration.upd]
         ==
       ==
   ==

pkg/landscape/sur/migrate.hoon

@@ -0,0 +1,23 @@
+/-  met=metadata-store, gra=graph-store
+/-  *group
+|%
++$  flag  (pair ship term)
+++  graph
+  |%
+  +$  import
+    [writers=(set ship) =association:met =update-log:gra =graph:gra]
+  +$  imports  (map flag import)
+  --
+::
+++  groups
+  |%
+  +$  import   [=association:met chans=(map flag =association:met) roles=(set flag) =group]
+  +$  imports  (map flag import)
+  --
+++  club
+  |%
+  +$  import   [ships=(set ship) =association:met =graph:gra]
+  +$  imports  (map flag import)
+  --
+--
+    

pkg/landscape/sur/s3-0.hoon

@@ -0,0 +1,27 @@
+|%
++$  credentials
+  $:  endpoint=@t
+      access-key-id=@t
+      secret-access-key=@t
+  ==
+::
++$  configuration
+  $:  buckets=(set @t)
+      current-bucket=@t
+  ==
+::
++$  action
+  $%  [%set-endpoint endpoint=@t]
+      [%set-access-key-id access-key-id=@t]
+      [%set-secret-access-key secret-access-key=@t]
+      [%add-bucket bucket=@t]
+      [%remove-bucket bucket=@t]
+      [%set-current-bucket bucket=@t]
+  ==
+::
++$  update
+  $%  [%credentials =credentials]
+      [%configuration =configuration]
+      action
+  ==
+--

pkg/landscape/sur/s3.hoon

@@ -1,4 +1,9 @@
+/-  zer=s3-0
 |%
+++  past
+  |%
+  ++  zero  zer
+  --
 +$  credentials
   $:  endpoint=@t
       access-key-id=@t
@@ -8,6 +13,7 @@
 +$  configuration
   $:  buckets=(set @t)
       current-bucket=@t
+      region=@t
   ==
 ::
 +$  action
@@ -17,6 +23,7 @@
       [%add-bucket bucket=@t]
       [%remove-bucket bucket=@t]
       [%set-current-bucket bucket=@t]
+      [%set-region region=@t]
   ==
 ::
 +$  update

pkg/landscape/ted/graph/recover-archive.hoon

@@ -0,0 +1,19 @@
+/-  spider, graph=graph-store, met=metadata-store, *group, group-store, push-hook
+/+  strandio, resource, graph-view
+=>
+|%
+++  strand  strand:spider
+++  poke  poke:strandio
+++  poke-our   poke-our:strandio
+--
+=,  strand=strand:spider
+^-  thread:spider
+|=  arg=vase
+=/  m  (strand ,vase)
+^-  form:m
+=+  !<([~ pax=path] arg)
+;<  =bowl:spider  bind:m  get-bowl:strandio
+=+  .^([rid=resource mar=marked-graph:graph] %cx pax)
+;<  ~  bind:m 
+  (poke-our:strandio %graph-store %graph-update-3 !>([now.bowl %add-graph p.mar q.mar &]))
+(pure:m *vase)

pkg/landscape/ted/keep.hoon

@@ -0,0 +1,14 @@
+/-  spider, docket
+/+  strandio
+=,  strand=strand:spider
+^-  thread:spider
+|=  arg=vase
+=/  m  (strand ,vase)
+^-  form:m
+=+  !<(jim=@ arg)
+;<  =bowl:spider  bind:m  get-bowl:strandio
+=/  home=path  /(scot %p our.bowl)/[q.byk.bowl]/(scot %da now.bowl)
+=/  =path  /(cat 3 'backup-' (scot %da now.bowl))/noun
+  ::  XX remove prints
+;<  ~  bind:m  (poke-our:strandio %hood drum-put+!>([path jim]))
+(pure:m *vase)

pkg/landscape/ted/migrate.hoon

@@ -0,0 +1,91 @@
+:: Migrate scripts
+/-  spider
+/-  gra=graph-store
+/-  met=metadata-store
+/-  grp=group-store
+/-  i=migrate
+/-  *group
+/+  strandio
+=,  strand=strand:spider
+|%
+++  import-for-mark
+  |=  [=groups =associations:met =network:gra]
+  |=  =mark
+  ^-  imports:graph:i
+  %-  ~(gas by *imports:graph:i)
+  %+  murn  ~(tap by graphs.network)
+  |=  [=flag:i graph=graph:gra mar=(unit ^mark)]
+  ?.  =(mar `mark)  ::  XX: correct detection?
+    ~
+  ?~  assoc=(~(get by associations) [%graph flag])
+    ~&  missing-assoc/flag^mark
+    ~
+  ?~  group=(~(get by groups) group.u.assoc)
+    ~&  missing-group/[flag group.u.assoc]
+    ~
+  =/  writers=(set ship)  
+    (~(get ju tags.u.group) %graph flag %writers)
+  ?~  log=(~(get by update-logs.network) flag)
+    ~&  missing-log/flag  :: XX: doesn't need to fail, but suspect case
+    ~
+  `[flag writers u.assoc u.log graph]
+::
+--
+^-  thread:spider
+|=  arg=vase
+=/  m  (strand ,vase)
+^-  form:m
+;<  =bowl:spider  bind:m  get-bowl:strandio
+;<  [%2 =groups]  bind:m
+  (scry:strandio ,[%2 =groups] /gx/group-store/export/noun)
+;<  [%6 =network:gra]  bind:m
+  (scry:strandio ,[%6 =network:gra] /gx/graph-store/export/noun)
+;<  =associations:met  bind:m
+  (scry:strandio ,associations:met /gx/metadata-store/associations/noun)
+=/  import  (import-for-mark groups associations network)
+=/  chats=imports:graph:i
+  (import %graph-validator-chat)
+=/  diarys=imports:graph:i
+  (import %graph-validator-publish)
+=/  links=imports:graph:i
+  (import %graph-validator-link)
+=/  =imports:groups:i
+  %-  ~(gas by *imports:groups:i)
+  %+  murn  ~(tap by groups)
+  |=  [=flag:i =group]
+  ^-  (unit [_flag import:groups:i])
+  ?~  assoc=(~(get by associations) [%groups flag])
+    ~&  missing-group-assoc/flag
+    ~
+  =/  chans=(map flag:i association:met)
+    %-  ~(gas by *(map flag:i association:met))
+    %+  murn  ~(tap by associations)
+    |=  [res=md-resource:met ass=association:met]
+    ^-  (unit [flag:i association:met])
+    ?.  =(group.ass flag)  ~
+    `[resource.res ass]
+  =/  roles=(set flag:i)
+    %-  ~(gas in *(set flag:i))
+    %+  murn  ~(tap by chans)
+    |=  [=flag:i =association:met]
+    ^-  (unit flag:i)
+    ?^  link=(~(get by links) flag)
+      ?:  =(writers.u.link ~)  ~
+      `flag
+    ?^  diary=(~(get by diarys) flag)
+      ?:  =(writers.u.diary ~)  ~
+      `flag
+    ?^  chat=(~(get by chats) flag)
+      ?:  =(writers.u.chat ~)  ~
+      `flag
+    ~
+  `[flag u.assoc chans roles group]
+;<  ~  bind:m  (poke-our:strandio %groups group-import+!>(imports))
+;<  ~  bind:m  (poke-our:strandio %chat graph-imports+!>(chats))
+;<  ~  bind:m  (poke-our:strandio %diary graph-imports+!>(diarys))
+;<  ~  bind:m  (poke-our:strandio %heap graph-imports+!>(links))
+;<  ~  bind:m  
+  ?~  dms=(~(get by graphs.network) [our.bowl %dm-inbox])
+    (pure:(strand ,~) ~)
+  (poke-our:strandio %chat %dm-imports !>(p.u.dms))
+(pure:m *vase)

pkg/urbit/version

@@ -1 +1 @@
-1.13
+1.15