diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b7b9542271..166bc72afa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,7 +32,29 @@ name: build -on: [push, pull_request] +on: + push: + paths: + - 'pkg/arvo/**' + - 'pkg/docker-image/**' + - 'pkg/ent/**' + - 'pkg/ge-additions/**' + - 'pkg/hs/**' + - 'pkg/libaes_siv/**' + - 'pkg/urbit/**' + - 'bin/**' + - 'nix/**' + pull_request: + paths: + - 'pkg/arvo/**' + - 'pkg/docker-image/**' + - 'pkg/ent/**' + - 'pkg/ge-additions/**' + - 'pkg/hs/**' + - 'pkg/libaes_siv/**' + - 'pkg/urbit/**' + - 'bin/**' + - 'nix/**' jobs: urbit: diff --git a/.github/workflows/glob.yml b/.github/workflows/glob.yml index 3aabb75c9f..ba67d98cb8 100644 --- a/.github/workflows/glob.yml +++ b/.github/workflows/glob.yml @@ -2,7 +2,7 @@ name: glob on: push: branches: - - 'release/next-js' + - 'release/next-userspace' jobs: glob: runs-on: ubuntu-latest diff --git a/.github/workflows/merge-master.yml b/.github/workflows/merge-master.yml index 627d9e802d..48d0abf9fb 100644 --- a/.github/workflows/merge-master.yml +++ b/.github/workflows/merge-master.yml @@ -6,13 +6,13 @@ on: jobs: merge-to-next-js: runs-on: ubuntu-latest - name: "Merge master to release/next-js" + name: "Merge master to release/next-userspace" steps: - uses: actions/checkout@v2 - uses: devmasx/merge-branch@v1.3.1 with: type: now - target_branch: release/next-js + target_branch: release/next-userspace github_token: ${{ secrets.JANEWAY_BOT_TOKEN }} merge-to-group-timer: diff --git a/.github/workflows/publish-npm-packages.yml b/.github/workflows/publish-npm-packages.yml index 0f0fc021e4..44fb251f89 100644 --- a/.github/workflows/publish-npm-packages.yml +++ b/.github/workflows/publish-npm-packages.yml @@ -3,6 +3,8 @@ on: push: branches: - 'master' + paths: + - 'pkg/npm/**' jobs: publish-api: runs-on: ubuntu-latest diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 78b2eda4a9..eaca1e7bda 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -309,9 +309,9 @@ the new binary, and restarting the pier with it. #### Continuous deployment A subset of release branches are deployed continuously to the network. Thus far -this only includes `release/next-js`, which deploys livenet-compatible -JavaScript changes to select QA ships. Any push to master will automatically -merge master into `release/next-js` to keep the streams at parity. +this only includes `release/next-userspace`, which deploys livenet-compatible +changes to select QA ships. Any push to master will automatically +merge master into `release/next-userspace` to keep the streams at parity. ### Announce the update diff --git a/bin/solid.pill b/bin/solid.pill index 6b7c74910b..034ca34273 100644 --- a/bin/solid.pill +++ b/bin/solid.pill @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:0d5d569d5bb9abc36b515f9c0690974706ba615c624ff76fd299712695506863 -size 9346854 +oid sha256:d7b7cf24e56ab078cf1dcb82e4e7744f188c5221c08772d6cfb15f59ce81aaa5 +size 11198219 diff --git a/nix/overlays/native.nix b/nix/overlays/native.nix index 2ab55e3751..f7f60e0412 100644 --- a/nix/overlays/native.nix +++ b/nix/overlays/native.nix @@ -25,4 +25,11 @@ in { ldapSupport = false; brotliSupport = false; }; + + lmdb = prev.lmdb.overrideAttrs (attrs: { + patches = + optionalList attrs.patches ++ prev.lib.optional prev.stdenv.isDarwin [ + ../pkgs/lmdb/darwin-fsync.patch + ]; + }); } diff --git a/nix/pkgs/docker-image/default.nix b/nix/pkgs/docker-image/default.nix index 467c06ac47..6c200a00ec 100644 --- a/nix/pkgs/docker-image/default.nix +++ b/nix/pkgs/docker-image/default.nix @@ -1,10 +1,24 @@ -{ urbit, libcap, coreutils, bashInteractive, dockerTools, writeScriptBin, amesPort ? 34343 }: +{ urbit, curl, libcap, coreutils, bashInteractive, dockerTools, writeScriptBin, amesPort ? 34343 }: let startUrbit = writeScriptBin "start-urbit" '' #!${bashInteractive}/bin/bash set -eu + # set defaults + amesPort=${toString amesPort} + + # check args + for i in "$@" + do + case $i in + -p=*|--port=*) + amesPort="''${i#*=}" + shift + ;; + esac + done + # If the container is not started with the `-i` flag # then STDIN will be closed and we need to start # Urbit/vere with the `-t` flag. @@ -23,7 +37,7 @@ let mv $keyname /tmp # Boot urbit with the key, exit when done booting - urbit $ttyflag -w $(basename $keyname .key) -k /tmp/$keyname -c $(basename $keyname .key) -p ${toString amesPort} -x + urbit $ttyflag -w $(basename $keyname .key) -k /tmp/$keyname -c $(basename $keyname .key) -p $amesPort -x # Remove the keyfile for security rm /tmp/$keyname @@ -34,7 +48,7 @@ let cometname=''${comets[0]} rm *.comet - urbit $ttyflag -c $(basename $cometname .comet) -p ${toString amesPort} -x + urbit $ttyflag -c $(basename $cometname .comet) -p $amesPort -x fi # Find the first directory and start urbit with the ship therein @@ -42,14 +56,44 @@ let dirs=( $dirnames ) dirname=''${dirnames[0]} - urbit $ttyflag -p ${toString amesPort} $dirname + exec urbit $ttyflag -p $amesPort $dirname + ''; + + getUrbitCode = writeScriptBin "get-urbit-code" '' + #!${bashInteractive}/bin/bash + + raw=$(curl -s -X POST -H "Content-Type: application/json" \ + -d '{ "source": { "dojo": "+code" }, "sink": { "stdout": null } }' \ + http://127.0.0.1:12321) + + # trim \n" from the end + trim="''${raw%\\n\"}" + + # trim " from the start + code="''${trim#\"}" + + echo "$code" + ''; + + resetUrbitCode = writeScriptBin "reset-urbit-code" '' + #!${bashInteractive}/bin/bash + + curl=$(curl -s -X POST -H "Content-Type: application/json" \ + -d '{ "source": { "dojo": "+hood/code %reset" }, "sink": { "app": "hood" } }' \ + http://127.0.0.1:12321) + + if [[ $? -eq 0 ]] + then + echo "OK" + else + echo "Curl error: $?" + fi ''; - in dockerTools.buildImage { name = "urbit"; tag = "v${urbit.version}"; - contents = [ bashInteractive urbit startUrbit coreutils ]; + contents = [ bashInteractive urbit curl startUrbit getUrbitCode resetUrbitCode coreutils ]; runAsRoot = '' #!${bashInteractive} mkdir -p /urbit diff --git a/nix/pkgs/hs/default.nix b/nix/pkgs/hs/default.nix index 91a2102e5c..ea09883ba4 100644 --- a/nix/pkgs/hs/default.nix +++ b/nix/pkgs/hs/default.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, darwin, haskell-nix, gmp, zlib, libffi, brass +{ lib, stdenv, darwin, haskell-nix, lmdb, gmp, zlib, libffi, brass , enableStatic ? stdenv.hostPlatform.isStatic }: haskell-nix.stackProject { @@ -65,6 +65,7 @@ haskell-nix.stackProject { enableShared = !enableStatic; configureFlags = lib.optionals enableStatic [ + "--ghc-option=-optl=-L${lmdb}/lib" "--ghc-option=-optl=-L${gmp}/lib" "--ghc-option=-optl=-L${libffi}/lib" "--ghc-option=-optl=-L${zlib}/lib" @@ -81,6 +82,8 @@ haskell-nix.stackProject { urbit-king.components.tests.urbit-king-tests.testFlags = [ "--brass-pill=${brass.lfs}" ]; + + lmdb.components.library.libs = lib.mkForce [ lmdb ]; }; }]; } diff --git a/nix/pkgs/lmdb/darwin-fsync.patch b/nix/pkgs/lmdb/darwin-fsync.patch new file mode 100644 index 0000000000..961b8a1ffd --- /dev/null +++ b/nix/pkgs/lmdb/darwin-fsync.patch @@ -0,0 +1,13 @@ +diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c +index fe65e30..0070215 100644 +--- a/libraries/liblmdb/mdb.c ++++ b/libraries/liblmdb/mdb.c +@@ -2526,7 +2526,7 @@ mdb_env_sync(MDB_env *env, int force) + rc = ErrCode(); + } else + #endif +- if (MDB_FDATASYNC(env->me_fd)) ++ if (fcntl(env->me_fd, F_FULLFSYNC, 0)) + rc = ErrCode(); + } + } diff --git a/pkg/arvo/README.md b/pkg/arvo/README.md index 3074d5ba78..206e47dffc 100644 --- a/pkg/arvo/README.md +++ b/pkg/arvo/README.md @@ -45,17 +45,16 @@ Most parts of Arvo have dedicated maintainers. * `/sys/vane/ames`: @belisarius222 (~rovnys-ricfer) & @philipcmonk (~wicdev-wisryt) * `/sys/vane/behn`: @belisarius222 (~rovnys-ricfer) * `/sys/vane/clay`: @philipcmonk (~wicdev-wisryt) & @belisarius222 (~rovnys-ricfer) -* `/sys/vane/dill`: @joemfb (~master-morzod) -* `/sys/vane/eyre`: @eglaysher (~littel-ponnys) +* `/sys/vane/dill`: @fang- (~palfun-foslup) +* `/sys/vane/eyre`: @fang- (~palfun-foslup) * `/sys/vane/gall`: @philipcmonk (~wicdev-wisryt) * `/sys/vane/jael`: @fang- (~palfun-foslup) & @philipcmonk (~wicdev-wisryt) * `/app/acme`: @joemfb (~master-morzod) * `/app/dns`: @joemfb (~master-morzod) * `/app/aqua`: @philipcmonk (~wicdev-wisryt) * `/app/hood`: @belisarius222 (~rovnys-ricfer) -* `/lib/hood/drum`: @philipcmonk (~wicdev-wisryt) +* `/lib/hood/drum`: @fang- (~palfun-foslup) * `/lib/hood/kiln`: @philipcmonk (~wicdev-wisryt) -* `/lib/test`: @eglaysher (~littel-ponnys) ## Contributing diff --git a/pkg/arvo/app/acme.hoon b/pkg/arvo/app/acme.hoon index bbb53b8eb4..7d64a69f3f 100644 --- a/pkg/arvo/app/acme.hoon +++ b/pkg/arvo/app/acme.hoon @@ -1394,8 +1394,6 @@ ^+ this ?: =(~ dom) ~|(%acme-empty-certificate-order !!) - ?: ?=(?(%earl %pawn) (clan:title our.bow)) - this =. ..emit (queue-next-order 1 | dom) =. ..emit cancel-current-order :: notify %dill diff --git a/pkg/arvo/app/chat-cli.hoon b/pkg/arvo/app/chat-cli.hoon index 49e6b2ff78..c13141a96d 100644 --- a/pkg/arvo/app/chat-cli.hoon +++ b/pkg/arvo/app/chat-cli.hoon @@ -169,7 +169,7 @@ :: %fact ?+ p.cage.sign ~|([dap.bowl %bad-sub-mark wire p.cage.sign] !!) - %graph-update + %graph-update-2 %- on-graph-update:tc !<(update:graph q.cage.sign) == @@ -401,12 +401,16 @@ :: +read-post: add envelope to state and show it to user :: ++ read-post - |= [=target =index:post =post:post] + |= [=target =index:post =maybe-post:graph] ^- (quip card _session) - :- (show-post:sh-out target post) - %_ session - history [[target index] history.session] - count +(count.session) + ?- -.maybe-post + %| [~ session] + %& + :- (show-post:sh-out target p.maybe-post) + %_ session + history [[target index] history.session] + count +(count.session) + == == :: ++ notice-remove @@ -734,7 +738,8 @@ :: ?. (is-chat-graph target) [[(note:sh-out "no such chat")]~ put-ses] - =. viewing (~(put in viewing) target) + =. audience target + =. viewing (~(put in viewing) target) =^ cards state ?: (~(has by bound) target) [~ state] @@ -758,15 +763,15 @@ ::TODO move creation into lib? %^ act %out-message %graph-push-hook - :- %graph-update + :- %graph-update-2 !> ^- update:graph - :+ %0 now.bowl + :- now.bowl :+ %add-nodes audience %- ~(put by *(map index:post node:graph)) :- ~[now.bowl] :_ *internal-graph:graph - ^- post:post - [our-self ~[now.bowl] now.bowl [msg]~ ~ ~] + ^- maybe-post:graph + [%& `post:post`[our-self ~[now.bowl] now.bowl [msg]~ ~ ~]] :: +eval: run hoon, send code and result as message :: :: this double-virtualizes and clams to disable .^ for security reasons @@ -890,10 +895,12 @@ =/ =uid:post (snag index history) =/ =node:graph (got-node:libgraph uid) =. audience resource.uid + ?: ?=(%| -.post.node) + [~ state] :_ put-ses ^- (list card) :~ (print:sh-out ['?' ' ' number]) - (effect:sh-out ~(render-activate mr resource.uid post.node)) + (effect:sh-out ~(render-activate mr resource.uid p.post.node)) prompt:sh-out == -- @@ -1185,7 +1192,15 @@ ?- -.content %text txt+(trip text.content) %url url+url.content - %reference txt+"[reference to msg in {~(phat tr resource.uid.content)}]" + :: + %reference + ?- -.reference.content + %graph + txt+"[reference to msg in {~(phat tr resource.uid.reference.content)}]" + :: + %group + txt+"[reference to msg in {~(phat tr group.reference.content)}]" + == :: %mention ?. =(ship.content our-self) txt+(scow %p ship.content) diff --git a/pkg/arvo/app/chat-hook.hoon b/pkg/arvo/app/chat-hook.hoon index 622ce92275..ce44a73ea5 100644 --- a/pkg/arvo/app/chat-hook.hoon +++ b/pkg/arvo/app/chat-hook.hoon @@ -154,7 +154,7 @@ ++ poke-graph-store |= =update:graph-store ^- card - (poke-our %graph-store %graph-update !>(update)) + (poke-our %graph-store %graph-update-2 !>(update)) :: ++ nobody ^- @p @@ -190,7 +190,7 @@ cards :_ cards %- poke-graph-store - :+ %0 now.bol + :- now.bol archive-graph+rid == ?: =(our.bol ship) diff --git a/pkg/arvo/app/chat-store.hoon b/pkg/arvo/app/chat-store.hoon index c993d37e4b..9406360b47 100644 --- a/pkg/arvo/app/chat-store.hoon +++ b/pkg/arvo/app/chat-store.hoon @@ -1,334 +1,28 @@ -:: chat-store [landscape]: +:: chat-store [landscape]: deprecated :: -:: data store that holds linear sequences of chat messages -:: -/- *group, store=chat-store -/+ default-agent, verb, dbug, group-store, - graph-store, resource, *migrate, grpl=group, mdl=metadata -~% %chat-store-top ..part ~ +/- store=chat-store +/+ default-agent |% +$ card card:agent:gall -+$ versioned-state - $% state-0 - state-1 - state-2 - state-3 - state-4 - == -:: -+$ state-0 [%0 =inbox:store] -+$ state-1 [%1 =inbox:store] -+$ state-2 [%2 =inbox:store] -+$ state-3 [%3 =inbox:store] -+$ state-4 [%4 =inbox:store] -+$ admin-action - $% [%trim ~] - [%migrate-graph ~] - == -- :: -=| state-4 -=* state - -:: -%- agent:dbug -%+ verb | ^- agent:gall -=< - ~% %chat-store-agent-core ..peek-x-envelopes ~ - |_ =bowl:gall - +* this . - chat-core +> - cc ~(. chat-core bowl) - def ~(. (default-agent this %|) bowl) - :: - ++ on-init on-init:def - ++ on-save !>(state) - ++ on-load - |= old-vase=vase - ^- (quip card _this) - |^ - =/ old !<(versioned-state old-vase) - =| cards=(list card) - |- - ^- (quip card _this) - ?- -.old - %4 [cards this(state old)] - :: - %3 - =. cards :_(cards (poke-admin %migrate-graph ~)) - $(old [%4 inbox.old]) - :: - %2 - =/ =inbox:store - (migrate-path-map:group-store inbox.old) - =/ kick-paths - %~ tap in - %+ roll - ~(val by sup.bowl) - |= [[=ship sub=path] subs=(set path)] - ^- (set path) - ?. ?=([@ @ *] sub) - subs - ?. &(=(%mailbox i.sub) =('~' i.t.sub)) - subs - (~(put in subs) sub) - =? cards ?=(^ kick-paths) - :_ cards - [%give %kick kick-paths ~] - $(old [%3 inbox]) - :: - ?(%0 %1) $(old (old-to-2 inbox.old)) - :: - == - ++ poke-admin - |= =admin-action - ^- card - [%pass / %agent [our dap]:bowl %poke noun+!>(admin-action)] - :: - ++ old-to-2 - |= =inbox:store - ^- state-2 - :- %2 - %- ~(run by inbox) - |= =mailbox:store - ^- mailbox:store - [config.mailbox (flop envelopes.mailbox)] - -- - :: - ++ on-poke - ~/ %chat-store-poke - |= [=mark =vase] - ^- (quip card _this) - ?> (team:title our.bowl src.bowl) - =^ cards state - ?+ mark (on-poke:def mark vase) - %noun (poke-noun:cc !<(admin-action vase)) - %import (poke-import:cc q.vase) - == - [cards this] - :: - ++ on-watch on-watch:def - ++ on-leave on-leave:def - ++ on-peek - ~/ %chat-store-peek - |= =path - ^- (unit (unit cage)) - ?+ path (on-peek:def path) - [%x %all ~] ``noun+!>(inbox) - [%x %keys ~] ``noun+!>(~(key by inbox)) - [%x %envelopes *] (peek-x-envelopes:cc t.t.path) - [%x %mailbox *] - ?~ t.t.path - ~ - ``noun+!>((~(get by inbox) t.t.path)) - :: - [%x %config *] - ?~ t.t.path - ~ - =/ mailbox (~(get by inbox) t.t.path) - ?~ mailbox - ~ - ``noun+!>(config.u.mailbox) - :: - [%x %export ~] - ``noun+!>(state) - == - :: - ++ on-agent on-agent:def - ++ on-arvo on-arvo:def - ++ on-fail on-fail:def - -- +|_ =bowl:gall ++* this . + def ~(. (default-agent this %|) bowl) :: -~% %chat-store-library ..card ~ -|_ bol=bowl:gall -++ met ~(. mdl bol) -++ grp ~(. grpl bol) +++ on-init on-init:def +++ on-save !>(~) +++ on-load + |= old-vase=vase + ^- (quip card _this) + [~ this] :: -++ peek-x-envelopes - |= pax=path - ^- (unit (unit [%noun vase])) - ?+ pax ~ - [@ @ *] - =/ mail-path t.t.pax - =/ mailbox (~(get by inbox) mail-path) - ?~ mailbox - [~ ~ %noun !>(~)] - =* envelopes envelopes.u.mailbox - =/ sign-test=[?(%neg %pos) @] - %- need - %+ rush i.pax - ;~ pose - %+ cook - |= n=@ - [%neg n] - ;~(pfix hep dem:ag) - :: - %+ cook - |= n=@ - [%pos n] - dem:ag - == - =* length length.config.u.mailbox - =* start +.sign-test - ?: =(-.sign-test %neg) - ?: (gth start length) - [~ ~ %noun !>(envelopes)] - [~ ~ %noun !>((swag [(sub length start) start] envelopes))] - :: - =/ end (slav %ud i.t.pax) - ?. (lte start end) - ~ - =. end ?:((lth end length) end length) - [~ ~ %noun !>((swag [start (sub end start)] envelopes))] - == -:: -++ poke-noun - |= nou=admin-action - ^- (quip card _state) - ?: ?=([%migrate-graph ~] nou) - :_ state - (migrate-inbox inbox) - ~& %trimming-chat-store - :- ~ - %_ state - inbox - %- ~(urn by inbox) - |= [=path mailbox:store] - ^- mailbox:store - =/ [a=* out=(list envelope:store)] - %+ roll envelopes - |= $: =envelope:store - o=[[hav=(set serial:store) curr=@] out=(list envelope:store)] - == - ?: (~(has in hav.o) uid.envelope) - [[hav.o curr.o] out.o] - :- - ^- [(set serial:store) @] - [(~(put in hav.o) uid.envelope) +(curr.o)] - ^- (list envelope:store) - [envelope(number curr.o) out.o] - =/ len (lent out) - ~? !=(len (lent envelopes)) [path [%old (lent envelopes)] [%new len]] - [[len len] (flop out)] - == -:: -++ poke-import - |= arc=* - ^- (quip card _state) - =/ sty=state-4 [%4 (remake-map ;;((tree [path mailbox:store]) +.arc))] - :_ sty - (migrate-inbox inbox.sty) -:: -++ update-subscribers - |= [pax=path =update:store] - ^- (list card) - [%give %fact ~[pax] %chat-update !>(update)]~ -:: -++ send-diff - |= [pax=path upd=update:store] - ^- (list card) - %- zing - :~ (update-subscribers /all upd) - (update-subscribers /updates upd) - (update-subscribers [%mailbox pax] upd) - ?. |(|(=(%read -.upd) =(%message -.upd)) =(%messages -.upd)) - ~ - ?. |(=(%create -.upd) =(%delete -.upd)) - ~ - (update-subscribers /keys upd) - == -:: -++ migrate-inbox - |= =inbox:store - ^- (list card) - %- zing - (turn ~(tap by inbox) mailbox-to-updates) -:: -++ add-graph - |= [rid=resource =mailbox:store] - %- poke-graph-store - :+ %0 now.bol - :+ %add-graph rid - :- (mailbox-to-graph mailbox) - [`%graph-validator-chat %.y] -:: -++ archive-graph - |= rid=resource - %- poke-graph-store - [%0 now.bol %archive-graph rid] -:: -++ nobody - ^- @p - (bex 128) -:: -++ path-to-resource - |= =path - ^- resource - ?. ?=([@ @ ~] path) - nobody^(spat path) - =/ m-ship=(unit ship) - (slaw %p i.path) - ?~ m-ship - nobody^(spat path) - [u.m-ship i.t.path] -:: -++ mailbox-to-updates - |= [=path =mailbox:store] - ^- (list card) - =/ app-rid=resource - (path-to-resource path) - =/ group-rid=resource - (fall (peek-group:met %graph app-rid) [nobody %bad-group]) - =/ group=(unit group) - (scry-group:grp group-rid) - :- (add-graph app-rid mailbox) - ?~ group (archive-graph app-rid)^~ - ?. &(=(~ members.u.group) hidden.u.group) ~ - ~& >>> "archiving {}" - :~ (archive-graph app-rid) - (remove-group group-rid) - == -:: -++ remove-group - |= group=resource - ^- card - =- [%pass / %agent [our.bol %group-store] %poke -] - group-update+!>([%remove-group group ~]) -:: -++ poke-graph-store - |= =update:graph-store - ^- card - [%pass / %agent [our.bol %graph-store] %poke %graph-update !>(update)] -:: -++ letter-to-contents - |= =letter:store - ^- (list content:graph-store) - :_ ~ - ?. ?=(%me -.letter) - letter - [%text narrative.letter] -:: -++ envelope-to-node - |= =envelope:store - ^- [atom:graph-store node:graph-store] - =/ contents=(list content:graph-store) - (letter-to-contents letter.envelope) - =/ =index:graph-store - [when.envelope ~] - =, envelope - :- when.envelope - :_ [%empty ~] - ^- post:graph-store - :* author - index - when - contents - ~ ~ - == -:: -++ mailbox-to-graph - |= =mailbox:store - ^- graph:graph-store - %+ gas:orm:graph-store *graph:graph-store - (turn envelopes.mailbox envelope-to-node) +++ on-poke on-poke:def +++ on-watch on-watch:def +++ on-leave on-leave:def +++ on-peek on-peek:def +++ on-agent on-agent:def +++ on-arvo on-arvo:def +++ on-fail on-fail:def -- diff --git a/pkg/arvo/app/contact-pull-hook.hoon b/pkg/arvo/app/contact-pull-hook.hoon index 9c468d0f0e..5d992a12dd 100644 --- a/pkg/arvo/app/contact-pull-hook.hoon +++ b/pkg/arvo/app/contact-pull-hook.hoon @@ -10,6 +10,7 @@ update:store %contact-update %contact-push-hook + 0 0 %.y :: necessary to enable p2p == -- diff --git a/pkg/arvo/app/contact-push-hook.hoon b/pkg/arvo/app/contact-push-hook.hoon index f363e196e9..bb8a1a5262 100644 --- a/pkg/arvo/app/contact-push-hook.hoon +++ b/pkg/arvo/app/contact-push-hook.hoon @@ -11,6 +11,7 @@ update:store %contact-update %contact-pull-hook + 0 0 == :: +$ agent (push-hook:push-hook config) @@ -69,10 +70,11 @@ :: ++ transform-proxy-update |= vas=vase - ^- (unit vase) + ^- (quip card (unit vase)) :: TODO: should check if user is allowed to %add, %remove, %edit :: contact =/ =update:store !<(update:store vas) + :- ~ ?- -.update %initial ~ %add `vas diff --git a/pkg/arvo/app/contact-store.hoon b/pkg/arvo/app/contact-store.hoon index 85ebc4afbf..8f39e75404 100644 --- a/pkg/arvo/app/contact-store.hoon +++ b/pkg/arvo/app/contact-store.hoon @@ -71,7 +71,7 @@ ++ give |= =update:store ^- (list card) - [%give %fact ~ [%contact-update !>(update)]]~ + [%give %fact ~ [%contact-update-0 !>(update)]]~ -- :: ++ on-poke @@ -81,7 +81,7 @@ |^ =^ cards state ?+ mark (on-poke:def mark vase) - %contact-update (update !<(update:store vase)) + %contact-update-0 (update !<(update:store vase)) %import (import q.vase) == [cards this] @@ -126,6 +126,14 @@ !=(contact(last-updated *@da) u.old(last-updated *@da)) == [~ state] + ~| "cannot add a data url to cover!" + ?> ?| ?=(~ cover.contact) + !=('data:' (cut 3 [0 5] u.cover.contact)) + == + ~| "cannot add a data url to avatar!" + ?> ?| ?=(~ avatar.contact) + !=('data:' (cut 3 [0 5] u.avatar.contact)) + == :- (send-diff [%add ship contact] =(ship our.bowl)) state(rolodex (~(put by rolodex) ship contact)) :: @@ -149,6 +157,14 @@ =/ contact (edit-contact old edit-field) ?: =(old contact) [~ state] + ~| "cannot add a data url to cover!" + ?> ?| ?=(~ cover.contact) + !=('data:' (cut 3 [0 5] u.cover.contact)) + == + ~| "cannot add a data url to avatar!" + ?> ?| ?=(~ avatar.contact) + !=('data:' (cut 3 [0 5] u.avatar.contact)) + == =. last-updated.contact timestamp :- (send-diff [%edit ship edit-field timestamp] =(ship our.bowl)) state(rolodex (~(put by rolodex) ship contact)) @@ -203,7 +219,7 @@ ?: our [/updates /our /all ~] [/updates /all ~] - [%give %fact paths %contact-update !>(update)]~ + [%give %fact paths %contact-update-0 !>(update)]~ -- :: ++ import @@ -223,7 +239,7 @@ =/ =ship (slav %p i.t.t.path) =/ contact=(unit contact:store) (~(get by rolodex) ship) ?~ contact [~ ~] - :- ~ :- ~ :- %contact-update + :- ~ :- ~ :- %contact-update-0 !> ^- update:store [%add ship u.contact] :: diff --git a/pkg/arvo/app/demo-pull-hook.hoon b/pkg/arvo/app/demo-pull-hook.hoon new file mode 100644 index 0000000000..86a17b369e --- /dev/null +++ b/pkg/arvo/app/demo-pull-hook.hoon @@ -0,0 +1,60 @@ +/- store=demo +/+ default-agent, verb, dbug, pull-hook, agentio, resource +~% %demo-pull-hook-top ..part ~ +|% ++$ card card:agent:gall +:: +++ config + ^- config:pull-hook + :* %demo-store + update:store + %demo-update + %demo-push-hook + :: do not change spacing, required by tests + 0 + 0 + %.n + == +:: +-- +:: +:: +%- agent:dbug +%+ verb | +^- agent:gall +%- (agent:pull-hook config) +^- (pull-hook:pull-hook config) +|_ =bowl:gall ++* this . + def ~(. (default-agent this %|) bowl) + dep ~(. (default:pull-hook this config) bowl) +:: +++ on-init on-init:def +++ on-save !>(~) +++ on-load on-load:def +++ on-poke on-poke:def +++ on-agent on-agent:def +++ on-watch on-watch:def +++ on-leave on-leave:def +++ on-peek on-peek:def +++ on-arvo on-arvo:def +++ on-fail on-fail:def +++ on-pull-nack + |= [=resource =tang] + ^- (quip card _this) + ~& "{}: nacked" + %- (slog tang) + `this +:: +++ on-pull-kick + |= =resource + ^- (unit path) + ~& "{}: kicked" + `/ +:: +++ resource-for-update + |= =vase + =+ !<(=update:store vase) + ~[p.update] +-- + diff --git a/pkg/arvo/app/demo-push-hook.hoon b/pkg/arvo/app/demo-push-hook.hoon new file mode 100644 index 0000000000..988a31b2a4 --- /dev/null +++ b/pkg/arvo/app/demo-push-hook.hoon @@ -0,0 +1,65 @@ +/- store=demo +/+ default-agent, verb, dbug, push-hook, resource, agentio +|% ++$ card card:agent:gall +:: +++ config + ^- config:push-hook + :* %demo-store + /updates + update:store + %demo-update + %demo-pull-hook + :: + 0 + 0 + == +:: ++$ agent (push-hook:push-hook config) +-- +:: +:: +%- agent:dbug +%+ verb | +^- agent:gall +%- (agent:push-hook config) +^- agent +|_ =bowl:gall ++* this . + def ~(. (default-agent this %|) bowl) + grp ~(. grpl bowl) + io ~(. agentio bowl) +:: +++ on-init on-init:def +++ on-save !>(~) +++ on-load on-load:def +++ on-poke on-poke:def +++ on-agent on-agent:def +++ on-watch on-watch:def +++ on-leave on-leave:def +++ on-peek on-peek:def +++ on-arvo on-arvo:def +++ on-fail on-fail:def +:: +++ transform-proxy-update + |= vas=vase + ^- (quip card (unit vase)) + ``vas +:: +++ resource-for-update + |= =vase + =+ !<(=update:store vase) + ~[p.update] +:: +++ take-update + |= =vase + ^- [(list card) agent] + `this +:: +++ initial-watch + |= [=path rid=resource] + ^- vase + =+ .^(=update:store %gx (scry:io %demo-store (snoc `^path`log+(en-path:resource rid) %noun))) + !>(update) +:: +-- diff --git a/pkg/arvo/app/demo-store.hoon b/pkg/arvo/app/demo-store.hoon new file mode 100644 index 0000000000..705a68863f --- /dev/null +++ b/pkg/arvo/app/demo-store.hoon @@ -0,0 +1,100 @@ +/- store=demo +/+ default-agent, verb, dbug, resource, agentio +|% ++$ card card:agent:gall ++$ state-0 + [%0 log=(jar resource update:store) counters=(map resource @ud)] +-- +=| state-0 +=* state - +:: +%- agent:dbug +%+ verb | +^- agent:gall +|_ =bowl:gall ++* this . + def ~(. (default-agent this %|) bowl) + io ~(. agentio bowl) +++ on-init + `this +:: +++ on-save + !>(state) +:: +++ on-load + |= =vase + =+ !<(old=state-0 vase) + `this(state old) +:: +++ on-poke + |= [=mark =vase] + ^- (quip card _this) + ?. =(%demo-update-0 mark) + (on-poke:def mark vase) + ~& mark + =+ !<(=update:store vase) + |^ + =. log + (~(add ja log) p.update update) + =^ cards state + (upd update) + [cards this] + :: + ++ upd + |= up=update:store + ^- (quip card _state) + ?- -.up + %ini (upd-ini +.up) + %add (upd-add +.up) + %sub (upd-sub +.up) + %run (upd-run +.up) + == + :: + ++ upd-ini + |= [rid=resource ~] + :- (fact:io mark^!>([%ini +<]) /updates ~)^~ + state(counters (~(put by counters) rid 0)) + :: + ++ upd-add + |= [rid=resource count=@ud] + :- (fact:io mark^!>([%add +<]) /updates ~)^~ + state(counters (~(jab by counters) rid (cury add count))) + :: + ++ upd-sub + |= [rid=resource count=@ud] + :- (fact:io mark^!>([%sub +<]) /updates ~)^~ + state(counters (~(jab by counters) rid (cury sub count))) + :: + ++ upd-run + =| cards=(list card) + |= [rid=resource =(list update:store)] + ?~ list [cards state] + =^ caz state + (upd i.list) + $(list t.list, cards (weld cards caz)) + -- +:: +++ on-watch + |= =path + ?. ?=([%updates ~] path) + (on-watch:def path) + `this +:: +++ on-peek + |= =path + ?. ?=([%x %log @ @ @ ~] path) + (on-peek:def path) + =/ rid=resource + (de-path:resource t.t.path) + =/ =update:store + [%run rid (flop (~(get ja log) rid))] + ``noun+!>(update) +:: +++ on-agent on-agent:def +:: +++ on-arvo on-arvo:def +:: +++ on-leave on-leave:def +:: +++ on-fail on-fail:def +-- diff --git a/pkg/arvo/app/dojo.hoon b/pkg/arvo/app/dojo.hoon index 70e07d618d..c06b86e50e 100644 --- a/pkg/arvo/app/dojo.hoon +++ b/pkg/arvo/app/dojo.hoon @@ -1077,7 +1077,12 @@ :: %thread-done ?> ?=(^ poy) - (~(dy-hand dy u.poy(pux ~)) %noun q.cage.sign) + :: print the vase as a tang if it nests in tang + =/ =mark + ?: (~(nest ut -:!>(*tang)) | p.q.cage.sign) + %tang + %noun + (~(dy-hand dy u.poy(pux ~)) mark q.cage.sign) == :: %kick +>.$ diff --git a/pkg/arvo/app/file-server.hoon b/pkg/arvo/app/file-server.hoon index 121005b290..244f851c68 100644 --- a/pkg/arvo/app/file-server.hoon +++ b/pkg/arvo/app/file-server.hoon @@ -188,8 +188,11 @@ ?: ?=([%'~landscape' %js %session ~] site.req-line) %+ require-authorization-simple:app inbound-request - %- js-response:gen - (as-octt:mimes:html "window.ship = '{+:(scow %p our.bowl)}';") + %. %- as-octs:mimes:html + (rap 3 'window.ship = "' (rsh 3 (scot %p our.bowl)) '";' ~) + %* . js-response:gen + cache %.n + == :: =/ [payload=simple-payload:http public=?] (get-file req-line is-file) ?: public payload @@ -222,6 +225,7 @@ [~ %js] (js-response:gen file) [~ %css] (css-response:gen file) [~ %png] (png-response:gen file) + [~ %ico] (ico-response:gen file) :: [~ %html] %. file @@ -238,12 +242,10 @@ [not-found:gen %.n] :_ public.u.content =/ mime-type=@t (rsh 3 (crip )) - :: Should maybe inspect to see how long cache should hold - :: =/ headers :~ content-type+mime-type - max-1-da:gen - 'Service-Worker-Allowed'^'/' + max-1-wk:gen + 'service-worker-allowed'^'/' == [[200 headers] `q.u.data] == diff --git a/pkg/arvo/app/glob.hoon b/pkg/arvo/app/glob.hoon index 3ffcd16eb9..d0acf23371 100644 --- a/pkg/arvo/app/glob.hoon +++ b/pkg/arvo/app/glob.hoon @@ -5,7 +5,7 @@ /- glob /+ default-agent, verb, dbug |% -++ hash 0v3.o81b7.9dkd7.6ubrn.ebhmi.dtree +++ hash 0v4.vrvkt.4gcnm.dgg5o.e73d6.kqnaq +$ state-0 [%0 hash=@uv glob=(unit (each glob:glob tid=@ta))] +$ all-states $% state-0 diff --git a/pkg/arvo/app/graph-pull-hook.hoon b/pkg/arvo/app/graph-pull-hook.hoon index 431ac1c1df..1f9f9fc0e4 100644 --- a/pkg/arvo/app/graph-pull-hook.hoon +++ b/pkg/arvo/app/graph-pull-hook.hoon @@ -9,6 +9,7 @@ update:store %graph-update %graph-push-hook + 2 2 %.n == -- @@ -40,9 +41,9 @@ %- (slog leaf+"nacked {}" tang) :_ this ?. (~(has in get-keys:gra) resource) ~ - =- [%pass /pull-nack %agent [our.bowl %graph-store] %poke %graph-update -]~ + =- [%pass /pull-nack %agent [our.bowl %graph-store] %poke %graph-update-2 -]~ !> ^- update:store - [%0 now.bowl [%archive-graph resource]] + [now.bowl [%archive-graph resource]] :: ++ on-pull-kick |= =resource diff --git a/pkg/arvo/app/graph-push-hook.hoon b/pkg/arvo/app/graph-push-hook.hoon index 0ac5026377..9d4a5c72b5 100644 --- a/pkg/arvo/app/graph-push-hook.hoon +++ b/pkg/arvo/app/graph-push-hook.hoon @@ -1,6 +1,6 @@ /- *group, metadata=metadata-store /+ store=graph-store, mdl=metadata, res=resource, graph, group, default-agent, - dbug, verb, push-hook + dbug, verb, push-hook, agentio :: ~% %graph-push-hook-top ..part ~ |% @@ -12,43 +12,88 @@ update:store %graph-update %graph-pull-hook + 2 2 == :: +$ agent (push-hook:push-hook config) :: +$ state-null ~ +$ state-zero [%0 marks=(set mark)] ++$ state-one [%1 ~] +$ versioned-state $@ state-null - state-zero + $% state-zero + state-one + == +:: +:: TODO: come back to this and potentially use send a %t +:: to be notified of validator changes ++$ cache + $: graph-to-mark=(map resource:res (unit mark)) + perm-marks=(map [mark @tas] tube:clay) + transform-marks=(map mark tube:clay) + == +:: ++$ inflated-state + $: state-one + cache + == +:: ++$ cache-action + $% [%graph-to-mark (pair resource:res (unit mark))] + [%perm-marks (pair (pair mark @tas) tube:clay)] + [%transform-marks (pair mark tube:clay)] + == -- :: -=| state-zero -=* state - %- agent:dbug %+ verb | ^- agent:gall %- (agent:push-hook config) ^- agent -=< +=- +~% %graph-push-hook-agent ..scry.hook-core ~ +=| inflated-state +=* state - |_ =bowl:gall +* this . def ~(. (default-agent this %|) bowl) grp ~(. group bowl) gra ~(. graph bowl) - hc ~(. +> bowl) + met ~(. mdl bowl) + hc ~(. hook-core bowl +.state) + io ~(. agentio bowl) :: ++ on-init on-init:def -++ on-save !>(state) +++ on-save !>(-.state) ++ on-load |= =vase =+ !<(old=versioned-state vase) =? old ?=(~ old) [%0 ~] - ?> ?=(%0 -.old) - `this(state old) + =? old ?=(%0 -.old) + [%1 ~] + ?> ?=(%1 -.old) + `this(-.state old, +.state *cache) +:: +++ on-poke + |= [=mark =vase] + ^- (quip card _this) + ?. =(mark %graph-cache-hook) + [~ this] + =/ a=cache-action !<(cache-action vase) + =* c +.state + =* graph-to-mark graph-to-mark.c + =* perm-marks perm-marks.c + =* transform-marks transform-marks.c + =. c + ?- -.a + %graph-to-mark c(graph-to-mark (~(put by graph-to-mark) p.a q.a)) + %perm-marks c(perm-marks (~(put by perm-marks) p.a q.a)) + %transform-marks c(transform-marks (~(put by transform-marks) p.a q.a)) + == + [~ this(+.state c)] :: -++ on-poke on-poke:def ++ on-agent on-agent:def ++ on-watch on-watch:def ++ on-leave on-leave:def @@ -57,47 +102,72 @@ |= [=wire =sign-arvo] ^- (quip card _this) ?+ wire (on-arvo:def wire sign-arvo) + :: XX: no longer necessary :: - [%perms @ @ ~] - ?> ?=(?(%add %remove) i.t.t.wire) - =* mark i.t.wire - :_ this - (build-permissions mark i.t.t.wire %next)^~ - :: - [%transform-add @ ~] - =* mark i.t.wire - :_ this - (build-transform-add mark %next)^~ + [%perms @ @ ~] [~ this] + [%transform-add @ ~] [~ this] == :: ++ on-fail on-fail:def ++ transform-proxy-update + ~/ %transform-proxy-update |= vas=vase - ^- (unit vase) + ^- (quip card (unit vase)) =/ =update:store !<(update:store vas) =* rid resource.q.update =. p.update now.bowl ?- -.q.update %add-nodes - ?. (is-allowed-add:hc rid nodes.q.update) - ~ - =/ mark (get-mark:gra rid) - ?~ mark `vas - |^ - =/ transform - !< $-([index:store post:store atom ?] [index:store post:store]) - %. !>(*indexed-post:store) - .^(tube:clay (scry:hc %cc %home /[u.mark]/transform-add-nodes)) - =/ [* result=(list [index:store node:store])] - %+ roll - (flatten-node-map ~(tap by nodes.q.update)) - (transform-list transform) - =. nodes.q.update - %- ~(gas by *(map index:store node:store)) - result - [~ !>(update)] + =| cards=(list card) + =^ allowed cards (is-allowed-add:hc rid nodes.q.update) + ?. allowed + [cards ~] + =/ mark-cached (~(has by graph-to-mark) rid) + =/ mark + ?: mark-cached + (~(got by graph-to-mark) rid) + (get-mark:gra rid) + ?~ mark + [cards `vas] + =< $ + ~% %transform-add-nodes ..transform-proxy-update ~ + |% + ++ $ + ^- (quip card (unit vase)) + =/ transform-cached (~(has by transform-marks) u.mark) + =/ =tube:clay + ?: transform-cached + (~(got by transform-marks) u.mark) + .^(tube:clay (scry:hc %cc %home /[u.mark]/transform-add-nodes)) + =/ transform + !< $-([index:store post:store atom ?] [index:store post:store]) + %. !>(*indexed-post:store) + tube + =/ [* result=(list [index:store node:store])] + %+ roll + (flatten-node-map ~(tap by nodes.q.update)) + (transform-list transform) + =. nodes.q.update + %- ~(gas by *(map index:store node:store)) + result + :_ [~ !>(update)] + %+ weld cards + %- zing + :~ ?: mark-cached ~ + :_ ~ + %+ poke-self:pass:io %graph-cache-hook + !> ^- cache-action + [%graph-to-mark rid mark] + :: + ?: transform-cached ~ + :_ ~ + %+ poke-self:pass:io %graph-cache-hook + !> ^- cache-action + [%transform-marks u.mark tube] + == :: ++ flatten-node-map + ~/ %flatten-node-map |= lis=(list [index:store node:store]) ^- (list [index:store node:store]) |^ @@ -129,10 +199,13 @@ -- :: ++ transform-list + ~/ %transform-list |= transform=$-([index:store post:store atom ?] [index:store post:store]) |= $: [=index:store =node:store] [indices=(set index:store) lis=(list [index:store node:store])] == + ~| "cannot put a deleted post into %add-nodes {}" + ?> ?=(%& -.post.node) =/ l (lent index) =/ parent-modified=? %- ~(rep in indices) @@ -143,36 +216,42 @@ %.n =((swag [0 k] index) i) =/ [ind=index:store =post:store] - (transform index post.node now.bowl parent-modified) + (transform index p.post.node now.bowl parent-modified) :- (~(put in indices) index) - (snoc lis [ind node(post post)]) + (snoc lis [ind node(p.post post)]) -- :: - %remove-nodes - ?. (is-allowed-remove:hc resource.q.update indices.q.update) + %remove-posts + =| cards=(list card) + =^ allowed cards + (is-allowed-remove:hc rid indices.q.update) + :- cards + ?. allowed ~ `vas :: - %add-graph ~ - %remove-graph ~ - %add-signatures ~ - %remove-signatures ~ - %archive-graph ~ - %unarchive-graph ~ - %add-tag ~ - %remove-tag ~ - %keys ~ - %tags ~ - %tag-queries ~ - %run-updates ~ + %add-graph [~ ~] + %remove-graph [~ ~] + %add-signatures [~ ~] + %remove-signatures [~ ~] + %archive-graph [~ ~] + %unarchive-graph [~ ~] + %add-tag [~ ~] + %remove-tag [~ ~] + %keys [~ ~] + %tags [~ ~] + %tag-queries [~ ~] + %run-updates [~ ~] == :: ++ resource-for-update resource-for-update:gra :: ++ initial-watch + ~/ %initial-watch |= [=path =resource:res] ^- vase - ?> (is-allowed:hc resource) + |^ + ?> (is-allowed resource) !> ^- update:store ?~ path :: new subscribe @@ -184,23 +263,20 @@ (get-graph:gra resource) =/ =time (slav %da i.path) =/ =update-log:store (get-update-log-subset:gra resource time) - [%0 now.bowl [%run-updates resource update-log]] + [now.bowl [%run-updates resource update-log]] + :: + ++ is-allowed + |= =resource:res + =/ group-res=resource:res + (need (peek-group:met %graph resource)) + (is-member:grp src.bowl group-res) + -- :: ++ take-update |= =vase ^- [(list card) agent] =/ =update:store !<(update:store vase) ?+ -.q.update [~ this] - %add-graph - ?~ mark.q.update `this - =* mark u.mark.q.update - ?: (~(has in marks) mark) `this - :_ this(marks (~(put in marks) mark)) - :~ (build-permissions:hc mark %add %sing) - (build-permissions:hc mark %remove %sing) - (build-transform-add:hc mark %sing) - == - :: %remove-graph :_ this [%give %kick ~[resource+(en-path:res resource.q.update)] ~]~ @@ -210,10 +286,14 @@ [%give %kick ~[resource+(en-path:res resource.q.update)] ~]~ == -- -|_ =bowl:gall +:: +~% %graph-push-hook-helper ..card.hook-core ~ +^= hook-core +|_ [=bowl:gall =cache] +* grp ~(. group bowl) met ~(. mdl bowl) gra ~(. graph bowl) + io ~(. agentio bowl) :: ++ scry |= [care=@t desk=@t =path] @@ -221,28 +301,46 @@ /[care]/(scot %p our.bowl)/[desk]/(scot %da now.bowl) path :: -++ perm-mark-name - |= perm=@t - ^- @t - (cat 3 'graph-permissions-' perm) -:: ++ perm-mark |= [=resource:res perm=@t vip=vip-metadata:metadata =indexed-post:store] - ^- permissions:store - =- (check vip) - !< check=$-(vip-metadata:metadata permissions:store) - %. !>(indexed-post) - =/ mark (get-mark:gra resource) - ?~ mark |=(=vase !>([%no %no %no])) - .^(tube:clay (scry %cc %home /[u.mark]/(perm-mark-name perm))) -:: -++ add-mark - |= [=resource:res vip=vip-metadata:metadata =indexed-post:store] - (perm-mark resource %add vip indexed-post) -:: -++ remove-mark - |= [=resource:res vip=vip-metadata:metadata =indexed-post:store] - (perm-mark resource %remove vip indexed-post) + ^- [permissions:store (list card)] + |^ + =/ mark-cached (~(has by graph-to-mark.cache) resource) + =/ mark + ?: mark-cached + (~(got by graph-to-mark.cache) resource) + (get-mark:gra resource) + ?~ mark + [[%no %no %no] ~] + =/ key [u.mark (perm-mark-name perm)] + =/ perms-cached (~(has by perm-marks.cache) key) + =/ =tube:clay + ?: perms-cached + (~(got by perm-marks.cache) key) + .^(tube:clay (scry %cc %home /[u.mark]/(perm-mark-name perm))) + =/ check + !< $-(vip-metadata:metadata permissions:store) + (tube !>(indexed-post)) + :- (check vip) + %- zing + :~ ?: mark-cached ~ + :_ ~ + %+ poke-self:pass:io %graph-cache-hook + !> ^- cache-action + [%graph-to-mark resource mark] + :: + ?: perms-cached ~ + :_ ~ + %+ poke-self:pass:io %graph-cache-hook + !> ^- cache-action + [%perm-marks [u.mark (perm-mark-name perm)] tube] + == + :: + ++ perm-mark-name + |= perm=@t + ^- @t + (cat 3 'graph-permissions-' perm) + -- :: ++ get-permission |= [=permissions:store is-admin=? writers=(set ship)] @@ -255,22 +353,23 @@ writer.permissions reader.permissions :: -++ is-allowed - |= =resource:res - =/ group-res=resource:res - (need (peek-group:met %graph resource)) - (is-member:grp src.bowl group-res) -:: ++ get-roles-writers-variation + ~/ %get-roles-writers-variation |= =resource:res ^- (unit [is-admin=? writers=(set ship) vip=vip-metadata:metadata]) =/ assoc=(unit association:metadata) - (peek-association:met %graph resource) + (peek-association:met %graph resource) ?~ assoc ~ + =/ group=(unit group:grp) + (scry-group:grp group.u.assoc) + ?~ group ~ =/ role=(unit (unit role-tag)) - (role-for-ship:grp group.u.assoc src.bowl) + (role-for-ship-with-group:grp u.group group.u.assoc src.bowl) =/ writers=(set ship) - (get-tagged-ships:grp group.u.assoc [%graph resource %writers]) + %^ get-tagged-ships-with-group:grp + u.group + group.u.assoc + [%graph resource %writers] ?~ role ~ =/ is-admin=? ?=(?([~ %admin] [~ %moderator]) u.role) @@ -279,73 +378,111 @@ ++ node-to-indexed-post |= =node:store ^- indexed-post:store - =* index index.post.node - [(snag (dec (lent index)) index) post.node] + ?> ?=(%& -.post.node) + =* index index.p.post.node + [(snag (dec (lent index)) index) p.post.node] :: ++ is-allowed-add + ~/ %is-allowed-add |= [=resource:res nodes=(map index:store node:store)] - ^- ? - %- (bond |.(%.n)) + ^- [? (list card)] + |^ + %- (bond |.([%.n ~])) %+ biff (get-roles-writers-variation resource) |= [is-admin=? writers=(set ship) vip=vip-metadata:metadata] + ^- (unit [? (list card)]) %- some - %+ levy ~(tap by nodes) - |= [=index:store =node:store] - ?. =(author.post.node src.bowl) - %.n - =/ =permissions:store - %^ add-mark resource vip - (node-to-indexed-post node) - =/ =permission-level:store - (get-permission permissions is-admin writers) - ?- permission-level - %yes %.y - %no %.n - :: - %self - =/ parent-index=index:store - (scag (dec (lent index)) index) - =/ parent-node=node:store - (got-node:gra resource parent-index) - =(author.post.parent-node src.bowl) - == + =/ a ~(tap by nodes) + =| cards=(list card) + |- ^- [? (list card)] + ?~ a [& cards] + =/ c (check i.a is-admin writers vip) + ?. -.c + [| (weld cards +.c)] + $(a t.a, cards (weld cards +.c)) + :: + ++ check + |= $: [=index:store =node:store] + is-admin=? + writers=(set ship) + vip=vip-metadata:metadata + == + ^- [? (list card)] + =/ parent-index=index:store + (scag (dec (lent index)) index) + ?: (~(has by nodes) parent-index) + [%.y ~] + ?: ?=(%| -.post.node) + [%.n ~] + ?. =(author.p.post.node src.bowl) + [%.n ~] + =/ added + %^ add-mark resource vip + (node-to-indexed-post node) + =* permissions -.added + =* cards +.added + =/ =permission-level:store + (get-permission permissions is-admin writers) + :_ cards + ?- permission-level + %yes %.y + %no %.n + :: + %self + =/ parent-node=node:store + (got-node:gra resource parent-index) + ?: ?=(%| -.post.parent-node) + %.n + =(author.p.post.parent-node src.bowl) + == + :: + ++ add-mark + |= [=resource:res vip=vip-metadata:metadata =indexed-post:store] + (perm-mark resource %add vip indexed-post) + -- :: ++ is-allowed-remove + ~/ %is-allowed-remove |= [=resource:res indices=(set index:store)] - ^- ? - %- (bond |.(%.n)) + ^- [? (list card)] + |^ + %- (bond |.([%.n ~])) %+ biff (get-roles-writers-variation resource) |= [is-admin=? writers=(set ship) vip=vip-metadata:metadata] - %- some - %+ levy ~(tap by indices) - |= =index:store - =/ =node:store - (got-node:gra resource index) - =/ =permissions:store - %^ remove-mark resource vip - (node-to-indexed-post node) - =/ =permission-level:store - (get-permission permissions is-admin writers) - ?- permission-level - %yes %.y - %no %.n - %self =(author.post.node src.bowl) - == -:: -++ build-permissions - |= [=mark kind=?(%add %remove) mode=?(%sing %next)] - ^- card - =/ =wire /perms/[mark]/[kind] - =/ =mood:clay [%c da+now.bowl /[mark]/(perm-mark-name kind)] - =/ =rave:clay ?:(?=(%sing mode) [mode mood] [mode mood]) - [%pass wire %arvo %c %warp our.bowl %home `rave] -:: -++ build-transform-add - |= [=mark mode=?(%sing %next)] - ^- card - =/ =wire /transform-add/[mark] - =/ =mood:clay [%c da+now.bowl /[mark]/transform-add-nodes] - =/ =rave:clay ?:(?=(%sing mode) [mode mood] [mode mood]) - [%pass wire %arvo %c %warp our.bowl %home `rave] + %- some + =/ a ~(tap by indices) + =| cards=(list card) + |- ^- [? (list card)] + ?~ a [& cards] + =/ c (check i.a is-admin writers vip) + ?. -.c + [| (weld cards +.c)] + $(a t.a, cards (weld cards +.c)) + :: + ++ check + |= [=index:store is-admin=? writers=(set ship) vip=vip-metadata:metadata] + ^- [? (list card)] + =/ =node:store + (got-node:gra resource index) + ?: ?=(%| -.post.node) + [%.n ~] + =/ removed + %^ remove-mark resource vip + (node-to-indexed-post node) + =* permissions -.removed + =* cards +.removed + =/ =permission-level:store + (get-permission permissions is-admin writers) + :_ cards + ?- permission-level + %yes %.y + %no %.n + %self =(author.p.post.node src.bowl) + == + :: + ++ remove-mark + |= [=resource:res vip=vip-metadata:metadata =indexed-post:store] + (perm-mark resource %remove vip indexed-post) + -- -- diff --git a/pkg/arvo/app/graph-store.hoon b/pkg/arvo/app/graph-store.hoon index 87f4644803..e4cbca335a 100644 --- a/pkg/arvo/app/graph-store.hoon +++ b/pkg/arvo/app/graph-store.hoon @@ -1,27 +1,37 @@ :: graph-store [landscape] :: :: -/+ store=graph-store, sigs=signatures, res=resource, default-agent, dbug, verb, - *migrate +/+ store=graph-store, sigs=signatures, res=resource, default-agent, dbug, verb ~% %graph-store-top ..part ~ |% +$ card card:agent:gall +$ versioned-state - $% state-0 - state-1 - state-2 + $% [%0 network:zero:store] + [%1 network:zero:store] + [%2 network:zero:store] + [%3 network:one:store] + [%4 network:store] + state-5 == :: -+$ state-0 [%0 network:store] -+$ state-1 [%1 network:store] -+$ state-2 [%2 network:store] -:: ++$ state-5 [%5 network:store] ++ orm orm:store ++ orm-log orm-log:store +$ debug-input [%validate-graph =resource:store] +:: ++$ cache + $: validators=(map mark dais:clay) + == +:: +:: TODO: come back to this and potentially use ford runes or otherwise +:: send a %t to be notified of validator changes ++$ inflated-state + $: state-5 + cache + == -- :: -=| state-2 +=| inflated-state =* state - :: %- agent:dbug @@ -33,162 +43,84 @@ def ~(. (default-agent this %|) bowl) :: ++ on-init [~ this] -++ on-save !>(state) +++ on-save !>(-.state) ++ on-load |= =old=vase ^- (quip card _this) =+ !<(old=versioned-state old-vase) =| cards=(list card) - |^ + |- ?- -.old - %0 + %0 + =* zro zero-load:upgrade:store %_ $ -.old %1 - :: - validators.old - (~(put in validators.old) %graph-validator-link) - :: - cards - %+ weld cards - %+ turn - ~(tap in (~(put in validators.old) %graph-validator-link)) - |= validator=@t - ^- card - =/ =wire /validator/[validator] - =/ =rave:clay [%sing %b [%da now.bowl] /[validator]] - [%pass wire %arvo %c %warp our.bowl [%home `rave]] :: graphs.old %- ~(run by graphs.old) - |= [=graph:store q=(unit mark)] - ^- [graph:store (unit mark)] - :- (convert-unix-timestamped-graph graph) + |= [=graph:zero:store q=(unit mark)] + ^- [graph:zero:store (unit mark)] + :- (convert-unix-timestamped-graph:zro graph) ?^ q q `%graph-validator-link :: update-logs.old %- ~(run by update-logs.old) - |=(a=* *update-log:store) + |=(a=* *update-log:zero:store) == :: %1 + =* zro zero-load:upgrade:store %_ $ -.old %2 - graphs.old (~(run by graphs.old) change-revision-graph) + graphs.old (~(run by graphs.old) change-revision-graph:zro) + :: + update-logs.old + %- ~(run by update-logs.old) + |=(a=* *update-log:zero:store) + == + :: + %2 + =* upg upgrade:store + %_ $ + -.old %3 + update-logs.old (~(run by update-logs.old) update-log-to-one:upg) + graphs.old (~(run by graphs.old) marked-graph-to-one:upg) + archive.old (~(run by archive.old) marked-graph-to-one:upg) + == + :: + %3 + =* upg upgrade:store + %_ $ + -.old %4 + graphs.old (~(run by graphs.old) marked-graph-to-two:upg) + archive.old (~(run by archive.old) marked-graph-to-two:upg) + tag-queries.old *tag-queries:store + validators.old ~ :: update-logs.old %- ~(run by update-logs.old) |=(a=* *update-log:store) == :: - %2 [cards this(state old)] - == - :: - ++ change-revision-graph - |= [=graph:store q=(unit mark)] - ^- [graph:store (unit mark)] - |^ - :_ q - ?+ q graph - [~ %graph-validator-link] convert-links - [~ %graph-validator-publish] convert-publish + %4 + %_ $ + -.old %5 + :: + update-logs.old + %- ~(gas by *update-logs:store) + %+ turn ~(tap by graphs.old) + |= [=resource:store =graph:store mar=(unit mark)] + :- resource + =/ log (~(got by update-logs.old) resource) + ?. =(~ log) log + =/ =logged-update:store + [now.bowl %add-graph resource graph mar %.y] + (gas:orm-log ~ [now.bowl logged-update] ~) == - :: - ++ convert-links - %+ gas:orm *graph:store - %+ turn (tap:orm graph) - |= [=atom =node:store] - ^- [^atom node:store] - :: top-level - :: - :+ atom post.node - ?: ?=(%empty -.children.node) - [%empty ~] - :- %graph - %+ gas:orm *graph:store - %+ turn (tap:orm p.children.node) - |= [=^atom =node:store] - ^- [^^atom node:store] - :: existing comments get turned into containers for revisions - :: - :^ atom - post.node(contents ~, hash ~) - %graph - %+ gas:orm *graph:store - :_ ~ :- %1 - :_ [%empty ~] - post.node(index (snoc index.post.node atom), hash ~) - :: - ++ convert-publish - %+ gas:orm *graph:store - %+ turn (tap:orm graph) - |= [=atom =node:store] - ^- [^atom node:store] - :: top-level - :: - :+ atom post.node - ?: ?=(%empty -.children.node) - [%empty ~] - :- %graph - %+ gas:orm *graph:store - %+ turn (tap:orm p.children.node) - |= [=^atom =node:store] - ^- [^^atom node:store] - :: existing container for publish note revisions - :: - ?+ atom !! - %1 [atom node] - %2 - :+ atom post.node - ?: ?=(%empty -.children.node) - [%empty ~] - :- %graph - %+ gas:orm *graph:store - %+ turn (tap:orm p.children.node) - |= [=^^atom =node:store] - ^- [^^^atom node:store] - :+ atom post.node(contents ~, hash ~) - :- %graph - %+ gas:orm *graph:store - :_ ~ :- %1 - :_ [%empty ~] - post.node(index (snoc index.post.node atom), hash ~) - == - -- - :: - ++ maybe-unix-to-da - |= =atom - ^- @ - :: (bex 127) is roughly 226AD - ?. (lte atom (bex 127)) - atom - (add ~1970.1.1 (div (mul ~s1 atom) 1.000)) :: - ++ convert-unix-timestamped-node - |= =node:store - ^- node:store - =. index.post.node - (convert-unix-timestamped-index index.post.node) - ?. ?=(%graph -.children.node) - node - :+ post.node - %graph - (convert-unix-timestamped-graph p.children.node) - :: - ++ convert-unix-timestamped-index - |= =index:store - (turn index maybe-unix-to-da) - :: - ++ convert-unix-timestamped-graph - |= =graph:store - %+ gas:orm *graph:store - %+ turn - (tap:orm graph) - |= [=atom =node:store] - ^- [^atom node:store] - :- (maybe-unix-to-da atom) - (convert-unix-timestamped-node node) - -- + %5 [cards this(-.state old, +.state *cache)] + == :: ++ on-watch ~/ %graph-store-watch @@ -205,9 +137,9 @@ [cards this] :: ++ give - |= =update-0:store + |= =action:store ^- (list card) - [%give %fact ~ [%graph-update !>([%0 now.bowl update-0])]]~ + [%give %fact ~ [%graph-update-2 !>([now.bowl action])]]~ -- :: ++ on-poke @@ -217,10 +149,10 @@ |^ ?> (team:title our.bowl src.bowl) =^ cards state - ?+ mark (on-poke:def mark vase) - %graph-update (graph-update !<(update:store vase)) - %noun (debug !<(debug-input vase)) - %import (poke-import q.vase) + ?+ mark (on-poke:def mark vase) + %graph-update-2 (graph-update !<(update:store vase)) + %noun (debug !<(debug-input vase)) + %import (poke-import q.vase) == [cards this] :: @@ -228,13 +160,12 @@ |= =update:store ^- (quip card _state) |^ - ?> ?=(%0 -.update) =? p.update =(p.update *time) now.bowl ?- -.q.update %add-graph (add-graph p.update +.q.update) %remove-graph (remove-graph +.q.update) %add-nodes (add-nodes p.update +.q.update) - %remove-nodes (remove-nodes p.update +.q.update) + %remove-posts (remove-posts p.update +.q.update) %add-signatures (add-signatures p.update +.q.update) %remove-signatures (remove-signatures p.update +.q.update) %add-tag (add-tag +.q.update) @@ -259,28 +190,22 @@ ?& !(~(has by archive) resource) !(~(has by graphs) resource) == == - ?> (validate-graph graph mark) + ~| "validation of graph {} failed using mark {}" + =^ is-valid state + (validate-graph graph mark) + ?> is-valid =/ =logged-update:store - [%0 time %add-graph resource graph mark overwrite] + [time %add-graph resource graph mark overwrite] =/ =update-log:store (gas:orm-log ~ [time logged-update] ~) :_ %_ state graphs (~(put by graphs) resource [graph mark]) update-logs (~(put by update-logs) resource update-log) archive (~(del by archive) resource) - :: - validators - ?~ mark validators - (~(put in validators) u.mark) == %- zing :~ (give [/keys ~] %keys (~(put in ~(key by graphs)) resource)) (give [/updates ~] %add-graph resource *graph:store mark overwrite) - ?~ mark ~ - ?: (~(has in validators) u.mark) ~ - =/ wire /validator/[u.mark] - =/ =rave:clay [%sing %b [%da now.bowl] /[u.mark]] - [%pass wire %arvo %c %warp our.bowl [%home `rave]]~ == :: ++ remove-graph @@ -305,9 +230,13 @@ (~(got by graphs) resource) ~| "cannot add duplicate nodes to {}" ?< (check-for-duplicates graph ~(key by nodes)) + ~| "validation of nodes failed using mark {}" + =^ is-valid state + (check-validity ~(tap by nodes) mark) + ?> is-valid =/ =update-log:store (~(got by update-logs) resource) =. update-log - (put:orm-log update-log time [%0 time [%add-nodes resource nodes]]) + (put:orm-log update-log time [time [%add-nodes resource nodes]]) :: :- (give [/updates]~ [%add-nodes resource nodes]) %_ state @@ -319,30 +248,41 @@ (add-node-list resource graph mark (sort-nodes nodes)) == :: + ++ check-validity + |= [lis=(list (pair index:store node:store)) mark=(unit ^mark)] + ^- [? _state] + |- + ?~ lis [& state] + =^ is-valid state + (validate-graph (gas:orm ~ [(rear p.i.lis) q.i.lis]~) mark) + ?. is-valid + [| state] + $(lis t.lis) + :: ++ check-for-duplicates |= [=graph:store nodes=(set index:store)] ^- ? - =/ node-list ~(tap in nodes) - |- - ?~ node-list %.n - ?: (has-node graph i.node-list) %.y - $(node-list t.node-list) - :: - ++ has-node - |= [=graph:store =index:store] - ^- ? - =/ node=(unit node:store) ~ - |- - ?~ index - ?=(^ node) - ?~ t.index - ?=(^ (get:orm graph i.index)) - =. node (get:orm graph i.index) - ?~ node %.n - ?- -.children.u.node - %empty %.n - %graph $(graph p.children.u.node, index t.index) - == + |^ + %+ lien ~(tap in nodes) + |= =index:store + (has-node graph index) + :: + ++ has-node + |= [=graph:store =index:store] + ^- ? + =/ node=(unit node:store) ~ + |- + ?~ index + ?=(^ node) + ?~ t.index + ?=(^ (get:orm graph i.index)) + =. node (get:orm graph i.index) + ?~ node %.n + ?- -.children.u.node + %empty %.n + %graph $(graph p.children.u.node, index t.index) + == + -- :: ++ sort-nodes |= nodes=(map index:store node:store) @@ -376,8 +316,6 @@ == ^- graph:store ?< ?=(~ index) - ~| "validation of node failed using mark {}" - ?> (validate-graph (gas:orm ~ [i.index node]~) mark) =* atom i.index %^ put:orm graph @@ -385,8 +323,10 @@ :: add child :: ?~ t.index - =* p post.node - ?~ hash.p node(signatures.post *signatures:store) + ~| "cannot add deleted post" + ?> ?=(%& -.post.node) + =* p p.post.node + ?~ hash.p node(signatures.p.post *signatures:store) =/ =validated-portion:store [parent-hash author.p time-sent.p contents.p] =/ =hash:store `@ux`(sham validated-portion) @@ -405,8 +345,14 @@ ^- internal-graph:store :- %graph %_ $ - index t.index - parent-hash hash.post.parent + index t.index + :: + parent-hash + ?- -.post.parent + %| `p.post.parent + %& hash.p.post.parent + == + :: graph ?: ?=(%graph -.children.parent) p.children.parent @@ -415,7 +361,7 @@ == -- :: - ++ remove-nodes + ++ remove-posts |= [=time =resource:store indices=(set index:store)] ^- (quip card _state) |^ @@ -423,82 +369,83 @@ (~(got by graphs) resource) =/ =update-log:store (~(got by update-logs) resource) =. update-log - (put:orm-log update-log time [%0 time [%remove-nodes resource indices]]) - =/ [affected-indices=(set index:store) new-graph=graph:store] - (remove-indices resource graph (sort ~(tap in indices) by-lent)) - :: - :- (give [/updates]~ [%remove-nodes resource (~(uni in indices) affected-indices)]) + (put:orm-log update-log time [time [%remove-posts resource indices]]) + :- (give [/updates]~ [%remove-posts resource indices]) %_ state - update-logs (~(put by update-logs) resource update-log) + update-logs (~(put by update-logs) resource update-log) + :: graphs %+ ~(put by graphs) resource - [new-graph mark] + :_ mark + %^ remove-indices + resource + graph + (sort ~(tap in indices) by-lent) == :: - :: we always want to remove the deepest node first, - :: so we don't remove parents before children ++ by-lent |* [a=(list) b=(list)] ^- ? (gth (lent a) (lent b)) :: ++ remove-indices - =| affected=(set index:store) |= [=resource:store =graph:store indices=(list index:store)] - ^- [(set index:store) graph:store] - ?~ indices [affected graph] - =^ new-affected graph - (remove-index graph i.indices) + ^- graph:store + ?~ indices graph %_ $ - indices t.indices - affected (~(uni in affected) new-affected) - == - :: - ++ get-descendants - |= =graph:store - =| indices=(list index:store) - =/ nodes (tap:orm:store graph) - %- ~(gas in *(set index:store)) - |- =* tap-nodes $ - ^+ indices - %- zing - %+ turn nodes - |= [atom =node:store] - ^- (list index:store) - %+ welp - index.post.node^~ - ?. ?=(%graph -.children.node) - ~ - %_ tap-nodes - nodes (tap:orm p.children.node) + indices t.indices + graph (remove-index graph i.indices) == :: ++ remove-index - =| indices=(set index:store) + =| parent-hash=(unit hash:store) |= [=graph:store =index:store] - ^- [(set index:store) graph:store] - ?~ index [indices graph] + ^- graph:store + ?~ index graph =* atom i.index + %^ put:orm + graph + atom :: last index in list :: ?~ t.index - =^ rm-node graph (del:orm graph atom) - ?~ rm-node `graph - ?. ?=(%graph -.children.u.rm-node) - `graph - =/ new-indices - (get-descendants p.children.u.rm-node) - [(~(uni in indices) new-indices) graph] - =/ =node:store + =/ =node:store + ~| "cannot remove index that does not exist {}" + (need (get:orm graph atom)) + %_ node + post + ~| "cannot remove post that has already been removed" + ?> ?=(%& -.post.node) + =* p p.post.node + ^- maybe-post:store + :- %| + ?~ hash.p + =/ =validated-portion:store + [parent-hash author.p time-sent.p contents.p] + `@ux`(sham validated-portion) + u.hash.p + == + :: recurse children + :: + =/ parent=node:store ~| "parent index does not exist to remove a node from!" (need (get:orm graph atom)) ~| "child index does not exist to remove a node from!" - ?> ?=(%graph -.children.node) - =^ new-indices p.children.node - $(graph p.children.node, index t.index) - :- (~(uni in indices) new-indices) - (put:orm graph atom node) + ?> ?=(%graph -.children.parent) + %_ parent + p.children + %_ $ + index t.index + graph p.children.parent + :: + parent-hash + ?- -.post.parent + %| `p.post.parent + %& hash.p.post.parent + == + == + == -- :: ++ add-signatures @@ -510,7 +457,7 @@ (~(got by graphs) resource) =/ =update-log:store (~(got by update-logs) resource) =. update-log - (put:orm-log update-log time [%0 time [%add-signatures uid signatures]]) + (put:orm-log update-log time [time [%add-signatures uid signatures]]) :: :- (give [/updates]~ [%add-signatures uid signatures]) %_ state @@ -534,11 +481,13 @@ graph atom ?~ t.index + ~| "cannot add signatures to a deleted post" + ?> ?=(%& -.post.node) ~| "cannot add signatures to a node missing a hash" - ?> ?=(^ hash.post.node) + ?> ?=(^ hash.p.post.node) ~| "signatures did not match public keys!" - ?> (are-signatures-valid:sigs our.bowl signatures u.hash.post.node now.bowl) - node(signatures.post (~(uni in signatures) signatures.post.node)) + ?> (are-signatures-valid:sigs our.bowl signatures u.hash.p.post.node now.bowl) + node(signatures.p.post (~(uni in signatures) signatures.p.post.node)) ~| "child graph does not exist to add signatures to!" ?> ?=(%graph -.children.node) node(p.children $(graph p.children.node, index t.index)) @@ -555,7 +504,7 @@ =. update-log %^ put:orm-log update-log time - [%0 time [%remove-signatures uid signatures]] + [time [%remove-signatures uid signatures]] :: :- (give [/updates]~ [%remove-signatures uid signatures]) %_ state @@ -579,28 +528,29 @@ graph atom ?~ t.index - node(signatures.post (~(dif in signatures) signatures.post.node)) + ~| "cannot add signatures to a deleted post" + ?> ?=(%& -.post.node) + node(signatures.p.post (~(dif in signatures) signatures.p.post.node)) ~| "child graph does not exist to add signatures to!" ?> ?=(%graph -.children.node) node(p.children $(graph p.children.node, index t.index)) -- :: ++ add-tag - |= [=term =resource:store] + |= [=term =uid:store] ^- (quip card _state) - ?> (~(has by graphs) resource) - :- (give [/updates /tags ~] [%add-tag term resource]) + ?> (~(has by graphs) resource.uid) + :- (give [/updates /tags ~] [%add-tag term uid]) %_ state - tag-queries (~(put ju tag-queries) term resource) + tag-queries (~(put ju tag-queries) term uid) == :: ++ remove-tag - |= [=term =resource:store] + |= [=term =uid:store] ^- (quip card _state) - ?> (~(has by graphs) resource) - :- (give [/updates /tags ~] [%remove-tag term resource]) + :- (give [/updates /tags ~] [%remove-tag term uid]) %_ state - tag-queries (~(del ju tag-queries) term resource) + tag-queries (~(del ju tag-queries) term uid) == :: ++ archive-graph @@ -613,10 +563,6 @@ archive (~(put by archive) resource (~(got by graphs) resource)) graphs (~(del by graphs) resource) update-logs (~(del by update-logs) resource) - tag-queries - %- ~(run by tag-queries) - |= =resources:store - (~(del in resources) resource) == :: ++ unarchive-graph @@ -646,21 +592,21 @@ [cards state] =* update upd.i.updates =^ crds state - %- graph-update + %- graph-update ^- update:store ?- -.q.update %add-graph update(resource.q resource) %add-nodes update(resource.q resource) - %remove-nodes update(resource.q resource) + %remove-posts update(resource.q resource) %add-signatures update(resource.uid.q resource) %remove-signatures update(resource.uid.q resource) == $(cards (weld cards crds), updates t.updates) :: ++ give - |= [paths=(list path) update=update-0:store] + |= [paths=(list path) update=action:store] ^- (list card) - [%give %fact paths [%graph-update !>([%0 now.bowl update])]]~ + [%give %fact paths [%graph-update-2 !>([now.bowl update])]]~ -- :: ++ debug @@ -668,182 +614,44 @@ ^- (quip card _state) =/ [=graph:store mark=(unit mark:store)] (~(got by graphs) resource.debug-input) - ?> (validate-graph graph mark) + =^ is-valid state + (validate-graph graph mark) + ?> is-valid [~ state] :: ++ validate-graph |= [=graph:store mark=(unit mark:store)] - ^- ? - ?~ mark %.y - ?~ graph %.y + ^- [? _state] + ?~ mark [%.y state] + =/ has-dais (~(has by validators) u.mark) =/ =dais:clay + ?: has-dais + (~(got by validators) u.mark) .^ =dais:clay %cb /(scot %p our.bowl)/[q.byk.bowl]/(scot %da now.bowl)/[u.mark] == + :_ state(validators (~(put by validators) u.mark dais)) + |- ^- ? + ?~ graph %.y %+ roll (tap:orm graph) |= [[=atom =node:store] out=?] - ?& out - =(%& -:(mule |.((vale:dais [atom post.node])))) - ?- -.children.node - %empty %.y - %graph ^$(graph p.children.node) + ^- ? + ?& ?| ?=(%| -.post.node) + ?=(^ (vale:dais [atom p.post.node])) == - == + :: + ?- -.children.node + %empty %.y + %graph ^$(graph p.children.node) + == == :: ++ poke-import |= arc=* ^- (quip card _state) - |^ - =/ sty=state-2 [%2 (remake-network ;;(tree-network +.arc))] - :_ sty - %+ turn ~(tap by graphs.sty) - |= [rid=resource:store =marked-graph:store] - ^- card - ?: =(our.bowl entity.rid) - =/ =cage [%push-hook-action !>([%add rid])] - [%pass / %agent [our.bowl %graph-push-hook] %poke cage] - (try-rejoin rid 0) - :: - +$ tree-network - $: graphs=tree-graphs - tag-queries=(tree [term (tree resource:store)]) - update-logs=tree-update-logs - archive=tree-graphs - validators=(tree ^mark) - == - +$ tree-graphs (tree [resource:store tree-marked-graph]) - +$ tree-marked-graph [p=tree-graph q=(unit ^mark)] - +$ tree-graph (tree [atom tree-node]) - +$ tree-node [post=tree-post children=tree-internal-graph] - +$ tree-internal-graph - $~ [%empty ~] - $% [%graph p=tree-graph] - [%empty ~] - == - +$ tree-update-logs (tree [resource:store tree-update-log]) - +$ tree-update-log (tree [time tree-logged-update]) - +$ tree-logged-update - $: %0 - p=time - $= q - $% [%add-graph =resource:store =tree-graph mark=(unit ^mark) ow=?] - [%add-nodes =resource:store nodes=(tree [index:store tree-node])] - [%remove-nodes =resource:store indices=(tree index:store)] - [%add-signatures =uid:store signatures=tree-signatures] - [%remove-signatures =uid:store signatures=tree-signatures] - == - == - +$ tree-signatures (tree signature:store) - +$ tree-post - $: author=ship - =index:store - time-sent=time - contents=(list content:store) - hash=(unit hash:store) - signatures=tree-signatures - == - :: - ++ remake-network - |= t=tree-network - ^- network:store - :* (remake-graphs graphs.t) - (remake-jug tag-queries.t) - (remake-update-logs update-logs.t) - (remake-graphs archive.t) - (remake-set validators.t) - == - :: - ++ remake-graphs - |= t=tree-graphs - ^- graphs:store - %- remake-map - (~(run by t) remake-marked-graph) - :: - ++ remake-marked-graph - |= t=tree-marked-graph - ^- marked-graph:store - [(remake-graph p.t) q.t] - :: - ++ remake-graph - |= t=tree-graph - ^- graph:store - %+ gas:orm *graph:store - %+ turn ~(tap by t) - |= [a=atom tn=tree-node] - ^- [atom node:store] - [a (remake-node tn)] - :: - ++ remake-internal-graph - |= t=tree-internal-graph - ^- internal-graph:store - ?: ?=(%empty -.t) - [%empty ~] - [%graph (remake-graph p.t)] - :: - ++ remake-node - |= t=tree-node - ^- node:store - :- (remake-post post.t) - (remake-internal-graph children.t) - :: - ++ remake-update-logs - |= t=tree-update-logs - ^- update-logs:store - %- remake-map - (~(run by t) remake-update-log) - :: - ++ remake-update-log - |= t=tree-update-log - ^- update-log:store - =/ ulm ((ordered-map time logged-update:store) gth) - %+ gas:ulm *update-log:store - %+ turn ~(tap by t) - |= [=time tlu=tree-logged-update] - ^- [^time logged-update:store] - [time (remake-logged-update tlu)] - :: - ++ remake-logged-update - |= t=tree-logged-update - ^- logged-update:store - :+ %0 p.t - ?- -.q.t - %add-graph - :* %add-graph - resource.q.t - (remake-graph tree-graph.q.t) - mark.q.t - ow.q.t - == - :: - %add-nodes - :- %add-nodes - :- resource.q.t - %- remake-map - (~(run by nodes.q.t) remake-node) - :: - %remove-nodes - [%remove-nodes resource.q.t (remake-set indices.q.t)] - :: - %add-signatures - [%add-signatures uid.q.t (remake-set signatures.q.t)] - :: - %remove-signatures - [%remove-signatures uid.q.t (remake-set signatures.q.t)] - == - :: - ++ remake-post - |= t=tree-post - ^- post:store - t(signatures (remake-set signatures.t)) - -- - :: - ++ try-rejoin - |= [rid=resource:store nack-count=@] - ^- card - =/ res-path (en-path:res rid) - =/ wire [%try-rejoin (scot %ud nack-count) res-path] - [%pass wire %agent [entity.rid %graph-push-hook] %watch resource+res-path] + =^ cards -.state + (import:store arc our.bowl) + [cards state] -- :: ++ on-peek @@ -862,16 +670,16 @@ ``noun+!>(q.u.result) :: [%x %keys ~] - :- ~ :- ~ :- %graph-update - !>(`update:store`[%0 now.bowl [%keys ~(key by graphs)]]) + :- ~ :- ~ :- %graph-update-2 + !>(`update:store`[now.bowl [%keys ~(key by graphs)]]) :: [%x %tags ~] - :- ~ :- ~ :- %graph-update - !>(`update:store`[%0 now.bowl [%tags ~(key by tag-queries)]]) + :- ~ :- ~ :- %graph-update-2 + !>(`update:store`[now.bowl [%tags ~(key by tag-queries)]]) :: [%x %tag-queries ~] - :- ~ :- ~ :- %graph-update - !>(`update:store`[%0 now.bowl [%tag-queries tag-queries]]) + :- ~ :- ~ :- %graph-update-2 + !>(`update:store`[now.bowl [%tag-queries tag-queries]]) :: [%x %graph @ @ ~] =/ =ship (slav %p i.t.t.path) @@ -879,10 +687,9 @@ =/ result=(unit marked-graph:store) (~(get by graphs) [ship term]) ?~ result [~ ~] - :- ~ :- ~ :- %graph-update + :- ~ :- ~ :- %graph-update-2 !> ^- update:store - :+ %0 - now.bowl + :- now.bowl [%add-graph [ship term] `graph:store`p.u.result q.u.result %.y] :: :: note: near-duplicate of /x/graph @@ -895,10 +702,9 @@ ?~ result ~& no-archived-graph+[ship term] [~ ~] - :- ~ :- ~ :- %graph-update + :- ~ :- ~ :- %graph-update-2 !> ^- update:store - :+ %0 - now.bowl + :- now.bowl [%add-graph [ship term] `graph:store`p.u.result q.u.result %.y] :: [%x %export ~] @@ -912,9 +718,9 @@ =/ graph=(unit marked-graph:store) (~(get by graphs) [ship term]) ?~ graph [~ ~] - :- ~ :- ~ :- %graph-update + :- ~ :- ~ :- %graph-update-2 !> ^- update:store - :+ %0 now.bowl + :- now.bowl :+ %add-nodes [ship term] %- ~(gas by *(map index:store node:store)) @@ -939,10 +745,9 @@ (turn t.t.t.t.path (cury slav %ud)) =/ node=(unit node:store) (get-node ship term index) ?~ node [~ ~] - :- ~ :- ~ :- %graph-update + :- ~ :- ~ :- %graph-update-2 !> ^- update:store - :+ %0 - now.bowl + :- now.bowl :+ %add-nodes [ship term] (~(gas by *(map index:store node:store)) [index u.node] ~) @@ -959,10 +764,9 @@ =/ graph (get-node-children ship term parent) ?~ graph [~ ~] - :- ~ :- ~ :- %graph-update + :- ~ :- ~ :- %graph-update-2 !> ^- update:store - :+ %0 - now.bowl + :- now.bowl :+ %add-nodes [ship term] %- ~(gas by *(map index:store node:store)) @@ -990,10 +794,9 @@ =/ children (get-node-children ship term index) ?~ children [~ ~] - :- ~ :- ~ :- %graph-update + :- ~ :- ~ :- %graph-update-2 !> ^- update:store - :+ %0 - now.bowl + :- now.bowl :+ %add-nodes [ship term] %- ~(gas by *(map index:store node:store)) @@ -1017,10 +820,9 @@ ?- -.children.u.node %empty [~ ~] %graph - :- ~ :- ~ :- %graph-update + :- ~ :- ~ :- %graph-update-2 !> ^- update:store - :+ %0 - now.bowl + :- now.bowl :+ %add-nodes [ship term] %- ~(gas by *(map index:store node:store)) @@ -1109,13 +911,8 @@ ?+ wire (on-arvo:def wire sign-arvo) :: :: old wire, do nothing - [%graph *] [~ this] - :: - [%validator @ ~] - :_ this - =* validator i.t.wire - =/ =rave:clay [%next %b [%da now.bowl] /[validator]] - [%pass wire %arvo %c %warp our.bowl [%home `rave]]~ + [%graph *] [~ this] + [%validator @ ~] [~ this] :: [%try-rejoin @ *] =/ rid=resource:store (de-path:res t.t.wire) diff --git a/pkg/arvo/app/group-pull-hook.hoon b/pkg/arvo/app/group-pull-hook.hoon index 5bc8208d81..f97582e70b 100644 --- a/pkg/arvo/app/group-pull-hook.hoon +++ b/pkg/arvo/app/group-pull-hook.hoon @@ -14,6 +14,7 @@ update:store %group-update %group-push-hook + 0 0 %.n == :: @@ -44,9 +45,10 @@ ++ on-pull-nack |= [=resource =tang] ^- (quip card _this) + %- (slog tang) :_ this =- [%pass / %agent [our.bowl %group-store] %poke -]~ - group-update+!>([%remove-group resource ~]) + group-update-0+!>([%remove-group resource ~]) :: ++ on-pull-kick |= =resource diff --git a/pkg/arvo/app/group-push-hook.hoon b/pkg/arvo/app/group-push-hook.hoon index e8736ccb5b..20decf8c3a 100644 --- a/pkg/arvo/app/group-push-hook.hoon +++ b/pkg/arvo/app/group-push-hook.hoon @@ -17,6 +17,7 @@ update:store %group-update %group-pull-hook + 0 0 == :: +$ agent (push-hook:push-hook config) @@ -36,84 +37,19 @@ ++ on-init on-init:def ++ on-save !>(~) ++ on-load on-load:def -++ on-poke - |= [=mark =vase] - ^- (quip card _this) - |^ - ?. =(mark %sane) - (on-poke:def mark vase) - [(sane !<(?(%check %fix) vase)) this] - :: - ++ scry-sharing - .^ (set resource) - %gx - (scot %p our.bowl) - %group-push-hook - (scot %da now.bowl) - /sharing/noun - == - :: - ++ sane - |= input=?(%check %fix) - ^- (list card) - =; cards=(list card) - ?: =(%check input) - ~&(cards ~) - cards - %+ murn - ~(tap in scry-sharing) - |= rid=resource - ^- (unit card) - =/ u-g=(unit group) - (scry-group:grp rid) - ?~ u-g - `(poke-us %remove rid) - =* group u.u-g - =/ subs=(set ship) - (get-subscribers-for-group rid) - =/ to-remove=(set ship) - (~(dif in members.group) (~(gas in subs) our.bowl ~)) - ?~ to-remove ~ - `(poke-store %remove-members rid to-remove) - :: - ++ poke-us - |= =action:push-hook - ^- card - =- [%pass / %agent [our.bowl %group-push-hook] %poke -] - push-hook-action+!>(action) - :: - ++ poke-store - |= =update:store - ^- card - =+ group-update+!>(update) - [%pass /sane %agent [our.bowl %group-store] %poke -] - :: - ++ get-subscribers-for-group - |= rid=resource - ^- (set ship) - =/ target=path - (en-path:resource rid) - %- ~(gas in *(set ship)) - %+ murn - ~(val by sup.bowl) - |= [her=ship =path] - ^- (unit ship) - ?. =(path resource+target) - ~ - `her - -- - -++ on-agent on-agent:def -++ on-watch on-watch:def -++ on-leave on-leave:def -++ on-peek on-peek:def -++ on-arvo on-arvo:def -++ on-fail on-fail:def +++ on-poke on-poke:def +++ on-agent on-agent:def +++ on-watch on-watch:def +++ on-leave on-leave:def +++ on-peek on-peek:def +++ on-arvo on-arvo:def +++ on-fail on-fail:def :: ++ transform-proxy-update |= vas=vase - ^- (unit vase) + ^- (quip card (unit vase)) =/ =update:store !<(update:store vas) + :- ~ ?: ?=(%initial -.update) ~ |^ diff --git a/pkg/arvo/app/group-store.hoon b/pkg/arvo/app/group-store.hoon index 10f6afb4bc..59ddc4dc94 100644 --- a/pkg/arvo/app/group-store.hoon +++ b/pkg/arvo/app/group-store.hoon @@ -113,7 +113,7 @@ ?+ mark (on-poke:def mark vase) %sane (poke-sane:gc !<(?(%check %fix) vase)) :: - ?(%group-update %group-action) + ?(%group-update-0 %group-action) (poke-group-update:gc !<(update:store vase)) :: %import @@ -127,7 +127,7 @@ ?> (team:title our.bowl src.bowl) ?> ?=([%groups ~] path) :_ this - [%give %fact ~ %group-update !>([%initial groups])]~ + [%give %fact ~ %group-update-0 !>([%initial groups])]~ :: ++ on-leave on-leave:def :: @@ -234,8 +234,8 @@ sane+(en-path:resource rid) =* poke-self ~(poke-self pass:io wire) %+ weld out - :~ (poke-self group-update+!>([%add-members rid (silt our.bol ~)])) - (poke-self group-update+!>([%add-tag rid %admin (silt our.bol ~)])) + :~ (poke-self group-update-0+!>([%add-members rid (silt our.bol ~)])) + (poke-self group-update-0+!>([%add-tag rid %admin (silt our.bol ~)])) == :: ++ poke-import @@ -298,7 +298,7 @@ |= [rid=resource nack-count=@ud] ^- card =/ =cage - :- %group-update + :- %group-update-0 !> ^- update:store [%add-members rid (sy our.bol ~)] =/ =wire @@ -414,11 +414,8 @@ ?> ?& ?=(~ (~(dif in ships) members)) (~(has by tags) tag) == - %= +< - :: - tags - %+ ~(jab by tags) tag - |=((set ship) (~(dif in +<) ships)) + %= +< + tags (dif-ju tags tag ships) == :_ state (send-diff %remove-tag rid tag ships) @@ -543,7 +540,15 @@ (send-diff %remove-group rid ~) :: -- - +:: TODO: move to +zuse +++ dif-ju + |= [=tags =tag remove=(set ship)] + =/ ships ~(tap in remove) + |- + ?~ ships + tags + $(tags (~(del ju tags) tag i.ships), ships t.ships) +:: ++ merge-tags |= [=tags ships=(set ship) new-tags=(set tag)] ^+ tags @@ -583,6 +588,6 @@ ++ send-diff |= =update:store ^- (list card) - [%give %fact ~[/groups] %group-update !>(update)]~ + [%give %fact ~[/groups] %group-update-0 !>(update)]~ :: -- diff --git a/pkg/arvo/app/group-view.hoon b/pkg/arvo/app/group-view.hoon index 51ad9b09f2..e7b2177b00 100644 --- a/pkg/arvo/app/group-view.hoon +++ b/pkg/arvo/app/group-view.hoon @@ -4,23 +4,30 @@ |% ++ card card:agent:gall :: -+$ base-state ++$ base-state-0 joining=(map rid=resource [=ship =progress:view]) :: ++$ base-state-1 + joining=(map rid=resource request:view) +:: +$ state-zero - [%0 base-state] + [%0 base-state-0] :: +$ state-one - [%1 base-state] + [%1 base-state-0] +:: ++$ state-two + [%2 base-state-1] :: +$ versioned-state $% state-zero state-one + state-two == :: ++ view view-sur -- -=| state-one +=| state-two =* state - :: %- agent:dbug @@ -41,10 +48,29 @@ |= =vase =+ !<(old=versioned-state vase) =| cards=(list card) - |- - ?: ?=(%1 -.old) - `this(state old) - $(-.old %1, cards :_(cards (poke-self:pass:io noun+!>(%cleanup)))) + |^ + ?- -.old + %2 [cards this(state old)] + %1 $(-.old %2, +.old (base-state-to-1 +.old)) + %0 $(-.old %1, cards :_(cards (poke-self:pass:io noun+!>(%cleanup)))) + == + :: + ++ base-state-to-1 + |= base-state-0 + %- ~(gas by *(map resource request:view)) + (turn ~(tap by joining) request-to-1) + :: + ++ request-to-1 + |= [rid=resource =ship =progress:view] + ^- [resource request:view] + :- rid + %* . *request:view + started now.bowl + hidden %.n + ship ship + progress progress + == + -- :: ++ on-poke |= [=mark =vase] @@ -56,9 +82,11 @@ ?. ?=(%group-view-action mark) (on-poke:def mark vase) =+ !<(=action:view vase) - ?> ?=(%join -.action) =^ cards state - jn-abet:(jn-start:join:gc +.action) + ?+ -.action !! + %join jn-abet:(jn-start:join:gc +.action) + %hide (hide:gc +.action) + == [cards this] :: ++ on-watch @@ -69,8 +97,7 @@ :_ ~ %+ fact:io :- %group-view-update - !> ^- update:view - [%initial (~(run by joining) |=([=ship =progress:view] progress))] + !>(`update:view`[%initial joining]) ~ == :: @@ -97,6 +124,11 @@ ++ grp ~(. grpl bowl) ++ io ~(. agentio bowl) ++ con ~(. conl bowl) +++ hide + |= rid=resource + ^- (quip card _state) + :- (fact:io group-view-update+!>([%hide rid]) /all ~)^~ + state(joining (~(jab by joining) rid |=(request:view +<(hidden %.y)))) :: ++ has-joined |= rid=resource @@ -107,10 +139,10 @@ :: ++ poke-noun ^- (quip card _state) - =; new-joining=(map resource [ship progress:view]) + =; new-joining=(map resource request:view) `state(joining new-joining) %+ roll ~(tap by joining) - |= [[rid=resource =ship =progress:view] out=_joining] + |= [[rid=resource =request:view] out=_joining] ?. (has-joined rid) out (~(del by out) rid) :: @@ -128,7 +160,7 @@ ++ tx-progress |= =progress:view =. joining - (~(put by joining) rid [ship progress]) + (~(jab by joining) rid |=(request:view +<(progress progress))) =; =cage (emit (fact:io cage /all tx+(en-path:resource rid) ~)) group-view-update+!>([%progress rid progress]) @@ -145,9 +177,9 @@ :: ++ jn-abed |= r=resource - =/ [s=^ship =progress:view] + =/ =request:view (~(got by joining) r) - jn-core(rid r, ship s) + jn-core(rid r, ship ship.request) :: ++ jn-abet ^- (quip card _state) @@ -158,15 +190,20 @@ ^+ jn-core ?< (~(has by joining) rid) =. joining - (~(put by joining) rid [ship %start]) + (~(put by joining) rid [%.n now.bowl ship %start]) =. jn-core (jn-abed rid) + =. jn-core + %- emit + %+ fact:io + group-view-update+!>([%started rid (~(got by joining) rid)]) + ~[/all] ?< ~|("already joined {}" (has-joined rid)) =. jn-core %- emit %+ poke:(jn-pass-io /add) [ship %group-push-hook] - group-update+!>([%add-members rid (silt our.bowl ~)]) + group-update-0+!>([%add-members rid (silt our.bowl ~)]) =. jn-core (tx-progress %start) => watch-md watch-groups @@ -227,7 +264,7 @@ :: ++ groups-fact |= =cage - ?. ?=(%group-update p.cage) jn-core + ?. ?=(%group-update-0 p.cage) jn-core =+ !<(=update:group-store q.cage) ?. ?=(%initial-group -.update) jn-core ?. =(rid resource.update) jn-core @@ -246,12 +283,27 @@ :: ++ md-fact |= [=mark =vase] - ?. ?=(%metadata-update mark) jn-core + ?. ?=(%metadata-update-1 mark) jn-core =+ !<(=update:metadata vase) ?. ?=(%initial-group -.update) jn-core ?. =(group.update rid) jn-core =. jn-core (cleanup %done) - ?. hidden:(need (scry-group:grp rid)) jn-core + ?. hidden:(need (scry-group:grp rid)) + =/ list-md=(list [=md-resource:metadata =association:metadata]) + %+ skim ~(tap by associations.update) + |= [=md-resource:metadata =association:metadata] + =(app-name.md-resource %groups) + ?> ?=(^ list-md) + =* metadatum metadatum.association.i.list-md + ?. ?& ?=(%group -.config.metadatum) + ?=(^ feed.config.metadatum) + ?=(^ u.feed.config.metadatum) + == + jn-core + =* feed resource.u.u.feed.config.metadatum + %- emit + %+ poke-our:(jn-pass-io /pull-feed) %graph-pull-hook + pull-hook-action+!>([%add [entity .]:feed]) %- emit-many %+ murn ~(tap by associations.update) |= [=md-resource:metadata =association:metadata] diff --git a/pkg/arvo/app/hark-graph-hook.hoon b/pkg/arvo/app/hark-graph-hook.hoon index 3e779a8bcf..828025d3d9 100644 --- a/pkg/arvo/app/hark-graph-hook.hoon +++ b/pkg/arvo/app/hark-graph-hook.hoon @@ -24,7 +24,6 @@ watch-on-self=_& == :: -:: ++ scry |* [[our=@p now=@da] =mold p=path] ?> ?=(^ p) @@ -37,7 +36,6 @@ %^ scry [our now] tube:clay /cc/[desk]/[mark]/notification-kind -:: -- :: =| state-1 @@ -126,15 +124,15 @@ :: ++ poke-noun |= non=* - ?> ?=(%rewatch-dms non) - =/ graphs=(list resource) - ~(tap in get-keys:gra) - :- ~ - %_ state - watching - %- ~(gas in watching) - (murn graphs |=(rid=resource ?:((should-watch:ha rid) `[rid ~] ~))) - == + [~ state] +:: ?> ?=(%rewatch-dms non) +:: =/ graphs=(list resource) +:: ~(tap in get-keys:gra) +:: %_ state +:: watching +:: %- ~(gas in watching) +:: (murn graphs |=(rid=resource ?:((should-watch:ha rid) `[rid ~] ~))) +:: == :: ++ hark-graph-hook-action |= =action:hook @@ -182,7 +180,7 @@ ~[watch-graph:ha] :: %fact - ?. ?=(%graph-update p.cage.sign) + ?. ?=(%graph-update-2 p.cage.sign) (on-agent:def wire sign) =^ cards state (graph-update !<(update:graph-store q.cage.sign)) @@ -197,18 +195,20 @@ :: ?(%remove-graph %archive-graph) (remove-graph resource.q.update) - :: - %remove-nodes - (remove-nodes resource.q.update indices.q.update) - :: + :: + %remove-posts + (remove-posts resource.q.update indices.q.update) + :: %add-nodes =* rid resource.q.update - (check-nodes ~(val by nodes.q.update) rid) + =/ assoc=(unit association:metadata) + (peek-association:met %graph rid) + (check-nodes ~(val by nodes.q.update) rid assoc) == :: this is awful, but notification kind should always switch :: on the index, so hopefully doesn't matter :: TODO: rethink this - ++ remove-nodes + ++ remove-posts |= [rid=resource indices=(set index:graph-store)] =/ to-remove %- ~(gas by *(set [resource index:graph-store])) @@ -257,27 +257,30 @@ (get-graph-mop:gra rid) =/ node=(unit node:graph-store) (bind (peek:orm:graph-store graph) |=([@ =node:graph-store] node)) + =/ assoc=(unit association:metadata) + (peek-association:met %graph rid) =^ cards state - (check-nodes (drop node) rid) - ?. (should-watch:ha rid) + (check-nodes (drop node) rid assoc) + ?. (should-watch:ha rid assoc) [cards state] :_ state(watching (~(put in watching) [rid ~])) (weld cards (give:ha ~[/updates] %listen [rid ~])) :: - :: ++ check-nodes |= $: nodes=(list node:graph-store) rid=resource + assoc=(unit association:metadata) == - =/ group=(unit resource) - (peek-group:met %graph rid) - ?~ group - ~& no-group+rid + ?~ assoc + ~& no-assoc+rid `state - =/ metadatum=(unit metadatum:metadata) - (peek-metadatum:met %graph rid) - ?~ metadatum `state - abet:check:(abed:handle-update:ha rid nodes u.group module.u.metadatum) + =* group group.u.assoc + =* metadatum metadatum.u.assoc + =/ module=term + ?: ?=(%empty -.config.metadatum) %$ + ?: ?=(%group -.config.metadatum) %$ + module.config.metadatum + abet:check:(abed:handle-update:ha rid nodes group module) -- :: ++ on-peek on-peek:def @@ -339,12 +342,11 @@ $(contents t.contents) :: ++ should-watch - |= rid=resource + |= [rid=resource assoc=(unit association:metadata)] ^- ? - =/ group-rid=(unit resource) - (peek-group:met %graph rid) - ?~ group-rid %.n - ?| !(is-managed:grp u.group-rid) + ?~ assoc + %.n + ?| !(is-managed:grp group.u.assoc) &(watch-on-self =(our.bowl entity.rid)) == :: @@ -363,7 +365,9 @@ update-core(rid r, updates upds, group grp, module mod) :: ++ get-conversion - (^get-conversion rid) + :: LA: this tube should be cached in %hark-graph-hook state + :: instead of just trying to keep it warm, as the scry overhead is large + ~+ (^get-conversion rid) :: ++ abet ^- (quip card _state) @@ -413,30 +417,34 @@ |= =node:graph-store ^+ update-core =. update-core (check-node-children node) + ?: ?=(%| -.post.node) + update-core + =* pos p.post.node =+ !< notif-kind=(unit notif-kind:hook) - (get-conversion !>([0 post.node])) + %- get-conversion + !>(`indexed-post:graph-store`[0 pos]) ?~ notif-kind update-core =/ desc=@t - ?: (is-mention contents.post.node) + ?: (is-mention contents.pos) %mention name.u.notif-kind =* not-kind u.notif-kind =/ parent=index:post - (scag parent.index-len.not-kind index.post.node) + (scag parent.index-len.not-kind index.pos) =/ notif-index=index:store [%graph group rid module desc parent] - ?: =(our.bowl author.post.node) + ?: =(our.bowl author.pos) (self-post node notif-index not-kind) =. update-core - (update-unread-count not-kind notif-index [time-sent index]:post.node) + (update-unread-count not-kind notif-index [time-sent index]:pos) =? update-core ?| =(desc %mention) (~(has in watching) [rid parent]) == =/ =contents:store - [%graph (limo post.node ~)] - (add-unread notif-index [time-sent.post.node %.n contents]) + [%graph (limo pos ~)] + (add-unread notif-index [time-sent.pos %.n contents]) update-core :: ++ update-unread-count @@ -455,20 +463,19 @@ =notif-kind:hook == ^+ update-core - ?: ?=(%none mode.notif-kind) update-core + ?> ?=(%& -.post.node) =/ =stats-index:store (to-stats-index:store index) =. update-core - (hark %seen-index time-sent.post.node stats-index) + (hark %seen-index time-sent.p.post.node stats-index) =? update-core ?=(%count mode.notif-kind) (hark %read-count stats-index) =? update-core watch-on-self - (new-watch index.post.node [watch-for index-len]:notif-kind) + (new-watch index.p.post.node [watch-for index-len]:notif-kind) update-core :: ++ add-unread |= [=index:store =notification:store] (hark %add-note index notification) - :: -- -- diff --git a/pkg/arvo/app/hark-group-hook.hoon b/pkg/arvo/app/hark-group-hook.hoon index ed7b888076..2123f8cb87 100644 --- a/pkg/arvo/app/hark-group-hook.hoon +++ b/pkg/arvo/app/hark-group-hook.hoon @@ -108,12 +108,12 @@ :: %fact ?+ p.cage.sign (on-agent:def wire sign) - %group-update + %group-update-0 =^ cards state (group-update !<(update:group-store q.cage.sign)) [cards this] :: - %metadata-update + %metadata-update-1 =^ cards state (metadata-update !<(update:metadata q.cage.sign)) [cards this] diff --git a/pkg/arvo/app/hark-store.hoon b/pkg/arvo/app/hark-store.hoon index fd1ab4f965..6c86fb9b4f 100644 --- a/pkg/arvo/app/hark-store.hoon +++ b/pkg/arvo/app/hark-store.hoon @@ -24,11 +24,12 @@ state-3 state-4 state-5 + state-6 == +$ unread-stats [indices=(set index:graph-store) last=@da] :: -+$ base-state ++$ base-state $: unreads-each=(jug stats-index:store index:graph-store) unreads-count=(map stats-index:store @ud) last-seen=(map stats-index:store @da) @@ -45,13 +46,16 @@ [%3 state-two:store] :: +$ state-4 - [%4 base-state] + [%4 state-three:store] :: +$ state-5 - [%5 base-state] + [%5 state-three:store] +:: ++$ state-6 + [%6 base-state] :: +$ inflated-state - $: state-5 + $: state-6 cache == :: $cache: useful to have precalculated, but can be derived from state @@ -92,9 +96,16 @@ =| cards=(list card) |^ ?- -.old - %5 + %6 :- (flop cards) this(-.state old, +.state (inflate-cache:ha old)) + :: + %5 + %_ $ + -.old %6 + notifications.old (convert-notifications-4 notifications.old) + archive.old (convert-notifications-4 archive.old) + == :: %4 %_ $ @@ -149,15 +160,59 @@ == == :: - ++ convert-notifications-3 - |= old=notifications:state-two:store + ++ convert-notifications-4 + |= old=notifications:state-three:store %+ gas:orm *notifications:store ^- (list [@da timebox:store]) %+ murn - (tap:orm:state-two:store old) - |= [time=@da =timebox:state-two:store] + (tap:orm:state-three:store old) + |= [time=@da =timebox:state-three:store] ^- (unit [@da timebox:store]) =/ new-timebox=timebox:store + (convert-timebox-4 timebox) + ?: =(0 ~(wyt by new-timebox)) + ~ + `[time new-timebox] + :: + ++ convert-timebox-4 + |= =timebox:state-three:store + ^- timebox:store + %- ~(gas by *timebox:store) + ^- (list [index:store notification:store]) + %+ murn + ~(tap by timebox) + |= [=index:store =notification:state-three:store] + ^- (unit [index:store notification:store]) + =/ new-notification=(unit notification:store) + (convert-notification-4 notification) + ?~ new-notification ~ + `[index u.new-notification] + :: + ++ convert-notification-4 + |= =notification:state-three:store + ^- (unit notification:store) + ?: ?=(%group -.contents.notification) + `notification + =/ con=(list post:post) + (convert-graph-contents-4 list.contents.notification) + ?: =(~ con) ~ + =, notification + `[date read %graph con] + :: + ++ convert-graph-contents-4 + |= con=(list post:post-zero:post) + ^- (list post:post) + (turn con post-to-one:upgrade:graph-store) + :: + ++ convert-notifications-3 + |= old=notifications:state-two:store + %+ gas:orm:state-three:store *notifications:state-three:store + ^- (list [@da timebox:state-three:store]) + %+ murn + (tap:orm:state-two:store old) + |= [time=@da =timebox:state-two:store] + ^- (unit [@da timebox:state-three:store]) + =/ new-timebox=timebox:state-three:store (convert-timebox-3 timebox) ?: =(0 ~(wyt by new-timebox)) ~ @@ -165,21 +220,21 @@ :: ++ convert-timebox-3 |= =timebox:state-two:store - ^- timebox:store - %- ~(gas by *timebox:store) - ^- (list [index:store notification:store]) + ^- timebox:state-three:store + %- ~(gas by *timebox:state-three:store) + ^- (list [index:state-three:store notification:state-three:store]) %+ murn ~(tap by timebox) |= [=index:store =notification:state-two:store] - ^- (unit [index:store notification:store]) - =/ new-notification=(unit notification:store) + ^- (unit [index:store notification:state-three:store]) + =/ new-notification=(unit notification:state-three:store) (convert-notification-3 notification) ?~ new-notification ~ `[index u.new-notification] :: ++ convert-notification-3 |= =notification:state-two:store - ^- (unit notification:store) + ^- (unit notification:state-three:store) ?: ?=(%graph -.contents.notification) `notification =/ con=(list group-contents:store) @@ -293,7 +348,7 @@ ~(tap by unreads-count) |= [=stats-index:store count=@ud] :* stats-index - ~(wyt in (~(gut by by-index) stats-index ~)) + (~(gut by by-index) stats-index ~) [%count count] (~(gut by last-seen) stats-index *time) == @@ -304,7 +359,7 @@ ~(tap by unreads-each) |= [=stats-index:store indices=(set index:graph-store)] :* stats-index - ~(wyt in (~(gut by by-index) stats-index ~)) + (~(gut by by-index) stats-index ~) [%each indices] (~(gut by last-seen) stats-index *time) == @@ -317,7 +372,7 @@ ~ :- ~ :* stats-index - ~(wyt in nots) + nots [%count 0] *time == @@ -778,7 +833,7 @@ == :: ++ inflate-cache - |= state-5 + |= state-6 ^+ +.state =. +.state *cache diff --git a/pkg/arvo/app/landscape/index.html b/pkg/arvo/app/landscape/index.html index 305f892bb2..2dcc4a987a 100644 --- a/pkg/arvo/app/landscape/index.html +++ b/pkg/arvo/app/landscape/index.html @@ -24,6 +24,6 @@
- + diff --git a/pkg/arvo/app/launch.hoon b/pkg/arvo/app/launch.hoon index 16997bfc0b..04df68453d 100644 --- a/pkg/arvo/app/launch.hoon +++ b/pkg/arvo/app/launch.hoon @@ -191,9 +191,14 @@ ^- (unit (unit cage)) ?. (team:title our.bowl src.bowl) ~ ?+ path [~ ~] - [%x %tiles ~] ``noun+!>([tiles tile-ordering]) - [%x %first-time ~] ``noun+!>(first-time) - [%x %keys ~] ``noun+!>(~(key by tiles)) + [%x %tiles ~] ``noun+!>([tiles tile-ordering]) + [%x %first-time ~] ``noun+!>(first-time) + [%x %keys ~] ``noun+!>(~(key by tiles)) + :: + [%x %runtime-lag ~] + :^ ~ ~ %json + !> ^- json + b+.^(? //(scot %p our.bowl)//(scot %da now.bowl)/zen/lag) == :: ++ on-arvo diff --git a/pkg/arvo/app/metadata-pull-hook.hoon b/pkg/arvo/app/metadata-pull-hook.hoon index 3e57c04b61..4766ff0a14 100644 --- a/pkg/arvo/app/metadata-pull-hook.hoon +++ b/pkg/arvo/app/metadata-pull-hook.hoon @@ -9,19 +9,49 @@ |% +$ card card:agent:gall :: ++$ group-preview-0 + $: group=resource + channels=associations-0 + members=@ud + channel-count=@ud + metadatum=metadatum-0 + == +:: ++$ associations-0 + (map md-resource:metadata [group=resource metadatum=metadatum-0]) +:: ++$ metadatum-0 + $: title=cord + description=cord + =color:metadata + date-created=time + creator=ship + module=term + picture=url:metadata + preview=? + vip=vip-metadata:metadata + == +:: ++ config ^- config:pull-hook :* %metadata-store update:metadata %metadata-update %metadata-push-hook + 1 1 %.n == +$ state-zero - [%0 previews=(map resource group-preview:metadata)] + [%0 previews=(map resource group-preview-0)] :: +$ state-one $: %1 + pending=(set resource) + previews=(map resource group-preview-0) + == +:: ++$ state-two + $: %2 pending=(set resource) previews=(map resource group-preview:metadata) == @@ -29,17 +59,16 @@ +$ versioned-state $% state-zero state-one + state-two == -:: -- :: -:: %- agent:dbug %+ verb | ^- agent:gall %- (agent:pull-hook config) ^- (pull-hook:pull-hook config) -=| state-one +=| state-two =* state - => |_ =bowl:gall ++ def ~(. (default-agent state %|) bowl) @@ -82,7 +111,7 @@ %kick [~[watch-contacts] state] :: %fact - ?> ?=(%contact-update p.cage.sign) + ?> ?=(%contact-update-0 p.cage.sign) =+ !<(=update:contact q.cage.sign) ?+ -.update `state %add @@ -151,7 +180,7 @@ %kick [watch-store^~ state] :: %fact - ?> ?=(%metadata-update p.cage.sign) + ?> ?=(%metadata-update-1 p.cage.sign) =+ !<(=update:metadata q.cage.sign) ?. ?=(%initial-group -.update) `state `state(previews (~(del by previews) group.update)) @@ -175,9 +204,17 @@ ++ on-load |= =vase =+ !<(old=versioned-state vase) - |- + |^ ?- -.old - %1 `this(state old) + %2 `this(state old) + :: + %1 + %_ $ + old + %* . *state-two + previews (~(run by previews.old) preview-to-1) + == + == :: %0 %_ $ @@ -187,6 +224,39 @@ == == == + :: + ++ metadatum-to-1 + |= m=metadatum-0 + %* . *metadatum:metadata + title title.m + description description.m + color color.m + date-created date-created.m + creator creator.m + preview preview.m + hidden %| + :: + config + ?: =(module.m %$) + [%group ~] + [%graph module.m] + == + :: + ++ preview-to-1 + |= preview=group-preview-0 + ^- group-preview:metadata + %= preview + metadatum (metadatum-to-1 metadatum.preview) + channels (associations-to-1 channels.preview) + == + :: + ++ associations-to-1 + |= a=associations-0 + ^- associations:metadata + %- ~(run by a) + |= [g=resource m=metadatum-0] + [g (metadatum-to-1 m)] + -- :: ++ on-poke |= [=mark =vase] @@ -255,7 +325,7 @@ %+ turn ~(tap by associations) |= [=md-resource:metadata =association:metadata] %+ poke-our:pass:io %metadata-store - :- %metadata-update + :- %metadata-update-1 !> ^- update:metadata [%remove resource md-resource] :: diff --git a/pkg/arvo/app/metadata-push-hook.hoon b/pkg/arvo/app/metadata-push-hook.hoon index 30dea7e233..65ca2e00a2 100644 --- a/pkg/arvo/app/metadata-push-hook.hoon +++ b/pkg/arvo/app/metadata-push-hook.hoon @@ -14,6 +14,7 @@ update:store %metadata-update %metadata-pull-hook + 1 1 == :: +$ agent (push-hook:push-hook config) @@ -35,7 +36,7 @@ ++ on-init on-init:def ++ on-save !>(~) ++ on-load on-load:def -++ on-poke +++ on-poke |= [=mark =vase] ?. ?=(%metadata-hook-update mark) (on-poke:def mark vase) @@ -58,22 +59,32 @@ :: ++ transform-proxy-update |= vas=vase - ^- (unit vase) + ^- (quip card (unit vase)) =/ =update:store !<(update:store vas) + :- ~ ?. ?=(?(%add %remove) -.update) ~ =/ role=(unit (unit role-tag)) (role-for-ship:grp group.update src.bowl) - =/ =metadatum:store - (need (peek-metadatum:met %groups group.update)) ?~ role ~ - ?^ u.role + =/ metadatum=(unit metadatum:store) + (peek-metadatum:met %groups group.update) + ?: ?& ?=(~ metadatum) + (is-managed:grp group.update) + == + ~ + ?: ?& ?=(^ metadatum) + !(is-managed:grp group.update) + == + ~ + ?^ u.role ?: ?=(?(%admin %moderator) u.u.role) `vas ~ ?. ?=(%add -.update) ~ - ?: ?& =(src.bowl entity.resource.resource.update) - ?=(%member-metadata vip.metadatum) + ?: ?& ?=(^ metadatum) + =(src.bowl entity.resource.resource.update) + ?=(%member-metadata vip.u.metadatum) == `vas ~ diff --git a/pkg/arvo/app/metadata-store.hoon b/pkg/arvo/app/metadata-store.hoon index 440d00c69d..2f430864c5 100644 --- a/pkg/arvo/app/metadata-store.hoon +++ b/pkg/arvo/app/metadata-store.hoon @@ -23,7 +23,7 @@ :: /app-name/%app-name associations for app :: /group/%path associations for group :: -/- store=metadata-store +/- store=metadata-store, pull-hook /+ default-agent, verb, dbug, resource, *migrate |% +$ card card:agent:gall @@ -64,6 +64,21 @@ resource-indices=(jug md-resource-1 path) == :: ++$ metadatum-2 + $: title=cord + description=cord + =color:store + date-created=time + creator=ship + module=term + picture=url:store + preview=? + vip=vip-metadata:store + == +:: ++$ association-2 [group=resource =metadatum-2] ++$ associations-2 (map md-resource:store association-2) +:: +$ cached-indices $: group-indices=(jug resource md-resource:store) app-indices=(jug app-name:store [group=resource =resource]) @@ -71,18 +86,26 @@ == :: +$ base-state-2 + $: associations=associations-2 + ~ + == +:: ++$ base-state-3 $: =associations:store ~ == :: -+$ state-0 [%0 base-state-0] -+$ state-1 [%1 base-state-0] -+$ state-2 [%2 base-state-0] -+$ state-3 [%3 base-state-1] -+$ state-4 [%4 base-state-1] -+$ state-5 [%5 base-state-1] -+$ state-6 [%6 base-state-1] -+$ state-7 [%7 base-state-2] ++$ state-0 [%0 base-state-0] ++$ state-1 [%1 base-state-0] ++$ state-2 [%2 base-state-0] ++$ state-3 [%3 base-state-1] ++$ state-4 [%4 base-state-1] ++$ state-5 [%5 base-state-1] ++$ state-6 [%6 base-state-1] ++$ state-7 [%7 base-state-2] ++$ state-8 [%8 base-state-3] ++$ state-9 [%9 base-state-3] ++$ state-10 [%10 base-state-3] +$ versioned-state $% state-0 state-1 @@ -92,10 +115,13 @@ state-5 state-6 state-7 + state-8 + state-9 + state-10 == :: +$ inflated-state - $: state-7 + $: state-10 cached-indices == -- @@ -126,7 +152,7 @@ ?> (team:title our.bowl src.bowl) =^ cards state ?+ mark (on-poke:def mark vase) - ?(%metadata-action %metadata-update) + ?(%metadata-action %metadata-update-1) (poke-metadata-update:mc !<(update:store vase)) :: %import @@ -144,7 +170,7 @@ =/ cards=(list card) ?+ path (on-watch:def path) [%all ~] - (give %metadata-update !>([%associations associations])) + (give %metadata-update-1 !>([%associations associations])) :: [%updates ~] ~ @@ -152,7 +178,7 @@ [%app-name @ ~] =/ =app-name:store i.t.path =/ app-indices (metadata-for-app:mc app-name) - (give %metadata-update !>([%associations app-indices])) + (give %metadata-update-1 !>([%associations app-indices])) == [cards this] :: @@ -208,21 +234,39 @@ =| cards=(list card) |^ =* loop $ - ?: ?=(%7 -.old) + ?: ?=(%10 -.old) :- cards %_ state - associations - associations.old - :: - resource-indices - (rebuild-resource-indices associations.old) - :: - group-indices - (rebuild-group-indices associations.old) - :: - app-indices - (rebuild-app-indices associations.old) + associations associations.old + resource-indices (rebuild-resource-indices associations.old) + group-indices (rebuild-group-indices associations.old) + app-indices (rebuild-app-indices associations.old) == + ?: ?=(%9 -.old) + =/ groups + (fall (~(get by (rebuild-app-indices associations.old)) %groups) ~) + =/ pokes=(list card) + %+ murn ~(tap in ~(key by groups)) + |= group=resource + ^- (unit card) + =/ =association:store (~(got by associations.old) [%groups group]) + =* met metadatum.association + ?. ?=([%group [~ [~ [@ [@ @]]]]] config.met) + ~ + =* res resource.u.u.feed.config.met + ?: =(our.bowl entity.res) ~ + =- `[%pass /fix-feed %agent [our.bowl %graph-pull-hook] %poke -] + :- %pull-hook-action + !> ^- action:pull-hook + [%add entity.res res] + %_ $ + cards (weld cards pokes) + -.old %10 + == + ?: ?=(%8 -.old) + $(-.old %9) + ?: ?=(%7 -.old) + $(old [%8 (associations-2-to-3 associations.old) ~]) ?: ?=(%6 -.old) =/ old-assoc=associations-1 (migrate-app-to-graph-store %chat associations.old) @@ -236,12 +280,37 @@ associations.old associations == :: pre-breach, can safely throw away - loop(old *state-7) + loop(old *state-8) + :: + ++ associations-2-to-3 + |= assoc=associations-2 + ^- associations:store + %- ~(gas by *associations:store) + %+ turn ~(tap by assoc) + |= [m=md-resource:store [g=resource met=metadatum-2]] + [m [g (metadatum-2-to-3 met)]] + :: + ++ metadatum-2-to-3 + |= m=metadatum-2 + %* . *metadatum:store + title title.m + description description.m + color color.m + date-created date-created.m + creator creator.m + preview preview.m + hidden %| + :: + config + ?: =(module.m %$) + [%group ~] + [%graph module.m] + == :: ++ associations-1-to-2 |= assoc=associations-1 - ^- associations:store - %- ~(gas by *associations:store) + ^- associations-2 + %- ~(gas by *associations-2) %+ murn ~(tap by assoc) |= [[group=path m=md-resource-1] met=metadata-1] @@ -262,7 +331,7 @@ :: ++ metadata-1-to-2 |= m=metadata-1 - %* . *metadatum:store + %* . *metadatum-2 title title.m description description.m color color.m @@ -309,6 +378,8 @@ ship+path.md-resource [[path [%graph new-path]] m(module app)] -- +:: +:: TODO: refactor into a |^ inside the agent core ++ poke-metadata-update |= upd=update:store ^- (quip card _state) @@ -319,12 +390,13 @@ %initial-group (handle-initial-group +.upd) == :: +:: TODO: refactor into a |^ inside the agent core ++ poke-import |= arc=* ^- (quip card _state) |^ =^ cards state - (on-load !>([%7 (remake-metadata ;;(tree-metadata +.arc))])) + (on-load !>([%9 (remake-metadata ;;(tree-metadata +.arc))])) :_ state %+ weld cards %+ turn ~(tap in ~(key by group-indices)) @@ -348,7 +420,7 @@ :: ++ remake-metadata |= tm=tree-metadata - ^- base-state-2 + ^- base-state-3 :* (remake-map associations.tm) ~ == @@ -443,6 +515,6 @@ ++ update-subscribers |= [pax=path =update:store] ^- (list card) - [%give %fact ~[pax] %metadata-update !>(update)]~ + [%give %fact ~[pax] %metadata-update-1 !>(update)]~ -- -- diff --git a/pkg/arvo/app/observe-hook.hoon b/pkg/arvo/app/observe-hook.hoon index b1d8b47ba2..07fadb5c2d 100644 --- a/pkg/arvo/app/observe-hook.hoon +++ b/pkg/arvo/app/observe-hook.hoon @@ -12,6 +12,9 @@ $% [%0 observers=(map serial observer:sur)] [%1 observers=(map serial observer:sur)] [%2 observers=(map serial observer:sur)] + [%3 observers=(map serial observer:sur)] + [%4 observers=(map serial observer:sur)] + [%5 observers=(map serial observer:sur) warm-cache=_|] == :: +$ serial @uv @@ -25,7 +28,7 @@ -- :: %- agent:dbug -=| [%2 observers=(map serial observer:sur)] +=| [%5 observers=(map serial observer:sur) warm-cache=_|] =* state - :: ^- agent:gall @@ -39,6 +42,8 @@ :~ (act [%watch %invite-store /invitatory/graph %invite-accepted-graph]) (act [%watch %group-store /groups %group-on-leave]) (act [%watch %group-store /groups %group-on-remove-member]) + (act [%watch %metadata-store /updates %md-on-add-group-feed]) + (act [%warm-cache-all ~]) == :: ++ act @@ -50,8 +55,7 @@ [our.bowl %observe-hook] %poke %observe-action - !> ^- action:sur - action + !>(action) == -- :: @@ -63,20 +67,36 @@ =/ old-state !<(versioned-state old-vase) =| cards=(list card) |- - ?: ?=(%2 -.old-state) - =. cards - :_ cards - (act [%watch %group-store /groups %group-on-leave]) + ?- -.old-state + %5 [cards this(state old-state)] - ?: ?=(%1 -.old-state) + %4 + =. cards + :_ cards + (act [%warm-cache-all ~]) + $(old-state [%5 observers.old-state %.n]) + :: + %3 + =. cards + :_ cards + (act [%watch %metadata-store /updates %md-on-add-group-feed]) + $(-.old-state %4) + :: + %2 =. cards :_ cards (act [%watch %group-store /groups %group-on-leave]) + $(-.old-state %3) + :: + %1 $(-.old-state %2) - =. cards - :_ cards - (act [%watch %group-store /groups %group-on-remove-member]) - $(-.old-state %1) + :: + %0 + =. cards + :_ cards + (act [%watch %group-store /groups %group-on-remove-member]) + $(-.old-state %1) + == :: ++ act |= =action:sur @@ -87,8 +107,7 @@ [our.bowl %observe-hook] %poke %observe-action - !> ^- action:sur - action + !>(action) == -- :: @@ -98,11 +117,19 @@ ?> (team:title our.bowl src.bowl) ?. ?=(%observe-action mark) (on-poke:def mark vase) + |^ =/ =action:sur !<(action:sur vase) =* observer observer.action =/ vals (silt ~(val by observers)) ?- -.action - %watch + %watch (watch observer vals) + %ignore (ignore observer vals) + %warm-cache-all warm-cache-all + %cool-cache-all cool-cache-all + == + :: + ++ watch + |= [=observer:sur vals=(set observer:sur)] ?: ?|(=(app.observer %spider) =(app.observer %observe-hook)) ~|('we avoid infinite loops' !!) ?: (~(has in vals) observer) @@ -117,7 +144,8 @@ path.observer == :: - %ignore + ++ ignore + |= [=observer:sur vals=(set observer:sur)] ?. (~(has in vals) observer) ~|('cannot remove nonexistent observer' !!) =/ key (got-by-val observers observer) @@ -130,7 +158,19 @@ %leave ~ == - == + :: + ++ warm-cache-all + ?: warm-cache + ~|('cannot warm up cache that is already warm' !!) + :_ this(warm-cache %.y) + =/ =rave:clay [%sing [%t da+now.bowl /mar]] + [%pass /warm-cache %arvo %c %warp our.bowl %home `rave]~ + :: + ++ cool-cache-all + ?. warm-cache + ~|('cannot cool down cache that is already cool' !!) + [~ this(warm-cache %.n)] + -- :: ++ on-agent |= [=wire =sign:agent:gall] @@ -248,9 +288,48 @@ == == -- :: +++ on-arvo + |= [=wire =sign-arvo] + ^- (quip card _this) + :_ this + ?+ wire (on-arvo:def wire sign-arvo) + [%warm-cache ~] + ?. warm-cache + ~ + ?> ?=([%clay %writ *] sign-arvo) + =* riot p.sign-arvo + ?~ riot + =/ =rave:clay [%next [%t da+now.bowl /mar]] + [%pass /warm-cache %arvo %c %warp our.bowl %home `rave]~ + :- =/ =rave:clay [%next [%t q.p.u.riot /mar]] + [%pass /warm-cache %arvo %c %warp our.bowl %home `rave] + %+ turn !<((list path) q.r.u.riot) + |= pax=path + ^- card + =. pax (snip (slag 1 pax)) + =/ mark=@ta + %+ roll pax + |= [=term mark=term] + ?: ?=(%$ mark) + term + :((cury cat 3) mark '-' term) + =/ =rave:clay [%sing %b da+now.bowl /[mark]] + [%pass [%mar mark ~] %arvo %c %warp our.bowl %home `rave] + :: + [%mar ^] + ?. warm-cache + ~ + ?> ?=([%clay %writ *] sign-arvo) + =* riot p.sign-arvo + =* mark t.wire + ?~ riot + ~ + =/ =rave:clay [%next %b q.p.u.riot mark] + [%pass wire %arvo %c %warp our.bowl %home `rave]~ + == +:: ++ on-watch on-watch:def ++ on-leave on-leave:def ++ on-peek on-peek:def -++ on-arvo on-arvo:def ++ on-fail on-fail:def -- diff --git a/pkg/arvo/gen/demo/add.hoon b/pkg/arvo/gen/demo/add.hoon new file mode 100644 index 0000000000..fe70ae11c5 --- /dev/null +++ b/pkg/arvo/gen/demo/add.hoon @@ -0,0 +1,8 @@ +/- *demo +:- %say +|= $: [now=@da eny=@uvJ =beak] + [[ver=@ud =term count=@ud ~] ~] + == +:- (cat 3 %demo-update- (scot %ud ver)) +^- update +[%add [p.beak term] count] diff --git a/pkg/arvo/gen/demo/ini.hoon b/pkg/arvo/gen/demo/ini.hoon new file mode 100644 index 0000000000..79389bb078 --- /dev/null +++ b/pkg/arvo/gen/demo/ini.hoon @@ -0,0 +1,8 @@ +/- *demo +:- %say +|= $: [now=@da eny=@uvJ =beak] + [[=term ~] ~] + == +:- %demo-update-0 +^- update +[%ini [p.beak term] ~] diff --git a/pkg/arvo/gen/demo/run.hoon b/pkg/arvo/gen/demo/run.hoon new file mode 100644 index 0000000000..d3a8cd99d9 --- /dev/null +++ b/pkg/arvo/gen/demo/run.hoon @@ -0,0 +1,8 @@ +/- *demo +:- %say +|= $: [now=@da eny=@uvJ =beak] + [[=term lst=(list update) ~] ~] + == +:- %demo-update-0 +^- update +[%run [p.beak term] lst] diff --git a/pkg/arvo/gen/demo/sub.hoon b/pkg/arvo/gen/demo/sub.hoon new file mode 100644 index 0000000000..c360d19931 --- /dev/null +++ b/pkg/arvo/gen/demo/sub.hoon @@ -0,0 +1,8 @@ +/- *demo +:- %say +|= $: [now=@da eny=@uvJ =beak] + [[=term count=@ud ~] ~] + == +:- %demo-update-0 +^- update +[%sub [p.beak term] count] diff --git a/pkg/arvo/gen/graph-store/add-graph.hoon b/pkg/arvo/gen/graph-store/add-graph.hoon index 0192ede89e..2f284f26a8 100644 --- a/pkg/arvo/gen/graph-store/add-graph.hoon +++ b/pkg/arvo/gen/graph-store/add-graph.hoon @@ -5,6 +5,6 @@ |= $: [now=@da eny=@uvJ =beak] [[=resource mark=(unit mark) overwrite=? ~] ~] == -:- %graph-update +:- %graph-update-2 ^- update -[%0 now [%add-graph resource (gas:orm ~ ~) mark overwrite]] +[now [%add-graph resource (gas:orm ~ ~) mark overwrite]] diff --git a/pkg/arvo/gen/graph-store/add-post.hoon b/pkg/arvo/gen/graph-store/add-post.hoon index 643e073c03..2f68f2af23 100644 --- a/pkg/arvo/gen/graph-store/add-post.hoon +++ b/pkg/arvo/gen/graph-store/add-post.hoon @@ -12,9 +12,9 @@ contents.post contents == :: -:- %graph-update +:- %graph-update-2 ^- update -:+ %0 now +:- now :+ %add-nodes [our name] %- ~(gas by *(map index node)) -~[[[now]~ [post [%empty ~]]]] +~[[[now]~ [[%& post] [%empty ~]]]] diff --git a/pkg/arvo/gen/graph-store/add-signatures.hoon b/pkg/arvo/gen/graph-store/add-signatures.hoon index 9a536d6d52..570fa97e70 100644 --- a/pkg/arvo/gen/graph-store/add-signatures.hoon +++ b/pkg/arvo/gen/graph-store/add-signatures.hoon @@ -5,6 +5,6 @@ |= $: [now=@da eny=@uvJ =beak] [[[=resource =index] =signatures ~] ~] == -:- %graph-update +:- %graph-update-2 ^- update -[%0 now [%add-signatures [resource index] signatures]] +[now [%add-signatures [resource index] signatures]] diff --git a/pkg/arvo/gen/graph-store/add-tag.hoon b/pkg/arvo/gen/graph-store/add-tag.hoon index 8c83f1c4eb..08366364f9 100644 --- a/pkg/arvo/gen/graph-store/add-tag.hoon +++ b/pkg/arvo/gen/graph-store/add-tag.hoon @@ -3,8 +3,8 @@ /- *graph-store :- %say |= $: [now=@da eny=@uvJ =beak] - [[=term =resource ~] ~] + [[=term =uid ~] ~] == -:- %graph-update +:- %graph-update-2 ^- update -[%0 now [%add-tag term resource]] +[now [%add-tag term uid]] diff --git a/pkg/arvo/gen/graph-store/archive-graph.hoon b/pkg/arvo/gen/graph-store/archive-graph.hoon index a40a7a5336..a6f9ae2989 100644 --- a/pkg/arvo/gen/graph-store/archive-graph.hoon +++ b/pkg/arvo/gen/graph-store/archive-graph.hoon @@ -5,6 +5,6 @@ |= $: [now=@da eny=@uvJ =beak] [[=resource ~] ~] == -:- %graph-update +:- %graph-update-2 ^- update -[%0 now [%archive-graph resource]] +[now [%archive-graph resource]] diff --git a/pkg/arvo/gen/graph-store/export-graph.hoon b/pkg/arvo/gen/graph-store/export-graph.hoon index 9a52862d6c..4330d01eb2 100644 --- a/pkg/arvo/gen/graph-store/export-graph.hoon +++ b/pkg/arvo/gen/graph-store/export-graph.hoon @@ -4,7 +4,7 @@ |= $: [now=@da eny=@uvJ bec=beak] [[=ship graph=term ~] ~] == -:- %graph-update +:- %graph-update-2 =/ our (scot %p p.bec) =/ wen (scot %da now) =/ who (scot %p ship) diff --git a/pkg/arvo/gen/graph-store/import-graph.hoon b/pkg/arvo/gen/graph-store/import-graph.hoon index 720c24e613..4d3e03aa0b 100644 --- a/pkg/arvo/gen/graph-store/import-graph.hoon +++ b/pkg/arvo/gen/graph-store/import-graph.hoon @@ -4,6 +4,6 @@ |= $: [now=@da eny=@uvJ bec=beak] [[graph=term =path ~] ~] == -:- %graph-update +:- %graph-update-2 =- ~& update=- - .^(=update:graph-store %cx path) diff --git a/pkg/arvo/gen/graph-store/remove-graph.hoon b/pkg/arvo/gen/graph-store/remove-graph.hoon index e06305981e..2bd979faa7 100644 --- a/pkg/arvo/gen/graph-store/remove-graph.hoon +++ b/pkg/arvo/gen/graph-store/remove-graph.hoon @@ -5,6 +5,6 @@ |= $: [now=@da eny=@uvJ =beak] [[=resource ~] ~] == -:- %graph-update +:- %graph-update-2 ^- update -[%0 now [%remove-graph resource]] +[now [%remove-graph resource]] diff --git a/pkg/arvo/gen/graph-store/remove-nodes.hoon b/pkg/arvo/gen/graph-store/remove-posts.hoon similarity index 50% rename from pkg/arvo/gen/graph-store/remove-nodes.hoon rename to pkg/arvo/gen/graph-store/remove-posts.hoon index 8c5e13603b..01f2345c30 100644 --- a/pkg/arvo/gen/graph-store/remove-nodes.hoon +++ b/pkg/arvo/gen/graph-store/remove-posts.hoon @@ -1,10 +1,10 @@ -:: graph-store|remove-nodes: remove nodes from a graph at indices +:: graph-store|remove-posts: remove nodes from a graph at indices :: /- *graph-store :- %say |= $: [now=@da eny=@uvJ =beak] [[=resource indices=(set index) ~] ~] == -:- %graph-update +:- %graph-update-2 ^- update -[%0 now [%remove-nodes resource indices]] +[now [%remove-posts resource indices]] diff --git a/pkg/arvo/gen/graph-store/remove-signatures.hoon b/pkg/arvo/gen/graph-store/remove-signatures.hoon index b9bd658fa7..8bd4b86958 100644 --- a/pkg/arvo/gen/graph-store/remove-signatures.hoon +++ b/pkg/arvo/gen/graph-store/remove-signatures.hoon @@ -6,6 +6,6 @@ |= $: [now=@da eny=@uvJ =beak] [[[=resource =index] =signatures ~] ~] == -:- %graph-update +:- %graph-update-2 ^- update -[%0 now [%remove-signatures [resource index] signatures]] +[now [%remove-signatures [resource index] signatures]] diff --git a/pkg/arvo/gen/graph-store/remove-tag.hoon b/pkg/arvo/gen/graph-store/remove-tag.hoon index 722d4af5c7..1a425a4bb2 100644 --- a/pkg/arvo/gen/graph-store/remove-tag.hoon +++ b/pkg/arvo/gen/graph-store/remove-tag.hoon @@ -3,8 +3,8 @@ /- *graph-store :- %say |= $: [now=@da eny=@uvJ =beak] - [[=term =resource ~] ~] + [[=term =uid ~] ~] == -:- %graph-update +:- %graph-update-2 ^- update -[%0 now [%remove-tag term resource]] +[now [%remove-tag term uid]] diff --git a/pkg/arvo/gen/graph-store/unarchive-graph.hoon b/pkg/arvo/gen/graph-store/unarchive-graph.hoon index 1d684de6a1..9402207af9 100644 --- a/pkg/arvo/gen/graph-store/unarchive-graph.hoon +++ b/pkg/arvo/gen/graph-store/unarchive-graph.hoon @@ -5,6 +5,6 @@ |= $: [now=@da eny=@uvJ =beak] [[=resource ~] ~] == -:- %graph-update +:- %graph-update-2 ^- update -[%0 now [%unarchive-graph resource]] +[now [%unarchive-graph resource]] diff --git a/pkg/arvo/gen/group-store/allow-ranks.hoon b/pkg/arvo/gen/group-store/allow-ranks.hoon index 087591350e..6515b5955a 100644 --- a/pkg/arvo/gen/group-store/allow-ranks.hoon +++ b/pkg/arvo/gen/group-store/allow-ranks.hoon @@ -5,6 +5,6 @@ |= $: [now=@da eny=@uvJ =beak] [[=ship =term ranks=(list rank:title) ~] ~] == -:- %group-update +:- %group-update-0 ^- action [%change-policy [ship term] %open %allow-ranks (sy ranks)] diff --git a/pkg/arvo/gen/group-store/allow-ships.hoon b/pkg/arvo/gen/group-store/allow-ships.hoon index 0878ed9724..61c2a7b2bb 100644 --- a/pkg/arvo/gen/group-store/allow-ships.hoon +++ b/pkg/arvo/gen/group-store/allow-ships.hoon @@ -5,6 +5,6 @@ |= $: [now=@da eny=@uvJ =beak] [[=ship =term ships=(list ship) ~] ~] == -:- %group-update +:- %group-update-0 ^- action [%change-policy [ship term] %open %allow-ships (sy ships)] diff --git a/pkg/arvo/gen/group-store/ban-ranks.hoon b/pkg/arvo/gen/group-store/ban-ranks.hoon index 84c7e39d3b..ce637c33f8 100644 --- a/pkg/arvo/gen/group-store/ban-ranks.hoon +++ b/pkg/arvo/gen/group-store/ban-ranks.hoon @@ -5,6 +5,6 @@ |= $: [now=@da eny=@uvJ =beak] [[=ship =term ranks=(list rank:title) ~] ~] == -:- %group-update +:- %group-update-0 ^- action [%change-policy [ship term] %open %ban-ranks (sy ranks)] diff --git a/pkg/arvo/gen/group-store/ban-ships.hoon b/pkg/arvo/gen/group-store/ban-ships.hoon index c3753c3d89..4c71cad0aa 100644 --- a/pkg/arvo/gen/group-store/ban-ships.hoon +++ b/pkg/arvo/gen/group-store/ban-ships.hoon @@ -5,6 +5,6 @@ |= $: [now=@da eny=@uvJ =beak] [[=ship =term ships=(list ship) ~] ~] == -:- %group-update +:- %group-update-0 ^- action [%change-policy [ship term] %open %ban-ships (sy ships)] diff --git a/pkg/arvo/gen/group-store/create.hoon b/pkg/arvo/gen/group-store/create.hoon index 85816472bf..4b859f4215 100644 --- a/pkg/arvo/gen/group-store/create.hoon +++ b/pkg/arvo/gen/group-store/create.hoon @@ -5,6 +5,6 @@ |= $: [now=@da eny=@uvJ =beak] [[=term ~] ~] == -:- %group-update +:- %group-action ^- action [%add-group [p.beak term] *open:policy %.n] diff --git a/pkg/arvo/gen/hood/code.hoon b/pkg/arvo/gen/hood/code.hoon index 3cb86c225f..3cd4eacae3 100644 --- a/pkg/arvo/gen/hood/code.hoon +++ b/pkg/arvo/gen/hood/code.hoon @@ -3,15 +3,14 @@ :::: /hoon/code/hood/gen :: /? 310 -:: -:::: - :: -:- %say +/- *sole +/+ *generators +:- %ask |= $: [now=@da eny=@uvJ bec=beak] [arg=?(~ [%reset ~]) ~] == =* our p.bec -:- %helm-code +^- (sole-result [%helm-code ?(~ %reset)]) ?~ arg =/ code=tape %+ slag 1 @@ -20,11 +19,23 @@ =/ step=tape %+ scow %ud .^(@ud %j /(scot %p our)/step/(scot %da now)/(scot %p our)) - %- %- slog - :~ [%leaf code] - [%leaf (weld "current step=" step)] - [%leaf "use |code %reset to invalidate this and generate a new code"] - == - ~ + :: + %+ print 'use |code %reset to invalidate this and generate a new code' + %+ print leaf+(weld "current step=" step) + %+ print leaf+code + (produce [%helm-code ~]) +:: ?> =(%reset -.arg) -%reset +%+ print 'continue?' +%+ print 'warning: resetting your code closes all web sessions' +%+ prompt + [%& %project "y/n: "] +%+ parse + ;~ pose + (cold %.y (mask "yY")) + (cold %.n (mask "nN")) + == +|= reset=? +?. reset + no-product +(produce [%helm-code %reset]) diff --git a/pkg/arvo/gen/hood/wash-gall.hoon b/pkg/arvo/gen/hood/wash-gall.hoon deleted file mode 100644 index 860afd7533..0000000000 --- a/pkg/arvo/gen/hood/wash-gall.hoon +++ /dev/null @@ -1,13 +0,0 @@ -:: Kiln: clear Gall compiler caches -:: -:::: /hoon/wash-gall/hood/gen - :: -/? 310 -:: -:::: - !: -:- %say -|= $: [now=@da eny=@uvJ bec=beak] - ~ ~ - == -[%kiln-wash-gall ~] diff --git a/pkg/arvo/gen/pull/add.hoon b/pkg/arvo/gen/pull/add.hoon new file mode 100644 index 0000000000..520efdca6a --- /dev/null +++ b/pkg/arvo/gen/pull/add.hoon @@ -0,0 +1,8 @@ +/- *pull-hook +:- %say +|= $: [now=@da eny=@uvJ =beak] + [[=ship =term ~] ~] + == +:- %pull-hook-action +^- action +[%add ship ship term] diff --git a/pkg/arvo/gen/push/add.hoon b/pkg/arvo/gen/push/add.hoon new file mode 100644 index 0000000000..7daf6c1fbe --- /dev/null +++ b/pkg/arvo/gen/push/add.hoon @@ -0,0 +1,8 @@ +/- *push-hook +:- %say +|= $: [now=@da eny=@uvJ =beak] + [[=term ~] ~] + == +:- %push-hook-action +^- action +[%add p.beak term] diff --git a/pkg/arvo/gen/tally.hoon b/pkg/arvo/gen/tally.hoon index fbaf04e0a9..9040da1d04 100644 --- a/pkg/arvo/gen/tally.hoon +++ b/pkg/arvo/gen/tally.hoon @@ -60,13 +60,14 @@ |= [m=md-resource:md association:md] ::NOTE we only count graphs for now ?. &(=(%graph app-name.m) =(our creator.metadatum)) ~ - `[module.metadatum resource.m] + ?. ?=(%graph -.config.metadatum) ~ + `[module.config.metadatum resource.m] :: for sanity checks :: =/ real=(set resource:re) =/ upd=update:ga %+ scry update:ga - [%x %graph-store /keys/graph-update] + [%x %graph-store /keys/graph-update-2] ?> ?=(%keys -.q.upd) resources.q.upd :: count activity per channel @@ -91,8 +92,11 @@ :- (lent week) %~ wyt in %+ roll week - |= [[* [author=ship *] *] a=(set ship)] - (~(put in a) author) + |= [[* mp=maybe-post:ga *] a=(set ship)] + ?- -.mp + %| a + %& (~(put in a) author.p.mp) + == :: render results :: :- (tac 'the date is ' (scot %da now)) diff --git a/pkg/arvo/lib/agentio.hoon b/pkg/arvo/lib/agentio.hoon index 8af4935fef..bd09905f72 100644 --- a/pkg/arvo/lib/agentio.hoon +++ b/pkg/arvo/lib/agentio.hoon @@ -5,12 +5,10 @@ :: |_ =bowl:gall ++ scry - |* [desk=@tas =path] - ?> ?=(^ path) - ?> ?=(^ t.path) + |= [desk=@tas =path] %+ weld /(scot %p our.bowl)/[desk]/(scot %da now.bowl) - t.t.path + path :: ++ pass |_ =wire @@ -105,6 +103,13 @@ ^- card [%give %fact ~ cage] :: +++ fact-init-kick + |= =cage + ^- (list card) + :~ (fact cage ~) + (kick ~) + == +:: ++ fact |= [=cage paths=(list path)] ^- card diff --git a/pkg/arvo/lib/graph-store.hoon b/pkg/arvo/lib/graph-store.hoon index f943882198..5c64c9350f 100644 --- a/pkg/arvo/lib/graph-store.hoon +++ b/pkg/arvo/lib/graph-store.hoon @@ -1,5 +1,5 @@ /- sur=graph-store, pos=post -/+ res=resource +/+ res=resource, migrate =< [sur .] =< [pos .] =, sur @@ -50,18 +50,17 @@ == :: ++ index - |= i=^index + |= ind=^index ^- json - ?: =(~ i) s+'/' - =/ j=^tape "" - |- - ?~ i [%s (crip j)] - =/ k=json (numb i.i) - ?> ?=(%n -.k) - %_ $ - i t.i - j (weld j (weld "/" (trip +.k))) - == + :- %s + ?: =(~ ind) + '/' + %+ roll ind + |= [cur=@ acc=@t] + ^- @t + =/ num (numb cur) + ?> ?=(%n -.num) + (rap 3 acc '/' p.num ~) :: ++ uid |= u=^uid @@ -78,7 +77,7 @@ %mention (frond %mention (ship ship.c)) %text (frond %text s+text.c) %url (frond %url s+url.c) - %reference (frond %reference (uid uid.c)) + %reference (frond %reference (reference +.c)) %code %+ frond %code %- pairs @@ -95,6 +94,36 @@ == == :: + ++ reference + |= ref=^reference + |^ + %+ frond -.ref + ?- -.ref + %graph (graph +.ref) + %group (group +.ref) + == + :: + ++ graph + |= [grp=res gra=res idx=^index] + %- pairs + :~ graph+s+(enjs-path:res gra) + group+s+(enjs-path:res grp) + index+(index idx) + == + :: + ++ group + |= grp=res + s+(enjs-path:res grp) + -- + :: + ++ maybe-post + |= mp=^maybe-post + ^- json + ?- -.mp + %| s+(scot %ux p.mp) + %& (post p.mp) + == + :: ++ post |= p=^post ^- json @@ -110,11 +139,10 @@ ++ update |= upd=^update ^- json - ?> ?=(%0 -.upd) |^ (frond %graph-update (pairs ~[(encode q.upd)])) :: ++ encode - |= upd=update-0 + |= upd=action ^- [cord json] ?- -.upd %add-graph @@ -136,8 +164,8 @@ [%nodes (nodes nodes.upd)] == :: - %remove-nodes - :- %remove-nodes + %remove-posts + :- %remove-posts %- pairs :~ [%resource (enjs:res resource.upd)] [%indices (indices indices.upd)] @@ -161,14 +189,14 @@ :- %add-tag %- pairs :~ [%term s+term.upd] - [%resource (enjs:res resource.upd)] + [%uid (uid uid.upd)] == :: %remove-tag :- %remove-tag %- pairs :~ [%term s+term.upd] - [%resource (enjs:res resource.upd)] + [%uid (uid uid.upd)] == :: %archive-graph @@ -190,9 +218,9 @@ :- %tag-queries %- pairs %+ turn ~(tap by tag-queries.upd) - |= [=term =resources] + |= [=term uids=(set ^uid)] ^- [cord json] - [term [%a (turn ~(tap in resources) enjs:res)]] + [term [%a (turn ~(tap in uids) uid)]] == :: ++ graph @@ -212,7 +240,7 @@ |= n=^node ^- json %- pairs - :~ [%post (post post.n)] + :~ [%post (maybe-post post.n)] :- %children ?- -.children.n %empty ~ @@ -220,7 +248,6 @@ == == :: - :: ++ nodes |= m=(map ^index ^node) ^- json @@ -247,15 +274,14 @@ ++ update |= jon=json ^- ^update - :- %0 :- *time - ^- update-0 + ^- action =< (decode jon) |% ++ decode %- of :~ [%add-nodes add-nodes] - [%remove-nodes remove-nodes] + [%remove-posts remove-posts] [%add-signatures add-signatures] [%remove-signatures remove-signatures] :: @@ -307,7 +333,7 @@ :: ++ node %- ot - :~ [%post post] + :~ [%post maybe-post] [%children internal-graph] == :: @@ -318,6 +344,15 @@ [%empty ~] [%graph (graph jon)] :: + ++ maybe-post + |= jon=json + ^- ^maybe-post + ?~ jon !! + ?+ -.jon !! + %s [%| (nu jon)] + %o [%& (post jon)] + == + :: ++ post %- ot :~ [%author (su ;~(pfix sig fed:ag))] @@ -333,10 +368,25 @@ :~ [%mention (su ;~(pfix sig fed:ag))] [%text so] [%url so] - [%reference uid] + [%reference reference] [%code eval] == :: + ++ reference + |^ + %- of + :~ graph+graph + group+dejs-path:res + == + :: + ++ graph + %- ot + :~ group+dejs-path:res + graph+dejs-path:res + index+index + == + -- + :: ++ tang |= jon=^json ^- ^tang @@ -359,9 +409,8 @@ :~ expression+so output+tang == - :: - ++ remove-nodes + ++ remove-posts %- ot :~ [%resource dejs:res] [%indices (as index)] @@ -397,13 +446,13 @@ ++ add-tag %- ot :~ [%term so] - [%resource dejs:res] + [%uid uid] == :: ++ remove-tag %- ot :~ [%term so] - [%resource dejs:res] + [%uid uid] == :: ++ keys @@ -438,4 +487,391 @@ *signatures == -- +:: +++ upgrade + |% + :: + :: +two + :: + ++ marked-graph-to-two + |= [=graph:one m=(unit mark)] + [(graph-to-two graph) m] + :: + ++ graph-to-two + |= =graph:one + (graph:(upgrade ,post:one ,maybe-post) graph post-to-two) + :: + ++ post-to-two + |= p=post:one + ^- maybe-post + [%& p] + :: + :: + :: +one + :: + ++ update-log-to-one + |= =update-log:zero + ^- update-log:one + %+ gas:orm-log:one *update-log:one + %+ turn (tap:orm-log:zero update-log) + |= [=time =logged-update:zero] + ^- [^time logged-update:one] + :- time + :- p.logged-update + (logged-update-to-one q.logged-update) + :: + ++ logged-update-to-one + |= upd=logged-update-0:zero + ^- logged-action:one + ?+ -.upd upd + %add-graph upd(graph (graph-to-one graph.upd)) + %add-nodes upd(nodes (~(run by nodes.upd) node-to-one)) + == + :: + ++ node-to-one + |= =node:zero + (node:(upgrade ,post:zero ,post) node post-to-one) + :: + ++ graph-to-one + |= =graph:zero + (graph:(upgrade ,post:zero ,post) graph post-to-one) + :: + ++ marked-graph-to-one + |= [=graph:zero m=(unit mark)] + [(graph-to-one graph) m] + :: + ++ post-to-one + |= p=post:zero + ^- post + p(contents (contents-to-one contents.p)) + :: + ++ contents-to-one + |= cs=(list content:zero) + ^- (list content) + %+ murn cs + |= =content:zero + ^- (unit ^content) + ?: ?=(%reference -.content) ~ + `content + :: + ++ upgrade + |* [in-pst=mold out-pst=mold] + => + |% + ++ in-orm + ((ordered-map atom in-node) gth) + +$ in-node + [post=in-pst children=in-internal-graph] + +$ in-graph + ((mop atom in-node) gth) + +$ in-internal-graph + $~ [%empty ~] + $% [%graph p=in-graph] + [%empty ~] + == + :: + ++ out-orm + ((ordered-map atom out-node) gth) + +$ out-node + [post=out-pst children=out-internal-graph] + +$ out-graph + ((mop atom out-node) gth) + +$ out-internal-graph + $~ [%empty ~] + $% [%graph p=out-graph] + [%empty ~] + == + -- + |% + :: + ++ graph + |= $: gra=in-graph + fn=$-(in-pst out-pst) + == + ^- out-graph + %+ gas:out-orm *out-graph + ^- (list [atom out-node]) + %+ turn (tap:in-orm gra) + |= [a=atom n=in-node] + ^- [atom out-node] + [a (node n fn)] + :: + ++ node + |= [nod=in-node fn=$-(in-pst out-pst)] + ^- out-node + :- (fn post.nod) + ^- out-internal-graph + ?: ?=(%empty -.children.nod) + [%empty ~] + [%graph (graph p.children.nod fn)] + -- + :: + ++ zero-load + :: =* infinitely recurses + =, store=zero + =, orm=orm:zero + =, orm-log=orm-log:zero + |% + ++ change-revision-graph + |= [=graph:store q=(unit mark)] + ^- [graph:store (unit mark)] + |^ + :_ q + ?+ q graph + [~ %graph-validator-link] convert-links + [~ %graph-validator-publish] convert-publish + == + :: + ++ convert-links + %+ gas:orm *graph:store + %+ turn (tap:orm graph) + |= [=atom =node:store] + ^- [^atom node:store] + :: top-level + :: + :+ atom post.node + ?: ?=(%empty -.children.node) + [%empty ~] + :- %graph + %+ gas:orm *graph:store + %+ turn (tap:orm p.children.node) + |= [=^atom =node:store] + ^- [^^atom node:store] + :: existing comments get turned into containers for revisions + :: + :^ atom + post.node(contents ~, hash ~) + %graph + %+ gas:orm *graph:store + :_ ~ :- %0 + :_ [%empty ~] + post.node(index (snoc index.post.node atom), hash ~) + :: + ++ convert-publish + %+ gas:orm *graph:store + %+ turn (tap:orm graph) + |= [=atom =node:store] + ^- [^atom node:store] + :: top-level + :: + :+ atom post.node + ?: ?=(%empty -.children.node) + [%empty ~] + :- %graph + %+ gas:orm *graph:store + %+ turn (tap:orm p.children.node) + |= [=^atom =node:store] + ^- [^^atom node:store] + :: existing container for publish note revisions + :: + ?+ atom !! + %1 [atom node] + %2 + :+ atom post.node + ?: ?=(%empty -.children.node) + [%empty ~] + :- %graph + %+ gas:orm *graph:store + %+ turn (tap:orm p.children.node) + |= [=^^atom =node:store] + ^- [^^^atom node:store] + :+ atom post.node(contents ~, hash ~) + :- %graph + %+ gas:orm *graph:store + :_ ~ :- %1 + :_ [%empty ~] + post.node(index (snoc index.post.node atom), hash ~) + == + -- + :: + ++ maybe-unix-to-da + |= =atom + ^- @ + :: (bex 127) is roughly 226AD + ?. (lte atom (bex 127)) + atom + (add ~1970.1.1 (div (mul ~s1 atom) 1.000)) + :: + ++ convert-unix-timestamped-node + |= =node:store + ^- node:store + =. index.post.node + (convert-unix-timestamped-index index.post.node) + ?. ?=(%graph -.children.node) + node + :+ post.node + %graph + (convert-unix-timestamped-graph p.children.node) + :: + ++ convert-unix-timestamped-index + |= =index:store + (turn index maybe-unix-to-da) + :: + ++ convert-unix-timestamped-graph + |= =graph:store + %+ gas:orm *graph:store + %+ turn + (tap:orm graph) + |= [=atom =node:store] + ^- [^atom node:store] + :- (maybe-unix-to-da atom) + (convert-unix-timestamped-node node) + -- + -- +++ import + |= [arc=* our=ship] + ^- (quip card:agent:gall [%5 network]) + |^ + =/ sty [%5 (remake-network ;;(tree-network +.arc))] + :_ sty + %+ turn ~(tap by graphs.sty) + |= [rid=resource =marked-graph] + ^- card:agent:gall + ?: =(our entity.rid) + =/ =cage [%push-hook-action !>([%add rid])] + [%pass / %agent [our %graph-push-hook] %poke cage] + (try-rejoin rid 0) + :: + +$ tree-network + $: graphs=tree-graphs + tag-queries=(tree [term (tree uid)]) + update-logs=tree-update-logs + archive=tree-graphs + ~ + == + +$ tree-graphs (tree [resource tree-marked-graph]) + +$ tree-marked-graph [p=tree-graph q=(unit ^mark)] + +$ tree-graph (tree [atom tree-node]) + +$ tree-node [post=tree-maybe-post children=tree-internal-graph] + +$ tree-internal-graph + $~ [%empty ~] + $% [%graph p=tree-graph] + [%empty ~] + == + +$ tree-update-logs (tree [resource tree-update-log]) + +$ tree-update-log (tree [time tree-logged-update]) + +$ tree-logged-update + $: p=time + $= q + $% [%add-graph =resource =tree-graph mark=(unit ^mark) ow=?] + [%add-nodes =resource nodes=(tree [index tree-node])] + [%remove-posts =resource indices=(tree index)] + [%add-signatures =uid signatures=tree-signatures] + [%remove-signatures =uid signatures=tree-signatures] + == + == + +$ tree-signatures (tree signature) + +$ tree-maybe-post (each tree-post hash) + +$ tree-post + $: author=ship + =index + time-sent=time + contents=(list content) + hash=(unit hash) + signatures=tree-signatures + == + :: + ++ remake-network + |= t=tree-network + ^- network + :* (remake-graphs graphs.t) + (remake-jug:migrate tag-queries.t) + (remake-update-logs update-logs.t) + (remake-graphs archive.t) + ~ + == + :: + ++ remake-graphs + |= t=tree-graphs + ^- graphs + %- remake-map:migrate + (~(run by t) remake-marked-graph) + :: + ++ remake-marked-graph + |= t=tree-marked-graph + ^- marked-graph + [(remake-graph p.t) q.t] + :: + ++ remake-graph + |= t=tree-graph + ^- graph + %+ gas:orm *graph + %+ turn ~(tap by t) + |= [a=atom tn=tree-node] + ^- [atom node] + [a (remake-node tn)] + :: + ++ remake-internal-graph + |= t=tree-internal-graph + ^- internal-graph + ?: ?=(%empty -.t) + [%empty ~] + [%graph (remake-graph p.t)] + :: + ++ remake-node + |= t=tree-node + ^- node + :- (remake-post post.t) + (remake-internal-graph children.t) + :: + ++ remake-update-logs + |= t=tree-update-logs + ^- update-logs + %- remake-map:migrate + (~(run by t) remake-update-log) + :: + ++ remake-update-log + |= t=tree-update-log + ^- update-log + =/ ulm ((ordered-map time logged-update) gth) + %+ gas:ulm *update-log + %+ turn ~(tap by t) + |= [=time tlu=tree-logged-update] + ^- [^time logged-update] + [time (remake-logged-update tlu)] + :: + ++ remake-logged-update + |= t=tree-logged-update + ^- logged-update + :- p.t + ?- -.q.t + %add-graph + :* %add-graph + resource.q.t + (remake-graph tree-graph.q.t) + mark.q.t + ow.q.t + == + :: + %add-nodes + :- %add-nodes + :- resource.q.t + %- remake-map:migrate + (~(run by nodes.q.t) remake-node) + :: + %remove-posts + [%remove-posts resource.q.t (remake-set:migrate indices.q.t)] + :: + %add-signatures + [%add-signatures uid.q.t (remake-set:migrate signatures.q.t)] + :: + %remove-signatures + [%remove-signatures uid.q.t (remake-set:migrate signatures.q.t)] + == + :: + ++ remake-post + |= t=tree-maybe-post + ^- maybe-post + ?- -.t + %| t + %& t(signatures.p (remake-set:migrate signatures.p.t)) + == + :: + ++ try-rejoin + |= [rid=resource nack-count=@] + ^- card:agent:gall + =/ res-path (en-path:res rid) + =/ wire [%try-rejoin (scot %ud nack-count) res-path] + [%pass wire %agent [entity.rid %graph-push-hook] %watch resource+res-path] + -- -- diff --git a/pkg/arvo/lib/graph-view.hoon b/pkg/arvo/lib/graph-view.hoon index 22e876a713..d2663a508d 100644 --- a/pkg/arvo/lib/graph-view.hoon +++ b/pkg/arvo/lib/graph-view.hoon @@ -1,5 +1,5 @@ /- sur=graph-view, store=graph-store -/+ resource, group-store +/+ resource, group-store, metadata-store ^? =< [sur .] =, sur @@ -18,6 +18,8 @@ groupify+groupify eval+so pending-indices+pending-indices + create-group-feed+create-group-feed + disable-group-feed+disable-group-feed ::invite+invite == :: @@ -62,6 +64,17 @@ :~ group+dejs:resource policy+policy:dejs:group-store == + :: + ++ create-group-feed + %- ot + :~ resource+dejs:resource + vip+vip:dejs:metadata-store + == + :: + ++ disable-group-feed + %- ot + :~ resource+dejs:resource + == -- -- :: diff --git a/pkg/arvo/lib/graph.hoon b/pkg/arvo/lib/graph.hoon index efdb38f2a0..89f34370b2 100644 --- a/pkg/arvo/lib/graph.hoon +++ b/pkg/arvo/lib/graph.hoon @@ -19,7 +19,7 @@ %add-graph ~[resource.q.update] %remove-graph ~[resource.q.update] %add-nodes ~[resource.q.update] - %remove-nodes ~[resource.q.update] + %remove-posts ~[resource.q.update] %add-signatures ~[resource.uid.q.update] %remove-signatures ~[resource.uid.q.update] %archive-graph ~[resource.q.update] @@ -32,9 +32,51 @@ %run-updates ~[resource.q.update] == :: +++ upgrade + |* [pst=mold out-pst=mold] + => + |% + ++ orm + ((ordered-map atom node) gth) + +$ node + [post=pst children=internal-graph] + +$ graph + ((mop atom node) gth) + +$ internal-graph + $~ [%empty ~] + $% [%graph p=graph] + [%empty ~] + == + :: + ++ out-orm + ((ordered-map atom out-node) gth) + +$ out-node + [post=out-pst children=out-internal-graph] + +$ out-graph + ((mop atom out-node) gth) + +$ out-internal-graph + $~ [%empty ~] + $% [%graph p=out-graph] + [%empty ~] + == + -- + + |= $: gra=graph + fn=$-(pst out-pst) + == + ^- out-graph + %- gas:out-orm + %+ turn (tap:orm gra) + |= [=atom =node] + :- (fn post.node) + ?: ?=(%empty -.children.node) + [%empty ~] + $(gra p.children.node) +:: ++ get-graph |= res=resource ^- update:store + =- -(p *time) %+ scry-for update:store /graph/(scot %p entity.res)/[name.res] :: @@ -43,7 +85,6 @@ ^- graph:store =/ =update:store (get-graph res) - ?> ?=(%0 -.update) ?> ?=(%add-graph -.q.update) graph.q.update :: @@ -54,7 +95,6 @@ %+ weld /node-siblings/younger/(scot %p entity.res)/[name.res]/all (turn index (cury scot %ud)) - ?> ?=(%0 -.update) ?> ?=(%add-nodes -.q.update) nodes.q.update :: @@ -65,7 +105,6 @@ %+ weld /node/(scot %p entity.res)/[name.res] (turn index (cury scot %ud)) - ?> ?=(%0 -.update) ?> ?=(%add-nodes -.q.update) ?> ?=(^ nodes.q.update) q.n.nodes.q.update @@ -99,7 +138,6 @@ ^- resources =+ %+ scry-for ,=update:store /keys - ?> ?=(%0 -.update) ?> ?=(%keys -.q.update) resources.q.update :: diff --git a/pkg/arvo/lib/group-view.hoon b/pkg/arvo/lib/group-view.hoon index f3919891a8..9341243ae5 100644 --- a/pkg/arvo/lib/group-view.hoon +++ b/pkg/arvo/lib/group-view.hoon @@ -15,6 +15,7 @@ join+join leave+leave invite+invite + hide+dejs-path:resource == :: ++ create @@ -53,6 +54,15 @@ ?- -.upd %initial (initial +.upd) %progress (progress +.upd) + %started (started +.upd) + %hide s+(enjs-path:resource +.upd) + == + :: + ++ started + |= [rid=resource req=^request] + %- pairs + :~ resource+s+(enjs-path:resource rid) + request+(request req) == :: ++ progress @@ -61,13 +71,21 @@ :~ resource+s+(enjs-path:resource rid) progress+s+prog == + ++ request + |= req=^request + %- pairs + :~ hidden+b+hidden.req + started+(time started.req) + ship+(ship ship.req) + progress+s+progress.req + == :: ++ initial - |= init=(map resource ^progress) + |= init=(map resource ^request) %- pairs %+ turn ~(tap by init) - |= [rid=resource prog=^progress] - :_ s+prog + |= [rid=resource req=^request] + :_ (request req) (enjs-path:resource rid) -- ++ cleanup-md diff --git a/pkg/arvo/lib/group.hoon b/pkg/arvo/lib/group.hoon index 436bffdef5..2e1b613327 100644 --- a/pkg/arvo/lib/group.hoon +++ b/pkg/arvo/lib/group.hoon @@ -75,7 +75,12 @@ =/ grp=(unit group) (scry-group rid) ?~ grp ~ - =* group u.grp + (role-for-ship-with-group u.grp rid ship) +:: +++ role-for-ship-with-group + |= [grp=group rid=resource =ship] + ^- (unit (unit role-tag)) + =* group grp =* policy policy.group =* tags tags.group =/ admins=(set ^ship) @@ -106,8 +111,13 @@ ^- (set ship) =/ grp=(unit group) (scry-group rid) - ?~ grp ~ - (~(get ju tags.u.grp) tag) + ?~ grp ~ + (get-tagged-ships-with-group u.grp rid tag) +:: +++ get-tagged-ships-with-group + |= [grp=group rid=resource =tag] + ^- (set ship) + (~(get ju tags.grp) tag) :: ++ is-managed |= rid=resource diff --git a/pkg/arvo/lib/hark/store.hoon b/pkg/arvo/lib/hark/store.hoon index 0edc658f04..99b2f38df6 100644 --- a/pkg/arvo/lib/hark/store.hoon +++ b/pkg/arvo/lib/hark/store.hoon @@ -151,7 +151,7 @@ ^- json %- pairs :~ unreads+(unread unreads.s) - notifications+(numb notifications.s) + notifications+a+(turn ~(tap in notifications.s) notif-ref) last+(time last-seen.s) == ++ added diff --git a/pkg/arvo/lib/metadata-store.hoon b/pkg/arvo/lib/metadata-store.hoon index f207d566bf..304c988c57 100644 --- a/pkg/arvo/lib/metadata-store.hoon +++ b/pkg/arvo/lib/metadata-store.hoon @@ -23,13 +23,13 @@ %+ turn ~(tap by associations) |= [=md-resource [group=resource =^metadatum]] ^- [cord json] - :- - %- crip - ;: weld - (trip (spat (en-path:resource group))) - (weld "/" (trip app-name.md-resource)) - (trip (spat (en-path:resource resource.md-resource))) - == + :- %: rap 3 + (spat (en-path:resource group)) + '/' + app-name.md-resource + (spat (en-path:resource resource.md-resource)) + ~ + == %- pairs :~ [%group s+(enjs-path:resource group)] [%app-name s+app-name.md-resource] @@ -46,9 +46,28 @@ [%color s+(scot %ux color.met)] [%date-created s+(scot %da date-created.met)] [%creator s+(scot %p creator.met)] - [%module s+module.met] + :: + :- %config + ?+ -.config.met o+~ + %graph + %+ frond %graph + s+module.config.met + :: + %group + %+ frond %group + ?~ feed.config.met + ~ + ?~ u.feed.config.met + o+~ + %- pairs + :~ [%app-name s+app-name.u.u.feed.config.met] + [%resource s+(enjs-path:resource resource.u.u.feed.config.met)] + == + == + :: [%picture s+picture.met] [%preview b+preview.met] + [%hidden b+hidden.met] [%vip s+`@t`vip.met] == :: @@ -145,6 +164,8 @@ %- perk :~ %reader-comments %member-metadata + %admin-feed + %host-feed %$ == :: @@ -156,12 +177,41 @@ [%color nu] [%date-created (se %da)] [%creator (su ;~(pfix sig fed:ag))] - [%module so] + [%config config] [%picture so] [%preview bo] + [%hidden bo] [%vip vip] == :: + ++ config + |= jon=^json + ^- md-config + ?~ jon + [%group ~] + ?> ?=(%o -.jon) + ?: (~(has by p.jon) %graph) + =/ mod + (~(got by p.jon) %graph) + ?> ?=(%s -.mod) + [%graph p.mod] + =/ jin=json + (~(got by p.jon) %group) + :+ %group ~ + ?~ jin + ~ + ?> ?=(%o -.jin) + ?. ?& (~(has by p.jin) 'app-name') + (~(has by p.jin) 'resource') + == + ~ + =/ app-name=^json (~(got by p.jin) 'app-name') + ?> ?=(%s -.app-name) + :+ ~ + p.app-name + =/ res=^json (~(got by p.jin) 'resource') + (dejs-path:resource res) + :: ++ md-resource ^- $-(json ^md-resource) %- ot diff --git a/pkg/arvo/lib/pull-hook-virt.hoon b/pkg/arvo/lib/pull-hook-virt.hoon new file mode 100644 index 0000000000..07a844c28e --- /dev/null +++ b/pkg/arvo/lib/pull-hook-virt.hoon @@ -0,0 +1,52 @@ +:: pull-hook-virt: virtualisation for pull-hook +/- *resource +|_ =bowl:gall +++ mule-scry + |= [ref=* raw=*] + =/ pax=(unit path) + ((soft path) raw) + ?~ pax ~ + ?. ?=([@ @ @ @ *] u.pax) ~ + =/ ship + (slaw %p i.t.u.pax) + =/ ved + (slay i.t.t.t.u.pax) + =/ dat + ?~ ved now.bowl + =/ cas=(unit case) + ((soft case) p.u.ved) + ?~ cas now.bowl + ?: ?=(%da -.u.cas) + p.u.cas + now.bowl + :: catch bad gall scries early + ?: ?& =((end 3 i.u.pax) %g) + ?| !=(`our.bowl ship) + !=(dat now.bowl) + == + == + ~ + ``.^(* u.pax) +:: +++ kick-mule + |= [rid=resource trp=(trap *)] + ^- (unit (unit path)) + =/ res=toon + (mock [trp %9 2 %0 1] mule-scry) + =/ pax=(unit path) + !< (unit path) + :- -:!>(*(unit path)) + ?:(?=(%0 -.res) p.res ~) + ?: !?=(%0 -.res) + =/ =tang + :+ leaf+"failed kick handler, please report" + leaf+"{} in {(trip dap.bowl)}" + ?: ?=(%2 -.res) + p.res + ?> ?=(%1 -.res) + =/ maybe-path=(unit path) ((soft path) p.res) + ?~ maybe-path ~ + [(smyt u.maybe-path) ~] + ((slog tang) ~) + `pax +-- diff --git a/pkg/arvo/lib/pull-hook.hoon b/pkg/arvo/lib/pull-hook.hoon index 31b6d9a7a3..8b94e96f36 100644 --- a/pkg/arvo/lib/pull-hook.hoon +++ b/pkg/arvo/lib/pull-hook.hoon @@ -19,7 +19,7 @@ :: %pull-hook-action: Add/remove a resource from pulling. :: /- *pull-hook -/+ default-agent, resource +/+ default-agent, resource, versioning, agentio, pull-hook-virt |% :: JSON conversions ++ dejs @@ -44,7 +44,8 @@ :: $config: configuration for the pull hook :: :: .store-name: name of the store to send subscription updates to. -:: .update-mark: mark that updates will be tagged with +:: .update-mark: mark that updates will be tagged with, without +:: version number :: .push-hook-name: name of the corresponding push-hook :: .no-validate: If true, don't validate that resource/wire/src match :: up @@ -54,6 +55,8 @@ update=mold update-mark=term push-hook-name=term + version=@ud + min-version=@ud no-validate=_| == :: @@ -73,17 +76,52 @@ failed-kicks=(map resource ship) == :: ++$ track + [=ship =status] +:: ++$ status + $% [%active ~] + [%failed-kick ~] + [%pub-ver ver=@ud] + [%sub-ver ver=@ud] + == +:: ++$ base-state-2 + $: tracking=(map resource track) + inner-state=vase + == +:: ++$ base-state-3 + $: prev-version=@ud + prev-min-version=@ud + base-state-2 + == +:: +$ state-0 [%0 base-state-0] :: +$ state-1 [%1 base-state-0] :: +$ state-2 [%2 base-state-1] :: ++$ state-3 [%3 base-state-2] +:: ++$ state-4 [%4 base-state-3] +:: +$ versioned-state $% state-0 state-1 state-2 + state-3 + state-4 == +:: +diplomatic: only renegotiate if versions changed +:: +:: If %.n please leave note as to why renegotiation necessary +:: +:: +++ diplomatic + ^- ? + %.y :: ++ default |* [pull-hook=* =config] @@ -176,7 +214,7 @@ ++ agent |* =config |= =(pull-hook config) - =| state-2 + =| state-4 =* state - ^- agent:gall =< @@ -185,6 +223,9 @@ og ~(. pull-hook bowl) hc ~(. +> bowl) def ~(. (default-agent this %|) bowl) + ver ~(. versioning [bowl [update-mark version min-version]:config]) + io ~(. agentio bowl) + pass pass:io :: ++ on-init ^- [(list card:agent:gall) agent:gall] @@ -199,61 +240,49 @@ =| cards=(list card:agent:gall) |^ ?- -.old - %2 + %4 =^ og-cards pull-hook (on-load:og inner-state.old) =. state old - =^ retry-cards state - retry-failed-kicks + =/ kick=(list card) + ?: ?& =(min-version.config prev-min-version.old) + =(version.config prev-version.old) + diplomatic + == + ~ + (poke-self:pass kick+!>(%kick))^~ :_ this - :(weld cards og-cards retry-cards) - :: + :(weld cards og-cards kick) + :: + %3 $(old [%4 0 0 +.old]) + %2 $(old (state-to-3 old)) %1 $(old [%2 +.old ~]) - :: - %0 - %_ $ - -.old %1 - :: - cards - (weld cards (missing-subscriptions tracking.old)) - == + %0 !! :: pre-breach == :: - ++ retry-failed-kicks - =| acc-cards=(list card) - =/ failures=(list [rid=resource =ship]) - ~(tap by failed-kicks) - =. tracking - (~(uni by tracking) failed-kicks) - =. failed-kicks ~ - |- ^- (quip card _state) - ?~ failures - [acc-cards state] - =, failures - =^ crds state - (handle-kick:hc i) - $(failures t, acc-cards (weld acc-cards crds)) + ++ state-to-3 + |= old=state-2 + %* . *state-3 + tracking (tracking-to-3 tracking.old) + inner-state inner-state.old + == + :: + ++ tracking-to-3 + |= trk=(map resource ship) + %- ~(gas by *(map resource track)) + %+ turn ~(tap by trk) + |= [=resource =ship] + :- resource + [ship %active ~] :: - ++ missing-subscriptions - |= tracking=(map resource ship) - ^- (list card:agent:gall) - %+ murn - ~(tap by tracking) - |= [rid=resource =ship] - ^- (unit card:agent:gall) - =/ =path - resource+(en-path:resource rid) - =/ =wire - (make-wire pull+path) - ?: (~(has by wex.bowl) [wire ship push-hook-name.config]) - ~ - `[%pass wire %agent [ship push-hook-name.config] %watch path] -- :: ++ on-save ^- vase - =. inner-state - on-save:og + =: inner-state on-save:og + prev-min-version min-version.config + prev-version version.config + == !>(state) :: ++ on-poke @@ -263,6 +292,13 @@ =^ cards pull-hook (on-poke:og mark vase) [cards this] + :: + %kick + ?> (team:title [our src]:bowl) + =^ [cards=(list card:agent:gall) hook=_pull-hook] state + restart-subs:hc + =. pull-hook hook + [cards this] :: %sane ?> (team:title [our src]:bowl) @@ -272,8 +308,9 @@ :: %pull-hook-action ?> (team:title [our src]:bowl) - =^ cards state - (poke-hook-action:hc !<(action vase)) + =^ [cards=(list card) hook=_pull-hook] state + tr-abet:(tr-hook-act:track-engine:hc !<(action vase)) + =. pull-hook hook [cards this] == :: @@ -295,72 +332,333 @@ =^ cards pull-hook (on-agent:og wire sign) [cards this] - ?. ?=([%pull %resource *] t.t.wire) + ?: ?=([%version ~] t.t.wire) + =^ [cards=(list card) hook=_pull-hook] state + (take-version:hc src.bowl sign) + =. pull-hook hook + [cards this] + ?. ?=([%pull ?(%unver-resource %resource) *] t.t.wire) (on-agent:def wire sign) =/ rid=resource (de-path:resource t.t.t.t.wire) - ?+ -.sign (on-agent:def wire sign) - %kick - =^ cards state - (handle-kick:hc rid src.bowl) - [cards this] - :: - %watch-ack - ?~ p.sign - [~ this] - =. tracking - (~(del by tracking) rid) - =^ cards pull-hook - (on-pull-nack:og rid u.p.sign) - :_ this - [give-update cards] - :: - %fact - ?. =(update-mark.config p.cage.sign) - =^ cards pull-hook - (on-agent:og wire sign) - [cards this] - :_ this - ~[(update-store:hc rid q.cage.sign)] - == - ++ on-leave - |= =path - ^- [(list card:agent:gall) agent:gall] - =^ cards pull-hook - (on-leave:og path) - [cards this] - :: - ++ on-arvo - |= [=wire =sign-arvo] - ^- [(list card:agent:gall) agent:gall] - =^ cards pull-hook - (on-arvo:og wire sign-arvo) - [cards this] - ++ on-fail - |= [=term =tang] - ^- [(list card:agent:gall) agent:gall] - =^ cards pull-hook - (on-fail:og term tang) - [cards this] - ++ on-peek - |= =path - ^- (unit (unit cage)) - ?: =(/x/dbug/state path) - ``noun+(slop !>(state(inner-state *vase)) on-save:og) - ?. =(/x/tracking path) - (on-peek:og path) - ``noun+!>(~(key by tracking)) + =/ versioned=? + ?=(%resource i.t.t.t.wire) + =^ [cards=(list card) hook=_pull-hook] state + tr-abet:(tr-sign:(tr-abed:track-engine:hc rid) sign versioned) + =. pull-hook hook + [cards this] + :: + ++ on-leave + |= =path + ^- [(list card:agent:gall) agent:gall] + =^ cards pull-hook + (on-leave:og path) + [cards this] + :: + ++ on-arvo + |= [=wire =sign-arvo] + ^- [(list card:agent:gall) agent:gall] + =^ cards pull-hook + (on-arvo:og wire sign-arvo) + [cards this] + :: + ++ on-fail + |= [=term =tang] + ^- [(list card:agent:gall) agent:gall] + =^ cards pull-hook + (on-fail:og term tang) + [cards this] + :: + ++ on-peek + |= =path + ^- (unit (unit cage)) + ?: =(/x/dbug/state path) + ``noun+(slop !>(state(inner-state *vase)) on-save:og) + ?. =(/x/tracking path) + (on-peek:og path) + ``noun+!>(~(key by tracking)) -- |_ =bowl:gall +* og ~(. pull-hook bowl) + io ~(. agentio bowl) + pass pass:io + virt ~(. pull-hook-virt bowl) + ver ~(. versioning [bowl [update-mark version min-version]:config]) + :: + ++ restart-subs + =| acc-cards=(list card) + =/ subs=(list resource) + ~(tap in ~(key by tracking)) + |- ^- [[(list card) _pull-hook] _state] + ?~ subs + [[acc-cards pull-hook] state] + =* rid i.subs + =^ [crds=(list card) hook=_pull-hook] state + tr-abet:tr-on-load:(tr-abed:track-engine rid) + =. pull-hook hook + $(subs t.subs, acc-cards (weld acc-cards crds)) + + :: + ++ track-engine + |_ [cards=(list card) rid=resource =ship =status gone=_|] + :: +| %init: state machine setup and manipulation + :: + ++ tr-core . + ++ tr-abed + |= r=resource + =/ [s=^ship sta=^status] + (~(got by tracking) r) + tr-core(rid r, ship s, status sta) + :: + ++ tr-abet + ^- [[(list card) _pull-hook] _state] + =. tracking + ?: gone + (~(del by tracking) rid) + (~(put by tracking) rid [ship status]) + [[(flop cards) pull-hook] state] + :: + ++ tr-emit + |= =card + tr-core(cards [card cards]) + :: + ++ tr-emis + |= caz=(list card) + tr-core(cards (welp (flop cards) cards)) + :: + ++ tr-ap-og + |= ap=_^?(|.(*(quip card _pull-hook))) + =^ caz pull-hook + (ap) + (tr-emis caz) + :: +| %sign: sign handling + :: + :: + ++ tr-sign + |= [=sign:agent:gall versioned=?] + |^ + ?+ -.sign !! + %kick tr-kick + %watch-ack (tr-wack +.sign) + %fact (tr-fact +.sign) + == + :: + ++ tr-wack + |= tan=(unit tang) + ?~ tan tr-core + ?. versioned + (tr-ap-og:tr-cleanup |.((on-pull-nack:og rid u.tan))) + %- (slog leaf+"versioned nack for {} in {}" u.tan) + =/ pax + (kick-mule:virt rid |.((on-pull-kick:og rid))) + ?~ pax tr-failed-kick + ?~ u.pax tr-cleanup + (tr-watch-unver u.u.pax) + :: + ++ tr-fact + |= =cage + ?: ?=(%version p.cage) + =/ req-ver=@ud + !<(@ud q.cage) + ?: (lth req-ver min-version.config) + (tr-suspend-pub-ver min-version.config) + (tr-suspend-sub-ver req-ver) + ?> (is-root:ver p.cage) + =/ fact-ver=@ud + (read-version:ver p.cage) + ?. (gte fact-ver min-version.config) + ?. versioned + :: don't process unversioned, unsupported facts + :: just wait for publisher to upgrade and kick the + :: subscription + tr-core + (tr-suspend-pub-ver min-version.config) + =/ =^cage + (convert-to:ver cage) + =/ =wire + (make-wire /store) + =+ resources=(~(gas in *(set resource)) (resource-for-update:og q.cage)) + ?> ?| no-validate.config + ?& (check-src resources) + (~(has in resources) rid) + == == + =/ =mark + (append-version:ver version.config) + (tr-emit (~(poke-our pass wire) store-name.config cage)) + -- + :: + ++ tr-kick + ?. ?=(%active -.status) tr-core + =/ pax + (kick-mule:virt rid |.((on-pull-kick:og rid))) + ?~ pax tr-failed-kick + ?~ u.pax tr-cleanup + (tr-watch u.u.pax) + :: +| %lifecycle: lifecycle management for tracked resource + :: + :: + ++ tr-add + |= [s=^ship r=resource] + ?< =(s our.bowl) + =: ship s + rid r + status [%active ~] + == + (tr-watch /) + :: + ++ tr-remove + tr-leave:tr-cleanup + :: + ++ tr-hook-act + |= =action + ^+ tr-core + ?- -.action + %add (tr-add +.action) + %remove tr-remove:(tr-abed resource.action) + == + :: + ++ tr-cleanup + =. gone %.y + (tr-emit give-update) + :: + ++ tr-failed-kick + tr-core(status [%failed-kick ~]) + :: + ++ tr-suspend-pub-ver + |= ver=@ud + =. status [%pub-ver ver] + tr-leave:tr-watch-ver + :: + :: + ++ tr-suspend-sub-ver + |= ver=@ud + tr-core(status [%sub-ver ver]) + :: + ++ tr-on-load + ?+ -.status tr-core + %failed-kick tr-restart + %active tr-rewatch + :: + %sub-ver + ?. (supported:ver (append-version:ver ver.status)) + tr-core + tr-restart + == + :: + ++ tr-restart + =. status [%active ~] + tr-kick + :: + ++ tr-rewatch + tr-kick:tr-leave + :: + :: + :: +| %subscription: subscription cards + :: + :: + ++ tr-ver-wire + (make-wire /version) + :: + ++ tr-watch-ver + (tr-emit (watch-version ship)) + :: + ++ tr-leave-ver + (tr-emit (~(leave pass tr-ver-wire) tr-sub-dock)) + ++ tr-sub-wire + (make-wire pull+resource+(en-path:resource rid)) + ++ tr-unver-sub-wire + (make-wire pull+unver-resource+(en-path:resource rid)) + :: + ++ tr-sub-dock + ^- dock + [ship push-hook-name.config] + :: + ++ tr-check-sub + ?: (~(has by wex.bowl) [tr-sub-wire tr-sub-dock]) + tr-core + tr-kick + :: + ++ tr-watch-unver + |= pax=path + =/ =path + :- %resource + (weld (en-path:resource rid) pax) + (tr-emit (~(watch pass tr-unver-sub-wire) tr-sub-dock path)) + :: + ++ tr-watch + |= pax=path + ^+ tr-core + =/ =path + :+ %resource %ver + %+ weld + (snoc (en-path:resource rid) (scot %ud version.config)) + pax + (tr-emit (~(watch pass tr-sub-wire) tr-sub-dock path)) + :: + ++ tr-leave + (tr-emit (~(leave pass tr-sub-wire) tr-sub-dock)) + -- + :: + ++ take-version + |= [who=ship =sign:agent:gall] + ^- [[(list card) _pull-hook] _state] + ?+ -.sign !! + %watch-ack + ?~ p.sign [~^pull-hook state] + =/ =tank leaf+"subscribe failed from {} on wire {}" + %- (slog tank u.p.sign) + [~^pull-hook state] + :: + %kick + :_ state + [(watch-version who)^~ pull-hook] + :: + %fact + ?. =(%version p.cage.sign) + [~^pull-hook state] + =+ !<(version=@ud q.cage.sign) + =/ tracks=(list [rid=resource =track]) + ~(tap by tracking) + =| cards=(list card) + =| leave=_& + |- + ?~ tracks + =? cards leave + :_(cards (leave-version who)) + [[cards pull-hook] state] + ?. ?=(%pub-ver -.status.track.i.tracks) + $(tracks t.tracks) + ?. =(who ship.track.i.tracks) + $(tracks t.tracks) + ?. =(ver.status.track.i.tracks version) + =. leave %.n + $(tracks t.tracks) + =^ [caz=(list card) hook=_pull-hook] state + tr-abet:tr-restart:(tr-abed:track-engine rid.i.tracks) + =. pull-hook hook + $(tracks t.tracks, cards (weld cards caz)) + == + :: + ++ version-wir + (make-wire /version) + :: + ++ watch-version + |= =ship + (~(watch pass version-wir) [ship push-hook-name.config] /version) + :: + ++ leave-version + |= =ship + (~(leave pass version-wir) [ship push-hook-name.config]) + :: ++ poke-sane ^- (quip card:agent:gall _state) =/ cards - restart-subscriptions + :: TODO revive + ~ :: restart-subscriptions ~? > ?=(^ cards) "Fixed subscriptions in {}" - :_ state - restart-subscriptions + [cards state] + :: ++ check-subscription |= [rid=resource =ship] @@ -375,122 +673,6 @@ =(`rid (de-path-soft:resource (slag 4 wire))) == :: - ++ restart-subscriptions - ^- (list card:agent:gall) - %- zing - %+ turn - ~(tap by tracking) - |= [rid=resource =ship] - ^- (list card:agent:gall) - ?: (check-subscription rid ship) ~ - ~& >> "restarting: {}" - =/ pax=(unit path) - (on-pull-kick:og rid) - ?~ pax ~ - (watch-resource rid u.pax) - :: - ++ mule-scry - |= [ref=* raw=*] - =/ pax=(unit path) - ((soft path) raw) - ?~ pax ~ - ?. ?=([@ @ @ @ *] u.pax) ~ - =/ ship - (slaw %p i.t.u.pax) - =/ ved - (slay i.t.t.t.u.pax) - =/ dat - ?~ ved now.bowl - =/ cas=(unit case) - ((soft case) p.u.ved) - ?~ cas now.bowl - ?: ?=(%da -.u.cas) - p.u.cas - now.bowl - :: catch bad gall scries early - ?: ?& =((end 3 i.u.pax) %g) - ?| !=(`our.bowl ship) - !=(dat now.bowl) - == - == - ~ - ``.^(* u.pax) - :: - ++ handle-kick - |= [rid=resource =ship] - ^- (quip card _state) - =/ res=toon - (mock [|.((on-pull-kick:og rid)) %9 2 %0 1] mule-scry) - =/ pax=(unit path) - !< (unit path) - :- -:!>(*(unit path)) - ?:(?=(%0 -.res) p.res ~) - =? failed-kicks !?=(%0 -.res) - =/ =tang - :+ leaf+"failed kick handler, please report" - leaf+"{} in {(trip dap.bowl)}" - ?: ?=(%2 -.res) - p.res - ?> ?=(%1 -.res) - =/ maybe-path=(unit path) ((soft path) p.res) - ?~ maybe-path ~ - [(smyt u.maybe-path) ~] - %- (slog tang) - (~(put by failed-kicks) rid ship) - ?^ pax - :_ state - (watch-resource rid u.pax) - =. tracking - (~(del by tracking) rid) - :_ state - ~[give-update] - :: - ++ poke-hook-action - |= =action - ^- [(list card:agent:gall) _state] - |^ - ?- -.action - %add (add +.action) - %remove (remove +.action) - == - ++ add - |= [=ship =resource] - ~| resource - ?< |(=(our.bowl ship) =(our.bowl entity.resource)) - ?: (~(has by tracking) resource) - [~ state] - =. tracking - (~(put by tracking) resource ship) - :_ state - (watch-resource resource /) - :: - ++ remove - |= =resource - :- (leave-resource resource) - state(tracking (~(del by tracking) resource)) - -- - :: - ++ leave-resource - |= rid=resource - ^- (list card) - =/ ship=(unit ship) - (~(get by tracking) rid) - ?~ ship ~ - =/ =wire - (make-wire pull+resource+(en-path:resource rid)) - [%pass wire %agent [u.ship push-hook-name.config] %leave ~]~ - :: - ++ watch-resource - |= [rid=resource pax=path] - ^- (list card) - =/ ship=(unit ship) - (~(get by tracking) rid) - ?~ ship ~ - =/ =path - (welp resource+(en-path:resource rid) pax) - =/ =wire - (make-wire pull+resource+(en-path:resource rid)) - [%pass wire %agent [u.ship push-hook-name.config] %watch path]~ :: ++ make-wire |= =wire @@ -509,20 +691,8 @@ %+ roll ~(tap in resources) |= [rid=resource out=_|] ?: out %.y - ?~ ship=(~(get by tracking) rid) + ?~ status=(~(get by tracking) rid) %.n - =(src.bowl u.ship) - :: - ++ update-store - |= [wire-rid=resource =vase] - ^- card - =/ =wire - (make-wire /store) - =+ resources=(~(gas in *(set resource)) (resource-for-update:og vase)) - ?> ?| no-validate.config - ?& (check-src resources) - (~(has in resources) wire-rid) - == == - [%pass wire %agent [our.bowl store-name.config] %poke update-mark.config vase] + =(src.bowl ship.u.status) -- -- diff --git a/pkg/arvo/lib/push-hook.hoon b/pkg/arvo/lib/push-hook.hoon index 107a9e3d2a..d2e4e5c9e6 100644 --- a/pkg/arvo/lib/push-hook.hoon +++ b/pkg/arvo/lib/push-hook.hoon @@ -25,7 +25,8 @@ :: foreign push-hook :: /- *push-hook -/+ default-agent, resource, verb +/+ default-agent, resource, verb, versioning, agentio +~% %push-hook-top ..part ~ |% +$ card card:agent:gall :: @@ -43,6 +44,8 @@ update=mold update-mark=term pull-hook-name=term + version=@ud + min-version=@ud == :: :: $base-state-0: state for the push hook @@ -55,15 +58,32 @@ inner-state=vase == :: ++$ base-state-1 + $: prev-version=@ud + prev-min-version=@ud + base-state-0 + == +:: +$ state-0 [%0 base-state-0] :: +$ state-1 [%1 base-state-0] ++$ state-2 [%2 base-state-1] :: +$ versioned-state $% state-0 state-1 + state-2 == +:: +diplomatic: only renegotiate if versions changed +:: +:: If %.n please leave note as to why renegotiation necessary +:: +++ diplomatic + ^- ? + %.y +:: ++ push-hook + ~/ %push-hook |* =config $_ ^| |_ bowl:gall @@ -93,7 +113,7 @@ :: ++ transform-proxy-update |~ vase - *(unit vase) + *[(list card) (unit vase)] :: +initial-watch: produce initial state for a subscription :: :: .resource is the resource being subscribed to. @@ -151,15 +171,19 @@ ++ agent |* =config |= =(push-hook config) - =| state-1 + =| state-2 =* state - ^- agent:gall =< + ~% %push-agent-lib ..poke-hook-action ~ |_ =bowl:gall +* this . og ~(. push-hook bowl) hc ~(. +> bowl) def ~(. (default-agent this %|) bowl) + io ~(. agentio bowl) + pass pass:io + ver ~(. versioning [bowl [update-mark version min-version]:config]) :: ++ on-init =^ cards push-hook @@ -174,10 +198,22 @@ =| cards=(list card:agent:gall) |^ ?- -.old - %1 + %2 =^ og-cards push-hook (on-load:og inner-state.old) - [(weld cards og-cards) this(state old)] + =/ old-subs + (find-old-subs [prev-version prev-min-version]:old) + =/ version-cards + :- (fact:io version+!>(version.config) /version ~) + ?~ old-subs ~ + (kick:io old-subs)^~ + [:(weld cards og-cards version-cards) this(state old)] + :: + %1 + %_ $ + old [%2 0 0 +.old] + == + :: :: %0 %_ $ @@ -192,6 +228,26 @@ == == :: + ++ find-old-subs + |= [prev-min-version=@ud prev-version=@ud] + ?: ?& =(min-version.config prev-min-version) + =(prev-version version.config) + diplomatic + == + :: bail on kick if we didn't change versions + ~ + %~ tap in + %+ roll + ~(val by sup.bowl) + |= [[=ship =path] out=(set path)] + ?. ?=([%resource *] path) out + ?. ?=([%resource %ver] path) + (~(put in out) path) + =/ path-ver=@ud + (ver-from-path:hc path) + ?: (supported:ver (append-version:ver path-ver)) out + (~(put in out) path) + :: ++ kicked-watches ^- (list path) %~ tap in @@ -205,11 +261,14 @@ -- :: ++ on-save - =. inner-state - on-save:og + =: prev-version version.config + prev-min-version min-version.config + inner-state on-save:og + == !>(state) :: ++ on-poke + ~/ %on-poke |= [=mark =vase] ^- (quip card:agent:gall agent:gall) ?: =(mark %push-hook-action) @@ -218,34 +277,53 @@ (poke-hook-action:hc !<(action vase)) [cards this] :: - ?: =(mark update-mark.config) - ?: (team:title [our src]:bowl) - :_ this - (forward-update:hc vase) - =^ cards state - (poke-update:hc vase) - [cards this] - :: + ?: (is-root:ver mark) + :_ this + (forward-update:hc mark vase) =^ cards push-hook (on-poke:og mark vase) [cards this] :: ++ on-watch + ~/ %on-watch |= =path ^- (quip card:agent:gall agent:gall) + ?: ?=([%version ~] path) + :_ this + (fact-init:io version+!>(min-version.config))^~ ?. ?=([%resource *] path) =^ cards push-hook (on-watch:og path) [cards this] - ?> ?=([%ship @ @ *] t.path) + |^ + ?. ?=([%ver %ship @ @ @ *] t.path) + unversioned =/ =resource - (de-path:resource t.path) - =/ =vase - (initial-watch:og t.t.t.t.path resource) + (de-path:resource t.t.path) + =/ =mark + (append-version:ver (slav %ud i.t.t.t.t.t.path)) + ?. (supported:ver mark) + :_ this + (fact-init-kick:io version+!>(min-version.config)) :_ this - [%give %fact ~ update-mark.config vase]~ + =- [%give %fact ~ -]~ + (convert-to:ver mark (initial-watch:og t.t.t.t.t.t.path resource)) + :: + ++ unversioned + ?> ?=([%ship @ @ *] t.path) + =/ =resource + (de-path:resource t.path) + =/ =vase + (initial-watch:og t.t.t.t.path resource) + :_ this + ?. =(min-version.config 0) + ~& >>> "unversioned req from: {}, nooping" + ~ + [%give %fact ~ (convert-to:ver update-mark.config vase)]~ + -- :: ++ on-agent + ~/ %on-agent |= [=wire =sign:agent:gall] ^- (quip card:agent:gall agent:gall) ?. ?=([%helper %push-hook @ *] wire) @@ -258,7 +336,7 @@ %kick [~[watch-store:hc] this] :: %fact - ?. =(update-mark.config p.cage.sign) + ?. (is-root:ver p.cage.sign) =^ cards push-hook (on-agent:og wire sign) [cards this] @@ -266,7 +344,7 @@ (take-update:og q.cage.sign) :_ this %+ weld - (push-updates:hc q.cage.sign) + (push-updates:hc cage.sign) cards == :: @@ -293,23 +371,21 @@ ^- (unit (unit cage)) ?: =(/x/dbug/state path) ``noun+(slop !>(state(inner-state *vase)) on-save:og) - ?. =(/x/sharing path) - (on-peek:og path) - ``noun+!>(sharing) + ?+ path (on-peek:og path) + [%x %sharing ~] ``noun+!>(sharing) + [%x %version ~] ``version+!>(version.config) + [%x %min-version ~] ``version+!>(version.config) + == -- + ~% %push-helper-lib ..card ~ |_ =bowl:gall +* og ~(. push-hook bowl) - :: - ++ poke-update - |= vas=vase - ^- (quip card:agent:gall _state) - =/ vax=(unit vase) (transform-proxy-update:og vas) - ?> ?=(^ vax) - =/ wire (make-wire /store) - :_ state - [%pass wire %agent [our.bowl store-name.config] %poke update-mark.config u.vax]~ + ver ~(. versioning [bowl [update-mark version min-version]:config]) + io ~(. agentio bowl) + pass pass:io :: ++ poke-hook-action + ~/ %poke-hook-action |= =action ^- (quip card:agent:gall _state) |^ @@ -378,48 +454,90 @@ [%pass wire %agent [our.bowl store-name.config] %watch store-path.config] :: ++ push-updates - |= =vase + ~/ %push-updates + |= =cage ^- (list card:agent:gall) - =/ rids=(list resource) (resource-for-update vase) - =| cards=(list card:agent:gall) - |- - ?~ rids cards - =/ prefix=path - resource+(en-path:resource i.rids) - =/ paths=(list path) - %~ tap in - %- silt - %+ turn - (incoming-subscriptions prefix) - |=([ship pax=path] pax) - ?~ paths $(rids t.rids) - %_ $ - rids t.rids - cards (snoc cards [%give %fact paths update-mark.config vase]) - == + %+ roll (resource-for-update q.cage) + |= [rid=resource cards=(list card)] + |^ + :(weld cards versioned unversioned) + :: + ++ versioned + ^- (list card:agent:gall) + =/ prefix=path + resource+ver+(en-path:resource rid) + =/ paths=(jug @ud path) + %+ roll + (incoming-subscriptions prefix) + |= [[ship =path] out=(jug @ud path)] + =/ path-ver=@ud + (ver-from-path path) + (~(put ju out) path-ver path) + %+ turn ~(tap by paths) + |= [fact-ver=@ud paths=(set path)] + =/ =mark + (append-version:ver fact-ver) + (fact:io (convert-from:ver mark q.cage) ~(tap in paths)) + :: TODO: deprecate + ++ unversioned + ?. =(min-version.config 0) ~ + =/ prefix=path + resource+(en-path:resource rid) + =/ unversioned=(set path) + %- ~(gas in *(set path)) + (turn (incoming-subscriptions prefix) tail) + ?: =(0 ~(wyt in unversioned)) ~ + (fact:io (convert-from:ver update-mark.config q.cage) ~(tap in unversioned))^~ + -- :: ++ forward-update - |= =vase + ~/ %forward-update + |= =cage ^- (list card:agent:gall) - =/ rids=(list resource) (resource-for-update vase) - =| cards=(list card:agent:gall) - |- - ?~ rids cards + =- lis + =/ vas=vase + q:(convert-to:ver cage) + %+ roll (resource-for-update q.cage) + |= [rid=resource [lis=(list card:agent:gall) tf-vas=(unit vase)]] + ^- [(list card:agent:gall) (unit vase)] =/ =path - resource+(en-path:resource i.rids) - =/ =wire - (make-wire resource+(en-path:resource i.rids)) - =/ dap=term - ?:(=(our.bowl entity.i.rids) store-name.config dap.bowl) - %_ $ - rids t.rids + resource+(en-path:resource rid) + =* ship entity.rid + =/ out=(pair (list card:agent:gall) (unit vase)) + ?. =(our.bowl ship) + :: do not transform before forwarding + :: + ``vas + :: use cached transform + :: + ?^ tf-vas `tf-vas + :: transform before poking store + :: + (transform-proxy-update:og vas) + ~| "forwarding failed during transform. mark: {} rid: {}" + ?> ?=(^ q.out) + :_ q.out + :_ (weld lis p.out) + =/ =wire (make-wire path) + =- [%pass wire %agent - %poke [current-version:ver u.q.out]] + :- ship + ?. =(our.bowl ship) + :: forward to host + :: + dap.bowl + :: poke our store :: - cards - %+ snoc cards - [%pass wire %agent [entity.i.rids dap] %poke update-mark.config vase] - == + store-name.config + :: + ++ ver-from-path + |= =path + =/ extra=^path + (slag 5 path) + ?> ?=(^ extra) + (slav %ud i.extra) :: ++ resource-for-update + ~/ %resource-for-update |= =vase ^- (list resource) %~ tap in diff --git a/pkg/arvo/lib/server.hoon b/pkg/arvo/lib/server.hoon index 1f450c3e50..52f2180303 100644 --- a/pkg/arvo/lib/server.hoon +++ b/pkg/arvo/lib/server.hoon @@ -39,10 +39,10 @@ ~! +:*handler (handler inbound-request) :: - =/ redirect=cord - %- crip - "/~/login?redirect={(trip url.request.inbound-request)}" - [[307 ['location' redirect]~] ~] + =- [[307 ['location' -]~] ~] + %^ cat 3 + '/~/login?redirect=' + url.request.inbound-request :: :: +require-authorization-simple: :: redirect to the login page when unauthenticated @@ -56,10 +56,10 @@ ~! this simple-payload :: - =/ redirect=cord - %- crip - "/~/login?redirect={(trip url.request.inbound-request)}" - [[307 ['location' redirect]~] ~] + =- [[307 ['location' -]~] ~] + %^ cat 3 + '/~/login?redirect=' + url.request.inbound-request :: ++ give-simple-payload |= [eyre-id=@ta =simple-payload:http] @@ -86,36 +86,52 @@ :_ `octs [200 [['content-type' 'text/html'] ?:(cache [max-1-wk ~] ~)]] :: - ++ js-response - |= =octs - ^- simple-payload:http - [[200 [['content-type' 'text/javascript'] max-1-da ~]] `octs] - :: - ++ json-response - |= =json - ^- simple-payload:http - [[200 ['content-type' 'application/json']~] `(json-to-octs json)] - :: ++ css-response + =| cache=? |= =octs ^- simple-payload:http - [[200 [['content-type' 'text/css'] max-1-da ~]] `octs] + :_ `octs + [200 [['content-type' 'text/css'] ?:(cache [max-1-wk ~] ~)]] :: - ++ manx-response - |= man=manx + ++ js-response + =| cache=? + |= =octs ^- simple-payload:http - [[200 ['content-type' 'text/html']~] `(manx-to-octs man)] + :_ `octs + [200 [['content-type' 'text/javascript'] ?:(cache [max-1-wk ~] ~)]] :: ++ png-response + =| cache=? |= =octs ^- simple-payload:http - [[200 [['content-type' 'image/png'] max-1-wk ~]] `octs] + :_ `octs + [200 [['content-type' 'image/png'] ?:(cache [max-1-wk ~] ~)]] + :: + ++ ico-response + |= =octs + ^- simple-payload:http + [[200 [['content-type' 'image/x-icon'] max-1-wk ~]] `octs] :: ++ woff2-response + =| cache=? |= =octs ^- simple-payload:http [[200 [['content-type' 'font/woff2'] max-1-wk ~]] `octs] :: + ++ json-response + =| cache=_| + |= =json + ^- simple-payload:http + :_ `(json-to-octs json) + [200 [['content-type' 'application/json'] ?:(cache [max-1-da ~] ~)]] + :: + ++ manx-response + =| cache=_| + |= man=manx + ^- simple-payload:http + :_ `(manx-to-octs man) + [200 [['content-type' 'text/html'] ?:(cache [max-1-da ~] ~)]] + :: ++ not-found ^- simple-payload:http [[404 ~] ~] @@ -123,10 +139,10 @@ ++ login-redirect |= =request:http ^- simple-payload:http - =/ redirect=cord - %- crip - "/~/login?redirect={(trip url.request)}" - [[307 ['location' redirect]~] ~] + =- [[307 ['location' -]~] ~] + %^ cat 3 + '/~/login?redirect=' + url.request :: ++ redirect |= redirect=cord diff --git a/pkg/arvo/lib/signatures.hoon b/pkg/arvo/lib/signatures.hoon index 7c4de27add..4206638e83 100644 --- a/pkg/arvo/lib/signatures.hoon +++ b/pkg/arvo/lib/signatures.hoon @@ -30,7 +30,7 @@ ?~ lyf %.y =+ %: jael-scry ,deed=[a=life b=pass c=(unit @ux)] - our %deed now /(scot %p q.signature)/(scot %ud p.signature) + our %deed now /(scot %p q.signature)/(scot %ud r.signature) == ?. =(a.deed r.signature) %.y :: verify signature from ship at life diff --git a/pkg/arvo/lib/strandio.hoon b/pkg/arvo/lib/strandio.hoon index a8b52ede1f..7ccbfcbca7 100644 --- a/pkg/arvo/lib/strandio.hoon +++ b/pkg/arvo/lib/strandio.hoon @@ -719,7 +719,7 @@ (pure:m tid) :: +$ thread-result - (each vase [term (list tang)]) + (each vase [term tang]) :: ++ await-thread |= [file=term args=vase] @@ -727,14 +727,14 @@ ^- form:m ;< =bowl:spider bind:m get-bowl =/ tid (scot %ta (cat 3 'strand_' (scot %uv (sham file eny.bowl)))) - =/ tid (scot %ta (cat 3 'strand_' (scot %uv (sham file eny.bowl)))) =/ poke-vase !>([`tid.bowl `tid file args]) ;< ~ bind:m (watch-our /awaiting/[tid] %spider /thread-result/[tid]) ;< ~ bind:m (poke-our %spider %spider-start poke-vase) ;< ~ bind:m (sleep ~s0) :: wait for thread to start ;< =cage bind:m (take-fact /awaiting/[tid]) + ;< ~ bind:m (take-kick /awaiting/[tid]) ?+ p.cage ~|([%strange-thread-result p.cage file tid] !!) %thread-done (pure:m %& q.cage) - %thread-fail (pure:m %| !<([term (list tang)] q.cage)) + %thread-fail (pure:m %| !<([term tang] q.cage)) == -- diff --git a/pkg/arvo/lib/versioning.hoon b/pkg/arvo/lib/versioning.hoon new file mode 100644 index 0000000000..bddb10d574 --- /dev/null +++ b/pkg/arvo/lib/versioning.hoon @@ -0,0 +1,55 @@ +/+ agentio +|_ [=bowl:gall root=mark version=@ud min=@ud] ++* io ~(. agentio bowl) +++ is-root + |= =mark + ?~ (rush mark mark-parser) + %.n + %.y +:: +++ mark-parser + ;~(pfix (jest root) ;~(pose ;~(pfix hep dum:ag) (easy `@ud`0))) +:: +++ read-version + |= =mark + (rash mark mark-parser) +:: +++ append-version + |= ver=@ud + :((cury cat 3) root '-' (scot %ud ver)) +:: +++ current-version + ^- mark + (append-version version) +:: +++ supported + |= =mark + =/ ver + (read-version mark) + &((gte ver min) (lte ver version)) +:: +++ convert-to + |= [=mark =vase] + ^- cage + :- current-version + ?: =(mark current-version) + vase + ((tube-to mark) vase) +:: +++ tube-to + |= =mark + .^(tube:clay %cc (scry:io %home /[mark]/[current-version])) +:: +++ tube-from + |= =mark + .^(tube:clay %cc (scry:io %home /[current-version]/[mark])) +:: +++ convert-from + |= [=mark =vase] + ^- cage + :- mark + ?: =(mark current-version) + vase + ((tube-from mark) vase) +-- + diff --git a/pkg/arvo/mar/contact/update-0.hoon b/pkg/arvo/mar/contact/update-0.hoon new file mode 100644 index 0000000000..8a87d72340 --- /dev/null +++ b/pkg/arvo/mar/contact/update-0.hoon @@ -0,0 +1,17 @@ +/+ *contact-store +:: +|_ upd=update +++ grad %noun +++ grow + |% + ++ noun upd + ++ contact-update upd + ++ json (update:enjs upd) + -- +:: +++ grab + |% + ++ noun update + ++ json update:dejs + -- +-- diff --git a/pkg/arvo/mar/contact/update.hoon b/pkg/arvo/mar/contact/update.hoon index 733d30f48e..2a18a859dc 100644 --- a/pkg/arvo/mar/contact/update.hoon +++ b/pkg/arvo/mar/contact/update.hoon @@ -5,6 +5,7 @@ ++ grow |% ++ noun upd + ++ contact-update-0 upd ++ json (update:enjs upd) -- :: diff --git a/pkg/arvo/mar/demo/update-0.hoon b/pkg/arvo/mar/demo/update-0.hoon new file mode 100644 index 0000000000..615821ef18 --- /dev/null +++ b/pkg/arvo/mar/demo/update-0.hoon @@ -0,0 +1,16 @@ +/- *demo +:: +|_ upd=update +++ grad %noun +++ grow + |% + ++ noun upd + ++ demo-update-1 upd + ++ demo-update upd + -- +:: +++ grab + |% + ++ noun update + -- +-- diff --git a/pkg/arvo/mar/demo/update-1.hoon b/pkg/arvo/mar/demo/update-1.hoon new file mode 100644 index 0000000000..b495a4051e --- /dev/null +++ b/pkg/arvo/mar/demo/update-1.hoon @@ -0,0 +1,15 @@ +/- *demo +:: +|_ upd=update +++ grad %noun +++ grow + |% + ++ noun upd + ++ demo-update-0 upd + -- +:: +++ grab + |% + ++ noun update + -- +-- diff --git a/pkg/arvo/mar/demo/update.hoon b/pkg/arvo/mar/demo/update.hoon new file mode 100644 index 0000000000..f1a88f1eb6 --- /dev/null +++ b/pkg/arvo/mar/demo/update.hoon @@ -0,0 +1,16 @@ +/- *demo +:: +|_ upd=update +++ grad %noun +++ grow + |% + ++ noun upd + ++ demo-update-1 upd + ++ demo-update-0 upd + -- +:: +++ grab + |% + ++ noun update + -- +-- diff --git a/pkg/arvo/mar/graph/cache/hook.hoon b/pkg/arvo/mar/graph/cache/hook.hoon new file mode 100644 index 0000000000..1b9395a0f6 --- /dev/null +++ b/pkg/arvo/mar/graph/cache/hook.hoon @@ -0,0 +1,20 @@ +/- metadata=metadata-store, res=resource +|% ++$ cache-action + $% [%graph-to-mark (pair resource:res (unit mark))] + [%perm-marks (pair (pair mark @tas) tube:clay)] + [%transform-marks (pair mark tube:clay)] + == +-- +:: +|_ act=cache-action +++ grad %noun +++ grow + |% + ++ noun act + -- +++ grab + |% + ++ noun cache-action + -- +-- diff --git a/pkg/arvo/mar/graph/update-0.hoon b/pkg/arvo/mar/graph/update-0.hoon new file mode 100644 index 0000000000..a9e9458a2f --- /dev/null +++ b/pkg/arvo/mar/graph/update-0.hoon @@ -0,0 +1,18 @@ +/+ *graph-store +=* as-octs as-octs:mimes:html +:: +|_ upd=update:zero +++ grad %noun +++ grow + |% + ++ noun upd + ++ graph-update upd + ++ mime [/application/x-urb-graph-update (as-octs (jam upd))] + -- +:: +++ grab + |% + ++ noun update:zero + ++ mime |=([* =octs] ;;(update:zero (cue q.octs))) + -- +-- diff --git a/pkg/arvo/mar/graph/update-1.hoon b/pkg/arvo/mar/graph/update-1.hoon new file mode 100644 index 0000000000..be29d7e8fa --- /dev/null +++ b/pkg/arvo/mar/graph/update-1.hoon @@ -0,0 +1,17 @@ +/+ *graph-store +=* as-octs as-octs:mimes:html +:: +|_ upd=update:one +++ grad %noun +++ grow + |% + ++ noun upd + ++ mime [/application/x-urb-graph-update (as-octs (jam upd))] + -- +:: +++ grab + |% + ++ noun update:one + ++ mime |=([* =octs] ;;(update (cue q.octs))) + -- +-- diff --git a/pkg/arvo/mar/graph/update-2.hoon b/pkg/arvo/mar/graph/update-2.hoon new file mode 100644 index 0000000000..e6766edb51 --- /dev/null +++ b/pkg/arvo/mar/graph/update-2.hoon @@ -0,0 +1,19 @@ +/+ *graph-store +=* as-octs as-octs:mimes:html +:: +|_ upd=update +++ grad %noun +++ grow + |% + ++ noun upd + ++ json (update:enjs upd) + ++ mime [/application/x-urb-graph-update (as-octs (jam upd))] + -- +:: +++ grab + |% + ++ noun update + ++ json update:dejs + ++ mime |=([* =octs] ;;(update (cue q.octs))) + -- +-- diff --git a/pkg/arvo/mar/graph/update.hoon b/pkg/arvo/mar/graph/update.hoon index e6766edb51..0da7052b4f 100644 --- a/pkg/arvo/mar/graph/update.hoon +++ b/pkg/arvo/mar/graph/update.hoon @@ -1,19 +1,18 @@ /+ *graph-store =* as-octs as-octs:mimes:html :: -|_ upd=update +|_ upd=update:zero ++ grad %noun ++ grow |% ++ noun upd - ++ json (update:enjs upd) + ++ graph-update-0 upd ++ mime [/application/x-urb-graph-update (as-octs (jam upd))] -- :: ++ grab |% - ++ noun update - ++ json update:dejs - ++ mime |=([* =octs] ;;(update (cue q.octs))) + ++ noun update:zero + ++ mime |=([* =octs] ;;(update:zero (cue q.octs))) -- -- diff --git a/pkg/arvo/mar/graph/validator/chat.hoon b/pkg/arvo/mar/graph/validator/chat.hoon index 20c9c82c1d..53c10c9f0e 100644 --- a/pkg/arvo/mar/graph/validator/chat.hoon +++ b/pkg/arvo/mar/graph/validator/chat.hoon @@ -1,4 +1,4 @@ -/- *post, met=metadata-store +/- *post, met=metadata-store, graph=graph-store, hark=hark-graph-hook |_ i=indexed-post ++ grow |% @@ -6,17 +6,20 @@ :: ++ graph-permissions-add |= vip=vip-metadata:met + ^- permissions:graph ?+ index.p.i !! [@ ~] [%yes %yes %no] == :: ++ graph-permissions-remove |= vip=vip-metadata:met + ^- permissions:graph ?+ index.p.i !! [@ ~] [%self %self %no] == :: ++ notification-kind + ^- (unit notif-kind:hark) ?+ index.p.i ~ [@ ~] `[%message [0 1] %count %none] == diff --git a/pkg/arvo/mar/graph/validator/link.hoon b/pkg/arvo/mar/graph/validator/link.hoon index 6026867452..d877018648 100644 --- a/pkg/arvo/mar/graph/validator/link.hoon +++ b/pkg/arvo/mar/graph/validator/link.hoon @@ -1,4 +1,4 @@ -/- *post, met=metadata-store +/- *post, met=metadata-store, graph=graph-store, hark=hark-graph-hook |_ i=indexed-post ++ grow |% @@ -6,6 +6,7 @@ :: ++ graph-permissions-add |= vip=vip-metadata:met + ^- permissions:graph =/ reader ?=(%reader-comments vip) ?+ index.p.i !! @@ -16,6 +17,7 @@ :: ++ graph-permissions-remove |= vip=vip-metadata:met + ^- permissions:graph =/ reader ?=(%reader-comments vip) ?+ index.p.i !! @@ -25,10 +27,10 @@ == :: ++ notification-kind + ^- (unit notif-kind:hark) ?+ index.p.i ~ [@ ~] `[%link [0 1] %each %children] [@ @ %1 ~] `[%comment [1 2] %count %siblings] - [@ @ @ ~] `[%edit-comment [1 2] %none %none] == :: ++ transform-add-nodes @@ -53,7 +55,7 @@ :: top-level link post; title and url :: [@ ~] - ?> ?=([[%text @] [%url @] ~] contents.p.ip) + ?> ?=([[%text @] $%([%url @] [%reference *]) ~] contents.p.ip) ip :: :: comment on link post; container structure diff --git a/pkg/arvo/mar/graph/validator/post.hoon b/pkg/arvo/mar/graph/validator/post.hoon new file mode 100644 index 0000000000..a12e62e721 --- /dev/null +++ b/pkg/arvo/mar/graph/validator/post.hoon @@ -0,0 +1,53 @@ +/- *post, met=metadata-store, graph=graph-store, hark=hark-graph-hook +|_ i=indexed-post +++ grow + |% + ++ noun i + ++ graph-permissions-add + |= vip=vip-metadata:met + ^- permissions:graph + ?. ?=([@ ~] index.p.i) + [%yes %yes %yes] + ?+ vip [%yes %yes %yes] + %admin-feed [%yes %no %no] + %host-feed [%no %no %no] + == + :: + ++ graph-permissions-remove + |= vip=vip-metadata:met + ^- permissions:graph + [%yes %self %self] + :: +notification-kind: don't track unreads, notify on replies + :: + ++ notification-kind + ^- (unit notif-kind:hark) + =/ len (lent index.p.i) + =/ =mode:hark + ?:(=(1 len) %count %none) + `[%post [(dec len) len] mode %children] + :: + ++ transform-add-nodes + |= [=index =post =atom was-parent-modified=?] + ^- [^index ^post] + =- [- post(index -)] + ?~ index !! + ?: ?=([@ ~] index) + [atom ~] + ?: was-parent-modified + ~|(%cannot-submit-parents-with-prepopulated-children !!) + =/ ind=^index index + (snoc (snip ind) atom) + -- +++ grab + |% + :: +noun: validate post + :: + ++ noun + |= p=* + =/ ip ;;(indexed-post p) + ?> ?=(^ contents.p.ip) + ip + -- +:: +++ grad %noun +-- diff --git a/pkg/arvo/mar/graph/validator/publish.hoon b/pkg/arvo/mar/graph/validator/publish.hoon index f888d06514..c8da4405ca 100644 --- a/pkg/arvo/mar/graph/validator/publish.hoon +++ b/pkg/arvo/mar/graph/validator/publish.hoon @@ -1,10 +1,11 @@ -/- *post, met=metadata-store +/- *post, met=metadata-store, graph=graph-store, hark=hark-graph-hook |_ i=indexed-post ++ grow |% ++ noun i ++ graph-permissions-add |= vip=vip-metadata:met + ^- permissions:graph ?+ index.p.i !! [@ ~] [%yes %yes %no] :: new note [@ %1 @ ~] [%self %self %no] @@ -14,6 +15,7 @@ :: ++ graph-permissions-remove |= vip=vip-metadata:met + ^- permissions:graph ?+ index.p.i !! [@ ~] [%yes %self %self] [@ %1 @ @ ~] [%yes %self %self] @@ -24,11 +26,10 @@ :: ignore all containers, only notify on content :: ++ notification-kind + ^- (unit notif-kind:hark) ?+ index.p.i ~ [@ %1 %1 ~] `[%note [0 1] %each %children] - [@ %1 @ ~] `[%edit-note [0 1] %none %none] [@ %2 @ %1 ~] `[%comment [1 3] %count %siblings] - [@ %2 @ @ ~] `[%edit-comment [1 3] %none %none] == :: ++ transform-add-nodes @@ -54,7 +55,7 @@ -- ++ grab |% - :: +noun: validate publish post + :: +noun: validate publish note :: ++ noun |= p=* diff --git a/pkg/arvo/mar/group/update-0.hoon b/pkg/arvo/mar/group/update-0.hoon new file mode 100644 index 0000000000..e50fe62c4e --- /dev/null +++ b/pkg/arvo/mar/group/update-0.hoon @@ -0,0 +1,17 @@ +/+ *group-store +|_ upd=update +++ grad %noun +++ grow + |% + ++ noun upd + ++ group-update upd + ++ json + %+ frond:enjs:format 'groupUpdate' + (update:enjs upd) + -- +++ grab + |% + ++ noun update + ++ json update:dejs + -- +-- diff --git a/pkg/arvo/mar/group/update.hoon b/pkg/arvo/mar/group/update.hoon index d4ea2abd95..5159fae4a6 100644 --- a/pkg/arvo/mar/group/update.hoon +++ b/pkg/arvo/mar/group/update.hoon @@ -4,6 +4,7 @@ ++ grow |% ++ noun upd + ++ group-update-0 upd ++ json %+ frond:enjs:format 'groupUpdate' (update:enjs upd) diff --git a/pkg/arvo/mar/ico.hoon b/pkg/arvo/mar/ico.hoon new file mode 100644 index 0000000000..e862b9b9a9 --- /dev/null +++ b/pkg/arvo/mar/ico.hoon @@ -0,0 +1,12 @@ +|_ dat=@ +++ grow + |% + ++ mime [/image/x-icon (as-octs:mimes:html dat)] + -- +++ grab + |% + ++ mime |=([p=mite q=octs] q.q) + ++ noun @ + -- +++ grad %mime +-- diff --git a/pkg/arvo/mar/metadata/update-0.hoon b/pkg/arvo/mar/metadata/update-0.hoon new file mode 100644 index 0000000000..7b79126545 --- /dev/null +++ b/pkg/arvo/mar/metadata/update-0.hoon @@ -0,0 +1,14 @@ +/+ store=metadata-store +|_ =update:zero:store +++ grad %noun +++ grow + |% + ++ noun update + ++ metadata-update update + -- +:: +++ grab + |% + ++ noun update:zero:store + -- +-- diff --git a/pkg/arvo/mar/metadata/update-1.hoon b/pkg/arvo/mar/metadata/update-1.hoon new file mode 100644 index 0000000000..36036d1098 --- /dev/null +++ b/pkg/arvo/mar/metadata/update-1.hoon @@ -0,0 +1,15 @@ +/+ store=metadata-store +|_ =update:store +++ grad %noun +++ grow + |% + ++ noun update + ++ json (update:enjs:store update) + -- +:: +++ grab + |% + ++ noun update:store + ++ json action:dejs:store + -- +-- diff --git a/pkg/arvo/mar/metadata/update.hoon b/pkg/arvo/mar/metadata/update.hoon index 9f894cf841..4554864feb 100644 --- a/pkg/arvo/mar/metadata/update.hoon +++ b/pkg/arvo/mar/metadata/update.hoon @@ -1,15 +1,14 @@ /+ store=metadata-store -|_ =update:store +|_ =update:zero:store ++ grad %noun ++ grow |% ++ noun update - ++ json (update:enjs:store update) + ++ metadata-update-0 update -- :: ++ grab |% - ++ noun update:store - ++ json action:dejs:store + ++ noun update:zero:store -- -- diff --git a/pkg/arvo/mar/version.hoon b/pkg/arvo/mar/version.hoon new file mode 100644 index 0000000000..2ea3b405f4 --- /dev/null +++ b/pkg/arvo/mar/version.hoon @@ -0,0 +1,12 @@ +|_ ver=@ud +++ grad %noun +++ grow + |% + ++ noun ver + ++ json (numb:enjs:format ver) + -- +++ grab + |% + ++ noun @ud + -- +-- diff --git a/pkg/arvo/sur/demo.hoon b/pkg/arvo/sur/demo.hoon new file mode 100644 index 0000000000..47592fbfe7 --- /dev/null +++ b/pkg/arvo/sur/demo.hoon @@ -0,0 +1,10 @@ +/+ resource +|% ++$ update + $~ [%add *resource 0] + $% [%add p=resource q=@ud] + [%sub p=resource q=@ud] + [%ini p=resource ~] + [%run p=resource q=(list update)] + == +-- diff --git a/pkg/arvo/sur/graph-store.hoon b/pkg/arvo/sur/graph-store.hoon index 845885ed16..2217038f0b 100644 --- a/pkg/arvo/sur/graph-store.hoon +++ b/pkg/arvo/sur/graph-store.hoon @@ -1,29 +1,17 @@ /- *post |% -:: -+$ permissions - [admin=permission-level writer=permission-level reader=permission-level] -:: -:: $permission-level: levels of permissions in increasing order -:: -:: %no: May not add/remove node -:: %self: May only nodes beneath nodes that were added by -:: the same pilot, may remove nodes that the pilot 'owns' -:: %yes: May add a node or remove node -+$ permission-level - ?(%no %self %yes) +$ graph ((mop atom node) gth) +$ marked-graph [p=graph q=(unit mark)] :: -+$ node [=post children=internal-graph] ++$ maybe-post (each post hash) ++$ node [post=maybe-post children=internal-graph] +$ graphs (map resource marked-graph) :: -+$ tag-queries (jug term resource) ++$ tag-queries (jug term uid) :: +$ update-log ((mop time logged-update) gth) +$ update-logs (map resource update-log) :: -:: +$ internal-graph $~ [%empty ~] $% [%graph p=graph] @@ -35,31 +23,28 @@ =tag-queries =update-logs archive=graphs - validators=(set mark) + ~ == :: -+$ update - $% [%0 p=time q=update-0] - == ++$ update [p=time q=action] :: -+$ logged-update - $% [%0 p=time q=logged-update-0] - == ++$ logged-update [p=time q=logged-action] + :: -+$ logged-update-0 ++$ logged-action $% [%add-graph =resource =graph mark=(unit mark) overwrite=?] [%add-nodes =resource nodes=(map index node)] - [%remove-nodes =resource indices=(set index)] + [%remove-posts =resource indices=(set index)] [%add-signatures =uid =signatures] [%remove-signatures =uid =signatures] == :: -+$ update-0 - $% logged-update-0 ++$ action + $% logged-action [%remove-graph =resource] :: - [%add-tag =term =resource] - [%remove-tag =term =resource] + [%add-tag =term =uid] + [%remove-tag =term =uid] :: [%archive-graph =resource] [%unarchive-graph =resource] @@ -71,4 +56,149 @@ [%tags tags=(set term)] [%tag-queries =tag-queries] == +:: ++$ permissions + [admin=permission-level writer=permission-level reader=permission-level] +:: +:: $permission-level: levels of permissions in increasing order +:: +:: %no: May not add/remove node +:: %self: May only nodes beneath nodes that were added by +:: the same pilot, may remove nodes that the pilot 'owns' +:: %yes: May add a node or remove node ++$ permission-level + ?(%no %self %yes) +:: +:: %graph-store types version 1 +:: +++ one + |% + ++ orm ((ordered-map atom node) gth) + ++ orm-log ((ordered-map time logged-update) gth) + :: + +$ graph ((mop atom node) gth) + +$ marked-graph [p=graph q=(unit mark)] + :: + +$ node [=post children=internal-graph] + +$ graphs (map resource marked-graph) + :: + +$ tag-queries (jug term resource) + :: + +$ update-log ((mop time logged-update) gth) + +$ update-logs (map resource update-log) + :: + +$ internal-graph + $~ [%empty ~] + $% [%graph p=graph] + [%empty ~] + == + :: + +$ network + $: =graphs + =tag-queries + =update-logs + archive=graphs + validators=(set mark) + == + :: + +$ update [p=time q=action] + :: + +$ logged-update [p=time q=logged-action] + :: + +$ logged-action + $% [%add-graph =resource =graph mark=(unit mark) overwrite=?] + [%add-nodes =resource nodes=(map index node)] + [%remove-nodes =resource indices=(set index)] + [%add-signatures =uid =signatures] + [%remove-signatures =uid =signatures] + == + :: + +$ action + $% logged-action + [%remove-graph =resource] + :: + [%add-tag =term =resource] + [%remove-tag =term =resource] + :: + [%archive-graph =resource] + [%unarchive-graph =resource] + [%run-updates =resource =update-log] + :: + :: NOTE: cannot be sent as pokes + :: + [%keys =resources] + [%tags tags=(set term)] + [%tag-queries =tag-queries] + == + -- +:: +:: %graph-store types version 0 +:: +++ zero + =< [. post-zero] + =, post-zero + |% + ++ orm ((ordered-map atom node) gth) + ++ orm-log ((ordered-map time logged-update) gth) + :: + +$ graph ((mop atom node) gth) + +$ marked-graph [p=graph q=(unit mark)] + :: + +$ node [=post children=internal-graph] + +$ graphs (map resource marked-graph) + :: + +$ tag-queries (jug term resource) + :: + +$ update-log ((mop time logged-update) gth) + +$ update-logs (map resource update-log) + :: + :: + +$ internal-graph + $~ [%empty ~] + $% [%graph p=graph] + [%empty ~] + == + :: + +$ network + $: =graphs + =tag-queries + =update-logs + archive=graphs + validators=(set mark) + == + :: + +$ update + $% [%0 p=time q=update-0] + == + :: + +$ logged-update + $% [%0 p=time q=logged-update-0] + == + :: + +$ logged-update-0 + $% [%add-graph =resource =graph mark=(unit mark) overwrite=?] + [%add-nodes =resource nodes=(map index node)] + [%remove-nodes =resource indices=(set index)] + [%add-signatures =uid =signatures] + [%remove-signatures =uid =signatures] + == + :: + +$ update-0 + $% logged-update-0 + [%remove-graph =resource] + :: + [%add-tag =term =resource] + [%remove-tag =term =resource] + :: + [%archive-graph =resource] + [%unarchive-graph =resource] + [%run-updates =resource =update-log] + :: + :: NOTE: cannot be sent as pokes + :: + [%keys =resources] + [%tags tags=(set term)] + [%tag-queries =tag-queries] + == + -- -- diff --git a/pkg/arvo/sur/graph-view.hoon b/pkg/arvo/sur/graph-view.hoon index 939ac433a7..42818ac38b 100644 --- a/pkg/arvo/sur/graph-view.hoon +++ b/pkg/arvo/sur/graph-view.hoon @@ -1,4 +1,4 @@ -/- *group, store=graph-store +/- *group, store=graph-store, met=metadata-store /+ resource ^? |% @@ -43,6 +43,8 @@ [%forward rid=resource =update:store] [%eval =cord] [%pending-indices pending=(map hash:store index:store)] + [%create-group-feed group=resource vip=vip-metadata:met] + [%disable-group-feed group=resource] == -- diff --git a/pkg/arvo/sur/group-view.hoon b/pkg/arvo/sur/group-view.hoon index 549ae54307..82a0a51a4a 100644 --- a/pkg/arvo/sur/group-view.hoon +++ b/pkg/arvo/sur/group-view.hoon @@ -2,6 +2,13 @@ ^? |% :: ++$ request + $: hidden=? + started=time + =ship + =progress + == +:: +$ action $% :: host side [%create name=term =policy title=@t description=@t] @@ -11,6 +18,8 @@ [%leave =resource] :: [%invite =resource ships=(set ship) description=@t] + :: pending ops + [%hide =resource] == :: @@ -21,7 +30,9 @@ ?(%no-perms %strange %done) :: +$ update - $% [%initial initial=(map resource progress)] + $% [%initial initial=(map resource request)] + [%started =resource =request] [%progress =resource =progress] + [%hide =resource] == -- diff --git a/pkg/arvo/sur/hark-store.hoon b/pkg/arvo/sur/hark-store.hoon index e9e1abbaef..82a966f4e7 100644 --- a/pkg/arvo/sur/hark-store.hoon +++ b/pkg/arvo/sur/hark-store.hoon @@ -33,7 +33,7 @@ == :: +$ contents - $% [%graph =(list post:post)] + $% [%graph =(list post:post-zero:post)] [%group =(list group-contents)] [%chat =(list envelope:chat-store)] == @@ -75,7 +75,7 @@ [date=@da read=? =contents] :: +$ contents - $% [%graph =(list post:post)] + $% [%graph =(list post:post-zero:post)] [%group =(list group-contents)] == :: @@ -90,6 +90,38 @@ :: -- :: +++ state-three + =< state + |% + +$ state + $: unreads-each=(jug stats-index index:graph-store) + unreads-count=(map stats-index @ud) + last-seen=(map stats-index @da) + =notifications + archive=notifications + current-timebox=@da + dnd=_| + == + :: + ++ orm + ((ordered-map @da timebox) gth) + :: + +$ notification + [date=@da read=? =contents] + :: + +$ contents + $% [%graph =(list post:post-zero:post)] + [%group =(list group-contents)] + == + :: + +$ timebox + (map index notification) + :: + +$ notifications + ((mop @da timebox) gth) + :: + -- +:: +$ index $% $: %graph group=resource @@ -150,7 +182,7 @@ [index notification] :: +$ stats - [notifications=@ud =unreads last-seen=@da] + [notifications=(set [time index]) =unreads last-seen=@da] :: +$ unreads $% [%count num=@ud] diff --git a/pkg/arvo/sur/metadata-store.hoon b/pkg/arvo/sur/metadata-store.hoon index 0f788933a1..d568c13ece 100644 --- a/pkg/arvo/sur/metadata-store.hoon +++ b/pkg/arvo/sur/metadata-store.hoon @@ -25,18 +25,35 @@ :: %reader-comments: Allow readers to comment, regardless :: of whether they can write. (notebook, collections) :: %member-metadata: Allow members to add channels (groups) +:: %host-feed: Only host can post to group feed +:: %admin-feed: Only admins and host can post to group feed :: %$: No variation :: -+$ vip-metadata ?(%reader-comments %member-metadata %$) ++$ vip-metadata + $? %reader-comments + %member-metadata + %host-feed + %admin-feed + %$ + == +:: ++$ md-config + $~ [%empty ~] + $% [%group feed=(unit (unit md-resource))] + [%graph module=term] + [%empty ~] + == +:: +$ metadatum $: title=cord description=cord =color date-created=time creator=ship - module=term + config=md-config picture=url preview=? + hidden=? vip=vip-metadata == :: @@ -61,4 +78,38 @@ =metadatum == == +:: historical +++ zero + |% + :: + +$ association [group=resource =metadatum] + :: + +$ associations (map md-resource association) + :: + +$ metadatum + $: title=cord + description=cord + =color + date-created=time + creator=ship + module=term + picture=url + preview=? + vip=vip-metadata + == + :: + +$ update + $% [%add group=resource resource=md-resource =metadatum] + [%remove group=resource resource=md-resource] + [%initial-group group=resource =associations] + [%associations =associations] + $: %updated-metadata + group=resource + resource=md-resource + before=metadatum + =metadatum + == + == + :: + -- -- diff --git a/pkg/arvo/sur/observe-hook.hoon b/pkg/arvo/sur/observe-hook.hoon index a424c96229..60030ad5de 100644 --- a/pkg/arvo/sur/observe-hook.hoon +++ b/pkg/arvo/sur/observe-hook.hoon @@ -1,7 +1,14 @@ |% +$ observer [app=term =path thread=term] +$ action - $% [%watch =observer] + $% :: %gall actions + :: + [%watch =observer] [%ignore =observer] + :: + :: %clay actions + :: + [%warm-cache-all ~] + [%cool-cache-all ~] == -- diff --git a/pkg/arvo/sur/post.hoon b/pkg/arvo/sur/post.hoon index 4855a3a684..6f4b33c167 100644 --- a/pkg/arvo/sur/post.hoon +++ b/pkg/arvo/sur/post.hoon @@ -26,13 +26,37 @@ contents=(list content) == :: ++$ reference + $% [%graph group=resource =uid] + [%group group=resource] + == +:: +$ content $% [%text text=cord] [%mention =ship] [%url url=cord] [%code expression=cord output=(list tank)] - [%reference =uid] - :: TODO: maybe use a cask? - ::[%cage =cage] + [%reference =reference] == +:: +++ post-zero + |% + :: + +$ content + $% [%text text=cord] + [%mention =ship] + [%url url=cord] + [%code expression=cord output=(list tank)] + [%reference =uid] + == + :: + +$ post + $: author=ship + =index + time-sent=time + contents=(list content) + hash=(unit hash) + =signatures + == + -- -- diff --git a/pkg/arvo/sys/lull.hoon b/pkg/arvo/sys/lull.hoon index fce261bb4a..feaf17a92c 100644 --- a/pkg/arvo/sys/lull.hoon +++ b/pkg/arvo/sys/lull.hoon @@ -1005,7 +1005,7 @@ |= suffix=@tas ^- (list path) =/ parser - (most hep (cook crip ;~(plug low (star ;~(pose low nud))))) + (most hep (cook crip ;~(plug ;~(pose low nud) (star ;~(pose low nud))))) =/ torn=(list @tas) (fall (rush suffix parser) ~[suffix]) %- flop |- ^- (list (list @tas)) diff --git a/pkg/arvo/sys/vane/eyre.hoon b/pkg/arvo/sys/vane/eyre.hoon index b509e71314..998bd659a3 100644 --- a/pkg/arvo/sys/vane/eyre.hoon +++ b/pkg/arvo/sys/vane/eyre.hoon @@ -249,6 +249,7 @@ font-family: "Source Code Pro"; src: url("https://storage.googleapis.com/media.urbit.org/fonts/scp-regular.woff"); font-weight: 400; + font-display: swap; } :root { --red05: rgba(255,65,54,0.05); diff --git a/pkg/arvo/ted/diff.hoon b/pkg/arvo/ted/diff.hoon index b69fcb72d1..e74caa15c5 100644 --- a/pkg/arvo/ted/diff.hoon +++ b/pkg/arvo/ted/diff.hoon @@ -1,30 +1,299 @@ /- spider /+ strandio =, strand=strand:spider +=, clay ^- thread:spider |= arg=vase =/ m (strand ,vase) ^- form:m |^ -=+ !<([~ =a=path =b=path] arg) -=/ a-mark=mark -:(flop a-path) -=/ b-mark=mark -:(flop b-path) -?. =(a-mark b-mark) - (strand-fail:strandio %files-not-same-type ~) +:: workaround to make the shallow flag optional. if it's specified we +:: do require a non-empty path - however this shouldn't be called with +:: empty paths to begin with. +=+ !<([~ =a=path b=$~([/hi &] $^([(lest @ta) flag] path))] arg) +=/ [b-path=path shallow=flag] ?:(?=([^ *] b) b [`path`b |]) =/ a-beam (need (de-beam a-path)) -;< =a=cage bind:m (get-file a-path) -;< =b=cage bind:m (get-file b-path) -;< =dais:clay bind:m (build-mark:strandio -.a-beam a-mark) -(pure:m (~(diff dais q.a-cage) q.b-cage)) +=/ b-beam (need (de-beam b-path)) +;< a-dome=dome bind:m (get-from-clay a-beam dome %v) +;< b-dome=dome bind:m (get-from-clay b-beam dome %v) +;< diffs=(list diff-type) bind:m (diff-beams a-beam b-beam) +%- pure:m +!> ^- tang +:: our tang is built in reverse order +%- flop +?: shallow + (format-shallow diffs a-beam b-beam) +(format-deep diffs a-beam b-beam) +:: +:: $diff-type: type for diffs produced. +:: 1. %txt-diff is a standard diff (i.e. for hoon and txt files) +:: 2. %directory-diff shows unique files between two directories +:: 3. %other is for files that don't use txt-diff - we just take +:: the mug of the files ++$ diff-type + :: paths of the diffed beams + $: a=path + b=path + $% [%txt-diff diff=(urge cord)] + [%directory-diff p=(list path) q=(list path)] + [%other p=@ q=@] + == + == +:: +diff-is-empty: check if a diff is empty (for two identical files) +:: +++ diff-is-empty + |= d=diff-type + ^- flag + ?: ?=([%txt-diff *] +.+.d) + :: levy produces & on empty lists + %+ levy + diff.d + |= u=(unce cord) + ^- flag + -:u + ?: ?=([%directory-diff *] +.+.d) + =(p.d q.d) + =(p.d q.d) +:: +get-file: retrieve a cage of a file from clay :: ++ get-file - |= =path + |= =beam =/ m (strand ,cage) ^- form:m - =/ beam (need (de-beam path)) ;< =riot:clay bind:m (warp:strandio p.beam q.beam ~ %sing %x r.beam s.beam) ?~ riot - (strand-fail:strandio %file-not-found >path< ~) + (strand-fail:strandio %file-not-found >s.beam< ~) (pure:m r.u.riot) +:: +get-from-clay: retrieve other data from clay based on care +:: +++ get-from-clay + |* [=beam mol=mold =care] + =/ m (strand ,mol) + ^- form:m + ;< =riot:clay bind:m + (warp:strandio p.beam q.beam ~ %sing care r.beam s.beam) + ?~ riot + (strand-fail:strandio %file-not-found >s.beam< ~) + (pure:m !<(mol q.r.u.riot)) +:: +diff-beams: recursively diff two beams. produces a vase +:: of (list diff-type) +:: +++ diff-beams + =< + |= [a=beam b=beam] + =/ m (strand ,(list diff-type)) + ^- form:m + ;< hash-a=@ bind:m (get-from-clay a @ %z) + ;< hash-b=@ bind:m (get-from-clay b @ %z) + :: if the recursive hashes for each beam are the same we bail early + ?: =(hash-a hash-b) + (pure:m *(list diff-type)) + ;< a-arch=arch bind:m (get-from-clay a arch %y) + ;< b-arch=arch bind:m (get-from-clay b arch %y) + ;< file-diff=(unit diff-type) bind:m (diff-file-contents a a-arch b b-arch) + :: get distinct files along with shared files + =/ a-keys=(set @t) ~(key by dir.a-arch) + =/ b-keys=(set @t) ~(key by dir.b-arch) + :: unique children + =/ a-unique-children=(set @t) (~(dif in a-keys) b-keys) + =/ b-unique-children=(set @t) (~(dif in b-keys) a-keys) + ;< a-unique=(list path) bind:m (format-unique-children a a-arch a-unique-children) + ;< b-unique=(list path) bind:m (format-unique-children b b-arch b-unique-children) + =/ unique-diff=diff-type [s.a s.b %directory-diff a-unique b-unique] + :: shared children + =/ find-common-diffs + |. + ^- form:m + =| acc=(list diff-type) + =/ common-children=(list @t) ~(tap in (~(int in a-keys) b-keys)) + |- + =* loop $ + ^- form:m + ?~ common-children + (pure:m acc) + =/ child=@t i.common-children + =/ new-a=beam a(s (snoc s.a child)) + =/ new-b=beam b(s (snoc s.b child)) + ;< diffs=(list diff-type) bind:m + (diff-beams new-a new-b) + :: ;< introduces another $ so we use "loop" instead. + %= loop + acc (weld diffs acc) + common-children t.common-children + == + ;< common-diffs=(list diff-type) bind:m (find-common-diffs) + %- pure:m + ^- (list diff-type) + %+ skip + ;: weld + (drop file-diff) + [unique-diff ~] + common-diffs + == + diff-is-empty + |% + :: +format-unique-children: produce list of paths representing + :: files that are unique within a directory. + :: + ++ format-unique-children + |= [bem=beam ark=arch children=(set @t)] + =/ m (strand ,(list path)) + ^- form:m + =/ children=(list @t) ~(tap in children) + =| acc=(list path) + |- + =* loop $ + ^- form:m + ?~ children + (pure:m acc) + :: the %t care gives all paths with the specified prefix + ;< res=(list path) bind:m (get-from-clay bem(s (snoc s.bem i.children)) (list path) %t) + %= loop + acc (weld res acc) + children t.children + == + :: +diff-file-contents: diff two files at specified beams, + :: producing a vase of (unit diff-type) + ++ diff-file-contents + =< + |= [a=beam a-arch=arch b=beam b-arch=arch] + =/ m (strand ,(unit diff-type)) + ^- form:m + ?: =(fil.a-arch fil.b-arch) + (pure:m *(unit diff-type)) + ?~ fil.a-arch + :: only b has contents + %- pure:m + ^- (unit diff-type) + %- some + :^ s.a + s.b + %txt-diff + :_ ~ + ^- (unce cord) + :+ %| + ~ + ~[(format-file-content-missing s.b q.b)] + ?~ fil.b-arch + :: only a has contents + %- pure:m + ^- (unit diff-type) + %- some + :^ s.a + s.b + %txt-diff + :_ ~ + ^- (unce cord) + :+ %| + ~[(format-file-content-missing s.a q.a)] + ~ + :: have two file contents - check that they have + :: the same mark. + =/ mar=mark -:(flop s.a) + ?: !=(mar -:(flop s.b)) + (strand-fail:strandio %files-not-same-type >s.a< >s.b< ~) + ;< =a=cage bind:m (get-file a) + ;< =b=cage bind:m (get-file b) + ;< =dais:clay bind:m (build-mark:strandio -.a mar) + :: for txt-diff we produce an actual diff with type (urge cord). + :: for all other marks we just take the mug) + %- pure:m + ?: =(form:dais %txt-diff) + ^- (unit diff-type) + %- some + :^ s.a + s.b + %txt-diff + !<((urge cord) (~(diff dais q.a-cage) q.b-cage)) + ^- (unit diff-type) + %- some + :^ s.a + s.b + %other + :: For some reason, vases for identical files on different desks + :: can sometimes have different types. for this reason, we only + :: take the mug of the data. + [(mug q.q.a-cage) (mug q.q.b-cage)] + |% + ++ format-file-content-missing + |= [p=path d=desk] + ^- cord + %- crip + ;: weld + "only " +

+ " in desk " + + " has file contents" + == + -- + -- +:: +format-beams: helper to combine two beams into a tank +:: +++ format-beams + |= [a=beam b=beam] + ^- tank + [%rose [" " ~ ~] ~[(smyt (en-beam a)) (smyt (en-beam b))]] +:: +format-directory-diff: helper for producing a tank based on +:: a %directory-diff +:: +++ format-directory-diff + |= [paths=(list path) =beam] + ^- tang + =/ prefix=tape (weld "only in " <(en-beam beam)>) + %+ turn + paths + |= p=path + ^- tank + [%rose [": " ~ ~] [leaf+prefix (smyt p) ~]] +:: +format-shallow: converts a list of diff-type generated +:: between desks a and b into a tang in a shallow manner (just +:: listing files that differ). +:: +++ format-shallow +|= [diffs=(list diff-type) a=beam b=beam] + ^- tang + %+ reel + diffs + |= [d=diff-type acc=tang] + ^- tang + ?: ?=([%txt-diff *] +.+.d) + [(format-beams a(s a.d) b(s b.d)) acc] + ?: ?=([%other *] +.+.d) + [(format-beams a(s a.d) b(s b.d)) acc] + ?: ?=([%directory-diff *] +.+.d) + ;: weld + (format-directory-diff p.d a) + (format-directory-diff q.d b) + acc + == + !! +:: +format-deep: converts a list of diff-type generated +:: between desks a and b into a tang in a deep manner (preserving +:: diff information for files) +++ format-deep +|= [diffs=(list diff-type) a=beam b=beam] + ^- tang + %+ reel + diffs + |= [d=diff-type acc=tang] + ^- tang + ?: ?=([%txt-diff *] +.+.d) + :+ (format-beams a(s a.d) b(s b.d)) + >diff.d< + acc + ?: ?=([%directory-diff *] +.+.d) + ;: weld + (format-directory-diff p.d a) + (format-directory-diff q.d b) + acc + == + ?: ?=([%other *] +.+.d) + =/ a-tank=tank (smyt (en-beam a(s a.d))) + =/ b-tank=tank (smyt (en-beam b(s b.d))) + :+ [%rose [" " "files " ~] ~[a-tank b-tank]] + [%rose [" and " "have mugs: " ~] ~[leaf+ leaf+]] + acc + !! -- diff --git a/pkg/arvo/ted/gcp/is-configured.hoon b/pkg/arvo/ted/gcp/is-configured.hoon index 02b67e60b3..8c32e7d41d 100644 --- a/pkg/arvo/ted/gcp/is-configured.hoon +++ b/pkg/arvo/ted/gcp/is-configured.hoon @@ -6,7 +6,6 @@ /- gcp, spider, settings /+ strandio =, strand=strand:spider -=, enjs:format ^- thread:spider |^ |= * @@ -22,7 +21,7 @@ == %- pure:m !> -%+ frond %gcp-configured +^- json b+has :: ++ has-settings diff --git a/pkg/arvo/ted/graph/add-nodes.hoon b/pkg/arvo/ted/graph/add-nodes.hoon index 3041eb61cd..e313eb1adc 100644 --- a/pkg/arvo/ted/graph/add-nodes.hoon +++ b/pkg/arvo/ted/graph/add-nodes.hoon @@ -10,7 +10,6 @@ ;< =update:store bind:m %+ scry:strandio update:store /gx/graph-store/graph/(scot %p entity.rid)/[name.rid]/noun - ?> ?=(%0 -.update) ?> ?=(%add-graph -.q.update) (pure:m graph.q.update) -- @@ -33,7 +32,7 @@ =/ hashes (nodes-to-pending-indices nodes.q.update) ;< ~ bind:m %^ poke-our %graph-push-hook - %graph-update + %graph-update-2 !>(update) (pure:m !>(`action:graph-view`[%pending-indices hashes])) :: @@ -76,7 +75,8 @@ ^- [index:store node:store] =* loop $ :- index - =* p post.node + ?> ?=(%& -.post.node) + =* p p.post.node =/ =hash:store =- `@ux`(sham -) :^ ?^ parent-hash @@ -86,12 +86,11 @@ time-sent.p contents.p %_ node - hash.post `hash + hash.p.post `hash :: - :: TODO: enable signing our own post as soon as we're ready - :: signatures.post - :: %- ~(gas in *signatures:store) - :: [(sign:sig our.bowl now.bowl hash)]~ + signatures.p.post + %- ~(gas in *signatures:store) + [(sign:sig our.bowl now.bowl hash)]~ :: children ?: ?=(%empty -.children.node) @@ -117,7 +116,8 @@ ?: ?=([@ ~] index) ~ =/ node (got-deep:gra graph (snip `(list atom)`index)) - hash.post.node + ?> ?=(%& -.post.node) + hash.p.post.node :: ++ nodes-to-pending-indices |= nodes=(map index:store node:store) @@ -126,6 +126,7 @@ %+ turn ~(tap by nodes) |= [=index:store =node:store] ^- [hash:store index:store] - ?> ?=(^ hash.post.node) - [u.hash.post.node index] + ?> ?=(%& -.post.node) + ?> ?=(^ hash.p.post.node) + [u.hash.p.post.node index] -- diff --git a/pkg/arvo/ted/graph/create-group-feed.hoon b/pkg/arvo/ted/graph/create-group-feed.hoon new file mode 100644 index 0000000000..83590f5928 --- /dev/null +++ b/pkg/arvo/ted/graph/create-group-feed.hoon @@ -0,0 +1,71 @@ +/- spider, + graph=graph-store, + met=metadata-store, + push-hook +/+ strandio, resource, graph-view +:: +=* strand strand:spider +=* poke poke:strandio +=* poke-our poke-our:strandio +:: +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +=+ !<([~ =action:graph-view] arg) +?> ?=(%create-group-feed -.action) +;< =bowl:spider bind:m get-bowl:strandio +?. =(our.bowl entity.group.action) + (strand-fail:strandio %bad-request ~) +=/ feed-rid=resource + :- entity.group.action + %- crip + %+ weld (trip name.group.action) + %+ weld "-" + (trip (scot %ud (mod eny.bowl 10.000))) +;< association=(unit association:met) bind:m + %+ scry:strandio (unit association:met) + %- zing + :~ /gx/metadata-store/metadata/groups + (en-path:resource group.action) + /noun + == +?~ association + ~|('No group exists, cannot make group feed.' !!) +=* metadatum metadatum.u.association +?> ?=(%group -.config.metadatum) +?> ?| ?=(~ feed.config.metadatum) + ?=([~ ~] feed.config.metadatum) + == +;< ~ bind:m + %+ poke-our %graph-store + :- %graph-update-2 + !> ^- update:graph + [now.bowl %add-graph feed-rid *graph:graph `%graph-validator-post %&] +;< ~ bind:m + (poke-our %graph-push-hook %push-hook-action !>([%add feed-rid])) +;< ~ bind:m + %+ poke-our %metadata-push-hook + :- %metadata-update-1 + !> ^- action:met + :^ %add + group.action + groups+group.action + metadatum(feed.config ``[%graph feed-rid]) +;< ~ bind:m + %+ poke-our %metadata-push-hook + :- %metadata-update-1 + !> ^- action:met + :^ %add + group.action + graph+feed-rid + %* . *metadatum:met + title 'Group Feed' + date-created now.bowl + creator our.bowl + config [%graph %post] + preview %.n + hidden %.y + vip vip.action + == +(pure:m !>(feed-rid)) diff --git a/pkg/arvo/ted/graph/create.hoon b/pkg/arvo/ted/graph/create.hoon index e437b08fdf..44f9cd098e 100644 --- a/pkg/arvo/ted/graph/create.hoon +++ b/pkg/arvo/ted/graph/create.hoon @@ -25,12 +25,12 @@ (poke-our %metadata-push-hook push-hook-act) ;< ~ bind:m %+ poke-our %group-store - :- %group-update + :- %group-update-0 !> ^- update:group-store [%add-group rid policy.associated %.y] ;< =bowl:spider bind:m get-bowl:strandio ;< ~ bind:m - (poke-our %group-store group-update+!>([%add-members rid (sy our.bowl ~)])) + (poke-our %group-store group-update-0+!>([%add-members rid (sy our.bowl ~)])) ;< ~ bind:m (poke-our %group-push-hook push-hook-act) (pure:m rid) @@ -52,9 +52,9 @@ =/ overwrite=? ?=(%policy -.associated.action) =/ =update:graph - [%0 now.bowl %add-graph rid.action *graph:graph mark.action overwrite] + [now.bowl %add-graph rid.action *graph:graph mark.action overwrite] ;< ~ bind:m - (poke-our %graph-store graph-update+!>(update)) + (poke-our %graph-store graph-update-2+!>(update)) ;< ~ bind:m (poke-our %graph-push-hook %push-hook-action !>([%add rid.action])) :: @@ -71,13 +71,14 @@ description description.action date-created now.bowl creator our.bowl - module module.action + config [%graph module.action] preview %.n + hidden %.n == =/ met-action=action:met [%add group graph+rid.action metadatum] ;< ~ bind:m - (poke-our %metadata-push-hook metadata-update+!>(met-action)) + (poke-our %metadata-push-hook metadata-update-1+!>(met-action)) :: :: Send invites :: diff --git a/pkg/arvo/ted/graph/delete.hoon b/pkg/arvo/ted/graph/delete.hoon index f95fd096ef..bab899d6ef 100644 --- a/pkg/arvo/ted/graph/delete.hoon +++ b/pkg/arvo/ted/graph/delete.hoon @@ -36,12 +36,12 @@ ^- form:m ;< =bowl:spider bind:m get-bowl:strandio ;< ~ bind:m - (poke-our %graph-store %graph-update !>([%0 now.bowl %remove-graph rid])) + (poke-our %graph-store %graph-update-2 !>([now.bowl %remove-graph rid])) ;< ~ bind:m (poke-our %graph-push-hook %push-hook-action !>([%remove rid])) ;< ~ bind:m %+ poke-our %metadata-push-hook - :- %metadata-update + :- %metadata-update-1 !> ^- action:metadata [%remove group-rid [%graph rid]] (pure:m ~) @@ -63,7 +63,7 @@ (pure:m ~) ;< ~ bind:m %+ poke [entity.grp-rid %group-push-hook] - :- %group-update + :- %group-update-0 !> ^- update:group-store [%remove-tag grp-rid tag.i.tags tagged.i.tags] loop(tags t.tags) diff --git a/pkg/arvo/ted/graph/disable-group-feed.hoon b/pkg/arvo/ted/graph/disable-group-feed.hoon new file mode 100644 index 0000000000..5b9d6555e8 --- /dev/null +++ b/pkg/arvo/ted/graph/disable-group-feed.hoon @@ -0,0 +1,47 @@ +/- spider, + met=metadata-store, + graph=graph-store +/+ strandio, resource, graph-view +:: +=* strand strand:spider +=* poke-our poke-our:strandio +:: +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +=+ !<([~ =action:graph-view] arg) +?> ?=(%disable-group-feed -.action) +;< =bowl:spider bind:m get-bowl:strandio +?. =(our.bowl entity.group.action) + (strand-fail:strandio %bad-request ~) +;< association=(unit association:met) bind:m + %+ scry:strandio (unit association:met) + %- zing + :~ /gx/metadata-store/metadata/groups + (en-path:resource group.action) + /noun + == +?~ association + ~|('No group exists, cannot make group feed.' !!) +=* metadatum metadatum.u.association +?> ?=(%group -.config.metadatum) +?> ?| ?=(~ feed.config.metadatum) + ?=([~ ^] feed.config.metadatum) + == +;< ~ bind:m + %+ poke-our %metadata-push-hook + :- %metadata-update-1 + !> ^- action:met + :^ %add + group.action + groups+group.action + metadatum(feed.config [~ ~]) +?: ?=([~ ^] feed.config.metadatum) + ;< ~ bind:m + %+ poke-our %graph-store + :- %graph-update-2 + !> ^- update:graph + [now.bowl [%archive-graph resource.u.u.feed.config.metadatum]] + (pure:m !>(~)) +(pure:m !>(~)) diff --git a/pkg/arvo/ted/graph/groupify.hoon b/pkg/arvo/ted/graph/groupify.hoon index b3681e0171..f7954bd257 100644 --- a/pkg/arvo/ted/graph/groupify.hoon +++ b/pkg/arvo/ted/graph/groupify.hoon @@ -69,5 +69,5 @@ !> ^- action:met [%remove rid.action [%graph rid.action]] ;< ~ bind:m - (poke-our %group-store %group-update !>([%remove-group rid.action ~])) + (poke-our %group-store %group-update-0 !>([%remove-group rid.action ~])) (pure:m !>(~)) diff --git a/pkg/arvo/ted/graph/leave.hoon b/pkg/arvo/ted/graph/leave.hoon index 1ff9a915f4..b84e53a164 100644 --- a/pkg/arvo/ted/graph/leave.hoon +++ b/pkg/arvo/ted/graph/leave.hoon @@ -39,7 +39,7 @@ ;< ~ bind:m (poke-our %graph-pull-hook %pull-hook-action !>([%remove rid])) ;< ~ bind:m - (poke-our %graph-store %graph-update !>([%0 now [%remove-graph rid]])) + (poke-our %graph-store %graph-update-2 !>([now [%remove-graph rid]])) (pure:m ~) -- :: diff --git a/pkg/arvo/ted/graph/restore.hoon b/pkg/arvo/ted/graph/restore.hoon index e399deb4da..3efac97241 100644 --- a/pkg/arvo/ted/graph/restore.hoon +++ b/pkg/arvo/ted/graph/restore.hoon @@ -17,7 +17,7 @@ ;< =bowl:spider bind:m get-bowl:strandio :: unarchive graph and share it ;< ~ bind:m - (poke-our %graph-store %graph-update !>([%0 now.bowl %unarchive-graph rid])) + (poke-our %graph-store %graph-update-2 !>([now.bowl %unarchive-graph rid])) ;< ~ bind:m (poke-our %graph-push-hook %push-hook-action !>([%add rid])) :: @@ -29,7 +29,7 @@ description description date-created now.bowl creator our.bowl - module module + config [%graph module] == ;< ~ bind:m %+ poke-our %metadata-push-hook diff --git a/pkg/arvo/ted/group/create.hoon b/pkg/arvo/ted/group/create.hoon index 1b7da9b838..e450f0c1b3 100644 --- a/pkg/arvo/ted/group/create.hoon +++ b/pkg/arvo/ted/group/create.hoon @@ -40,6 +40,7 @@ description description.action date-created now.bowl creator our.bowl + config [%group ~] == =/ met-action=action:metadata [%add rid groups+rid metadatum] diff --git a/pkg/arvo/ted/group/delete.hoon b/pkg/arvo/ted/group/delete.hoon index 08525780d9..28fcfb1dc2 100644 --- a/pkg/arvo/ted/group/delete.hoon +++ b/pkg/arvo/ted/group/delete.hoon @@ -24,7 +24,7 @@ !> ^- action:push-hook [%remove resource.action] ;< ~ bind:m (cleanup-md:view rid) -;< ~ bind:m (poke-our %group-store %group-update !>([%remove-group rid ~])) +;< ~ bind:m (poke-our %group-store %group-update-0 !>([%remove-group rid ~])) ;< ~ bind:m (poke-our %metadata-push-hook push-hook-act) ;< ~ bind:m (poke-our %contact-push-hook push-hook-act) ;< ~ bind:m (poke-our %group-push-hook push-hook-act) diff --git a/pkg/arvo/ted/group/invite.hoon b/pkg/arvo/ted/group/invite.hoon index ad42479ef3..afd9b37b27 100644 --- a/pkg/arvo/ted/group/invite.hoon +++ b/pkg/arvo/ted/group/invite.hoon @@ -36,7 +36,7 @@ ^- form:m =/ =action:store [%change-policy rid %invite %add-invites ships] - ;< ~ bind:m (poke-our %group-push-hook %group-update !>(action)) + ;< ~ bind:m (poke-our %group-push-hook %group-update-0 !>(action)) (pure:m ~) -- ^- thread:spider diff --git a/pkg/arvo/ted/group/leave.hoon b/pkg/arvo/ted/group/leave.hoon index 65ea051cbc..e2b4d9514b 100644 --- a/pkg/arvo/ted/group/leave.hoon +++ b/pkg/arvo/ted/group/leave.hoon @@ -25,6 +25,6 @@ ;< ~ bind:m (poke-our %contact-pull-hook pull-hook-act) ;< ~ bind:m (poke-our %metadata-pull-hook pull-hook-act) ;< ~ bind:m (poke-our %group-pull-hook pull-hook-act) -;< ~ bind:m (poke-our %group-store %group-update !>([%remove-group rid ~])) +;< ~ bind:m (poke-our %group-store %group-update-0 !>([%remove-group rid ~])) ;< ~ bind:m (cleanup-md:view rid) (pure:m !>(~)) diff --git a/pkg/arvo/ted/group/on-leave.hoon b/pkg/arvo/ted/group/on-leave.hoon index e308b7d6d9..34b2b6636e 100644 --- a/pkg/arvo/ted/group/on-leave.hoon +++ b/pkg/arvo/ted/group/on-leave.hoon @@ -19,7 +19,7 @@ ;< ~ bind:m %+ raw-poke [entity.resource.update %group-push-hook] - :- %group-update + :- %group-update-0 !> ^- update:grp [%remove-members resource.update (silt [our.bowl ~])] :: stop serving or syncing group updates @@ -70,9 +70,9 @@ ;< ~ bind:m %+ raw-poke [our.bowl %graph-store] - :- %graph-update + :- %graph-update-2 !> ^- update:gra - [%0 now.bowl [%archive-graph app-resource]] + [now.bowl [%archive-graph app-resource]] ;< ~ bind:m %+ raw-poke [our.bowl %graph-pull-hook] diff --git a/pkg/arvo/ted/md/on-add-group-feed.hoon b/pkg/arvo/ted/md/on-add-group-feed.hoon new file mode 100644 index 0000000000..e6ccdd2f5b --- /dev/null +++ b/pkg/arvo/ted/md/on-add-group-feed.hoon @@ -0,0 +1,31 @@ +/- spider, graph-view, met=metadata-store +/+ strandio +:: +=* strand strand:spider +=* poke-our poke-our:strandio +:: +^- thread:spider +|= arg=vase +=/ m (strand ,vase) +^- form:m +=+ !<([~ =update:met] arg) +?. ?=(%add -.update) + (pure:m !>(~)) +;< =bowl:spider bind:m get-bowl:strandio +?: =(our.bowl entity.group.update) + (pure:m !>(~)) +?. ?=(%group -.config.metadatum.update) + (pure:m !>(~)) +?~ feed.config.metadatum.update + (pure:m !>(~)) +?~ u.feed.config.metadatum.update + (pure:m !>(~)) +=* feed u.u.feed.config.metadatum.update +;< ~ bind:m + %+ poke-our %spider + =- spider-start+!>([`tid.bowl ~ %graph-join -]) + %+ slop !>(~) + !> ^- action:graph-view + [%join resource.feed entity.resource.feed] +(pure:m !>(~)) + diff --git a/pkg/arvo/ted/ph/migrate/make-graphs.hoon b/pkg/arvo/ted/ph/migrate/make-graphs.hoon index 0bbfe6dbf2..c24b70c8c9 100644 --- a/pkg/arvo/ted/ph/migrate/make-graphs.hoon +++ b/pkg/arvo/ted/ph/migrate/make-graphs.hoon @@ -12,9 +12,9 @@ |= [our=@p wen=@da rid=resource body=cord id=@] =/ =index:post [id]~ =/ =post:post [our index wen [%text body]~ ~ ~] - =/ =node:graph-store [post %empty ~] - =/ act=update:graph-store [%0 wen %add-nodes rid (my [index node] ~)] - (poke-app our %graph-push-hook %graph-update act) + =/ =node:graph-store [[%& post] %empty ~] + =/ act=update:graph-store [wen %add-nodes rid (my [index node] ~)] + (poke-app our %graph-push-hook %graph-update-2 act) -- :: ^- thread:spider diff --git a/pkg/arvo/ted/ph/migrate/post-import-graphs.hoon b/pkg/arvo/ted/ph/migrate/post-import-graphs.hoon index eed757263b..00b2309727 100644 --- a/pkg/arvo/ted/ph/migrate/post-import-graphs.hoon +++ b/pkg/arvo/ted/ph/migrate/post-import-graphs.hoon @@ -12,9 +12,9 @@ |= [our=@p wen=@da rid=resource body=cord id=@] =/ =index:post [id]~ =/ =post:post [our index wen [%text body]~ ~ ~] - =/ =node:graph-store [post %empty ~] - =/ act=update:graph-store [%0 wen %add-nodes rid (my [index node] ~)] - (poke-app our %graph-push-hook %graph-update act) + =/ =node:graph-store [[%& post] %empty ~] + =/ act=update:graph-store [wen %add-nodes rid (my [index node] ~)] + (poke-app our %graph-push-hook %graph-update-2 act) -- :: ^- thread:spider diff --git a/pkg/arvo/ted/sane.hoon b/pkg/arvo/ted/sane.hoon index e4207d2de0..c5e6be3412 100644 --- a/pkg/arvo/ted/sane.hoon +++ b/pkg/arvo/ted/sane.hoon @@ -7,9 +7,7 @@ :: ++ supported-apps ^- (list term) - :~ %group-push-hook - %group-store - == + ~[%group-store] :: ++ poke-all-sane |= =input diff --git a/pkg/arvo/tests/lib/pull-hook-virt.hoon b/pkg/arvo/tests/lib/pull-hook-virt.hoon new file mode 100644 index 0000000000..65a0ba3f57 --- /dev/null +++ b/pkg/arvo/tests/lib/pull-hook-virt.hoon @@ -0,0 +1,43 @@ +/+ pull-hook-virt, *test, resource +|% +++ bowl *bowl:gall +:: +++ virt ~(. pull-hook-virt bowl) +:: +++ test-mule-scry-bad-time + %+ expect-eq !>(~) + !> %+ mule-scry:virt ** + /gx/(scot %p ~zod)/graph-store/(scot %da ~2010.1.1)/keys/noun +:: +++ test-mule-scry-bad-ship + %+ expect-eq !>(~) + !> %+ mule-scry:virt ** + /gx/(scot %p ~bus)/graph-store/(scot %da *time)/keys/noun +:: +++ test-kick-mule + =/ rid=resource + [~zod %test] + =/ pax=path + /gx/(scot %p ~zod)/graph-store/(scot %da *time)/keys/noun + =/ test-trp=(trap *) + |. + :- ~ + .^(path pax) + =/ harness-trp=(trap *) + |.((kick-mule:virt rid test-trp)) + %+ expect-eq !>(``/foo) + !> + =/ res=toon + %+ mock [harness-trp %9 2 %0 1] + |= [ref=* raw=*] + =/ pox=(unit path) + ((soft path) raw) + ?~ pox ~ + ?: =(u.pox pax) + ``/foo + ``.^(* u.pox) + ?> ?=(%0 -.res) + ;;((unit (unit path)) p.res) +:: +-- + diff --git a/pkg/arvo/tests/lib/versioning.hoon b/pkg/arvo/tests/lib/versioning.hoon new file mode 100644 index 0000000000..e8cde1d48e --- /dev/null +++ b/pkg/arvo/tests/lib/versioning.hoon @@ -0,0 +1,49 @@ +/+ versioning, *test +|% +++ ver ~(. versioning [*bowl:gall %update 2 1]) +++ test-is-root + ;: weld + %+ expect-eq !> %.y + !> (is-root:ver %update-0) + :: + %+ expect-eq !> %.y + !> (is-root:ver %update) + + :: + %+ expect-eq !> %.n + !> (is-root:ver %not-update-0) + == +:: +++ test-read-version + ;: weld + %+ expect-eq !> 0 + !> (read-version:ver %update-0) + :: + %+ expect-eq !> 0 + !> (read-version:ver %update) + :: + %+ expect-eq !> 1 + !> (read-version:ver %update-1) + == +:: +++ test-append-version + ;: weld + %+ expect-eq !> %update-0 + !> (append-version:ver 0) + :: + %+ expect-eq !> %update-1 + !> (append-version:ver 1) + == +:: +++ test-current-version + %+ expect-eq !> %update-2 + !> current-version:ver +:: +++ test-supported + ;: weld + (expect !>((supported:ver %update-2))) + (expect !>((supported:ver %update-1))) + (expect !>(!(supported:ver %update-0))) + == +-- + diff --git a/pkg/arvo/tests/sys/hoon/mock.hoon b/pkg/arvo/tests/sys/nock.hoon similarity index 84% rename from pkg/arvo/tests/sys/hoon/mock.hoon rename to pkg/arvo/tests/sys/nock.hoon index 411ffa592f..ed2dedc301 100644 --- a/pkg/arvo/tests/sys/hoon/mock.hoon +++ b/pkg/arvo/tests/sys/nock.hoon @@ -29,4 +29,10 @@ %- expect-fail |. .*(~ [%6 [%1 2] [%1 42] %1 43]) == +:: nock 9 should support axis 1 +:: +++ test-call-one + %+ expect-eq + !> 0 + !> .*([3 0 1] [9 1 [0 1]]) -- diff --git a/pkg/docker-image/README.md b/pkg/docker-image/README.md index e5e2163eed..0e7e0a46aa 100644 --- a/pkg/docker-image/README.md +++ b/pkg/docker-image/README.md @@ -25,9 +25,55 @@ The first two options result in Urbit attempting to boot either the ship named b In consequence, it is safe to remove the container and start a new container which mounts the same volume, e.g. to upgrade the version of the urbit binary by running a later container version. It is also possible to stop the container and then move the pier away e.g. to a location where you will run it directly with the Urbit binary. ### Ports -The image includes `EXPOSE` directives for TCP port 80 and UDP port 34343. Port `80` is used for Urbit's HTTP interface for both [Landscape](https://urbit.org/docs/glossary/landscape/) and for [API calls](https://urbit.org/using/integrating-api/) to the ship. Port `34343` is used by [Ames](https://urbit.org/docs/glossary/ames/) for ship-to-ship communication. +The image includes `EXPOSE` directives for TCP port 80 and UDP port 34343. Port `80` is used for Urbit's HTTP interface for both [Landscape](https://urbit.org/docs/glossary/landscape/) and for [API calls](https://urbit.org/using/integrating-api/) to the ship. Port `34343` is set by default to be used by [Ames](https://urbit.org/docs/glossary/ames/) for ship-to-ship communication. -You can either pass the `-P` flag to docker to map ports directly to the corresponding ports on the host, or map them individually with `-p` flags. For local testing the latter is often convenient, for instance to remap port 80 to an unprivileged port. +You can either pass the `-P` flag to docker to map ports directly to the corresponding ports on the host, or map them individually with `-p` flags. For local testing the latter is often convenient, for instance to remap port 80 to an unprivileged port. + +For best performance, you must map the Ames UDP port to the *same* port on the host. If you map to a different port Ames will not be able to make direct connections and your network performance may suffer somewhat. Note that using the same port is required for direct connections but is not by itself sufficient for them. If you are behind a NAT router or the host is not on a public IP address or you are firewalled, you may not achive direct connections regardless. + +For this purpose you can force Ames to use a custom port. `/bin/start-urbit --port=$AMES_PORT` can be passed as an argument to the `docker start` command. Passing `/bin/start-urbit --port=13436` for example, would use port 13436. You must pass the name of the start script `/bin/start-urbit` in order to also pass arguments, if this is omitted your container will not start. + +### Examples +Creating a volume for ~sampel=palnet: +``` +docker volume create sampel-palnet +``` + +Copying key to sampel-palnet's volume (assumes default docker location): +``` +sudo cp ~/sampel-palnet.key /var/lib/docker/volumes/sampel-palnet/_data/sampel-palnet.key +``` + +Using that volume and launching ~sampel-palnet on host port 8080 with Ames talking on the default host port 34343: +``` +docker run -d -p 8080:80 -p 34343:34343/udp --name sampel-palnet \ + --mount type=volume,source=sampel-palnet,destination=/urbit \ + tloncorp/urbit +``` + +Using host port 8088 with Ames talking on host port 23232: +``` +docker run -d -p 8088:80 -p 23232:23232/udp --name sampel-palnet \ + --mount type=volume,source=sampel-palnet,destination=/urbit \ + tloncorp/urbit /bin/start-urbit --port=23232 +``` + +### Getting and resetting the Landscape +code +This docker image includes tools for retrieving and resetting the Landscape login code belonging to the planet, for programmatic use so the container does not need a tty. These scripts can be called using `docker container exec`. + +Getting the code: +``` +$ docker container exec sampel-palnet /bin/get-urbit-code +sampel-sampel-sampel-sampel +``` + +Resetting the code: +``` +$ docker container exec sampel-palnet /bin/reset-urbit-code +OK +``` + +Once the code has been reset the new code can be obtained from `/bin/get-urbit-code`. ## Extending diff --git a/pkg/hs/lmdb-static/.gitignore b/pkg/hs/lmdb-static/.gitignore deleted file mode 100644 index 6915ac3d4b..0000000000 --- a/pkg/hs/lmdb-static/.gitignore +++ /dev/null @@ -1,12 +0,0 @@ -dist -cabal-dev -*.o -*.hi -*.chi -*.chs.h -.virtualenv -.hsenv -.cabal-sandbox/ -cabal.sandbox.config -cabal.config -*~ diff --git a/pkg/hs/lmdb-static/LICENSE b/pkg/hs/lmdb-static/LICENSE deleted file mode 100644 index ecd0fa3504..0000000000 --- a/pkg/hs/lmdb-static/LICENSE +++ /dev/null @@ -1,24 +0,0 @@ -Copyright (c) 2014, David Barbour -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/pkg/hs/lmdb-static/README.md b/pkg/hs/lmdb-static/README.md deleted file mode 100644 index 7fb77f39c6..0000000000 --- a/pkg/hs/lmdb-static/README.md +++ /dev/null @@ -1,13 +0,0 @@ -This is a hack to avoid dynamic depencency on lmdb: - -This is a vendoring of `haskell-lmdb` and `lmdb` modified to include -the c-build of `lmdb` statically into `haskell-lmdb`. - -``` -haskell-lmdb: - repo: https://github.com/dmbarbour/haskell-lmdb.git - hash: 1e562429874919d445576c87cf118d7de5112b5b -lmdb: - repo: https://github.com/LMDB/lmdb.git - hash: c3e6b4209eed13af4a3670e5f04f42169c08e5c6 -``` diff --git a/pkg/hs/lmdb-static/Setup.hs b/pkg/hs/lmdb-static/Setup.hs deleted file mode 100644 index 200a2e51d0..0000000000 --- a/pkg/hs/lmdb-static/Setup.hs +++ /dev/null @@ -1,3 +0,0 @@ -import Distribution.Simple -main = defaultMain - diff --git a/pkg/hs/lmdb-static/cbits/lmdb.h b/pkg/hs/lmdb-static/cbits/lmdb.h deleted file mode 100644 index 1f4736ce2b..0000000000 --- a/pkg/hs/lmdb-static/cbits/lmdb.h +++ /dev/null @@ -1,1653 +0,0 @@ -/** @file lmdb.h - * @brief Lightning memory-mapped database library - * - * @mainpage Lightning Memory-Mapped Database Manager (LMDB) - * - * @section intro_sec Introduction - * LMDB is a Btree-based database management library modeled loosely on the - * BerkeleyDB API, but much simplified. The entire database is exposed - * in a memory map, and all data fetches return data directly - * from the mapped memory, so no malloc's or memcpy's occur during - * data fetches. As such, the library is extremely simple because it - * requires no page caching layer of its own, and it is extremely high - * performance and memory-efficient. It is also fully transactional with - * full ACID semantics, and when the memory map is read-only, the - * database integrity cannot be corrupted by stray pointer writes from - * application code. - * - * The library is fully thread-aware and supports concurrent read/write - * access from multiple processes and threads. Data pages use a copy-on- - * write strategy so no active data pages are ever overwritten, which - * also provides resistance to corruption and eliminates the need of any - * special recovery procedures after a system crash. Writes are fully - * serialized; only one write transaction may be active at a time, which - * guarantees that writers can never deadlock. The database structure is - * multi-versioned so readers run with no locks; writers cannot block - * readers, and readers don't block writers. - * - * Unlike other well-known database mechanisms which use either write-ahead - * transaction logs or append-only data writes, LMDB requires no maintenance - * during operation. Both write-ahead loggers and append-only databases - * require periodic checkpointing and/or compaction of their log or database - * files otherwise they grow without bound. LMDB tracks free pages within - * the database and re-uses them for new write operations, so the database - * size does not grow without bound in normal use. - * - * The memory map can be used as a read-only or read-write map. It is - * read-only by default as this provides total immunity to corruption. - * Using read-write mode offers much higher write performance, but adds - * the possibility for stray application writes thru pointers to silently - * corrupt the database. Of course if your application code is known to - * be bug-free (...) then this is not an issue. - * - * If this is your first time using a transactional embedded key/value - * store, you may find the \ref starting page to be helpful. - * - * @section caveats_sec Caveats - * Troubleshooting the lock file, plus semaphores on BSD systems: - * - * - A broken lockfile can cause sync issues. - * Stale reader transactions left behind by an aborted program - * cause further writes to grow the database quickly, and - * stale locks can block further operation. - * - * Fix: Check for stale readers periodically, using the - * #mdb_reader_check function or the \ref mdb_stat_1 "mdb_stat" tool. - * Stale writers will be cleared automatically on most systems: - * - Windows - automatic - * - BSD, systems using SysV semaphores - automatic - * - Linux, systems using POSIX mutexes with Robust option - automatic - * Otherwise just make all programs using the database close it; - * the lockfile is always reset on first open of the environment. - * - * - On BSD systems or others configured with MDB_USE_SYSV_SEM or - * MDB_USE_POSIX_SEM, - * startup can fail due to semaphores owned by another userid. - * - * Fix: Open and close the database as the user which owns the - * semaphores (likely last user) or as root, while no other - * process is using the database. - * - * Restrictions/caveats (in addition to those listed for some functions): - * - * - Only the database owner should normally use the database on - * BSD systems or when otherwise configured with MDB_USE_POSIX_SEM. - * Multiple users can cause startup to fail later, as noted above. - * - * - There is normally no pure read-only mode, since readers need write - * access to locks and lock file. Exceptions: On read-only filesystems - * or with the #MDB_NOLOCK flag described under #mdb_env_open(). - * - * - An LMDB configuration will often reserve considerable \b unused - * memory address space and maybe file size for future growth. - * This does not use actual memory or disk space, but users may need - * to understand the difference so they won't be scared off. - * - * - By default, in versions before 0.9.10, unused portions of the data - * file might receive garbage data from memory freed by other code. - * (This does not happen when using the #MDB_WRITEMAP flag.) As of - * 0.9.10 the default behavior is to initialize such memory before - * writing to the data file. Since there may be a slight performance - * cost due to this initialization, applications may disable it using - * the #MDB_NOMEMINIT flag. Applications handling sensitive data - * which must not be written should not use this flag. This flag is - * irrelevant when using #MDB_WRITEMAP. - * - * - A thread can only use one transaction at a time, plus any child - * transactions. Each transaction belongs to one thread. See below. - * The #MDB_NOTLS flag changes this for read-only transactions. - * - * - Use an MDB_env* in the process which opened it, not after fork(). - * - * - Do not have open an LMDB database twice in the same process at - * the same time. Not even from a plain open() call - close()ing it - * breaks fcntl() advisory locking. (It is OK to reopen it after - * fork() - exec*(), since the lockfile has FD_CLOEXEC set.) - * - * - Avoid long-lived transactions. Read transactions prevent - * reuse of pages freed by newer write transactions, thus the - * database can grow quickly. Write transactions prevent - * other write transactions, since writes are serialized. - * - * - Avoid suspending a process with active transactions. These - * would then be "long-lived" as above. Also read transactions - * suspended when writers commit could sometimes see wrong data. - * - * ...when several processes can use a database concurrently: - * - * - Avoid aborting a process with an active transaction. - * The transaction becomes "long-lived" as above until a check - * for stale readers is performed or the lockfile is reset, - * since the process may not remove it from the lockfile. - * - * This does not apply to write transactions if the system clears - * stale writers, see above. - * - * - If you do that anyway, do a periodic check for stale readers. Or - * close the environment once in a while, so the lockfile can get reset. - * - * - Do not use LMDB databases on remote filesystems, even between - * processes on the same host. This breaks flock() on some OSes, - * possibly memory map sync, and certainly sync between programs - * on different hosts. - * - * - Opening a database can fail if another process is opening or - * closing it at exactly the same time. - * - * @author Howard Chu, Symas Corporation. - * - * @copyright Copyright 2011-2019 Howard Chu, Symas Corp. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - * - * @par Derived From: - * This code is derived from btree.c written by Martin Hedenfalk. - * - * Copyright (c) 2009, 2010 Martin Hedenfalk - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -#ifndef _LMDB_H_ -#define _LMDB_H_ - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** Unix permissions for creating files, or dummy definition for Windows */ -#ifdef _MSC_VER -typedef int mdb_mode_t; -#else -typedef mode_t mdb_mode_t; -#endif - -#ifdef _WIN32 -# define MDB_FMT_Z "I" -#else -# define MDB_FMT_Z "z" /**< printf/scanf format modifier for size_t */ -#endif - -#ifndef MDB_VL32 -/** Unsigned type used for mapsize, entry counts and page/transaction IDs. - * - * It is normally size_t, hence the name. Defining MDB_VL32 makes it - * uint64_t, but do not try this unless you know what you are doing. - */ -typedef size_t mdb_size_t; -# define MDB_SIZE_MAX SIZE_MAX /**< max #mdb_size_t */ -/** #mdb_size_t printf formats, \b t = one of [diouxX] without quotes */ -# define MDB_PRIy(t) MDB_FMT_Z #t -/** #mdb_size_t scanf formats, \b t = one of [dioux] without quotes */ -# define MDB_SCNy(t) MDB_FMT_Z #t -#else -typedef uint64_t mdb_size_t; -# define MDB_SIZE_MAX UINT64_MAX -# define MDB_PRIy(t) PRI##t##64 -# define MDB_SCNy(t) SCN##t##64 -# define mdb_env_create mdb_env_create_vl32 /**< Prevent mixing with non-VL32 builds */ -#endif - -/** An abstraction for a file handle. - * On POSIX systems file handles are small integers. On Windows - * they're opaque pointers. - */ -#ifdef _WIN32 -typedef void *mdb_filehandle_t; -#else -typedef int mdb_filehandle_t; -#endif - -/** @defgroup mdb LMDB API - * @{ - * @brief OpenLDAP Lightning Memory-Mapped Database Manager - */ -/** @defgroup Version Version Macros - * @{ - */ -/** Library major version */ -#define MDB_VERSION_MAJOR 0 -/** Library minor version */ -#define MDB_VERSION_MINOR 9 -/** Library patch version */ -#define MDB_VERSION_PATCH 70 - -/** Combine args a,b,c into a single integer for easy version comparisons */ -#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) - -/** The full library version as a single integer */ -#define MDB_VERSION_FULL \ - MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) - -/** The release date of this library version */ -#define MDB_VERSION_DATE "December 19, 2015" - -/** A stringifier for the version info */ -#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" - -/** A helper for the stringifier macro */ -#define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) - -/** The full library version as a C string */ -#define MDB_VERSION_STRING \ - MDB_VERFOO(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH,MDB_VERSION_DATE) -/** @} */ - -/** @brief Opaque structure for a database environment. - * - * A DB environment supports multiple databases, all residing in the same - * shared-memory map. - */ -typedef struct MDB_env MDB_env; - -/** @brief Opaque structure for a transaction handle. - * - * All database operations require a transaction handle. Transactions may be - * read-only or read-write. - */ -typedef struct MDB_txn MDB_txn; - -/** @brief A handle for an individual database in the DB environment. */ -typedef unsigned int MDB_dbi; - -/** @brief Opaque structure for navigating through a database */ -typedef struct MDB_cursor MDB_cursor; - -/** @brief Generic structure used for passing keys and data in and out - * of the database. - * - * Values returned from the database are valid only until a subsequent - * update operation, or the end of the transaction. Do not modify or - * free them, they commonly point into the database itself. - * - * Key sizes must be between 1 and #mdb_env_get_maxkeysize() inclusive. - * The same applies to data sizes in databases with the #MDB_DUPSORT flag. - * Other data items can in theory be from 0 to 0xffffffff bytes long. - */ -typedef struct MDB_val { - size_t mv_size; /**< size of the data item */ - void *mv_data; /**< address of the data item */ -} MDB_val; - -/** @brief A callback function used to compare two keys in a database */ -typedef int (MDB_cmp_func)(const MDB_val *a, const MDB_val *b); - -/** @brief A callback function used to relocate a position-dependent data item - * in a fixed-address database. - * - * The \b newptr gives the item's desired address in - * the memory map, and \b oldptr gives its previous address. The item's actual - * data resides at the address in \b item. This callback is expected to walk - * through the fields of the record in \b item and modify any - * values based at the \b oldptr address to be relative to the \b newptr address. - * @param[in,out] item The item that is to be relocated. - * @param[in] oldptr The previous address. - * @param[in] newptr The new address to relocate to. - * @param[in] relctx An application-provided context, set by #mdb_set_relctx(). - * @todo This feature is currently unimplemented. - */ -typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *relctx); - -/** @defgroup mdb_env Environment Flags - * @{ - */ - /** mmap at a fixed address (experimental) */ -#define MDB_FIXEDMAP 0x01 - /** no environment directory */ -#define MDB_NOSUBDIR 0x4000 - /** don't fsync after commit */ -#define MDB_NOSYNC 0x10000 - /** read only */ -#define MDB_RDONLY 0x20000 - /** don't fsync metapage after commit */ -#define MDB_NOMETASYNC 0x40000 - /** use writable mmap */ -#define MDB_WRITEMAP 0x80000 - /** use asynchronous msync when #MDB_WRITEMAP is used */ -#define MDB_MAPASYNC 0x100000 - /** tie reader locktable slots to #MDB_txn objects instead of to threads */ -#define MDB_NOTLS 0x200000 - /** don't do any locking, caller must manage their own locks */ -#define MDB_NOLOCK 0x400000 - /** don't do readahead (no effect on Windows) */ -#define MDB_NORDAHEAD 0x800000 - /** don't initialize malloc'd memory before writing to datafile */ -#define MDB_NOMEMINIT 0x1000000 - /** use the previous snapshot rather than the latest one */ -#define MDB_PREVSNAPSHOT 0x2000000 -/** @} */ - -/** @defgroup mdb_dbi_open Database Flags - * @{ - */ - /** use reverse string keys */ -#define MDB_REVERSEKEY 0x02 - /** use sorted duplicates */ -#define MDB_DUPSORT 0x04 - /** numeric keys in native byte order, either unsigned int or #mdb_size_t. - * (lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.) - * The keys must all be of the same size. */ -#define MDB_INTEGERKEY 0x08 - /** with #MDB_DUPSORT, sorted dup items have fixed size */ -#define MDB_DUPFIXED 0x10 - /** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */ -#define MDB_INTEGERDUP 0x20 - /** with #MDB_DUPSORT, use reverse string dups */ -#define MDB_REVERSEDUP 0x40 - /** create DB if not already existing */ -#define MDB_CREATE 0x40000 -/** @} */ - -/** @defgroup mdb_put Write Flags - * @{ - */ -/** For put: Don't write if the key already exists. */ -#define MDB_NOOVERWRITE 0x10 -/** Only for #MDB_DUPSORT
- * For put: don't write if the key and data pair already exist.
- * For mdb_cursor_del: remove all duplicate data items. - */ -#define MDB_NODUPDATA 0x20 -/** For mdb_cursor_put: overwrite the current key/data pair */ -#define MDB_CURRENT 0x40 -/** For put: Just reserve space for data, don't copy it. Return a - * pointer to the reserved space. - */ -#define MDB_RESERVE 0x10000 -/** Data is being appended, don't split full pages. */ -#define MDB_APPEND 0x20000 -/** Duplicate data is being appended, don't split full pages. */ -#define MDB_APPENDDUP 0x40000 -/** Store multiple data items in one call. Only for #MDB_DUPFIXED. */ -#define MDB_MULTIPLE 0x80000 -/* @} */ - -/** @defgroup mdb_copy Copy Flags - * @{ - */ -/** Compacting copy: Omit free space from copy, and renumber all - * pages sequentially. - */ -#define MDB_CP_COMPACT 0x01 -/* @} */ - -/** @brief Cursor Get operations. - * - * This is the set of all operations for retrieving data - * using a cursor. - */ -typedef enum MDB_cursor_op { - MDB_FIRST, /**< Position at first key/data item */ - MDB_FIRST_DUP, /**< Position at first data item of current key. - Only for #MDB_DUPSORT */ - MDB_GET_BOTH, /**< Position at key/data pair. Only for #MDB_DUPSORT */ - MDB_GET_BOTH_RANGE, /**< position at key, nearest data. Only for #MDB_DUPSORT */ - MDB_GET_CURRENT, /**< Return key/data at current cursor position */ - MDB_GET_MULTIPLE, /**< Return up to a page of duplicate data items - from current cursor position. Move cursor to prepare - for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ - MDB_LAST, /**< Position at last key/data item */ - MDB_LAST_DUP, /**< Position at last data item of current key. - Only for #MDB_DUPSORT */ - MDB_NEXT, /**< Position at next data item */ - MDB_NEXT_DUP, /**< Position at next data item of current key. - Only for #MDB_DUPSORT */ - MDB_NEXT_MULTIPLE, /**< Return up to a page of duplicate data items - from next cursor position. Move cursor to prepare - for #MDB_NEXT_MULTIPLE. Only for #MDB_DUPFIXED */ - MDB_NEXT_NODUP, /**< Position at first data item of next key */ - MDB_PREV, /**< Position at previous data item */ - MDB_PREV_DUP, /**< Position at previous data item of current key. - Only for #MDB_DUPSORT */ - MDB_PREV_NODUP, /**< Position at last data item of previous key */ - MDB_SET, /**< Position at specified key */ - MDB_SET_KEY, /**< Position at specified key, return key + data */ - MDB_SET_RANGE, /**< Position at first key greater than or equal to specified key. */ - MDB_PREV_MULTIPLE /**< Position at previous page and return up to - a page of duplicate data items. Only for #MDB_DUPFIXED */ -} MDB_cursor_op; - -/** @defgroup errors Return Codes - * - * BerkeleyDB uses -30800 to -30999, we'll go under them - * @{ - */ - /** Successful result */ -#define MDB_SUCCESS 0 - /** key/data pair already exists */ -#define MDB_KEYEXIST (-30799) - /** key/data pair not found (EOF) */ -#define MDB_NOTFOUND (-30798) - /** Requested page not found - this usually indicates corruption */ -#define MDB_PAGE_NOTFOUND (-30797) - /** Located page was wrong type */ -#define MDB_CORRUPTED (-30796) - /** Update of meta page failed or environment had fatal error */ -#define MDB_PANIC (-30795) - /** Environment version mismatch */ -#define MDB_VERSION_MISMATCH (-30794) - /** File is not a valid LMDB file */ -#define MDB_INVALID (-30793) - /** Environment mapsize reached */ -#define MDB_MAP_FULL (-30792) - /** Environment maxdbs reached */ -#define MDB_DBS_FULL (-30791) - /** Environment maxreaders reached */ -#define MDB_READERS_FULL (-30790) - /** Too many TLS keys in use - Windows only */ -#define MDB_TLS_FULL (-30789) - /** Txn has too many dirty pages */ -#define MDB_TXN_FULL (-30788) - /** Cursor stack too deep - internal error */ -#define MDB_CURSOR_FULL (-30787) - /** Page has not enough space - internal error */ -#define MDB_PAGE_FULL (-30786) - /** Database contents grew beyond environment mapsize */ -#define MDB_MAP_RESIZED (-30785) - /** Operation and DB incompatible, or DB type changed. This can mean: - *

    - *
  • The operation expects an #MDB_DUPSORT / #MDB_DUPFIXED database. - *
  • Opening a named DB when the unnamed DB has #MDB_DUPSORT / #MDB_INTEGERKEY. - *
  • Accessing a data record as a database, or vice versa. - *
  • The database was dropped and recreated with different flags. - *
- */ -#define MDB_INCOMPATIBLE (-30784) - /** Invalid reuse of reader locktable slot */ -#define MDB_BAD_RSLOT (-30783) - /** Transaction must abort, has a child, or is invalid */ -#define MDB_BAD_TXN (-30782) - /** Unsupported size of key/DB name/data, or wrong DUPFIXED size */ -#define MDB_BAD_VALSIZE (-30781) - /** The specified DBI was changed unexpectedly */ -#define MDB_BAD_DBI (-30780) - /** Unexpected problem - txn should abort */ -#define MDB_PROBLEM (-30779) - /** The last defined error code */ -#define MDB_LAST_ERRCODE MDB_PROBLEM -/** @} */ - -/** @brief Statistics for a database in the environment */ -typedef struct MDB_stat { - unsigned int ms_psize; /**< Size of a database page. - This is currently the same for all databases. */ - unsigned int ms_depth; /**< Depth (height) of the B-tree */ - mdb_size_t ms_branch_pages; /**< Number of internal (non-leaf) pages */ - mdb_size_t ms_leaf_pages; /**< Number of leaf pages */ - mdb_size_t ms_overflow_pages; /**< Number of overflow pages */ - mdb_size_t ms_entries; /**< Number of data items */ -} MDB_stat; - -/** @brief Information about the environment */ -typedef struct MDB_envinfo { - void *me_mapaddr; /**< Address of map, if fixed */ - mdb_size_t me_mapsize; /**< Size of the data memory map */ - mdb_size_t me_last_pgno; /**< ID of the last used page */ - mdb_size_t me_last_txnid; /**< ID of the last committed transaction */ - unsigned int me_maxreaders; /**< max reader slots in the environment */ - unsigned int me_numreaders; /**< max reader slots used in the environment */ -} MDB_envinfo; - - /** @brief Return the LMDB library version information. - * - * @param[out] major if non-NULL, the library major version number is copied here - * @param[out] minor if non-NULL, the library minor version number is copied here - * @param[out] patch if non-NULL, the library patch version number is copied here - * @retval "version string" The library version as a string - */ -char *mdb_version(int *major, int *minor, int *patch); - - /** @brief Return a string describing a given error code. - * - * This function is a superset of the ANSI C X3.159-1989 (ANSI C) strerror(3) - * function. If the error code is greater than or equal to 0, then the string - * returned by the system function strerror(3) is returned. If the error code - * is less than 0, an error string corresponding to the LMDB library error is - * returned. See @ref errors for a list of LMDB-specific error codes. - * @param[in] err The error code - * @retval "error message" The description of the error - */ -char *mdb_strerror(int err); - - /** @brief Create an LMDB environment handle. - * - * This function allocates memory for a #MDB_env structure. To release - * the allocated memory and discard the handle, call #mdb_env_close(). - * Before the handle may be used, it must be opened using #mdb_env_open(). - * Various other options may also need to be set before opening the handle, - * e.g. #mdb_env_set_mapsize(), #mdb_env_set_maxreaders(), #mdb_env_set_maxdbs(), - * depending on usage requirements. - * @param[out] env The address where the new handle will be stored - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_create(MDB_env **env); - - /** @brief Open an environment handle. - * - * If this function fails, #mdb_env_close() must be called to discard the #MDB_env handle. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] path The directory in which the database files reside. This - * directory must already exist and be writable. - * @param[in] flags Special options for this environment. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - * Flags set by mdb_env_set_flags() are also used. - *
    - *
  • #MDB_FIXEDMAP - * use a fixed address for the mmap region. This flag must be specified - * when creating the environment, and is stored persistently in the environment. - * If successful, the memory map will always reside at the same virtual address - * and pointers used to reference data items in the database will be constant - * across multiple invocations. This option may not always work, depending on - * how the operating system has allocated memory to shared libraries and other uses. - * The feature is highly experimental. - *
  • #MDB_NOSUBDIR - * By default, LMDB creates its environment in a directory whose - * pathname is given in \b path, and creates its data and lock files - * under that directory. With this option, \b path is used as-is for - * the database main data file. The database lock file is the \b path - * with "-lock" appended. - *
  • #MDB_RDONLY - * Open the environment in read-only mode. No write operations will be - * allowed. LMDB will still modify the lock file - except on read-only - * filesystems, where LMDB does not use locks. - *
  • #MDB_WRITEMAP - * Use a writeable memory map unless MDB_RDONLY is set. This uses - * fewer mallocs but loses protection from application bugs - * like wild pointer writes and other bad updates into the database. - * This may be slightly faster for DBs that fit entirely in RAM, but - * is slower for DBs larger than RAM. - * Incompatible with nested transactions. - * Do not mix processes with and without MDB_WRITEMAP on the same - * environment. This can defeat durability (#mdb_env_sync etc). - *
  • #MDB_NOMETASYNC - * Flush system buffers to disk only once per transaction, omit the - * metadata flush. Defer that until the system flushes files to disk, - * or next non-MDB_RDONLY commit or #mdb_env_sync(). This optimization - * maintains database integrity, but a system crash may undo the last - * committed transaction. I.e. it preserves the ACI (atomicity, - * consistency, isolation) but not D (durability) database property. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
  • #MDB_NOSYNC - * Don't flush system buffers to disk when committing a transaction. - * This optimization means a system crash can corrupt the database or - * lose the last transactions if buffers are not yet flushed to disk. - * The risk is governed by how often the system flushes dirty buffers - * to disk and how often #mdb_env_sync() is called. However, if the - * filesystem preserves write order and the #MDB_WRITEMAP flag is not - * used, transactions exhibit ACI (atomicity, consistency, isolation) - * properties and only lose D (durability). I.e. database integrity - * is maintained, but a system crash may undo the final transactions. - * Note that (#MDB_NOSYNC | #MDB_WRITEMAP) leaves the system with no - * hint for when to write transactions to disk, unless #mdb_env_sync() - * is called. (#MDB_MAPASYNC | #MDB_WRITEMAP) may be preferable. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
  • #MDB_MAPASYNC - * When using #MDB_WRITEMAP, use asynchronous flushes to disk. - * As with #MDB_NOSYNC, a system crash can then corrupt the - * database or lose the last transactions. Calling #mdb_env_sync() - * ensures on-disk database integrity until next commit. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
  • #MDB_NOTLS - * Don't use Thread-Local Storage. Tie reader locktable slots to - * #MDB_txn objects instead of to threads. I.e. #mdb_txn_reset() keeps - * the slot reseved for the #MDB_txn object. A thread may use parallel - * read-only transactions. A read-only transaction may span threads if - * the user synchronizes its use. Applications that multiplex many - * user threads over individual OS threads need this option. Such an - * application must also serialize the write transactions in an OS - * thread, since LMDB's write locking is unaware of the user threads. - *
  • #MDB_NOLOCK - * Don't do any locking. If concurrent access is anticipated, the - * caller must manage all concurrency itself. For proper operation - * the caller must enforce single-writer semantics, and must ensure - * that no readers are using old transactions while a writer is - * active. The simplest approach is to use an exclusive lock so that - * no readers may be active at all when a writer begins. - *
  • #MDB_NORDAHEAD - * Turn off readahead. Most operating systems perform readahead on - * read requests by default. This option turns it off if the OS - * supports it. Turning it off may help random read performance - * when the DB is larger than RAM and system RAM is full. - * The option is not implemented on Windows. - *
  • #MDB_NOMEMINIT - * Don't initialize malloc'd memory before writing to unused spaces - * in the data file. By default, memory for pages written to the data - * file is obtained using malloc. While these pages may be reused in - * subsequent transactions, freshly malloc'd pages will be initialized - * to zeroes before use. This avoids persisting leftover data from other - * code (that used the heap and subsequently freed the memory) into the - * data file. Note that many other system libraries may allocate - * and free memory from the heap for arbitrary uses. E.g., stdio may - * use the heap for file I/O buffers. This initialization step has a - * modest performance cost so some applications may want to disable - * it using this flag. This option can be a problem for applications - * which handle sensitive data like passwords, and it makes memory - * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, - * which writes directly to the mmap instead of using malloc for pages. The - * initialization is also skipped if #MDB_RESERVE is used; the - * caller is expected to overwrite all of the memory that was - * reserved in that case. - * This flag may be changed at any time using #mdb_env_set_flags(). - *
  • #MDB_PREVSNAPSHOT - * Open the environment with the previous snapshot rather than the latest - * one. This loses the latest transaction, but may help work around some - * types of corruption. If opened with write access, this must be the - * only process using the environment. This flag is automatically reset - * after a write transaction is successfully committed. - *
- * @param[in] mode The UNIX permissions to set on created files and semaphores. - * This parameter is ignored on Windows. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • #MDB_VERSION_MISMATCH - the version of the LMDB library doesn't match the - * version that created the database environment. - *
  • #MDB_INVALID - the environment file headers are corrupted. - *
  • ENOENT - the directory specified by the path parameter doesn't exist. - *
  • EACCES - the user didn't have permission to access the environment files. - *
  • EAGAIN - the environment was locked by another process. - *
- */ -int mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode); - - /** @brief Copy an LMDB environment to the specified path. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] path The directory in which the copy will reside. This - * directory must already exist and be writable but must otherwise be - * empty. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copy(MDB_env *env, const char *path); - - /** @brief Copy an LMDB environment to the specified file descriptor. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] fd The filedescriptor to write the copy to. It must - * have already been opened for Write access. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copyfd(MDB_env *env, mdb_filehandle_t fd); - - /** @brief Copy an LMDB environment to the specified path, with options. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] path The directory in which the copy will reside. This - * directory must already exist and be writable but must otherwise be - * empty. - * @param[in] flags Special options for this operation. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *
    - *
  • #MDB_CP_COMPACT - Perform compaction while copying: omit free - * pages and sequentially renumber all pages in output. This option - * consumes more CPU and runs more slowly than the default. - * Currently it fails if the environment has suffered a page leak. - *
- * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags); - - /** @brief Copy an LMDB environment to the specified file descriptor, - * with options. - * - * This function may be used to make a backup of an existing environment. - * No lockfile is created, since it gets recreated at need. See - * #mdb_env_copy2() for further details. - * @note This call can trigger significant file size growth if run in - * parallel with write transactions, because it employs a read-only - * transaction. See long-lived transactions under @ref caveats_sec. - * @param[in] env An environment handle returned by #mdb_env_create(). It - * must have already been opened successfully. - * @param[in] fd The filedescriptor to write the copy to. It must - * have already been opened for Write access. - * @param[in] flags Special options for this operation. - * See #mdb_env_copy2() for options. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_copyfd2(MDB_env *env, mdb_filehandle_t fd, unsigned int flags); - - /** @brief Return statistics about the LMDB environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] stat The address of an #MDB_stat structure - * where the statistics will be copied - */ -int mdb_env_stat(MDB_env *env, MDB_stat *stat); - - /** @brief Return information about the LMDB environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] stat The address of an #MDB_envinfo structure - * where the information will be copied - */ -int mdb_env_info(MDB_env *env, MDB_envinfo *stat); - - /** @brief Flush the data buffers to disk. - * - * Data is always written to disk when #mdb_txn_commit() is called, - * but the operating system may keep it buffered. LMDB always flushes - * the OS buffers upon commit as well, unless the environment was - * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is - * not valid if the environment was opened with #MDB_RDONLY. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] force If non-zero, force a synchronous flush. Otherwise - * if the environment has the #MDB_NOSYNC flag set the flushes - * will be omitted, and with #MDB_MAPASYNC they will be asynchronous. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EACCES - the environment is read-only. - *
  • EINVAL - an invalid parameter was specified. - *
  • EIO - an error occurred during synchronization. - *
- */ -int mdb_env_sync(MDB_env *env, int force); - - /** @brief Close the environment and release the memory map. - * - * Only a single thread may call this function. All transactions, databases, - * and cursors must already be closed before calling this function. Attempts to - * use any such handles after calling this function will cause a SIGSEGV. - * The environment handle will be freed and must not be used again after this call. - * @param[in] env An environment handle returned by #mdb_env_create() - */ -void mdb_env_close(MDB_env *env); - - /** @brief Set environment flags. - * - * This may be used to set some flags in addition to those from - * #mdb_env_open(), or to unset these flags. If several threads - * change the flags at the same time, the result is undefined. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] flags The flags to change, bitwise OR'ed together - * @param[in] onoff A non-zero value sets the flags, zero clears them. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_env_set_flags(MDB_env *env, unsigned int flags, int onoff); - - /** @brief Get environment flags. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] flags The address of an integer to store the flags - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_env_get_flags(MDB_env *env, unsigned int *flags); - - /** @brief Return the path that was used in #mdb_env_open(). - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] path Address of a string pointer to contain the path. This - * is the actual string in the environment, not a copy. It should not be - * altered in any way. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_env_get_path(MDB_env *env, const char **path); - - /** @brief Return the filedescriptor for the given environment. - * - * This function may be called after fork(), so the descriptor can be - * closed before exec*(). Other LMDB file descriptors have FD_CLOEXEC. - * (Until LMDB 0.9.18, only the lockfile had that.) - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] fd Address of a mdb_filehandle_t to contain the descriptor. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *fd); - - /** @brief Set the size of the memory map to use for this environment. - * - * The size should be a multiple of the OS page size. The default is - * 10485760 bytes. The size of the memory map is also the maximum size - * of the database. The value should be chosen as large as possible, - * to accommodate future growth of the database. - * This function should be called after #mdb_env_create() and before #mdb_env_open(). - * It may be called at later times if no transactions are active in - * this process. Note that the library does not check for this condition, - * the caller must ensure it explicitly. - * - * The new size takes effect immediately for the current process but - * will not be persisted to any others until a write transaction has been - * committed by the current process. Also, only mapsize increases are - * persisted into the environment. - * - * If the mapsize is increased by another process, and data has grown - * beyond the range of the current mapsize, #mdb_txn_begin() will - * return #MDB_MAP_RESIZED. This function may be called with a size - * of zero to adopt the new size. - * - * Any attempt to set a size smaller than the space already consumed - * by the environment will be silently changed to the current size of the used space. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] size The size in bytes - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified, or the environment has - * an active write transaction. - *
- */ -int mdb_env_set_mapsize(MDB_env *env, mdb_size_t size); - - /** @brief Set the maximum number of threads/reader slots for the environment. - * - * This defines the number of slots in the lock table that is used to track readers in the - * the environment. The default is 126. - * Starting a read-only transaction normally ties a lock table slot to the - * current thread until the environment closes or the thread exits. If - * MDB_NOTLS is in use, #mdb_txn_begin() instead ties the slot to the - * MDB_txn object until it or the #MDB_env object is destroyed. - * This function may only be called after #mdb_env_create() and before #mdb_env_open(). - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] readers The maximum number of reader lock table slots - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified, or the environment is already open. - *
- */ -int mdb_env_set_maxreaders(MDB_env *env, unsigned int readers); - - /** @brief Get the maximum number of threads/reader slots for the environment. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] readers Address of an integer to store the number of readers - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers); - - /** @brief Set the maximum number of named databases for the environment. - * - * This function is only needed if multiple databases will be used in the - * environment. Simpler applications that use the environment as a single - * unnamed database can ignore this option. - * This function may only be called after #mdb_env_create() and before #mdb_env_open(). - * - * Currently a moderate number of slots are cheap but a huge number gets - * expensive: 7-120 words per transaction, and every #mdb_dbi_open() - * does a linear search of the opened slots. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] dbs The maximum number of databases - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified, or the environment is already open. - *
- */ -int mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs); - - /** @brief Get the maximum size of keys and #MDB_DUPSORT data we can write. - * - * Depends on the compile-time constant #MDB_MAXKEYSIZE. Default 511. - * See @ref MDB_val. - * @param[in] env An environment handle returned by #mdb_env_create() - * @return The maximum size of a key we can write - */ -int mdb_env_get_maxkeysize(MDB_env *env); - - /** @brief Set application information associated with the #MDB_env. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] ctx An arbitrary pointer for whatever the application needs. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_set_userctx(MDB_env *env, void *ctx); - - /** @brief Get the application information associated with the #MDB_env. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @return The pointer set by #mdb_env_set_userctx(). - */ -void *mdb_env_get_userctx(MDB_env *env); - - /** @brief A callback function for most LMDB assert() failures, - * called before printing the message and aborting. - * - * @param[in] env An environment handle returned by #mdb_env_create(). - * @param[in] msg The assertion message, not including newline. - */ -typedef void MDB_assert_func(MDB_env *env, const char *msg); - - /** Set or reset the assert() callback of the environment. - * Disabled if liblmdb is buillt with NDEBUG. - * @note This hack should become obsolete as lmdb's error handling matures. - * @param[in] env An environment handle returned by #mdb_env_create(). - * @param[in] func An #MDB_assert_func function, or 0. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func); - - /** @brief Create a transaction for use with the environment. - * - * The transaction handle may be discarded using #mdb_txn_abort() or #mdb_txn_commit(). - * @note A transaction and its cursors must only be used by a single - * thread, and a thread may only have a single transaction at a time. - * If #MDB_NOTLS is in use, this does not apply to read-only transactions. - * @note Cursors may not span transactions. - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] parent If this parameter is non-NULL, the new transaction - * will be a nested transaction, with the transaction indicated by \b parent - * as its parent. Transactions may be nested to any level. A parent - * transaction and its cursors may not issue any other operations than - * mdb_txn_commit and mdb_txn_abort while it has active child transactions. - * @param[in] flags Special options for this transaction. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *
    - *
  • #MDB_RDONLY - * This transaction will not perform any write operations. - *
  • #MDB_NOSYNC - * Don't flush system buffers to disk when committing this transaction. - *
  • #MDB_NOMETASYNC - * Flush system buffers but omit metadata flush when committing this transaction. - *
- * @param[out] txn Address where the new #MDB_txn handle will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • #MDB_PANIC - a fatal error occurred earlier and the environment - * must be shut down. - *
  • #MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's - * mapsize and this environment's map must be resized as well. - * See #mdb_env_set_mapsize(). - *
  • #MDB_READERS_FULL - a read-only transaction was requested and - * the reader lock table is full. See #mdb_env_set_maxreaders(). - *
  • ENOMEM - out of memory. - *
- */ -int mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **txn); - - /** @brief Returns the transaction's #MDB_env - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - */ -MDB_env *mdb_txn_env(MDB_txn *txn); - - /** @brief Return the transaction's ID. - * - * This returns the identifier associated with this transaction. For a - * read-only transaction, this corresponds to the snapshot being read; - * concurrent readers will frequently have the same transaction ID. - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @return A transaction ID, valid if input is an active transaction. - */ -mdb_size_t mdb_txn_id(MDB_txn *txn); - - /** @brief Commit all the operations of a transaction into the database. - * - * The transaction handle is freed. It and its cursors must not be used - * again after this call, except with #mdb_cursor_renew(). - * @note Earlier documentation incorrectly said all cursors would be freed. - * Only write-transactions free cursors. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
  • ENOSPC - no more disk space. - *
  • EIO - a low-level I/O error occurred while writing. - *
  • ENOMEM - out of memory. - *
- */ -int mdb_txn_commit(MDB_txn *txn); - - /** @brief Abandon all the operations of the transaction instead of saving them. - * - * The transaction handle is freed. It and its cursors must not be used - * again after this call, except with #mdb_cursor_renew(). - * @note Earlier documentation incorrectly said all cursors would be freed. - * Only write-transactions free cursors. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - */ -void mdb_txn_abort(MDB_txn *txn); - - /** @brief Reset a read-only transaction. - * - * Abort the transaction like #mdb_txn_abort(), but keep the transaction - * handle. #mdb_txn_renew() may reuse the handle. This saves allocation - * overhead if the process will start a new read-only transaction soon, - * and also locking overhead if #MDB_NOTLS is in use. The reader table - * lock is released, but the table slot stays tied to its thread or - * #MDB_txn. Use mdb_txn_abort() to discard a reset handle, and to free - * its lock table slot if MDB_NOTLS is in use. - * Cursors opened within the transaction must not be used - * again after this call, except with #mdb_cursor_renew(). - * Reader locks generally don't interfere with writers, but they keep old - * versions of database pages allocated. Thus they prevent the old pages - * from being reused when writers commit new data, and so under heavy load - * the database size may grow much more rapidly than otherwise. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - */ -void mdb_txn_reset(MDB_txn *txn); - - /** @brief Renew a read-only transaction. - * - * This acquires a new reader lock for a transaction handle that had been - * released by #mdb_txn_reset(). It must be called before a reset transaction - * may be used again. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • #MDB_PANIC - a fatal error occurred earlier and the environment - * must be shut down. - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_txn_renew(MDB_txn *txn); - -/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ -#define mdb_open(txn,name,flags,dbi) mdb_dbi_open(txn,name,flags,dbi) -/** Compat with version <= 0.9.4, avoid clash with libmdb from MDB Tools project */ -#define mdb_close(env,dbi) mdb_dbi_close(env,dbi) - - /** @brief Open a database in the environment. - * - * A database handle denotes the name and parameters of a database, - * independently of whether such a database exists. - * The database handle may be discarded by calling #mdb_dbi_close(). - * The old database handle is returned if the database was already open. - * The handle may only be closed once. - * - * The database handle will be private to the current transaction until - * the transaction is successfully committed. If the transaction is - * aborted the handle will be closed automatically. - * After a successful commit the handle will reside in the shared - * environment, and may be used by other transactions. - * - * This function must not be called from multiple concurrent - * transactions in the same process. A transaction that uses - * this function must finish (either commit or abort) before - * any other transaction in the process may use this function. - * - * To use named databases (with name != NULL), #mdb_env_set_maxdbs() - * must be called before opening the environment. Database names are - * keys in the unnamed database, and may be read but not written. - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] name The name of the database to open. If only a single - * database is needed in the environment, this value may be NULL. - * @param[in] flags Special options for this database. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *
    - *
  • #MDB_REVERSEKEY - * Keys are strings to be compared in reverse order, from the end - * of the strings to the beginning. By default, Keys are treated as strings and - * compared from beginning to end. - *
  • #MDB_DUPSORT - * Duplicate keys may be used in the database. (Or, from another perspective, - * keys may have multiple data items, stored in sorted order.) By default - * keys must be unique and may have only a single data item. - *
  • #MDB_INTEGERKEY - * Keys are binary integers in native byte order, either unsigned int - * or #mdb_size_t, and will be sorted as such. - * (lmdb expects 32-bit int <= size_t <= 32/64-bit mdb_size_t.) - * The keys must all be of the same size. - *
  • #MDB_DUPFIXED - * This flag may only be used in combination with #MDB_DUPSORT. This option - * tells the library that the data items for this database are all the same - * size, which allows further optimizations in storage and retrieval. When - * all data items are the same size, the #MDB_GET_MULTIPLE, #MDB_NEXT_MULTIPLE - * and #MDB_PREV_MULTIPLE cursor operations may be used to retrieve multiple - * items at once. - *
  • #MDB_INTEGERDUP - * This option specifies that duplicate data items are binary integers, - * similar to #MDB_INTEGERKEY keys. - *
  • #MDB_REVERSEDUP - * This option specifies that duplicate data items should be compared as - * strings in reverse order. - *
  • #MDB_CREATE - * Create the named database if it doesn't exist. This option is not - * allowed in a read-only transaction or a read-only environment. - *
- * @param[out] dbi Address where the new #MDB_dbi handle will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • #MDB_NOTFOUND - the specified database doesn't exist in the environment - * and #MDB_CREATE was not specified. - *
  • #MDB_DBS_FULL - too many databases have been opened. See #mdb_env_set_maxdbs(). - *
- */ -int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi); - - /** @brief Retrieve statistics for a database. - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[out] stat The address of an #MDB_stat structure - * where the statistics will be copied - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *stat); - - /** @brief Retrieve the DB flags for a database handle. - * - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[out] flags Address where the flags will be returned. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags); - - /** @brief Close a database handle. Normally unnecessary. Use with care: - * - * This call is not mutex protected. Handles should only be closed by - * a single thread, and only if no other threads are going to reference - * the database handle or one of its cursors any further. Do not close - * a handle if an existing transaction has modified its database. - * Doing so can cause misbehavior from database corruption to errors - * like MDB_BAD_VALSIZE (since the DB name is gone). - * - * Closing a database handle is not necessary, but lets #mdb_dbi_open() - * reuse the handle value. Usually it's better to set a bigger - * #mdb_env_set_maxdbs(), unless that value would be large. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - */ -void mdb_dbi_close(MDB_env *env, MDB_dbi dbi); - - /** @brief Empty or delete+close a database. - * - * See #mdb_dbi_close() for restrictions about closing the DB handle. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] del 0 to empty the DB, 1 to delete it from the - * environment and close the DB handle. - * @return A non-zero error value on failure and 0 on success. - */ -int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del); - - /** @brief Set a custom key comparison function for a database. - * - * The comparison function is called whenever it is necessary to compare a - * key specified by the application with a key currently stored in the database. - * If no comparison function is specified, and no special key flags were specified - * with #mdb_dbi_open(), the keys are compared lexically, with shorter keys collating - * before longer keys. - * @warning This function must be called before any data access functions are used, - * otherwise data corruption may occur. The same comparison function must be used by every - * program accessing the database, every time the database is used. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] cmp A #MDB_cmp_func function - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); - - /** @brief Set a custom data comparison function for a #MDB_DUPSORT database. - * - * This comparison function is called whenever it is necessary to compare a data - * item specified by the application with a data item currently stored in the database. - * This function only takes effect if the database was opened with the #MDB_DUPSORT - * flag. - * If no comparison function is specified, and no special key flags were specified - * with #mdb_dbi_open(), the data items are compared lexically, with shorter items collating - * before longer items. - * @warning This function must be called before any data access functions are used, - * otherwise data corruption may occur. The same comparison function must be used by every - * program accessing the database, every time the database is used. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] cmp A #MDB_cmp_func function - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp); - - /** @brief Set a relocation function for a #MDB_FIXEDMAP database. - * - * @todo The relocation function is called whenever it is necessary to move the data - * of an item to a different position in the database (e.g. through tree - * balancing operations, shifts as a result of adds or deletes, etc.). It is - * intended to allow address/position-dependent data items to be stored in - * a database in an environment opened with the #MDB_FIXEDMAP option. - * Currently the relocation feature is unimplemented and setting - * this function has no effect. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] rel A #MDB_rel_func function - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel); - - /** @brief Set a context pointer for a #MDB_FIXEDMAP database's relocation function. - * - * See #mdb_set_relfunc and #MDB_rel_func for more details. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] ctx An arbitrary pointer for whatever the application needs. - * It will be passed to the callback function set by #mdb_set_relfunc - * as its \b relctx parameter whenever the callback is invoked. - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx); - - /** @brief Get items from a database. - * - * This function retrieves key/data pairs from the database. The address - * and length of the data associated with the specified \b key are returned - * in the structure to which \b data refers. - * If the database supports duplicate keys (#MDB_DUPSORT) then the - * first data item for the key will be returned. Retrieval of other - * items requires the use of #mdb_cursor_get(). - * - * @note The memory pointed to by the returned values is owned by the - * database. The caller need not dispose of the memory, and may not - * modify it in any way. For values returned in a read-only transaction - * any modification attempts will cause a SIGSEGV. - * @note Values returned from the database are valid only until a - * subsequent update operation, or the end of the transaction. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] key The key to search for in the database - * @param[out] data The data corresponding to the key - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • #MDB_NOTFOUND - the key was not in the database. - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); - - /** @brief Store items into a database. - * - * This function stores key/data pairs in the database. The default behavior - * is to enter the new key/data pair, replacing any previously existing key - * if duplicates are disallowed, or adding a duplicate data item if - * duplicates are allowed (#MDB_DUPSORT). - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] key The key to store in the database - * @param[in,out] data The data to store - * @param[in] flags Special options for this operation. This parameter - * must be set to 0 or by bitwise OR'ing together one or more of the - * values described here. - *
    - *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not - * already appear in the database. This flag may only be specified - * if the database was opened with #MDB_DUPSORT. The function will - * return #MDB_KEYEXIST if the key/data pair already appears in the - * database. - *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key - * does not already appear in the database. The function will return - * #MDB_KEYEXIST if the key already appears in the database, even if - * the database supports duplicates (#MDB_DUPSORT). The \b data - * parameter will be set to point to the existing item. - *
  • #MDB_RESERVE - reserve space for data of the given size, but - * don't copy the given data. Instead, return a pointer to the - * reserved space, which the caller can fill in later - before - * the next update operation or the transaction ends. This saves - * an extra memcpy if the data is being generated later. - * LMDB does nothing else with this memory, the caller is expected - * to modify all of the space requested. This flag must not be - * specified if the database was opened with #MDB_DUPSORT. - *
  • #MDB_APPEND - append the given key/data pair to the end of the - * database. This option allows fast bulk loading when keys are - * already known to be in the correct order. Loading unsorted keys - * with this flag will cause a #MDB_KEYEXIST error. - *
  • #MDB_APPENDDUP - as above, but for sorted dup data. - *
- * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). - *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. - *
  • EACCES - an attempt was made to write in a read-only transaction. - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_put(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, - unsigned int flags); - - /** @brief Delete items from a database. - * - * This function removes key/data pairs from the database. - * If the database does not support sorted duplicate data items - * (#MDB_DUPSORT) the data parameter is ignored. - * If the database supports sorted duplicates and the data parameter - * is NULL, all of the duplicate data items for the key will be - * deleted. Otherwise, if the data parameter is non-NULL - * only the matching data item will be deleted. - * This function will return #MDB_NOTFOUND if the specified key/data - * pair is not in the database. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] key The key to delete from the database - * @param[in] data The data to delete - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EACCES - an attempt was made to write in a read-only transaction. - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_del(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); - - /** @brief Create a cursor handle. - * - * A cursor is associated with a specific transaction and database. - * A cursor cannot be used when its database handle is closed. Nor - * when its transaction has ended, except with #mdb_cursor_renew(). - * It can be discarded with #mdb_cursor_close(). - * A cursor in a write-transaction can be closed before its transaction - * ends, and will otherwise be closed when its transaction ends. - * A cursor in a read-only transaction must be closed explicitly, before - * or after its transaction ends. It can be reused with - * #mdb_cursor_renew() before finally closing it. - * @note Earlier documentation said that cursors in every transaction - * were closed when the transaction committed or aborted. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[out] cursor Address where the new #MDB_cursor handle will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **cursor); - - /** @brief Close a cursor handle. - * - * The cursor handle will be freed and must not be used again after this call. - * Its transaction must still be live if it is a write-transaction. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - */ -void mdb_cursor_close(MDB_cursor *cursor); - - /** @brief Renew a cursor handle. - * - * A cursor is associated with a specific transaction and database. - * Cursors that are only used in read-only - * transactions may be re-used, to avoid unnecessary malloc/free overhead. - * The cursor may be associated with a new read-only transaction, and - * referencing the same database handle as it was created with. - * This may be done whether the previous transaction is live or dead. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_cursor_renew(MDB_txn *txn, MDB_cursor *cursor); - - /** @brief Return the cursor's transaction handle. - * - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - */ -MDB_txn *mdb_cursor_txn(MDB_cursor *cursor); - - /** @brief Return the cursor's database handle. - * - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - */ -MDB_dbi mdb_cursor_dbi(MDB_cursor *cursor); - - /** @brief Retrieve by cursor. - * - * This function retrieves key/data pairs from the database. The address and length - * of the key are returned in the object to which \b key refers (except for the - * case of the #MDB_SET option, in which the \b key object is unchanged), and - * the address and length of the data are returned in the object to which \b data - * refers. - * See #mdb_get() for restrictions on using the output values. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[in,out] key The key for a retrieved item - * @param[in,out] data The data of a retrieved item - * @param[in] op A cursor operation #MDB_cursor_op - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • #MDB_NOTFOUND - no matching key found. - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data, - MDB_cursor_op op); - - /** @brief Store by cursor. - * - * This function stores key/data pairs into the database. - * The cursor is positioned at the new item, or on failure usually near it. - * @note Earlier documentation incorrectly said errors would leave the - * state of the cursor unchanged. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[in] key The key operated on. - * @param[in] data The data operated on. - * @param[in] flags Options for this operation. This parameter - * must be set to 0 or one of the values described here. - *
    - *
  • #MDB_CURRENT - replace the item at the current cursor position. - * The \b key parameter must still be provided, and must match it. - * If using sorted duplicates (#MDB_DUPSORT) the data item must still - * sort into the same place. This is intended to be used when the - * new data is the same size as the old. Otherwise it will simply - * perform a delete of the old record followed by an insert. - *
  • #MDB_NODUPDATA - enter the new key/data pair only if it does not - * already appear in the database. This flag may only be specified - * if the database was opened with #MDB_DUPSORT. The function will - * return #MDB_KEYEXIST if the key/data pair already appears in the - * database. - *
  • #MDB_NOOVERWRITE - enter the new key/data pair only if the key - * does not already appear in the database. The function will return - * #MDB_KEYEXIST if the key already appears in the database, even if - * the database supports duplicates (#MDB_DUPSORT). - *
  • #MDB_RESERVE - reserve space for data of the given size, but - * don't copy the given data. Instead, return a pointer to the - * reserved space, which the caller can fill in later - before - * the next update operation or the transaction ends. This saves - * an extra memcpy if the data is being generated later. This flag - * must not be specified if the database was opened with #MDB_DUPSORT. - *
  • #MDB_APPEND - append the given key/data pair to the end of the - * database. No key comparisons are performed. This option allows - * fast bulk loading when keys are already known to be in the - * correct order. Loading unsorted keys with this flag will cause - * a #MDB_KEYEXIST error. - *
  • #MDB_APPENDDUP - as above, but for sorted dup data. - *
  • #MDB_MULTIPLE - store multiple contiguous data elements in a - * single request. This flag may only be specified if the database - * was opened with #MDB_DUPFIXED. The \b data argument must be an - * array of two MDB_vals. The mv_size of the first MDB_val must be - * the size of a single data element. The mv_data of the first MDB_val - * must point to the beginning of the array of contiguous data elements. - * The mv_size of the second MDB_val must be the count of the number - * of data elements to store. On return this field will be set to - * the count of the number of elements actually written. The mv_data - * of the second MDB_val is unused. - *
- * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • #MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize(). - *
  • #MDB_TXN_FULL - the transaction has too many dirty pages. - *
  • EACCES - an attempt was made to write in a read-only transaction. - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data, - unsigned int flags); - - /** @brief Delete current key/data pair - * - * This function deletes the key/data pair to which the cursor refers. - * This does not invalidate the cursor, so operations such as MDB_NEXT - * can still be used on it. - * Both MDB_NEXT and MDB_GET_CURRENT will return the same record after - * this operation. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[in] flags Options for this operation. This parameter - * must be set to 0 or one of the values described here. - *
    - *
  • #MDB_NODUPDATA - delete all of the data items for the current key. - * This flag may only be specified if the database was opened with #MDB_DUPSORT. - *
- * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EACCES - an attempt was made to write in a read-only transaction. - *
  • EINVAL - an invalid parameter was specified. - *
- */ -int mdb_cursor_del(MDB_cursor *cursor, unsigned int flags); - - /** @brief Return count of duplicates for current key. - * - * This call is only valid on databases that support sorted duplicate - * data items #MDB_DUPSORT. - * @param[in] cursor A cursor handle returned by #mdb_cursor_open() - * @param[out] countp Address where the count will be stored - * @return A non-zero error value on failure and 0 on success. Some possible - * errors are: - *
    - *
  • EINVAL - cursor is not initialized, or an invalid parameter was specified. - *
- */ -int mdb_cursor_count(MDB_cursor *cursor, mdb_size_t *countp); - - /** @brief Compare two data items according to a particular database. - * - * This returns a comparison as if the two data items were keys in the - * specified database. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] a The first item to compare - * @param[in] b The second item to compare - * @return < 0 if a < b, 0 if a == b, > 0 if a > b - */ -int mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); - - /** @brief Compare two data items according to a particular database. - * - * This returns a comparison as if the two items were data items of - * the specified database. The database must have the #MDB_DUPSORT flag. - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - * @param[in] a The first item to compare - * @param[in] b The second item to compare - * @return < 0 if a < b, 0 if a == b, > 0 if a > b - */ -int mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b); - - /** @brief A callback function used to print a message from the library. - * - * @param[in] msg The string to be printed. - * @param[in] ctx An arbitrary context pointer for the callback. - * @return < 0 on failure, >= 0 on success. - */ -typedef int (MDB_msg_func)(const char *msg, void *ctx); - - /** @brief Dump the entries in the reader lock table. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[in] func A #MDB_msg_func function - * @param[in] ctx Anything the message function needs - * @return < 0 on failure, >= 0 on success. - */ -int mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx); - - /** @brief Check for stale entries in the reader lock table. - * - * @param[in] env An environment handle returned by #mdb_env_create() - * @param[out] dead Number of stale slots that were cleared - * @return 0 on success, non-zero on failure. - */ -int mdb_reader_check(MDB_env *env, int *dead); -/** @} */ - -#ifdef __cplusplus -} -#endif -/** @page tools LMDB Command Line Tools - The following describes the command line tools that are available for LMDB. - \li \ref mdb_copy_1 - \li \ref mdb_dump_1 - \li \ref mdb_load_1 - \li \ref mdb_stat_1 -*/ - -#endif /* _LMDB_H_ */ diff --git a/pkg/hs/lmdb-static/cbits/mdb.c b/pkg/hs/lmdb-static/cbits/mdb.c deleted file mode 100644 index f68542124e..0000000000 --- a/pkg/hs/lmdb-static/cbits/mdb.c +++ /dev/null @@ -1,11199 +0,0 @@ -/** @file mdb.c - * @brief Lightning memory-mapped database library - * - * A Btree-based database management library modeled loosely on the - * BerkeleyDB API, but much simplified. - */ -/* - * Copyright 2011-2019 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - * - * This code is derived from btree.c written by Martin Hedenfalk. - * - * Copyright (c) 2009, 2010 Martin Hedenfalk - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ -#ifndef _GNU_SOURCE -#define _GNU_SOURCE 1 -#endif -#if defined(MDB_VL32) || defined(__WIN64__) -#define _FILE_OFFSET_BITS 64 -#endif -#ifdef _WIN32 -#include -#include -#include /* get wcscpy() */ - -/* We use native NT APIs to setup the memory map, so that we can - * let the DB file grow incrementally instead of always preallocating - * the full size. These APIs are defined in and - * but those headers are meant for driver-level development and - * conflict with the regular user-level headers, so we explicitly - * declare them here. We get pointers to these functions from - * NTDLL.DLL at runtime, to avoid buildtime dependencies on any - * NTDLL import libraries. - */ -typedef NTSTATUS (WINAPI NtCreateSectionFunc) - (OUT PHANDLE sh, IN ACCESS_MASK acc, - IN void * oa OPTIONAL, - IN PLARGE_INTEGER ms OPTIONAL, - IN ULONG pp, IN ULONG aa, IN HANDLE fh OPTIONAL); - -static NtCreateSectionFunc *NtCreateSection; - -typedef enum _SECTION_INHERIT { - ViewShare = 1, - ViewUnmap = 2 -} SECTION_INHERIT; - -typedef NTSTATUS (WINAPI NtMapViewOfSectionFunc) - (IN PHANDLE sh, IN HANDLE ph, - IN OUT PVOID *addr, IN ULONG_PTR zbits, - IN SIZE_T cs, IN OUT PLARGE_INTEGER off OPTIONAL, - IN OUT PSIZE_T vs, IN SECTION_INHERIT ih, - IN ULONG at, IN ULONG pp); - -static NtMapViewOfSectionFunc *NtMapViewOfSection; - -typedef NTSTATUS (WINAPI NtCloseFunc)(HANDLE h); - -static NtCloseFunc *NtClose; - -/** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it - * as int64 which is wrong. MSVC doesn't define it at all, so just - * don't use it. - */ -#define MDB_PID_T int -#define MDB_THR_T DWORD -#include -#include -#ifdef __GNUC__ -# include -#else -# define LITTLE_ENDIAN 1234 -# define BIG_ENDIAN 4321 -# define BYTE_ORDER LITTLE_ENDIAN -# ifndef SSIZE_MAX -# define SSIZE_MAX INT_MAX -# endif -#endif -#else -#include -#include -#define MDB_PID_T pid_t -#define MDB_THR_T pthread_t -#include -#include -#include -#ifdef HAVE_SYS_FILE_H -#include -#endif -#include -#endif - -#if defined(__mips) && defined(__linux) -/* MIPS has cache coherency issues, requires explicit cache control */ -#include -extern int cacheflush(char *addr, int nbytes, int cache); -#define CACHEFLUSH(addr, bytes, cache) cacheflush(addr, bytes, cache) -#else -#define CACHEFLUSH(addr, bytes, cache) -#endif - -#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS) -/** fdatasync is broken on ext3/ext4fs on older kernels, see - * description in #mdb_env_open2 comments. You can safely - * define MDB_FDATASYNC_WORKS if this code will only be run - * on kernels 3.6 and newer. - */ -#define BROKEN_FDATASYNC -#endif - -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _MSC_VER -#include -typedef SSIZE_T ssize_t; -#else -#include -#endif - -#if defined(__sun) || defined(__ANDROID__) -/* Most platforms have posix_memalign, older may only have memalign */ -#define HAVE_MEMALIGN 1 -#include -/* On Solaris, we need the POSIX sigwait function */ -#if defined (__sun) -# define _POSIX_PTHREAD_SEMANTICS 1 -#endif -#endif - -#if !(defined(BYTE_ORDER) || defined(__BYTE_ORDER)) -#include -#include /* defines BYTE_ORDER on HPUX and Solaris */ -#endif - -#if defined(__APPLE__) || defined (BSD) || defined(__FreeBSD_kernel__) -# if !(defined(MDB_USE_POSIX_MUTEX) || defined(MDB_USE_POSIX_SEM)) -# define MDB_USE_SYSV_SEM 1 -# endif -# define MDB_FDATASYNC fsync -#elif defined(__ANDROID__) -# define MDB_FDATASYNC fsync -#endif - -#ifndef _WIN32 -#include -#include -#ifdef MDB_USE_POSIX_SEM -# define MDB_USE_HASH 1 -#include -#elif defined(MDB_USE_SYSV_SEM) -#include -#include -#ifdef _SEM_SEMUN_UNDEFINED -union semun { - int val; - struct semid_ds *buf; - unsigned short *array; -}; -#endif /* _SEM_SEMUN_UNDEFINED */ -#else -#define MDB_USE_POSIX_MUTEX 1 -#endif /* MDB_USE_POSIX_SEM */ -#endif /* !_WIN32 */ - -#if defined(_WIN32) + defined(MDB_USE_POSIX_SEM) + defined(MDB_USE_SYSV_SEM) \ - + defined(MDB_USE_POSIX_MUTEX) != 1 -# error "Ambiguous shared-lock implementation" -#endif - -#ifdef USE_VALGRIND -#include -#define VGMEMP_CREATE(h,r,z) VALGRIND_CREATE_MEMPOOL(h,r,z) -#define VGMEMP_ALLOC(h,a,s) VALGRIND_MEMPOOL_ALLOC(h,a,s) -#define VGMEMP_FREE(h,a) VALGRIND_MEMPOOL_FREE(h,a) -#define VGMEMP_DESTROY(h) VALGRIND_DESTROY_MEMPOOL(h) -#define VGMEMP_DEFINED(a,s) VALGRIND_MAKE_MEM_DEFINED(a,s) -#else -#define VGMEMP_CREATE(h,r,z) -#define VGMEMP_ALLOC(h,a,s) -#define VGMEMP_FREE(h,a) -#define VGMEMP_DESTROY(h) -#define VGMEMP_DEFINED(a,s) -#endif - -#ifndef BYTE_ORDER -# if (defined(_LITTLE_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(_LITTLE_ENDIAN) && defined(_BIG_ENDIAN)) -/* Solaris just defines one or the other */ -# define LITTLE_ENDIAN 1234 -# define BIG_ENDIAN 4321 -# ifdef _LITTLE_ENDIAN -# define BYTE_ORDER LITTLE_ENDIAN -# else -# define BYTE_ORDER BIG_ENDIAN -# endif -# else -# define BYTE_ORDER __BYTE_ORDER -# endif -#endif - -#ifndef LITTLE_ENDIAN -#define LITTLE_ENDIAN __LITTLE_ENDIAN -#endif -#ifndef BIG_ENDIAN -#define BIG_ENDIAN __BIG_ENDIAN -#endif - -#if defined(__i386) || defined(__x86_64) || defined(_M_IX86) -#define MISALIGNED_OK 1 -#endif - -#include "lmdb.h" -#include "midl.h" - -#if (BYTE_ORDER == LITTLE_ENDIAN) == (BYTE_ORDER == BIG_ENDIAN) -# error "Unknown or unsupported endianness (BYTE_ORDER)" -#elif (-6 & 5) || CHAR_BIT!=8 || UINT_MAX!=0xffffffff || MDB_SIZE_MAX%UINT_MAX -# error "Two's complement, reasonably sized integer types, please" -#endif - -#ifdef __GNUC__ -/** Put infrequently used env functions in separate section */ -# ifdef __APPLE__ -# define ESECT __attribute__ ((section("__TEXT,text_env"))) -# else -# define ESECT __attribute__ ((section("text_env"))) -# endif -#else -#define ESECT -#endif - -#ifdef _WIN32 -#define CALL_CONV WINAPI -#else -#define CALL_CONV -#endif - -/** @defgroup internal LMDB Internals - * @{ - */ -/** @defgroup compat Compatibility Macros - * A bunch of macros to minimize the amount of platform-specific ifdefs - * needed throughout the rest of the code. When the features this library - * needs are similar enough to POSIX to be hidden in a one-or-two line - * replacement, this macro approach is used. - * @{ - */ - - /** Features under development */ -#ifndef MDB_DEVEL -#define MDB_DEVEL 0 -#endif - - /** Wrapper around __func__, which is a C99 feature */ -#if __STDC_VERSION__ >= 199901L -# define mdb_func_ __func__ -#elif __GNUC__ >= 2 || _MSC_VER >= 1300 -# define mdb_func_ __FUNCTION__ -#else -/* If a debug message says (), update the #if statements above */ -# define mdb_func_ "" -#endif - -/* Internal error codes, not exposed outside liblmdb */ -#define MDB_NO_ROOT (MDB_LAST_ERRCODE + 10) -#ifdef _WIN32 -#define MDB_OWNERDEAD ((int) WAIT_ABANDONED) -#elif defined MDB_USE_SYSV_SEM -#define MDB_OWNERDEAD (MDB_LAST_ERRCODE + 11) -#elif defined(MDB_USE_POSIX_MUTEX) && defined(EOWNERDEAD) -#define MDB_OWNERDEAD EOWNERDEAD /**< #LOCK_MUTEX0() result if dead owner */ -#endif - -#ifdef __GLIBC__ -#define GLIBC_VER ((__GLIBC__ << 16 )| __GLIBC_MINOR__) -#endif -/** Some platforms define the EOWNERDEAD error code - * even though they don't support Robust Mutexes. - * Compile with -DMDB_USE_ROBUST=0, or use some other - * mechanism like -DMDB_USE_SYSV_SEM instead of - * -DMDB_USE_POSIX_MUTEX. (SysV semaphores are - * also Robust, but some systems don't support them - * either.) - */ -#ifndef MDB_USE_ROBUST -/* Android currently lacks Robust Mutex support. So does glibc < 2.4. */ -# if defined(MDB_USE_POSIX_MUTEX) && (defined(__ANDROID__) || \ - (defined(__GLIBC__) && GLIBC_VER < 0x020004)) -# define MDB_USE_ROBUST 0 -# else -# define MDB_USE_ROBUST 1 -# endif -#endif /* !MDB_USE_ROBUST */ - -#if defined(MDB_USE_POSIX_MUTEX) && (MDB_USE_ROBUST) -/* glibc < 2.12 only provided _np API */ -# if (defined(__GLIBC__) && GLIBC_VER < 0x02000c) || \ - (defined(PTHREAD_MUTEX_ROBUST_NP) && !defined(PTHREAD_MUTEX_ROBUST)) -# define PTHREAD_MUTEX_ROBUST PTHREAD_MUTEX_ROBUST_NP -# define pthread_mutexattr_setrobust(attr, flag) pthread_mutexattr_setrobust_np(attr, flag) -# define pthread_mutex_consistent(mutex) pthread_mutex_consistent_np(mutex) -# endif -#endif /* MDB_USE_POSIX_MUTEX && MDB_USE_ROBUST */ - -#if defined(MDB_OWNERDEAD) && (MDB_USE_ROBUST) -#define MDB_ROBUST_SUPPORTED 1 -#endif - -#ifdef _WIN32 -#define MDB_USE_HASH 1 -#define MDB_PIDLOCK 0 -#define THREAD_RET DWORD -#define pthread_t HANDLE -#define pthread_mutex_t HANDLE -#define pthread_cond_t HANDLE -typedef HANDLE mdb_mutex_t, mdb_mutexref_t; -#define pthread_key_t DWORD -#define pthread_self() GetCurrentThreadId() -#define pthread_key_create(x,y) \ - ((*(x) = TlsAlloc()) == TLS_OUT_OF_INDEXES ? ErrCode() : 0) -#define pthread_key_delete(x) TlsFree(x) -#define pthread_getspecific(x) TlsGetValue(x) -#define pthread_setspecific(x,y) (TlsSetValue(x,y) ? 0 : ErrCode()) -#define pthread_mutex_unlock(x) ReleaseMutex(*x) -#define pthread_mutex_lock(x) WaitForSingleObject(*x, INFINITE) -#define pthread_cond_signal(x) SetEvent(*x) -#define pthread_cond_wait(cond,mutex) do{SignalObjectAndWait(*mutex, *cond, INFINITE, FALSE); WaitForSingleObject(*mutex, INFINITE);}while(0) -#define THREAD_CREATE(thr,start,arg) \ - (((thr) = CreateThread(NULL, 0, start, arg, 0, NULL)) ? 0 : ErrCode()) -#define THREAD_FINISH(thr) \ - (WaitForSingleObject(thr, INFINITE) ? ErrCode() : 0) -#define LOCK_MUTEX0(mutex) WaitForSingleObject(mutex, INFINITE) -#define UNLOCK_MUTEX(mutex) ReleaseMutex(mutex) -#define mdb_mutex_consistent(mutex) 0 -#define getpid() GetCurrentProcessId() -#define MDB_FDATASYNC(fd) (!FlushFileBuffers(fd)) -#define MDB_MSYNC(addr,len,flags) (!FlushViewOfFile(addr,len)) -#define ErrCode() GetLastError() -#define GET_PAGESIZE(x) {SYSTEM_INFO si; GetSystemInfo(&si); (x) = si.dwPageSize;} -#define close(fd) (CloseHandle(fd) ? 0 : -1) -#define munmap(ptr,len) UnmapViewOfFile(ptr) -#ifdef PROCESS_QUERY_LIMITED_INFORMATION -#define MDB_PROCESS_QUERY_LIMITED_INFORMATION PROCESS_QUERY_LIMITED_INFORMATION -#else -#define MDB_PROCESS_QUERY_LIMITED_INFORMATION 0x1000 -#endif -#else -#define THREAD_RET void * -#define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) -#define THREAD_FINISH(thr) pthread_join(thr,NULL) - - /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ -#define MDB_PIDLOCK 1 - -#ifdef MDB_USE_POSIX_SEM - -typedef sem_t *mdb_mutex_t, *mdb_mutexref_t; -#define LOCK_MUTEX0(mutex) mdb_sem_wait(mutex) -#define UNLOCK_MUTEX(mutex) sem_post(mutex) - -static int -mdb_sem_wait(sem_t *sem) -{ - int rc; - while ((rc = sem_wait(sem)) && (rc = errno) == EINTR) ; - return rc; -} - -#elif defined MDB_USE_SYSV_SEM - -typedef struct mdb_mutex { - int semid; - int semnum; - int *locked; -} mdb_mutex_t[1], *mdb_mutexref_t; - -#define LOCK_MUTEX0(mutex) mdb_sem_wait(mutex) -#define UNLOCK_MUTEX(mutex) do { \ - struct sembuf sb = { 0, 1, SEM_UNDO }; \ - sb.sem_num = (mutex)->semnum; \ - *(mutex)->locked = 0; \ - semop((mutex)->semid, &sb, 1); \ -} while(0) - -static int -mdb_sem_wait(mdb_mutexref_t sem) -{ - int rc, *locked = sem->locked; - struct sembuf sb = { 0, -1, SEM_UNDO }; - sb.sem_num = sem->semnum; - do { - if (!semop(sem->semid, &sb, 1)) { - rc = *locked ? MDB_OWNERDEAD : MDB_SUCCESS; - *locked = 1; - break; - } - } while ((rc = errno) == EINTR); - return rc; -} - -#define mdb_mutex_consistent(mutex) 0 - -#else /* MDB_USE_POSIX_MUTEX: */ - /** Shared mutex/semaphore as the original is stored. - * - * Not for copies. Instead it can be assigned to an #mdb_mutexref_t. - * When mdb_mutexref_t is a pointer and mdb_mutex_t is not, then it - * is array[size 1] so it can be assigned to the pointer. - */ -typedef pthread_mutex_t mdb_mutex_t[1]; - /** Reference to an #mdb_mutex_t */ -typedef pthread_mutex_t *mdb_mutexref_t; - /** Lock the reader or writer mutex. - * Returns 0 or a code to give #mdb_mutex_failed(), as in #LOCK_MUTEX(). - */ -#define LOCK_MUTEX0(mutex) pthread_mutex_lock(mutex) - /** Unlock the reader or writer mutex. - */ -#define UNLOCK_MUTEX(mutex) pthread_mutex_unlock(mutex) - /** Mark mutex-protected data as repaired, after death of previous owner. - */ -#define mdb_mutex_consistent(mutex) pthread_mutex_consistent(mutex) -#endif /* MDB_USE_POSIX_SEM || MDB_USE_SYSV_SEM */ - - /** Get the error code for the last failed system function. - */ -#define ErrCode() errno - - /** An abstraction for a file handle. - * On POSIX systems file handles are small integers. On Windows - * they're opaque pointers. - */ -#define HANDLE int - - /** A value for an invalid file handle. - * Mainly used to initialize file variables and signify that they are - * unused. - */ -#define INVALID_HANDLE_VALUE (-1) - - /** Get the size of a memory page for the system. - * This is the basic size that the platform's memory manager uses, and is - * fundamental to the use of memory-mapped files. - */ -#define GET_PAGESIZE(x) ((x) = sysconf(_SC_PAGE_SIZE)) -#endif - -#define Z MDB_FMT_Z /**< printf/scanf format modifier for size_t */ -#define Yu MDB_PRIy(u) /**< printf format for #mdb_size_t */ -#define Yd MDB_PRIy(d) /**< printf format for 'signed #mdb_size_t' */ - -#ifdef MDB_USE_SYSV_SEM -#define MNAME_LEN (sizeof(int)) -#else -#define MNAME_LEN (sizeof(pthread_mutex_t)) -#endif - -/** Initial part of #MDB_env.me_mutexname[]. - * Changes to this code must be reflected in #MDB_LOCK_FORMAT. - */ -#ifdef _WIN32 -#define MUTEXNAME_PREFIX "Global\\MDB" -#elif defined MDB_USE_POSIX_SEM -#define MUTEXNAME_PREFIX "/MDB" -#endif - -/** @} */ - -#ifdef MDB_ROBUST_SUPPORTED - /** Lock mutex, handle any error, set rc = result. - * Return 0 on success, nonzero (not rc) on error. - */ -#define LOCK_MUTEX(rc, env, mutex) \ - (((rc) = LOCK_MUTEX0(mutex)) && \ - ((rc) = mdb_mutex_failed(env, mutex, rc))) -static int mdb_mutex_failed(MDB_env *env, mdb_mutexref_t mutex, int rc); -#else -#define LOCK_MUTEX(rc, env, mutex) ((rc) = LOCK_MUTEX0(mutex)) -#define mdb_mutex_failed(env, mutex, rc) (rc) -#endif - -#ifndef _WIN32 -/** A flag for opening a file and requesting synchronous data writes. - * This is only used when writing a meta page. It's not strictly needed; - * we could just do a normal write and then immediately perform a flush. - * But if this flag is available it saves us an extra system call. - * - * @note If O_DSYNC is undefined but exists in /usr/include, - * preferably set some compiler flag to get the definition. - */ -#ifndef MDB_DSYNC -# ifdef O_DSYNC -# define MDB_DSYNC O_DSYNC -# else -# define MDB_DSYNC O_SYNC -# endif -#endif -#endif - -/** Function for flushing the data of a file. Define this to fsync - * if fdatasync() is not supported. - */ -#ifndef MDB_FDATASYNC -# define MDB_FDATASYNC fdatasync -#endif - -#ifndef MDB_MSYNC -# define MDB_MSYNC(addr,len,flags) msync(addr,len,flags) -#endif - -#ifndef MS_SYNC -#define MS_SYNC 1 -#endif - -#ifndef MS_ASYNC -#define MS_ASYNC 0 -#endif - - /** A page number in the database. - * Note that 64 bit page numbers are overkill, since pages themselves - * already represent 12-13 bits of addressable memory, and the OS will - * always limit applications to a maximum of 63 bits of address space. - * - * @note In the #MDB_node structure, we only store 48 bits of this value, - * which thus limits us to only 60 bits of addressable data. - */ -typedef MDB_ID pgno_t; - - /** A transaction ID. - * See struct MDB_txn.mt_txnid for details. - */ -typedef MDB_ID txnid_t; - -/** @defgroup debug Debug Macros - * @{ - */ -#ifndef MDB_DEBUG - /** Enable debug output. Needs variable argument macros (a C99 feature). - * Set this to 1 for copious tracing. Set to 2 to add dumps of all IDLs - * read from and written to the database (used for free space management). - */ -#define MDB_DEBUG 0 -#endif - -#if MDB_DEBUG -static int mdb_debug; -static txnid_t mdb_debug_start; - - /** Print a debug message with printf formatting. - * Requires double parenthesis around 2 or more args. - */ -# define DPRINTF(args) ((void) ((mdb_debug) && DPRINTF0 args)) -# define DPRINTF0(fmt, ...) \ - fprintf(stderr, "%s:%d " fmt "\n", mdb_func_, __LINE__, __VA_ARGS__) -#else -# define DPRINTF(args) ((void) 0) -#endif - /** Print a debug string. - * The string is printed literally, with no format processing. - */ -#define DPUTS(arg) DPRINTF(("%s", arg)) - /** Debuging output value of a cursor DBI: Negative in a sub-cursor. */ -#define DDBI(mc) \ - (((mc)->mc_flags & C_SUB) ? -(int)(mc)->mc_dbi : (int)(mc)->mc_dbi) -/** @} */ - - /** @brief The maximum size of a database page. - * - * It is 32k or 64k, since value-PAGEBASE must fit in - * #MDB_page.%mp_upper. - * - * LMDB will use database pages < OS pages if needed. - * That causes more I/O in write transactions: The OS must - * know (read) the whole page before writing a partial page. - * - * Note that we don't currently support Huge pages. On Linux, - * regular data files cannot use Huge pages, and in general - * Huge pages aren't actually pageable. We rely on the OS - * demand-pager to read our data and page it out when memory - * pressure from other processes is high. So until OSs have - * actual paging support for Huge pages, they're not viable. - */ -#define MAX_PAGESIZE (PAGEBASE ? 0x10000 : 0x8000) - - /** The minimum number of keys required in a database page. - * Setting this to a larger value will place a smaller bound on the - * maximum size of a data item. Data items larger than this size will - * be pushed into overflow pages instead of being stored directly in - * the B-tree node. This value used to default to 4. With a page size - * of 4096 bytes that meant that any item larger than 1024 bytes would - * go into an overflow page. That also meant that on average 2-3KB of - * each overflow page was wasted space. The value cannot be lower than - * 2 because then there would no longer be a tree structure. With this - * value, items larger than 2KB will go into overflow pages, and on - * average only 1KB will be wasted. - */ -#define MDB_MINKEYS 2 - - /** A stamp that identifies a file as an LMDB file. - * There's nothing special about this value other than that it is easily - * recognizable, and it will reflect any byte order mismatches. - */ -#define MDB_MAGIC 0xBEEFC0DE - - /** The version number for a database's datafile format. */ -#define MDB_DATA_VERSION ((MDB_DEVEL) ? 999 : 1) - /** The version number for a database's lockfile format. */ -#define MDB_LOCK_VERSION ((MDB_DEVEL) ? 999 : 2) - /** Number of bits representing #MDB_LOCK_VERSION in #MDB_LOCK_FORMAT. - * The remaining bits must leave room for #MDB_lock_desc. - */ -#define MDB_LOCK_VERSION_BITS 12 - - /** @brief The max size of a key we can write, or 0 for computed max. - * - * This macro should normally be left alone or set to 0. - * Note that a database with big keys or dupsort data cannot be - * reliably modified by a liblmdb which uses a smaller max. - * The default is 511 for backwards compat, or 0 when #MDB_DEVEL. - * - * Other values are allowed, for backwards compat. However: - * A value bigger than the computed max can break if you do not - * know what you are doing, and liblmdb <= 0.9.10 can break when - * modifying a DB with keys/dupsort data bigger than its max. - * - * Data items in an #MDB_DUPSORT database are also limited to - * this size, since they're actually keys of a sub-DB. Keys and - * #MDB_DUPSORT data items must fit on a node in a regular page. - */ -#ifndef MDB_MAXKEYSIZE -#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511) -#endif - - /** The maximum size of a key we can write to the environment. */ -#if MDB_MAXKEYSIZE -#define ENV_MAXKEY(env) (MDB_MAXKEYSIZE) -#else -#define ENV_MAXKEY(env) ((env)->me_maxkey) -#endif - - /** @brief The maximum size of a data item. - * - * We only store a 32 bit value for node sizes. - */ -#define MAXDATASIZE 0xffffffffUL - -#if MDB_DEBUG - /** Key size which fits in a #DKBUF. - * @ingroup debug - */ -#define DKBUF_MAXKEYSIZE ((MDB_MAXKEYSIZE) > 0 ? (MDB_MAXKEYSIZE) : 511) - /** A key buffer. - * @ingroup debug - * This is used for printing a hex dump of a key's contents. - */ -#define DKBUF char kbuf[DKBUF_MAXKEYSIZE*2+1] - /** Display a key in hex. - * @ingroup debug - * Invoke a function to display a key in hex. - */ -#define DKEY(x) mdb_dkey(x, kbuf) -#else -#define DKBUF -#define DKEY(x) 0 -#endif - - /** An invalid page number. - * Mainly used to denote an empty tree. - */ -#define P_INVALID (~(pgno_t)0) - - /** Test if the flags \b f are set in a flag word \b w. */ -#define F_ISSET(w, f) (((w) & (f)) == (f)) - - /** Round \b n up to an even number. */ -#define EVEN(n) (((n) + 1U) & -2) /* sign-extending -2 to match n+1U */ - - /** Least significant 1-bit of \b n. n must be of an unsigned type. */ -#define LOW_BIT(n) ((n) & (-(n))) - - /** (log2(\b p2) % \b n), for p2 = power of 2 and 0 < n < 8. */ -#define LOG2_MOD(p2, n) (7 - 86 / ((p2) % ((1U<<(n))-1) + 11)) - /* Explanation: Let p2 = 2**(n*y + x), x> (CACHELINE>64), 5)) - + 6 * (sizeof(MDB_PID_T)/4 % 3) /* legacy(2) to word(4/8)? */ - + 18 * (sizeof(pthread_t)/4 % 5) /* can be struct{id, active data} */ - + 90 * (sizeof(MDB_txbody) / CACHELINE % 3) - + 270 * (MDB_LOCK_TYPE % 120) - /* The above is < 270*120 < 2**15 */ - + ((sizeof(txnid_t) == 8) << 15) /* 32bit/64bit */ - + ((sizeof(MDB_reader) > CACHELINE) << 16) - /* Not really needed - implied by MDB_LOCK_TYPE != (_WIN32 locking) */ - + (((MDB_PIDLOCK) != 0) << 17) - /* 18 bits total: Must be <= (32 - MDB_LOCK_VERSION_BITS). */ -}; -/** @} */ - -/** Common header for all page types. The page type depends on #mp_flags. - * - * #P_BRANCH and #P_LEAF pages have unsorted '#MDB_node's at the end, with - * sorted #mp_ptrs[] entries referring to them. Exception: #P_LEAF2 pages - * omit mp_ptrs and pack sorted #MDB_DUPFIXED values after the page header. - * - * #P_OVERFLOW records occupy one or more contiguous pages where only the - * first has a page header. They hold the real data of #F_BIGDATA nodes. - * - * #P_SUBP sub-pages are small leaf "pages" with duplicate data. - * A node with flag #F_DUPDATA but not #F_SUBDATA contains a sub-page. - * (Duplicate data can also go in sub-databases, which use normal pages.) - * - * #P_META pages contain #MDB_meta, the start point of an LMDB snapshot. - * - * Each non-metapage up to #MDB_meta.%mm_last_pg is reachable exactly once - * in the snapshot: Either used by a database or listed in a freeDB record. - */ -typedef struct MDB_page { -#define mp_pgno mp_p.p_pgno -#define mp_next mp_p.p_next - union { - pgno_t p_pgno; /**< page number */ - struct MDB_page *p_next; /**< for in-memory list of freed pages */ - } mp_p; - uint16_t mp_pad; /**< key size if this is a LEAF2 page */ -/** @defgroup mdb_page Page Flags - * @ingroup internal - * Flags for the page headers. - * @{ - */ -#define P_BRANCH 0x01 /**< branch page */ -#define P_LEAF 0x02 /**< leaf page */ -#define P_OVERFLOW 0x04 /**< overflow page */ -#define P_META 0x08 /**< meta page */ -#define P_DIRTY 0x10 /**< dirty page, also set for #P_SUBP pages */ -#define P_LEAF2 0x20 /**< for #MDB_DUPFIXED records */ -#define P_SUBP 0x40 /**< for #MDB_DUPSORT sub-pages */ -#define P_LOOSE 0x4000 /**< page was dirtied then freed, can be reused */ -#define P_KEEP 0x8000 /**< leave this page alone during spill */ -/** @} */ - uint16_t mp_flags; /**< @ref mdb_page */ -#define mp_lower mp_pb.pb.pb_lower -#define mp_upper mp_pb.pb.pb_upper -#define mp_pages mp_pb.pb_pages - union { - struct { - indx_t pb_lower; /**< lower bound of free space */ - indx_t pb_upper; /**< upper bound of free space */ - } pb; - uint32_t pb_pages; /**< number of overflow pages */ - } mp_pb; - indx_t mp_ptrs[1]; /**< dynamic size */ -} MDB_page; - - /** Size of the page header, excluding dynamic data at the end */ -#define PAGEHDRSZ ((unsigned) offsetof(MDB_page, mp_ptrs)) - - /** Address of first usable data byte in a page, after the header */ -#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ)) - - /** ITS#7713, change PAGEBASE to handle 65536 byte pages */ -#define PAGEBASE ((MDB_DEVEL) ? PAGEHDRSZ : 0) - - /** Number of nodes on a page */ -#define NUMKEYS(p) (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1) - - /** The amount of space remaining in the page */ -#define SIZELEFT(p) (indx_t)((p)->mp_upper - (p)->mp_lower) - - /** The percentage of space used in the page, in tenths of a percent. */ -#define PAGEFILL(env, p) (1000L * ((env)->me_psize - PAGEHDRSZ - SIZELEFT(p)) / \ - ((env)->me_psize - PAGEHDRSZ)) - /** The minimum page fill factor, in tenths of a percent. - * Pages emptier than this are candidates for merging. - */ -#define FILL_THRESHOLD 250 - - /** Test if a page is a leaf page */ -#define IS_LEAF(p) F_ISSET((p)->mp_flags, P_LEAF) - /** Test if a page is a LEAF2 page */ -#define IS_LEAF2(p) F_ISSET((p)->mp_flags, P_LEAF2) - /** Test if a page is a branch page */ -#define IS_BRANCH(p) F_ISSET((p)->mp_flags, P_BRANCH) - /** Test if a page is an overflow page */ -#define IS_OVERFLOW(p) F_ISSET((p)->mp_flags, P_OVERFLOW) - /** Test if a page is a sub page */ -#define IS_SUBP(p) F_ISSET((p)->mp_flags, P_SUBP) - - /** The number of overflow pages needed to store the given size. */ -#define OVPAGES(size, psize) ((PAGEHDRSZ-1 + (size)) / (psize) + 1) - - /** Link in #MDB_txn.%mt_loose_pgs list. - * Kept outside the page header, which is needed when reusing the page. - */ -#define NEXT_LOOSE_PAGE(p) (*(MDB_page **)((p) + 2)) - - /** Header for a single key/data pair within a page. - * Used in pages of type #P_BRANCH and #P_LEAF without #P_LEAF2. - * We guarantee 2-byte alignment for 'MDB_node's. - * - * #mn_lo and #mn_hi are used for data size on leaf nodes, and for child - * pgno on branch nodes. On 64 bit platforms, #mn_flags is also used - * for pgno. (Branch nodes have no flags). Lo and hi are in host byte - * order in case some accesses can be optimized to 32-bit word access. - * - * Leaf node flags describe node contents. #F_BIGDATA says the node's - * data part is the page number of an overflow page with actual data. - * #F_DUPDATA and #F_SUBDATA can be combined giving duplicate data in - * a sub-page/sub-database, and named databases (just #F_SUBDATA). - */ -typedef struct MDB_node { - /** part of data size or pgno - * @{ */ -#if BYTE_ORDER == LITTLE_ENDIAN - unsigned short mn_lo, mn_hi; -#else - unsigned short mn_hi, mn_lo; -#endif - /** @} */ -/** @defgroup mdb_node Node Flags - * @ingroup internal - * Flags for node headers. - * @{ - */ -#define F_BIGDATA 0x01 /**< data put on overflow page */ -#define F_SUBDATA 0x02 /**< data is a sub-database */ -#define F_DUPDATA 0x04 /**< data has duplicates */ - -/** valid flags for #mdb_node_add() */ -#define NODE_ADD_FLAGS (F_DUPDATA|F_SUBDATA|MDB_RESERVE|MDB_APPEND) - -/** @} */ - unsigned short mn_flags; /**< @ref mdb_node */ - unsigned short mn_ksize; /**< key size */ - char mn_data[1]; /**< key and data are appended here */ -} MDB_node; - - /** Size of the node header, excluding dynamic data at the end */ -#define NODESIZE offsetof(MDB_node, mn_data) - - /** Bit position of top word in page number, for shifting mn_flags */ -#define PGNO_TOPWORD ((pgno_t)-1 > 0xffffffffu ? 32 : 0) - - /** Size of a node in a branch page with a given key. - * This is just the node header plus the key, there is no data. - */ -#define INDXSIZE(k) (NODESIZE + ((k) == NULL ? 0 : (k)->mv_size)) - - /** Size of a node in a leaf page with a given key and data. - * This is node header plus key plus data size. - */ -#define LEAFSIZE(k, d) (NODESIZE + (k)->mv_size + (d)->mv_size) - - /** Address of node \b i in page \b p */ -#define NODEPTR(p, i) ((MDB_node *)((char *)(p) + (p)->mp_ptrs[i] + PAGEBASE)) - - /** Address of the key for the node */ -#define NODEKEY(node) (void *)((node)->mn_data) - - /** Address of the data for a node */ -#define NODEDATA(node) (void *)((char *)(node)->mn_data + (node)->mn_ksize) - - /** Get the page number pointed to by a branch node */ -#define NODEPGNO(node) \ - ((node)->mn_lo | ((pgno_t) (node)->mn_hi << 16) | \ - (PGNO_TOPWORD ? ((pgno_t) (node)->mn_flags << PGNO_TOPWORD) : 0)) - /** Set the page number in a branch node */ -#define SETPGNO(node,pgno) do { \ - (node)->mn_lo = (pgno) & 0xffff; (node)->mn_hi = (pgno) >> 16; \ - if (PGNO_TOPWORD) (node)->mn_flags = (pgno) >> PGNO_TOPWORD; } while(0) - - /** Get the size of the data in a leaf node */ -#define NODEDSZ(node) ((node)->mn_lo | ((unsigned)(node)->mn_hi << 16)) - /** Set the size of the data for a leaf node */ -#define SETDSZ(node,size) do { \ - (node)->mn_lo = (size) & 0xffff; (node)->mn_hi = (size) >> 16;} while(0) - /** The size of a key in a node */ -#define NODEKSZ(node) ((node)->mn_ksize) - - /** Copy a page number from src to dst */ -#ifdef MISALIGNED_OK -#define COPY_PGNO(dst,src) dst = src -#else -#if MDB_SIZE_MAX > 0xffffffffU -#define COPY_PGNO(dst,src) do { \ - unsigned short *s, *d; \ - s = (unsigned short *)&(src); \ - d = (unsigned short *)&(dst); \ - *d++ = *s++; \ - *d++ = *s++; \ - *d++ = *s++; \ - *d = *s; \ -} while (0) -#else -#define COPY_PGNO(dst,src) do { \ - unsigned short *s, *d; \ - s = (unsigned short *)&(src); \ - d = (unsigned short *)&(dst); \ - *d++ = *s++; \ - *d = *s; \ -} while (0) -#endif -#endif - /** The address of a key in a LEAF2 page. - * LEAF2 pages are used for #MDB_DUPFIXED sorted-duplicate sub-DBs. - * There are no node headers, keys are stored contiguously. - */ -#define LEAF2KEY(p, i, ks) ((char *)(p) + PAGEHDRSZ + ((i)*(ks))) - - /** Set the \b node's key into \b keyptr, if requested. */ -#define MDB_GET_KEY(node, keyptr) { if ((keyptr) != NULL) { \ - (keyptr)->mv_size = NODEKSZ(node); (keyptr)->mv_data = NODEKEY(node); } } - - /** Set the \b node's key into \b key. */ -#define MDB_GET_KEY2(node, key) { key.mv_size = NODEKSZ(node); key.mv_data = NODEKEY(node); } - - /** Information about a single database in the environment. */ -typedef struct MDB_db { - uint32_t md_pad; /**< also ksize for LEAF2 pages */ - uint16_t md_flags; /**< @ref mdb_dbi_open */ - uint16_t md_depth; /**< depth of this tree */ - pgno_t md_branch_pages; /**< number of internal pages */ - pgno_t md_leaf_pages; /**< number of leaf pages */ - pgno_t md_overflow_pages; /**< number of overflow pages */ - mdb_size_t md_entries; /**< number of data items */ - pgno_t md_root; /**< the root page of this tree */ -} MDB_db; - -#define MDB_VALID 0x8000 /**< DB handle is valid, for me_dbflags */ -#define PERSISTENT_FLAGS (0xffff & ~(MDB_VALID)) - /** #mdb_dbi_open() flags */ -#define VALID_FLAGS (MDB_REVERSEKEY|MDB_DUPSORT|MDB_INTEGERKEY|MDB_DUPFIXED|\ - MDB_INTEGERDUP|MDB_REVERSEDUP|MDB_CREATE) - - /** Handle for the DB used to track free pages. */ -#define FREE_DBI 0 - /** Handle for the default DB. */ -#define MAIN_DBI 1 - /** Number of DBs in metapage (free and main) - also hardcoded elsewhere */ -#define CORE_DBS 2 - - /** Number of meta pages - also hardcoded elsewhere */ -#define NUM_METAS 2 - - /** Meta page content. - * A meta page is the start point for accessing a database snapshot. - * Pages 0-1 are meta pages. Transaction N writes meta page #(N % 2). - */ -typedef struct MDB_meta { - /** Stamp identifying this as an LMDB file. It must be set - * to #MDB_MAGIC. */ - uint32_t mm_magic; - /** Version number of this file. Must be set to #MDB_DATA_VERSION. */ - uint32_t mm_version; -#ifdef MDB_VL32 - union { /* always zero since we don't support fixed mapping in MDB_VL32 */ - MDB_ID mmun_ull; - void *mmun_address; - } mm_un; -#define mm_address mm_un.mmun_address -#else - void *mm_address; /**< address for fixed mapping */ -#endif - mdb_size_t mm_mapsize; /**< size of mmap region */ - MDB_db mm_dbs[CORE_DBS]; /**< first is free space, 2nd is main db */ - /** The size of pages used in this DB */ -#define mm_psize mm_dbs[FREE_DBI].md_pad - /** Any persistent environment flags. @ref mdb_env */ -#define mm_flags mm_dbs[FREE_DBI].md_flags - /** Last used page in the datafile. - * Actually the file may be shorter if the freeDB lists the final pages. - */ - pgno_t mm_last_pg; - volatile txnid_t mm_txnid; /**< txnid that committed this page */ -} MDB_meta; - - /** Buffer for a stack-allocated meta page. - * The members define size and alignment, and silence type - * aliasing warnings. They are not used directly; that could - * mean incorrectly using several union members in parallel. - */ -typedef union MDB_metabuf { - MDB_page mb_page; - struct { - char mm_pad[PAGEHDRSZ]; - MDB_meta mm_meta; - } mb_metabuf; -} MDB_metabuf; - - /** Auxiliary DB info. - * The information here is mostly static/read-only. There is - * only a single copy of this record in the environment. - */ -typedef struct MDB_dbx { - MDB_val md_name; /**< name of the database */ - MDB_cmp_func *md_cmp; /**< function for comparing keys */ - MDB_cmp_func *md_dcmp; /**< function for comparing data items */ - MDB_rel_func *md_rel; /**< user relocate function */ - void *md_relctx; /**< user-provided context for md_rel */ -} MDB_dbx; - - /** A database transaction. - * Every operation requires a transaction handle. - */ -struct MDB_txn { - MDB_txn *mt_parent; /**< parent of a nested txn */ - /** Nested txn under this txn, set together with flag #MDB_TXN_HAS_CHILD */ - MDB_txn *mt_child; - pgno_t mt_next_pgno; /**< next unallocated page */ -#ifdef MDB_VL32 - pgno_t mt_last_pgno; /**< last written page */ -#endif - /** The ID of this transaction. IDs are integers incrementing from 1. - * Only committed write transactions increment the ID. If a transaction - * aborts, the ID may be re-used by the next writer. - */ - txnid_t mt_txnid; - MDB_env *mt_env; /**< the DB environment */ - /** The list of pages that became unused during this transaction. - */ - MDB_IDL mt_free_pgs; - /** The list of loose pages that became unused and may be reused - * in this transaction, linked through #NEXT_LOOSE_PAGE(page). - */ - MDB_page *mt_loose_pgs; - /** Number of loose pages (#mt_loose_pgs) */ - int mt_loose_count; - /** The sorted list of dirty pages we temporarily wrote to disk - * because the dirty list was full. page numbers in here are - * shifted left by 1, deleted slots have the LSB set. - */ - MDB_IDL mt_spill_pgs; - union { - /** For write txns: Modified pages. Sorted when not MDB_WRITEMAP. */ - MDB_ID2L dirty_list; - /** For read txns: This thread/txn's reader table slot, or NULL. */ - MDB_reader *reader; - } mt_u; - /** Array of records for each DB known in the environment. */ - MDB_dbx *mt_dbxs; - /** Array of MDB_db records for each known DB */ - MDB_db *mt_dbs; - /** Array of sequence numbers for each DB handle */ - unsigned int *mt_dbiseqs; -/** @defgroup mt_dbflag Transaction DB Flags - * @ingroup internal - * @{ - */ -#define DB_DIRTY 0x01 /**< DB was written in this txn */ -#define DB_STALE 0x02 /**< Named-DB record is older than txnID */ -#define DB_NEW 0x04 /**< Named-DB handle opened in this txn */ -#define DB_VALID 0x08 /**< DB handle is valid, see also #MDB_VALID */ -#define DB_USRVALID 0x10 /**< As #DB_VALID, but not set for #FREE_DBI */ -#define DB_DUPDATA 0x20 /**< DB is #MDB_DUPSORT data */ -/** @} */ - /** In write txns, array of cursors for each DB */ - MDB_cursor **mt_cursors; - /** Array of flags for each DB */ - unsigned char *mt_dbflags; -#ifdef MDB_VL32 - /** List of read-only pages (actually chunks) */ - MDB_ID3L mt_rpages; - /** We map chunks of 16 pages. Even though Windows uses 4KB pages, all - * mappings must begin on 64KB boundaries. So we round off all pgnos to - * a chunk boundary. We do the same on Linux for symmetry, and also to - * reduce the frequency of mmap/munmap calls. - */ -#define MDB_RPAGE_CHUNK 16 -#define MDB_TRPAGE_SIZE 4096 /**< size of #mt_rpages array of chunks */ -#define MDB_TRPAGE_MAX (MDB_TRPAGE_SIZE-1) /**< maximum chunk index */ - unsigned int mt_rpcheck; /**< threshold for reclaiming unref'd chunks */ -#endif - /** Number of DB records in use, or 0 when the txn is finished. - * This number only ever increments until the txn finishes; we - * don't decrement it when individual DB handles are closed. - */ - MDB_dbi mt_numdbs; - -/** @defgroup mdb_txn Transaction Flags - * @ingroup internal - * @{ - */ - /** #mdb_txn_begin() flags */ -#define MDB_TXN_BEGIN_FLAGS (MDB_NOMETASYNC|MDB_NOSYNC|MDB_RDONLY) -#define MDB_TXN_NOMETASYNC MDB_NOMETASYNC /**< don't sync meta for this txn on commit */ -#define MDB_TXN_NOSYNC MDB_NOSYNC /**< don't sync this txn on commit */ -#define MDB_TXN_RDONLY MDB_RDONLY /**< read-only transaction */ - /* internal txn flags */ -#define MDB_TXN_WRITEMAP MDB_WRITEMAP /**< copy of #MDB_env flag in writers */ -#define MDB_TXN_FINISHED 0x01 /**< txn is finished or never began */ -#define MDB_TXN_ERROR 0x02 /**< txn is unusable after an error */ -#define MDB_TXN_DIRTY 0x04 /**< must write, even if dirty list is empty */ -#define MDB_TXN_SPILLS 0x08 /**< txn or a parent has spilled pages */ -#define MDB_TXN_HAS_CHILD 0x10 /**< txn has an #MDB_txn.%mt_child */ - /** most operations on the txn are currently illegal */ -#define MDB_TXN_BLOCKED (MDB_TXN_FINISHED|MDB_TXN_ERROR|MDB_TXN_HAS_CHILD) -/** @} */ - unsigned int mt_flags; /**< @ref mdb_txn */ - /** #dirty_list room: Array size - \#dirty pages visible to this txn. - * Includes ancestor txns' dirty pages not hidden by other txns' - * dirty/spilled pages. Thus commit(nested txn) has room to merge - * dirty_list into mt_parent after freeing hidden mt_parent pages. - */ - unsigned int mt_dirty_room; -}; - -/** Enough space for 2^32 nodes with minimum of 2 keys per node. I.e., plenty. - * At 4 keys per node, enough for 2^64 nodes, so there's probably no need to - * raise this on a 64 bit machine. - */ -#define CURSOR_STACK 32 - -struct MDB_xcursor; - - /** Cursors are used for all DB operations. - * A cursor holds a path of (page pointer, key index) from the DB - * root to a position in the DB, plus other state. #MDB_DUPSORT - * cursors include an xcursor to the current data item. Write txns - * track their cursors and keep them up to date when data moves. - * Exception: An xcursor's pointer to a #P_SUBP page can be stale. - * (A node with #F_DUPDATA but no #F_SUBDATA contains a subpage). - */ -struct MDB_cursor { - /** Next cursor on this DB in this txn */ - MDB_cursor *mc_next; - /** Backup of the original cursor if this cursor is a shadow */ - MDB_cursor *mc_backup; - /** Context used for databases with #MDB_DUPSORT, otherwise NULL */ - struct MDB_xcursor *mc_xcursor; - /** The transaction that owns this cursor */ - MDB_txn *mc_txn; - /** The database handle this cursor operates on */ - MDB_dbi mc_dbi; - /** The database record for this cursor */ - MDB_db *mc_db; - /** The database auxiliary record for this cursor */ - MDB_dbx *mc_dbx; - /** The @ref mt_dbflag for this database */ - unsigned char *mc_dbflag; - unsigned short mc_snum; /**< number of pushed pages */ - unsigned short mc_top; /**< index of top page, normally mc_snum-1 */ -/** @defgroup mdb_cursor Cursor Flags - * @ingroup internal - * Cursor state flags. - * @{ - */ -#define C_INITIALIZED 0x01 /**< cursor has been initialized and is valid */ -#define C_EOF 0x02 /**< No more data */ -#define C_SUB 0x04 /**< Cursor is a sub-cursor */ -#define C_DEL 0x08 /**< last op was a cursor_del */ -#define C_UNTRACK 0x40 /**< Un-track cursor when closing */ -#define C_WRITEMAP MDB_TXN_WRITEMAP /**< Copy of txn flag */ -/** Read-only cursor into the txn's original snapshot in the map. - * Set for read-only txns, and in #mdb_page_alloc() for #FREE_DBI when - * #MDB_DEVEL & 2. Only implements code which is necessary for this. - */ -#define C_ORIG_RDONLY MDB_TXN_RDONLY -/** @} */ - unsigned int mc_flags; /**< @ref mdb_cursor */ - MDB_page *mc_pg[CURSOR_STACK]; /**< stack of pushed pages */ - indx_t mc_ki[CURSOR_STACK]; /**< stack of page indices */ -#ifdef MDB_VL32 - MDB_page *mc_ovpg; /**< a referenced overflow page */ -# define MC_OVPG(mc) ((mc)->mc_ovpg) -# define MC_SET_OVPG(mc, pg) ((mc)->mc_ovpg = (pg)) -#else -# define MC_OVPG(mc) ((MDB_page *)0) -# define MC_SET_OVPG(mc, pg) ((void)0) -#endif -}; - - /** Context for sorted-dup records. - * We could have gone to a fully recursive design, with arbitrarily - * deep nesting of sub-databases. But for now we only handle these - * levels - main DB, optional sub-DB, sorted-duplicate DB. - */ -typedef struct MDB_xcursor { - /** A sub-cursor for traversing the Dup DB */ - MDB_cursor mx_cursor; - /** The database record for this Dup DB */ - MDB_db mx_db; - /** The auxiliary DB record for this Dup DB */ - MDB_dbx mx_dbx; - /** The @ref mt_dbflag for this Dup DB */ - unsigned char mx_dbflag; -} MDB_xcursor; - - /** Check if there is an inited xcursor */ -#define XCURSOR_INITED(mc) \ - ((mc)->mc_xcursor && ((mc)->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) - - /** Update the xcursor's sub-page pointer, if any, in \b mc. Needed - * when the node which contains the sub-page may have moved. Called - * with leaf page \b mp = mc->mc_pg[\b top]. - */ -#define XCURSOR_REFRESH(mc, top, mp) do { \ - MDB_page *xr_pg = (mp); \ - MDB_node *xr_node; \ - if (!XCURSOR_INITED(mc) || (mc)->mc_ki[top] >= NUMKEYS(xr_pg)) break; \ - xr_node = NODEPTR(xr_pg, (mc)->mc_ki[top]); \ - if ((xr_node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) \ - (mc)->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(xr_node); \ -} while (0) - - /** State of FreeDB old pages, stored in the MDB_env */ -typedef struct MDB_pgstate { - pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ - txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ -} MDB_pgstate; - - /** The database environment. */ -struct MDB_env { - HANDLE me_fd; /**< The main data file */ - HANDLE me_lfd; /**< The lock file */ - HANDLE me_mfd; /**< For writing and syncing the meta pages */ -#if defined(MDB_VL32) && defined(_WIN32) - HANDLE me_fmh; /**< File Mapping handle */ -#endif - /** Failed to update the meta page. Probably an I/O error. */ -#define MDB_FATAL_ERROR 0x80000000U - /** Some fields are initialized. */ -#define MDB_ENV_ACTIVE 0x20000000U - /** me_txkey is set */ -#define MDB_ENV_TXKEY 0x10000000U - /** fdatasync is unreliable */ -#define MDB_FSYNCONLY 0x08000000U - uint32_t me_flags; /**< @ref mdb_env */ - unsigned int me_psize; /**< DB page size, inited from me_os_psize */ - unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */ - unsigned int me_maxreaders; /**< size of the reader table */ - /** Max #MDB_txninfo.%mti_numreaders of interest to #mdb_env_close() */ - volatile int me_close_readers; - MDB_dbi me_numdbs; /**< number of DBs opened */ - MDB_dbi me_maxdbs; /**< size of the DB table */ - MDB_PID_T me_pid; /**< process ID of this env */ - char *me_path; /**< path to the DB files */ - char *me_map; /**< the memory map of the data file */ - MDB_txninfo *me_txns; /**< the memory map of the lock file or NULL */ - MDB_meta *me_metas[NUM_METAS]; /**< pointers to the two meta pages */ - void *me_pbuf; /**< scratch area for DUPSORT put() */ - MDB_txn *me_txn; /**< current write transaction */ - MDB_txn *me_txn0; /**< prealloc'd write transaction */ - mdb_size_t me_mapsize; /**< size of the data memory map */ - off_t me_size; /**< current file size */ - pgno_t me_maxpg; /**< me_mapsize / me_psize */ - MDB_dbx *me_dbxs; /**< array of static DB info */ - uint16_t *me_dbflags; /**< array of flags from MDB_db.md_flags */ - unsigned int *me_dbiseqs; /**< array of dbi sequence numbers */ - pthread_key_t me_txkey; /**< thread-key for readers */ - txnid_t me_pgoldest; /**< ID of oldest reader last time we looked */ - MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ -# define me_pglast me_pgstate.mf_pglast -# define me_pghead me_pgstate.mf_pghead - MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ - /** IDL of pages that became unused in a write txn */ - MDB_IDL me_free_pgs; - /** ID2L of pages written during a write txn. Length MDB_IDL_UM_SIZE. */ - MDB_ID2L me_dirty_list; - /** Max number of freelist items that can fit in a single overflow page */ - int me_maxfree_1pg; - /** Max size of a node on a page */ - unsigned int me_nodemax; -#if !(MDB_MAXKEYSIZE) - unsigned int me_maxkey; /**< max size of a key */ -#endif - int me_live_reader; /**< have liveness lock in reader table */ -#ifdef _WIN32 - int me_pidquery; /**< Used in OpenProcess */ -#endif -#ifdef MDB_USE_POSIX_MUTEX /* Posix mutexes reside in shared mem */ -# define me_rmutex me_txns->mti_rmutex /**< Shared reader lock */ -# define me_wmutex me_txns->mti_wmutex /**< Shared writer lock */ -#else - mdb_mutex_t me_rmutex; - mdb_mutex_t me_wmutex; -# if defined(_WIN32) || defined(MDB_USE_POSIX_SEM) - /** Half-initialized name of mutexes, to be completed by #MUTEXNAME() */ - char me_mutexname[sizeof(MUTEXNAME_PREFIX) + 11]; -# endif -#endif -#ifdef MDB_VL32 - MDB_ID3L me_rpages; /**< like #mt_rpages, but global to env */ - pthread_mutex_t me_rpmutex; /**< control access to #me_rpages */ -#define MDB_ERPAGE_SIZE 16384 -#define MDB_ERPAGE_MAX (MDB_ERPAGE_SIZE-1) - unsigned int me_rpcheck; -#endif - void *me_userctx; /**< User-settable context */ - MDB_assert_func *me_assert_func; /**< Callback for assertion failures */ -}; - - /** Nested transaction */ -typedef struct MDB_ntxn { - MDB_txn mnt_txn; /**< the transaction */ - MDB_pgstate mnt_pgstate; /**< parent transaction's saved freestate */ -} MDB_ntxn; - - /** max number of pages to commit in one writev() call */ -#define MDB_COMMIT_PAGES 64 -#if defined(IOV_MAX) && IOV_MAX < MDB_COMMIT_PAGES -#undef MDB_COMMIT_PAGES -#define MDB_COMMIT_PAGES IOV_MAX -#endif - - /** max bytes to write in one call */ -#define MAX_WRITE (0x40000000U >> (sizeof(ssize_t) == 4)) - - /** Check \b txn and \b dbi arguments to a function */ -#define TXN_DBI_EXIST(txn, dbi, validity) \ - ((txn) && (dbi)<(txn)->mt_numdbs && ((txn)->mt_dbflags[dbi] & (validity))) - - /** Check for misused \b dbi handles */ -#define TXN_DBI_CHANGED(txn, dbi) \ - ((txn)->mt_dbiseqs[dbi] != (txn)->mt_env->me_dbiseqs[dbi]) - -static int mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp); -static int mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp); -static int mdb_page_touch(MDB_cursor *mc); - -#define MDB_END_NAMES {"committed", "empty-commit", "abort", "reset", \ - "reset-tmp", "fail-begin", "fail-beginchild"} -enum { - /* mdb_txn_end operation number, for logging */ - MDB_END_COMMITTED, MDB_END_EMPTY_COMMIT, MDB_END_ABORT, MDB_END_RESET, - MDB_END_RESET_TMP, MDB_END_FAIL_BEGIN, MDB_END_FAIL_BEGINCHILD -}; -#define MDB_END_OPMASK 0x0F /**< mask for #mdb_txn_end() operation number */ -#define MDB_END_UPDATE 0x10 /**< update env state (DBIs) */ -#define MDB_END_FREE 0x20 /**< free txn unless it is #MDB_env.%me_txn0 */ -#define MDB_END_SLOT MDB_NOTLS /**< release any reader slot if #MDB_NOTLS */ -static void mdb_txn_end(MDB_txn *txn, unsigned mode); - -static int mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **mp, int *lvl); -static int mdb_page_search_root(MDB_cursor *mc, - MDB_val *key, int modify); -#define MDB_PS_MODIFY 1 -#define MDB_PS_ROOTONLY 2 -#define MDB_PS_FIRST 4 -#define MDB_PS_LAST 8 -static int mdb_page_search(MDB_cursor *mc, - MDB_val *key, int flags); -static int mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst); - -#define MDB_SPLIT_REPLACE MDB_APPENDDUP /**< newkey is not new */ -static int mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, - pgno_t newpgno, unsigned int nflags); - -static int mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta); -static MDB_meta *mdb_env_pick_meta(const MDB_env *env); -static int mdb_env_write_meta(MDB_txn *txn); -#ifdef MDB_USE_POSIX_MUTEX /* Drop unused excl arg */ -# define mdb_env_close0(env, excl) mdb_env_close1(env) -#endif -static void mdb_env_close0(MDB_env *env, int excl); - -static MDB_node *mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp); -static int mdb_node_add(MDB_cursor *mc, indx_t indx, - MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags); -static void mdb_node_del(MDB_cursor *mc, int ksize); -static void mdb_node_shrink(MDB_page *mp, indx_t indx); -static int mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft); -static int mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data); -static size_t mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data); -static size_t mdb_branch_size(MDB_env *env, MDB_val *key); - -static int mdb_rebalance(MDB_cursor *mc); -static int mdb_update_key(MDB_cursor *mc, MDB_val *key); - -static void mdb_cursor_pop(MDB_cursor *mc); -static int mdb_cursor_push(MDB_cursor *mc, MDB_page *mp); - -static int mdb_cursor_del0(MDB_cursor *mc); -static int mdb_del0(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data, unsigned flags); -static int mdb_cursor_sibling(MDB_cursor *mc, int move_right); -static int mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); -static int mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op); -static int mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op, - int *exactp); -static int mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data); -static int mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data); - -static void mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx); -static void mdb_xcursor_init0(MDB_cursor *mc); -static void mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node); -static void mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int force); - -static int mdb_drop0(MDB_cursor *mc, int subs); -static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi); -static int mdb_reader_check0(MDB_env *env, int rlocked, int *dead); - -/** @cond */ -static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long; -/** @endcond */ - -/** Compare two items pointing at '#mdb_size_t's of unknown alignment. */ -#ifdef MISALIGNED_OK -# define mdb_cmp_clong mdb_cmp_long -#else -# define mdb_cmp_clong mdb_cmp_cint -#endif - -/** True if we need #mdb_cmp_clong() instead of \b cmp for #MDB_INTEGERDUP */ -#define NEED_CMP_CLONG(cmp, ksize) \ - (UINT_MAX < MDB_SIZE_MAX && \ - (cmp) == mdb_cmp_int && (ksize) == sizeof(mdb_size_t)) - -#ifdef _WIN32 -static SECURITY_DESCRIPTOR mdb_null_sd; -static SECURITY_ATTRIBUTES mdb_all_sa; -static int mdb_sec_inited; - -struct MDB_name; -static int utf8_to_utf16(const char *src, struct MDB_name *dst, int xtra); -#endif - -/** Return the library version info. */ -char * ESECT -mdb_version(int *major, int *minor, int *patch) -{ - if (major) *major = MDB_VERSION_MAJOR; - if (minor) *minor = MDB_VERSION_MINOR; - if (patch) *patch = MDB_VERSION_PATCH; - return MDB_VERSION_STRING; -} - -/** Table of descriptions for LMDB @ref errors */ -static char *const mdb_errstr[] = { - "MDB_KEYEXIST: Key/data pair already exists", - "MDB_NOTFOUND: No matching key/data pair found", - "MDB_PAGE_NOTFOUND: Requested page not found", - "MDB_CORRUPTED: Located page was wrong type", - "MDB_PANIC: Update of meta page failed or environment had fatal error", - "MDB_VERSION_MISMATCH: Database environment version mismatch", - "MDB_INVALID: File is not an LMDB file", - "MDB_MAP_FULL: Environment mapsize limit reached", - "MDB_DBS_FULL: Environment maxdbs limit reached", - "MDB_READERS_FULL: Environment maxreaders limit reached", - "MDB_TLS_FULL: Thread-local storage keys full - too many environments open", - "MDB_TXN_FULL: Transaction has too many dirty pages - transaction too big", - "MDB_CURSOR_FULL: Internal error - cursor stack limit reached", - "MDB_PAGE_FULL: Internal error - page has no more space", - "MDB_MAP_RESIZED: Database contents grew beyond environment mapsize", - "MDB_INCOMPATIBLE: Operation and DB incompatible, or DB flags changed", - "MDB_BAD_RSLOT: Invalid reuse of reader locktable slot", - "MDB_BAD_TXN: Transaction must abort, has a child, or is invalid", - "MDB_BAD_VALSIZE: Unsupported size of key/DB name/data, or wrong DUPFIXED size", - "MDB_BAD_DBI: The specified DBI handle was closed/changed unexpectedly", - "MDB_PROBLEM: Unexpected problem - txn should abort", -}; - -char * -mdb_strerror(int err) -{ -#ifdef _WIN32 - /** HACK: pad 4KB on stack over the buf. Return system msgs in buf. - * This works as long as no function between the call to mdb_strerror - * and the actual use of the message uses more than 4K of stack. - */ -#define MSGSIZE 1024 -#define PADSIZE 4096 - char buf[MSGSIZE+PADSIZE], *ptr = buf; -#endif - int i; - if (!err) - return ("Successful return: 0"); - - if (err >= MDB_KEYEXIST && err <= MDB_LAST_ERRCODE) { - i = err - MDB_KEYEXIST; - return mdb_errstr[i]; - } - -#ifdef _WIN32 - /* These are the C-runtime error codes we use. The comment indicates - * their numeric value, and the Win32 error they would correspond to - * if the error actually came from a Win32 API. A major mess, we should - * have used LMDB-specific error codes for everything. - */ - switch(err) { - case ENOENT: /* 2, FILE_NOT_FOUND */ - case EIO: /* 5, ACCESS_DENIED */ - case ENOMEM: /* 12, INVALID_ACCESS */ - case EACCES: /* 13, INVALID_DATA */ - case EBUSY: /* 16, CURRENT_DIRECTORY */ - case EINVAL: /* 22, BAD_COMMAND */ - case ENOSPC: /* 28, OUT_OF_PAPER */ - return strerror(err); - default: - ; - } - buf[0] = 0; - FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, err, 0, ptr, MSGSIZE, (va_list *)buf+MSGSIZE); - return ptr; -#else - return strerror(err); -#endif -} - -/** assert(3) variant in cursor context */ -#define mdb_cassert(mc, expr) mdb_assert0((mc)->mc_txn->mt_env, expr, #expr) -/** assert(3) variant in transaction context */ -#define mdb_tassert(txn, expr) mdb_assert0((txn)->mt_env, expr, #expr) -/** assert(3) variant in environment context */ -#define mdb_eassert(env, expr) mdb_assert0(env, expr, #expr) - -#ifndef NDEBUG -# define mdb_assert0(env, expr, expr_txt) ((expr) ? (void)0 : \ - mdb_assert_fail(env, expr_txt, mdb_func_, __FILE__, __LINE__)) - -static void ESECT -mdb_assert_fail(MDB_env *env, const char *expr_txt, - const char *func, const char *file, int line) -{ - char buf[400]; - sprintf(buf, "%.100s:%d: Assertion '%.200s' failed in %.40s()", - file, line, expr_txt, func); - if (env->me_assert_func) - env->me_assert_func(env, buf); - fprintf(stderr, "%s\n", buf); - abort(); -} -#else -# define mdb_assert0(env, expr, expr_txt) ((void) 0) -#endif /* NDEBUG */ - -#if MDB_DEBUG -/** Return the page number of \b mp which may be sub-page, for debug output */ -static pgno_t -mdb_dbg_pgno(MDB_page *mp) -{ - pgno_t ret; - COPY_PGNO(ret, mp->mp_pgno); - return ret; -} - -/** Display a key in hexadecimal and return the address of the result. - * @param[in] key the key to display - * @param[in] buf the buffer to write into. Should always be #DKBUF. - * @return The key in hexadecimal form. - */ -char * -mdb_dkey(MDB_val *key, char *buf) -{ - char *ptr = buf; - unsigned char *c = key->mv_data; - unsigned int i; - - if (!key) - return ""; - - if (key->mv_size > DKBUF_MAXKEYSIZE) - return "MDB_MAXKEYSIZE"; - /* may want to make this a dynamic check: if the key is mostly - * printable characters, print it as-is instead of converting to hex. - */ -#if 1 - buf[0] = '\0'; - for (i=0; imv_size; i++) - ptr += sprintf(ptr, "%02x", *c++); -#else - sprintf(buf, "%.*s", key->mv_size, key->mv_data); -#endif - return buf; -} - -static const char * -mdb_leafnode_type(MDB_node *n) -{ - static char *const tp[2][2] = {{"", ": DB"}, {": sub-page", ": sub-DB"}}; - return F_ISSET(n->mn_flags, F_BIGDATA) ? ": overflow page" : - tp[F_ISSET(n->mn_flags, F_DUPDATA)][F_ISSET(n->mn_flags, F_SUBDATA)]; -} - -/** Display all the keys in the page. */ -void -mdb_page_list(MDB_page *mp) -{ - pgno_t pgno = mdb_dbg_pgno(mp); - const char *type, *state = (mp->mp_flags & P_DIRTY) ? ", dirty" : ""; - MDB_node *node; - unsigned int i, nkeys, nsize, total = 0; - MDB_val key; - DKBUF; - - switch (mp->mp_flags & (P_BRANCH|P_LEAF|P_LEAF2|P_META|P_OVERFLOW|P_SUBP)) { - case P_BRANCH: type = "Branch page"; break; - case P_LEAF: type = "Leaf page"; break; - case P_LEAF|P_SUBP: type = "Sub-page"; break; - case P_LEAF|P_LEAF2: type = "LEAF2 page"; break; - case P_LEAF|P_LEAF2|P_SUBP: type = "LEAF2 sub-page"; break; - case P_OVERFLOW: - fprintf(stderr, "Overflow page %"Yu" pages %u%s\n", - pgno, mp->mp_pages, state); - return; - case P_META: - fprintf(stderr, "Meta-page %"Yu" txnid %"Yu"\n", - pgno, ((MDB_meta *)METADATA(mp))->mm_txnid); - return; - default: - fprintf(stderr, "Bad page %"Yu" flags 0x%X\n", pgno, mp->mp_flags); - return; - } - - nkeys = NUMKEYS(mp); - fprintf(stderr, "%s %"Yu" numkeys %d%s\n", type, pgno, nkeys, state); - - for (i=0; imp_pad; - key.mv_data = LEAF2KEY(mp, i, nsize); - total += nsize; - fprintf(stderr, "key %d: nsize %d, %s\n", i, nsize, DKEY(&key)); - continue; - } - node = NODEPTR(mp, i); - key.mv_size = node->mn_ksize; - key.mv_data = node->mn_data; - nsize = NODESIZE + key.mv_size; - if (IS_BRANCH(mp)) { - fprintf(stderr, "key %d: page %"Yu", %s\n", i, NODEPGNO(node), - DKEY(&key)); - total += nsize; - } else { - if (F_ISSET(node->mn_flags, F_BIGDATA)) - nsize += sizeof(pgno_t); - else - nsize += NODEDSZ(node); - total += nsize; - nsize += sizeof(indx_t); - fprintf(stderr, "key %d: nsize %d, %s%s\n", - i, nsize, DKEY(&key), mdb_leafnode_type(node)); - } - total = EVEN(total); - } - fprintf(stderr, "Total: header %d + contents %d + unused %d\n", - IS_LEAF2(mp) ? PAGEHDRSZ : PAGEBASE + mp->mp_lower, total, SIZELEFT(mp)); -} - -void -mdb_cursor_chk(MDB_cursor *mc) -{ - unsigned int i; - MDB_node *node; - MDB_page *mp; - - if (!mc->mc_snum || !(mc->mc_flags & C_INITIALIZED)) return; - for (i=0; imc_top; i++) { - mp = mc->mc_pg[i]; - node = NODEPTR(mp, mc->mc_ki[i]); - if (NODEPGNO(node) != mc->mc_pg[i+1]->mp_pgno) - printf("oops!\n"); - } - if (mc->mc_ki[i] >= NUMKEYS(mc->mc_pg[i])) - printf("ack!\n"); - if (XCURSOR_INITED(mc)) { - node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (((node->mn_flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA) && - mc->mc_xcursor->mx_cursor.mc_pg[0] != NODEDATA(node)) { - printf("blah!\n"); - } - } -} -#endif - -#if (MDB_DEBUG) > 2 -/** Count all the pages in each DB and in the freelist - * and make sure it matches the actual number of pages - * being used. - * All named DBs must be open for a correct count. - */ -static void mdb_audit(MDB_txn *txn) -{ - MDB_cursor mc; - MDB_val key, data; - MDB_ID freecount, count; - MDB_dbi i; - int rc; - - freecount = 0; - mdb_cursor_init(&mc, txn, FREE_DBI, NULL); - while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) - freecount += *(MDB_ID *)data.mv_data; - mdb_tassert(txn, rc == MDB_NOTFOUND); - - count = 0; - for (i = 0; imt_numdbs; i++) { - MDB_xcursor mx; - if (!(txn->mt_dbflags[i] & DB_VALID)) - continue; - mdb_cursor_init(&mc, txn, i, &mx); - if (txn->mt_dbs[i].md_root == P_INVALID) - continue; - count += txn->mt_dbs[i].md_branch_pages + - txn->mt_dbs[i].md_leaf_pages + - txn->mt_dbs[i].md_overflow_pages; - if (txn->mt_dbs[i].md_flags & MDB_DUPSORT) { - rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST); - for (; rc == MDB_SUCCESS; rc = mdb_cursor_sibling(&mc, 1)) { - unsigned j; - MDB_page *mp; - mp = mc.mc_pg[mc.mc_top]; - for (j=0; jmn_flags & F_SUBDATA) { - MDB_db db; - memcpy(&db, NODEDATA(leaf), sizeof(db)); - count += db.md_branch_pages + db.md_leaf_pages + - db.md_overflow_pages; - } - } - } - mdb_tassert(txn, rc == MDB_NOTFOUND); - } - } - if (freecount + count + NUM_METAS != txn->mt_next_pgno) { - fprintf(stderr, "audit: %"Yu" freecount: %"Yu" count: %"Yu" total: %"Yu" next_pgno: %"Yu"\n", - txn->mt_txnid, freecount, count+NUM_METAS, - freecount+count+NUM_METAS, txn->mt_next_pgno); - } -} -#endif - -int -mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) -{ - return txn->mt_dbxs[dbi].md_cmp(a, b); -} - -int -mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b) -{ - MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp; - if (NEED_CMP_CLONG(dcmp, a->mv_size)) - dcmp = mdb_cmp_clong; - return dcmp(a, b); -} - -/** Allocate memory for a page. - * Re-use old malloc'd pages first for singletons, otherwise just malloc. - * Set #MDB_TXN_ERROR on failure. - */ -static MDB_page * -mdb_page_malloc(MDB_txn *txn, unsigned num) -{ - MDB_env *env = txn->mt_env; - MDB_page *ret = env->me_dpages; - size_t psize = env->me_psize, sz = psize, off; - /* For ! #MDB_NOMEMINIT, psize counts how much to init. - * For a single page alloc, we init everything after the page header. - * For multi-page, we init the final page; if the caller needed that - * many pages they will be filling in at least up to the last page. - */ - if (num == 1) { - if (ret) { - VGMEMP_ALLOC(env, ret, sz); - VGMEMP_DEFINED(ret, sizeof(ret->mp_next)); - env->me_dpages = ret->mp_next; - return ret; - } - psize -= off = PAGEHDRSZ; - } else { - sz *= num; - off = sz - psize; - } - if ((ret = malloc(sz)) != NULL) { - VGMEMP_ALLOC(env, ret, sz); - if (!(env->me_flags & MDB_NOMEMINIT)) { - memset((char *)ret + off, 0, psize); - ret->mp_pad = 0; - } - } else { - txn->mt_flags |= MDB_TXN_ERROR; - } - return ret; -} -/** Free a single page. - * Saves single pages to a list, for future reuse. - * (This is not used for multi-page overflow pages.) - */ -static void -mdb_page_free(MDB_env *env, MDB_page *mp) -{ - mp->mp_next = env->me_dpages; - VGMEMP_FREE(env, mp); - env->me_dpages = mp; -} - -/** Free a dirty page */ -static void -mdb_dpage_free(MDB_env *env, MDB_page *dp) -{ - if (!IS_OVERFLOW(dp) || dp->mp_pages == 1) { - mdb_page_free(env, dp); - } else { - /* large pages just get freed directly */ - VGMEMP_FREE(env, dp); - free(dp); - } -} - -/** Return all dirty pages to dpage list */ -static void -mdb_dlist_free(MDB_txn *txn) -{ - MDB_env *env = txn->mt_env; - MDB_ID2L dl = txn->mt_u.dirty_list; - unsigned i, n = dl[0].mid; - - for (i = 1; i <= n; i++) { - mdb_dpage_free(env, dl[i].mptr); - } - dl[0].mid = 0; -} - -#ifdef MDB_VL32 -static void -mdb_page_unref(MDB_txn *txn, MDB_page *mp) -{ - pgno_t pgno; - MDB_ID3L tl = txn->mt_rpages; - unsigned x, rem; - if (mp->mp_flags & (P_SUBP|P_DIRTY)) - return; - rem = mp->mp_pgno & (MDB_RPAGE_CHUNK-1); - pgno = mp->mp_pgno ^ rem; - x = mdb_mid3l_search(tl, pgno); - if (x != tl[0].mid && tl[x+1].mid == mp->mp_pgno) - x++; - if (tl[x].mref) - tl[x].mref--; -} -#define MDB_PAGE_UNREF(txn, mp) mdb_page_unref(txn, mp) - -static void -mdb_cursor_unref(MDB_cursor *mc) -{ - int i; - if (mc->mc_txn->mt_rpages[0].mid) { - if (!mc->mc_snum || !mc->mc_pg[0] || IS_SUBP(mc->mc_pg[0])) - return; - for (i=0; imc_snum; i++) - mdb_page_unref(mc->mc_txn, mc->mc_pg[i]); - if (mc->mc_ovpg) { - mdb_page_unref(mc->mc_txn, mc->mc_ovpg); - mc->mc_ovpg = 0; - } - } - mc->mc_snum = mc->mc_top = 0; - mc->mc_pg[0] = NULL; - mc->mc_flags &= ~C_INITIALIZED; -} -#define MDB_CURSOR_UNREF(mc, force) \ - (((force) || ((mc)->mc_flags & C_INITIALIZED)) \ - ? mdb_cursor_unref(mc) \ - : (void)0) - -#else -#define MDB_PAGE_UNREF(txn, mp) -#define MDB_CURSOR_UNREF(mc, force) ((void)0) -#endif /* MDB_VL32 */ - -/** Loosen or free a single page. - * Saves single pages to a list for future reuse - * in this same txn. It has been pulled from the freeDB - * and already resides on the dirty list, but has been - * deleted. Use these pages first before pulling again - * from the freeDB. - * - * If the page wasn't dirtied in this txn, just add it - * to this txn's free list. - */ -static int -mdb_page_loose(MDB_cursor *mc, MDB_page *mp) -{ - int loose = 0; - pgno_t pgno = mp->mp_pgno; - MDB_txn *txn = mc->mc_txn; - - if ((mp->mp_flags & P_DIRTY) && mc->mc_dbi != FREE_DBI) { - if (txn->mt_parent) { - MDB_ID2 *dl = txn->mt_u.dirty_list; - /* If txn has a parent, make sure the page is in our - * dirty list. - */ - if (dl[0].mid) { - unsigned x = mdb_mid2l_search(dl, pgno); - if (x <= dl[0].mid && dl[x].mid == pgno) { - if (mp != dl[x].mptr) { /* bad cursor? */ - mc->mc_flags &= ~(C_INITIALIZED|C_EOF); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PROBLEM; - } - /* ok, it's ours */ - loose = 1; - } - } - } else { - /* no parent txn, so it's just ours */ - loose = 1; - } - } - if (loose) { - DPRINTF(("loosen db %d page %"Yu, DDBI(mc), mp->mp_pgno)); - NEXT_LOOSE_PAGE(mp) = txn->mt_loose_pgs; - txn->mt_loose_pgs = mp; - txn->mt_loose_count++; - mp->mp_flags |= P_LOOSE; - } else { - int rc = mdb_midl_append(&txn->mt_free_pgs, pgno); - if (rc) - return rc; - } - - return MDB_SUCCESS; -} - -/** Set or clear P_KEEP in dirty, non-overflow, non-sub pages watched by txn. - * @param[in] mc A cursor handle for the current operation. - * @param[in] pflags Flags of the pages to update: - * P_DIRTY to set P_KEEP, P_DIRTY|P_KEEP to clear it. - * @param[in] all No shortcuts. Needed except after a full #mdb_page_flush(). - * @return 0 on success, non-zero on failure. - */ -static int -mdb_pages_xkeep(MDB_cursor *mc, unsigned pflags, int all) -{ - enum { Mask = P_SUBP|P_DIRTY|P_LOOSE|P_KEEP }; - MDB_txn *txn = mc->mc_txn; - MDB_cursor *m3, *m0 = mc; - MDB_xcursor *mx; - MDB_page *dp, *mp; - MDB_node *leaf; - unsigned i, j; - int rc = MDB_SUCCESS, level; - - /* Mark pages seen by cursors: First m0, then tracked cursors */ - for (i = txn->mt_numdbs;; ) { - if (mc->mc_flags & C_INITIALIZED) { - for (m3 = mc;; m3 = &mx->mx_cursor) { - mp = NULL; - for (j=0; jmc_snum; j++) { - mp = m3->mc_pg[j]; - if ((mp->mp_flags & Mask) == pflags) - mp->mp_flags ^= P_KEEP; - } - mx = m3->mc_xcursor; - /* Proceed to mx if it is at a sub-database */ - if (! (mx && (mx->mx_cursor.mc_flags & C_INITIALIZED))) - break; - if (! (mp && (mp->mp_flags & P_LEAF))) - break; - leaf = NODEPTR(mp, m3->mc_ki[j-1]); - if (!(leaf->mn_flags & F_SUBDATA)) - break; - } - } - mc = mc->mc_next; - for (; !mc || mc == m0; mc = txn->mt_cursors[--i]) - if (i == 0) - goto mark_done; - } - -mark_done: - if (all) { - /* Mark dirty root pages */ - for (i=0; imt_numdbs; i++) { - if (txn->mt_dbflags[i] & DB_DIRTY) { - pgno_t pgno = txn->mt_dbs[i].md_root; - if (pgno == P_INVALID) - continue; - if ((rc = mdb_page_get(m0, pgno, &dp, &level)) != MDB_SUCCESS) - break; - if ((dp->mp_flags & Mask) == pflags && level <= 1) - dp->mp_flags ^= P_KEEP; - } - } - } - - return rc; -} - -static int mdb_page_flush(MDB_txn *txn, int keep); - -/** Spill pages from the dirty list back to disk. - * This is intended to prevent running into #MDB_TXN_FULL situations, - * but note that they may still occur in a few cases: - * 1) our estimate of the txn size could be too small. Currently this - * seems unlikely, except with a large number of #MDB_MULTIPLE items. - * 2) child txns may run out of space if their parents dirtied a - * lot of pages and never spilled them. TODO: we probably should do - * a preemptive spill during #mdb_txn_begin() of a child txn, if - * the parent's dirty_room is below a given threshold. - * - * Otherwise, if not using nested txns, it is expected that apps will - * not run into #MDB_TXN_FULL any more. The pages are flushed to disk - * the same way as for a txn commit, e.g. their P_DIRTY flag is cleared. - * If the txn never references them again, they can be left alone. - * If the txn only reads them, they can be used without any fuss. - * If the txn writes them again, they can be dirtied immediately without - * going thru all of the work of #mdb_page_touch(). Such references are - * handled by #mdb_page_unspill(). - * - * Also note, we never spill DB root pages, nor pages of active cursors, - * because we'll need these back again soon anyway. And in nested txns, - * we can't spill a page in a child txn if it was already spilled in a - * parent txn. That would alter the parent txns' data even though - * the child hasn't committed yet, and we'd have no way to undo it if - * the child aborted. - * - * @param[in] m0 cursor A cursor handle identifying the transaction and - * database for which we are checking space. - * @param[in] key For a put operation, the key being stored. - * @param[in] data For a put operation, the data being stored. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data) -{ - MDB_txn *txn = m0->mc_txn; - MDB_page *dp; - MDB_ID2L dl = txn->mt_u.dirty_list; - unsigned int i, j, need; - int rc; - - if (m0->mc_flags & C_SUB) - return MDB_SUCCESS; - - /* Estimate how much space this op will take */ - i = m0->mc_db->md_depth; - /* Named DBs also dirty the main DB */ - if (m0->mc_dbi >= CORE_DBS) - i += txn->mt_dbs[MAIN_DBI].md_depth; - /* For puts, roughly factor in the key+data size */ - if (key) - i += (LEAFSIZE(key, data) + txn->mt_env->me_psize) / txn->mt_env->me_psize; - i += i; /* double it for good measure */ - need = i; - - if (txn->mt_dirty_room > i) - return MDB_SUCCESS; - - if (!txn->mt_spill_pgs) { - txn->mt_spill_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX); - if (!txn->mt_spill_pgs) - return ENOMEM; - } else { - /* purge deleted slots */ - MDB_IDL sl = txn->mt_spill_pgs; - unsigned int num = sl[0]; - j=0; - for (i=1; i<=num; i++) { - if (!(sl[i] & 1)) - sl[++j] = sl[i]; - } - sl[0] = j; - } - - /* Preserve pages which may soon be dirtied again */ - if ((rc = mdb_pages_xkeep(m0, P_DIRTY, 1)) != MDB_SUCCESS) - goto done; - - /* Less aggressive spill - we originally spilled the entire dirty list, - * with a few exceptions for cursor pages and DB root pages. But this - * turns out to be a lot of wasted effort because in a large txn many - * of those pages will need to be used again. So now we spill only 1/8th - * of the dirty pages. Testing revealed this to be a good tradeoff, - * better than 1/2, 1/4, or 1/10. - */ - if (need < MDB_IDL_UM_MAX / 8) - need = MDB_IDL_UM_MAX / 8; - - /* Save the page IDs of all the pages we're flushing */ - /* flush from the tail forward, this saves a lot of shifting later on. */ - for (i=dl[0].mid; i && need; i--) { - MDB_ID pn = dl[i].mid << 1; - dp = dl[i].mptr; - if (dp->mp_flags & (P_LOOSE|P_KEEP)) - continue; - /* Can't spill twice, make sure it's not already in a parent's - * spill list. - */ - if (txn->mt_parent) { - MDB_txn *tx2; - for (tx2 = txn->mt_parent; tx2; tx2 = tx2->mt_parent) { - if (tx2->mt_spill_pgs) { - j = mdb_midl_search(tx2->mt_spill_pgs, pn); - if (j <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[j] == pn) { - dp->mp_flags |= P_KEEP; - break; - } - } - } - if (tx2) - continue; - } - if ((rc = mdb_midl_append(&txn->mt_spill_pgs, pn))) - goto done; - need--; - } - mdb_midl_sort(txn->mt_spill_pgs); - - /* Flush the spilled part of dirty list */ - if ((rc = mdb_page_flush(txn, i)) != MDB_SUCCESS) - goto done; - - /* Reset any dirty pages we kept that page_flush didn't see */ - rc = mdb_pages_xkeep(m0, P_DIRTY|P_KEEP, i); - -done: - txn->mt_flags |= rc ? MDB_TXN_ERROR : MDB_TXN_SPILLS; - return rc; -} - -/** Find oldest txnid still referenced. Expects txn->mt_txnid > 0. */ -static txnid_t -mdb_find_oldest(MDB_txn *txn) -{ - int i; - txnid_t mr, oldest = txn->mt_txnid - 1; - if (txn->mt_env->me_txns) { - MDB_reader *r = txn->mt_env->me_txns->mti_readers; - for (i = txn->mt_env->me_txns->mti_numreaders; --i >= 0; ) { - if (r[i].mr_pid) { - mr = r[i].mr_txnid; - if (oldest > mr) - oldest = mr; - } - } - } - return oldest; -} - -/** Add a page to the txn's dirty list */ -static void -mdb_page_dirty(MDB_txn *txn, MDB_page *mp) -{ - MDB_ID2 mid; - int rc, (*insert)(MDB_ID2L, MDB_ID2 *); - - if (txn->mt_flags & MDB_TXN_WRITEMAP) { - insert = mdb_mid2l_append; - } else { - insert = mdb_mid2l_insert; - } - mid.mid = mp->mp_pgno; - mid.mptr = mp; - rc = insert(txn->mt_u.dirty_list, &mid); - mdb_tassert(txn, rc == 0); - txn->mt_dirty_room--; -} - -/** Allocate page numbers and memory for writing. Maintain me_pglast, - * me_pghead and mt_next_pgno. Set #MDB_TXN_ERROR on failure. - * - * If there are free pages available from older transactions, they - * are re-used first. Otherwise allocate a new page at mt_next_pgno. - * Do not modify the freedB, just merge freeDB records into me_pghead[] - * and move me_pglast to say which records were consumed. Only this - * function can create me_pghead and move me_pglast/mt_next_pgno. - * When #MDB_DEVEL & 2, it is not affected by #mdb_freelist_save(): it - * then uses the transaction's original snapshot of the freeDB. - * @param[in] mc cursor A cursor handle identifying the transaction and - * database for which we are allocating. - * @param[in] num the number of pages to allocate. - * @param[out] mp Address of the allocated page(s). Requests for multiple pages - * will always be satisfied by a single contiguous chunk of memory. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) -{ -#ifdef MDB_PARANOID /* Seems like we can ignore this now */ - /* Get at most more freeDB records once me_pghead - * has enough pages. If not enough, use new pages from the map. - * If and mc is updating the freeDB, only get new - * records if me_pghead is empty. Then the freelist cannot play - * catch-up with itself by growing while trying to save it. - */ - enum { Paranoid = 1, Max_retries = 500 }; -#else - enum { Paranoid = 0, Max_retries = INT_MAX /*infinite*/ }; -#endif - int rc, retry = num * 60; - MDB_txn *txn = mc->mc_txn; - MDB_env *env = txn->mt_env; - pgno_t pgno, *mop = env->me_pghead; - unsigned i, j, mop_len = mop ? mop[0] : 0, n2 = num-1; - MDB_page *np; - txnid_t oldest = 0, last; - MDB_cursor_op op; - MDB_cursor m2; - int found_old = 0; - - /* If there are any loose pages, just use them */ - if (num == 1 && txn->mt_loose_pgs) { - np = txn->mt_loose_pgs; - txn->mt_loose_pgs = NEXT_LOOSE_PAGE(np); - txn->mt_loose_count--; - DPRINTF(("db %d use loose page %"Yu, DDBI(mc), np->mp_pgno)); - *mp = np; - return MDB_SUCCESS; - } - - *mp = NULL; - - /* If our dirty list is already full, we can't do anything */ - if (txn->mt_dirty_room == 0) { - rc = MDB_TXN_FULL; - goto fail; - } - - for (op = MDB_FIRST;; op = MDB_NEXT) { - MDB_val key, data; - MDB_node *leaf; - pgno_t *idl; - - /* Seek a big enough contiguous page range. Prefer - * pages at the tail, just truncating the list. - */ - if (mop_len > n2) { - i = mop_len; - do { - pgno = mop[i]; - if (mop[i-n2] == pgno+n2) - goto search_done; - } while (--i > n2); - if (--retry < 0) - break; - } - - if (op == MDB_FIRST) { /* 1st iteration */ - /* Prepare to fetch more and coalesce */ - last = env->me_pglast; - oldest = env->me_pgoldest; - mdb_cursor_init(&m2, txn, FREE_DBI, NULL); -#if (MDB_DEVEL) & 2 /* "& 2" so MDB_DEVEL=1 won't hide bugs breaking freeDB */ - /* Use original snapshot. TODO: Should need less care in code - * which modifies the database. Maybe we can delete some code? - */ - m2.mc_flags |= C_ORIG_RDONLY; - m2.mc_db = &env->me_metas[(txn->mt_txnid-1) & 1]->mm_dbs[FREE_DBI]; - m2.mc_dbflag = (unsigned char *)""; /* probably unnecessary */ -#endif - if (last) { - op = MDB_SET_RANGE; - key.mv_data = &last; /* will look up last+1 */ - key.mv_size = sizeof(last); - } - if (Paranoid && mc->mc_dbi == FREE_DBI) - retry = -1; - } - if (Paranoid && retry < 0 && mop_len) - break; - - last++; - /* Do not fetch more if the record will be too recent */ - if (oldest <= last) { - if (!found_old) { - oldest = mdb_find_oldest(txn); - env->me_pgoldest = oldest; - found_old = 1; - } - if (oldest <= last) - break; - } - rc = mdb_cursor_get(&m2, &key, NULL, op); - if (rc) { - if (rc == MDB_NOTFOUND) - break; - goto fail; - } - last = *(txnid_t*)key.mv_data; - if (oldest <= last) { - if (!found_old) { - oldest = mdb_find_oldest(txn); - env->me_pgoldest = oldest; - found_old = 1; - } - if (oldest <= last) - break; - } - np = m2.mc_pg[m2.mc_top]; - leaf = NODEPTR(np, m2.mc_ki[m2.mc_top]); - if ((rc = mdb_node_read(&m2, leaf, &data)) != MDB_SUCCESS) - goto fail; - - idl = (MDB_ID *) data.mv_data; - i = idl[0]; - if (!mop) { - if (!(env->me_pghead = mop = mdb_midl_alloc(i))) { - rc = ENOMEM; - goto fail; - } - } else { - if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0) - goto fail; - mop = env->me_pghead; - } - env->me_pglast = last; -#if (MDB_DEBUG) > 1 - DPRINTF(("IDL read txn %"Yu" root %"Yu" num %u", - last, txn->mt_dbs[FREE_DBI].md_root, i)); - for (j = i; j; j--) - DPRINTF(("IDL %"Yu, idl[j])); -#endif - /* Merge in descending sorted order */ - mdb_midl_xmerge(mop, idl); - mop_len = mop[0]; - } - - /* Use new pages from the map when nothing suitable in the freeDB */ - i = 0; - pgno = txn->mt_next_pgno; - if (pgno + num >= env->me_maxpg) { - DPUTS("DB size maxed out"); - rc = MDB_MAP_FULL; - goto fail; - } -#if defined(_WIN32) && !defined(MDB_VL32) - if (!(env->me_flags & MDB_RDONLY)) { - void *p; - p = (MDB_page *)(env->me_map + env->me_psize * pgno); - p = VirtualAlloc(p, env->me_psize * num, MEM_COMMIT, - (env->me_flags & MDB_WRITEMAP) ? PAGE_READWRITE: - PAGE_READONLY); - if (!p) { - DPUTS("VirtualAlloc failed"); - rc = ErrCode(); - goto fail; - } - } -#endif - -search_done: - if (env->me_flags & MDB_WRITEMAP) { - np = (MDB_page *)(env->me_map + env->me_psize * pgno); - } else { - if (!(np = mdb_page_malloc(txn, num))) { - rc = ENOMEM; - goto fail; - } - } - if (i) { - mop[0] = mop_len -= num; - /* Move any stragglers down */ - for (j = i-num; j < mop_len; ) - mop[++j] = mop[++i]; - } else { - txn->mt_next_pgno = pgno + num; - } - np->mp_pgno = pgno; - mdb_page_dirty(txn, np); - *mp = np; - - return MDB_SUCCESS; - -fail: - txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -/** Copy the used portions of a non-overflow page. - * @param[in] dst page to copy into - * @param[in] src page to copy from - * @param[in] psize size of a page - */ -static void -mdb_page_copy(MDB_page *dst, MDB_page *src, unsigned int psize) -{ - enum { Align = sizeof(pgno_t) }; - indx_t upper = src->mp_upper, lower = src->mp_lower, unused = upper-lower; - - /* If page isn't full, just copy the used portion. Adjust - * alignment so memcpy may copy words instead of bytes. - */ - if ((unused &= -Align) && !IS_LEAF2(src)) { - upper = (upper + PAGEBASE) & -Align; - memcpy(dst, src, (lower + PAGEBASE + (Align-1)) & -Align); - memcpy((pgno_t *)((char *)dst+upper), (pgno_t *)((char *)src+upper), - psize - upper); - } else { - memcpy(dst, src, psize - unused); - } -} - -/** Pull a page off the txn's spill list, if present. - * If a page being referenced was spilled to disk in this txn, bring - * it back and make it dirty/writable again. - * @param[in] txn the transaction handle. - * @param[in] mp the page being referenced. It must not be dirty. - * @param[out] ret the writable page, if any. ret is unchanged if - * mp wasn't spilled. - */ -static int -mdb_page_unspill(MDB_txn *txn, MDB_page *mp, MDB_page **ret) -{ - MDB_env *env = txn->mt_env; - const MDB_txn *tx2; - unsigned x; - pgno_t pgno = mp->mp_pgno, pn = pgno << 1; - - for (tx2 = txn; tx2; tx2=tx2->mt_parent) { - if (!tx2->mt_spill_pgs) - continue; - x = mdb_midl_search(tx2->mt_spill_pgs, pn); - if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { - MDB_page *np; - int num; - if (txn->mt_dirty_room == 0) - return MDB_TXN_FULL; - if (IS_OVERFLOW(mp)) - num = mp->mp_pages; - else - num = 1; - if (env->me_flags & MDB_WRITEMAP) { - np = mp; - } else { - np = mdb_page_malloc(txn, num); - if (!np) - return ENOMEM; - if (num > 1) - memcpy(np, mp, num * env->me_psize); - else - mdb_page_copy(np, mp, env->me_psize); - } - if (tx2 == txn) { - /* If in current txn, this page is no longer spilled. - * If it happens to be the last page, truncate the spill list. - * Otherwise mark it as deleted by setting the LSB. - */ - if (x == txn->mt_spill_pgs[0]) - txn->mt_spill_pgs[0]--; - else - txn->mt_spill_pgs[x] |= 1; - } /* otherwise, if belonging to a parent txn, the - * page remains spilled until child commits - */ - - mdb_page_dirty(txn, np); - np->mp_flags |= P_DIRTY; - *ret = np; - break; - } - } - return MDB_SUCCESS; -} - -/** Touch a page: make it dirty and re-insert into tree with updated pgno. - * Set #MDB_TXN_ERROR on failure. - * @param[in] mc cursor pointing to the page to be touched - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_touch(MDB_cursor *mc) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top], *np; - MDB_txn *txn = mc->mc_txn; - MDB_cursor *m2, *m3; - pgno_t pgno; - int rc; - - if (!F_ISSET(mp->mp_flags, P_DIRTY)) { - if (txn->mt_flags & MDB_TXN_SPILLS) { - np = NULL; - rc = mdb_page_unspill(txn, mp, &np); - if (rc) - goto fail; - if (np) - goto done; - } - if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) || - (rc = mdb_page_alloc(mc, 1, &np))) - goto fail; - pgno = np->mp_pgno; - DPRINTF(("touched db %d page %"Yu" -> %"Yu, DDBI(mc), - mp->mp_pgno, pgno)); - mdb_cassert(mc, mp->mp_pgno != pgno); - mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); - /* Update the parent page, if any, to point to the new page */ - if (mc->mc_top) { - MDB_page *parent = mc->mc_pg[mc->mc_top-1]; - MDB_node *node = NODEPTR(parent, mc->mc_ki[mc->mc_top-1]); - SETPGNO(node, pgno); - } else { - mc->mc_db->md_root = pgno; - } - } else if (txn->mt_parent && !IS_SUBP(mp)) { - MDB_ID2 mid, *dl = txn->mt_u.dirty_list; - pgno = mp->mp_pgno; - /* If txn has a parent, make sure the page is in our - * dirty list. - */ - if (dl[0].mid) { - unsigned x = mdb_mid2l_search(dl, pgno); - if (x <= dl[0].mid && dl[x].mid == pgno) { - if (mp != dl[x].mptr) { /* bad cursor? */ - mc->mc_flags &= ~(C_INITIALIZED|C_EOF); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PROBLEM; - } - return 0; - } - } - mdb_cassert(mc, dl[0].mid < MDB_IDL_UM_MAX); - /* No - copy it */ - np = mdb_page_malloc(txn, 1); - if (!np) - return ENOMEM; - mid.mid = pgno; - mid.mptr = np; - rc = mdb_mid2l_insert(dl, &mid); - mdb_cassert(mc, rc == 0); - } else { - return 0; - } - - mdb_page_copy(np, mp, txn->mt_env->me_psize); - np->mp_pgno = pgno; - np->mp_flags |= P_DIRTY; - -done: - /* Adjust cursors pointing to mp */ - mc->mc_pg[mc->mc_top] = np; - m2 = txn->mt_cursors[mc->mc_dbi]; - if (mc->mc_flags & C_SUB) { - for (; m2; m2=m2->mc_next) { - m3 = &m2->mc_xcursor->mx_cursor; - if (m3->mc_snum < mc->mc_snum) continue; - if (m3->mc_pg[mc->mc_top] == mp) - m3->mc_pg[mc->mc_top] = np; - } - } else { - for (; m2; m2=m2->mc_next) { - if (m2->mc_snum < mc->mc_snum) continue; - if (m2 == mc) continue; - if (m2->mc_pg[mc->mc_top] == mp) { - m2->mc_pg[mc->mc_top] = np; - if (IS_LEAF(np)) - XCURSOR_REFRESH(m2, mc->mc_top, np); - } - } - } - MDB_PAGE_UNREF(mc->mc_txn, mp); - return 0; - -fail: - txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_env_sync0(MDB_env *env, int force, pgno_t numpgs) -{ - int rc = 0; - if (env->me_flags & MDB_RDONLY) - return EACCES; - if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) { - if (env->me_flags & MDB_WRITEMAP) { - int flags = ((env->me_flags & MDB_MAPASYNC) && !force) - ? MS_ASYNC : MS_SYNC; - if (MDB_MSYNC(env->me_map, env->me_psize * numpgs, flags)) - rc = ErrCode(); -#ifdef _WIN32 - else if (flags == MS_SYNC && MDB_FDATASYNC(env->me_fd)) - rc = ErrCode(); -#endif - } else { -#ifdef BROKEN_FDATASYNC - if (env->me_flags & MDB_FSYNCONLY) { - if (fsync(env->me_fd)) - rc = ErrCode(); - } else -#endif - if (MDB_FDATASYNC(env->me_fd)) - rc = ErrCode(); - } - } - return rc; -} - -int -mdb_env_sync(MDB_env *env, int force) -{ - MDB_meta *m = mdb_env_pick_meta(env); - return mdb_env_sync0(env, force, m->mm_last_pg+1); -} - -/** Back up parent txn's cursors, then grab the originals for tracking */ -static int -mdb_cursor_shadow(MDB_txn *src, MDB_txn *dst) -{ - MDB_cursor *mc, *bk; - MDB_xcursor *mx; - size_t size; - int i; - - for (i = src->mt_numdbs; --i >= 0; ) { - if ((mc = src->mt_cursors[i]) != NULL) { - size = sizeof(MDB_cursor); - if (mc->mc_xcursor) - size += sizeof(MDB_xcursor); - for (; mc; mc = bk->mc_next) { - bk = malloc(size); - if (!bk) - return ENOMEM; - *bk = *mc; - mc->mc_backup = bk; - mc->mc_db = &dst->mt_dbs[i]; - /* Kill pointers into src to reduce abuse: The - * user may not use mc until dst ends. But we need a valid - * txn pointer here for cursor fixups to keep working. - */ - mc->mc_txn = dst; - mc->mc_dbflag = &dst->mt_dbflags[i]; - if ((mx = mc->mc_xcursor) != NULL) { - *(MDB_xcursor *)(bk+1) = *mx; - mx->mx_cursor.mc_txn = dst; - } - mc->mc_next = dst->mt_cursors[i]; - dst->mt_cursors[i] = mc; - } - } - } - return MDB_SUCCESS; -} - -/** Close this write txn's cursors, give parent txn's cursors back to parent. - * @param[in] txn the transaction handle. - * @param[in] merge true to keep changes to parent cursors, false to revert. - * @return 0 on success, non-zero on failure. - */ -static void -mdb_cursors_close(MDB_txn *txn, unsigned merge) -{ - MDB_cursor **cursors = txn->mt_cursors, *mc, *next, *bk; - MDB_xcursor *mx; - int i; - - for (i = txn->mt_numdbs; --i >= 0; ) { - for (mc = cursors[i]; mc; mc = next) { - next = mc->mc_next; - if ((bk = mc->mc_backup) != NULL) { - if (merge) { - /* Commit changes to parent txn */ - mc->mc_next = bk->mc_next; - mc->mc_backup = bk->mc_backup; - mc->mc_txn = bk->mc_txn; - mc->mc_db = bk->mc_db; - mc->mc_dbflag = bk->mc_dbflag; - if ((mx = mc->mc_xcursor) != NULL) - mx->mx_cursor.mc_txn = bk->mc_txn; - } else { - /* Abort nested txn */ - *mc = *bk; - if ((mx = mc->mc_xcursor) != NULL) - *mx = *(MDB_xcursor *)(bk+1); - } - mc = bk; - } - /* Only malloced cursors are permanently tracked. */ - free(mc); - } - cursors[i] = NULL; - } -} - -#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ -enum Pidlock_op { - Pidset, Pidcheck -}; -#else -enum Pidlock_op { - Pidset = F_SETLK, Pidcheck = F_GETLK -}; -#endif - -/** Set or check a pid lock. Set returns 0 on success. - * Check returns 0 if the process is certainly dead, nonzero if it may - * be alive (the lock exists or an error happened so we do not know). - * - * On Windows Pidset is a no-op, we merely check for the existence - * of the process with the given pid. On POSIX we use a single byte - * lock on the lockfile, set at an offset equal to the pid. - */ -static int -mdb_reader_pid(MDB_env *env, enum Pidlock_op op, MDB_PID_T pid) -{ -#if !(MDB_PIDLOCK) /* Currently the same as defined(_WIN32) */ - int ret = 0; - HANDLE h; - if (op == Pidcheck) { - h = OpenProcess(env->me_pidquery, FALSE, pid); - /* No documented "no such process" code, but other program use this: */ - if (!h) - return ErrCode() != ERROR_INVALID_PARAMETER; - /* A process exists until all handles to it close. Has it exited? */ - ret = WaitForSingleObject(h, 0) != 0; - CloseHandle(h); - } - return ret; -#else - for (;;) { - int rc; - struct flock lock_info; - memset(&lock_info, 0, sizeof(lock_info)); - lock_info.l_type = F_WRLCK; - lock_info.l_whence = SEEK_SET; - lock_info.l_start = pid; - lock_info.l_len = 1; - if ((rc = fcntl(env->me_lfd, op, &lock_info)) == 0) { - if (op == F_GETLK && lock_info.l_type != F_UNLCK) - rc = -1; - } else if ((rc = ErrCode()) == EINTR) { - continue; - } - return rc; - } -#endif -} - -/** Common code for #mdb_txn_begin() and #mdb_txn_renew(). - * @param[in] txn the transaction handle to initialize - * @return 0 on success, non-zero on failure. - */ -static int -mdb_txn_renew0(MDB_txn *txn) -{ - MDB_env *env = txn->mt_env; - MDB_txninfo *ti = env->me_txns; - MDB_meta *meta; - unsigned int i, nr, flags = txn->mt_flags; - uint16_t x; - int rc, new_notls = 0; - - if ((flags &= MDB_TXN_RDONLY) != 0) { - if (!ti) { - meta = mdb_env_pick_meta(env); - txn->mt_txnid = meta->mm_txnid; - txn->mt_u.reader = NULL; - } else { - MDB_reader *r = (env->me_flags & MDB_NOTLS) ? txn->mt_u.reader : - pthread_getspecific(env->me_txkey); - if (r) { - if (r->mr_pid != env->me_pid || r->mr_txnid != (txnid_t)-1) - return MDB_BAD_RSLOT; - } else { - MDB_PID_T pid = env->me_pid; - MDB_THR_T tid = pthread_self(); - mdb_mutexref_t rmutex = env->me_rmutex; - - if (!env->me_live_reader) { - rc = mdb_reader_pid(env, Pidset, pid); - if (rc) - return rc; - env->me_live_reader = 1; - } - - if (LOCK_MUTEX(rc, env, rmutex)) - return rc; - nr = ti->mti_numreaders; - for (i=0; imti_readers[i].mr_pid == 0) - break; - if (i == env->me_maxreaders) { - UNLOCK_MUTEX(rmutex); - return MDB_READERS_FULL; - } - r = &ti->mti_readers[i]; - /* Claim the reader slot, carefully since other code - * uses the reader table un-mutexed: First reset the - * slot, next publish it in mti_numreaders. After - * that, it is safe for mdb_env_close() to touch it. - * When it will be closed, we can finally claim it. - */ - r->mr_pid = 0; - r->mr_txnid = (txnid_t)-1; - r->mr_tid = tid; - if (i == nr) - ti->mti_numreaders = ++nr; - env->me_close_readers = nr; - r->mr_pid = pid; - UNLOCK_MUTEX(rmutex); - - new_notls = (env->me_flags & MDB_NOTLS); - if (!new_notls && (rc=pthread_setspecific(env->me_txkey, r))) { - r->mr_pid = 0; - return rc; - } - } - do /* LY: Retry on a race, ITS#7970. */ - r->mr_txnid = ti->mti_txnid; - while(r->mr_txnid != ti->mti_txnid); - txn->mt_txnid = r->mr_txnid; - txn->mt_u.reader = r; - meta = env->me_metas[txn->mt_txnid & 1]; - } - - } else { - /* Not yet touching txn == env->me_txn0, it may be active */ - if (ti) { - if (LOCK_MUTEX(rc, env, env->me_wmutex)) - return rc; - txn->mt_txnid = ti->mti_txnid; - meta = env->me_metas[txn->mt_txnid & 1]; - } else { - meta = mdb_env_pick_meta(env); - txn->mt_txnid = meta->mm_txnid; - } - txn->mt_txnid++; -#if MDB_DEBUG - if (txn->mt_txnid == mdb_debug_start) - mdb_debug = 1; -#endif - txn->mt_child = NULL; - txn->mt_loose_pgs = NULL; - txn->mt_loose_count = 0; - txn->mt_dirty_room = MDB_IDL_UM_MAX; - txn->mt_u.dirty_list = env->me_dirty_list; - txn->mt_u.dirty_list[0].mid = 0; - txn->mt_free_pgs = env->me_free_pgs; - txn->mt_free_pgs[0] = 0; - txn->mt_spill_pgs = NULL; - env->me_txn = txn; - memcpy(txn->mt_dbiseqs, env->me_dbiseqs, env->me_maxdbs * sizeof(unsigned int)); - } - - /* Copy the DB info and flags */ - memcpy(txn->mt_dbs, meta->mm_dbs, CORE_DBS * sizeof(MDB_db)); - - /* Moved to here to avoid a data race in read TXNs */ - txn->mt_next_pgno = meta->mm_last_pg+1; -#ifdef MDB_VL32 - txn->mt_last_pgno = txn->mt_next_pgno - 1; -#endif - - txn->mt_flags = flags; - - /* Setup db info */ - txn->mt_numdbs = env->me_numdbs; - for (i=CORE_DBS; imt_numdbs; i++) { - x = env->me_dbflags[i]; - txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS; - txn->mt_dbflags[i] = (x & MDB_VALID) ? DB_VALID|DB_USRVALID|DB_STALE : 0; - } - txn->mt_dbflags[MAIN_DBI] = DB_VALID|DB_USRVALID; - txn->mt_dbflags[FREE_DBI] = DB_VALID; - - if (env->me_flags & MDB_FATAL_ERROR) { - DPUTS("environment had fatal error, must shutdown!"); - rc = MDB_PANIC; - } else if (env->me_maxpg < txn->mt_next_pgno) { - rc = MDB_MAP_RESIZED; - } else { - return MDB_SUCCESS; - } - mdb_txn_end(txn, new_notls /*0 or MDB_END_SLOT*/ | MDB_END_FAIL_BEGIN); - return rc; -} - -int -mdb_txn_renew(MDB_txn *txn) -{ - int rc; - - if (!txn || !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY|MDB_TXN_FINISHED)) - return EINVAL; - - rc = mdb_txn_renew0(txn); - if (rc == MDB_SUCCESS) { - DPRINTF(("renew txn %"Yu"%c %p on mdbenv %p, root page %"Yu, - txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', - (void *)txn, (void *)txn->mt_env, txn->mt_dbs[MAIN_DBI].md_root)); - } - return rc; -} - -int -mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret) -{ - MDB_txn *txn; - MDB_ntxn *ntxn; - int rc, size, tsize; - - flags &= MDB_TXN_BEGIN_FLAGS; - flags |= env->me_flags & MDB_WRITEMAP; - - if (env->me_flags & MDB_RDONLY & ~flags) /* write txn in RDONLY env */ - return EACCES; - - if (parent) { - /* Nested transactions: Max 1 child, write txns only, no writemap */ - flags |= parent->mt_flags; - if (flags & (MDB_RDONLY|MDB_WRITEMAP|MDB_TXN_BLOCKED)) { - return (parent->mt_flags & MDB_TXN_RDONLY) ? EINVAL : MDB_BAD_TXN; - } - /* Child txns save MDB_pgstate and use own copy of cursors */ - size = env->me_maxdbs * (sizeof(MDB_db)+sizeof(MDB_cursor *)+1); - size += tsize = sizeof(MDB_ntxn); - } else if (flags & MDB_RDONLY) { - size = env->me_maxdbs * (sizeof(MDB_db)+1); - size += tsize = sizeof(MDB_txn); - } else { - /* Reuse preallocated write txn. However, do not touch it until - * mdb_txn_renew0() succeeds, since it currently may be active. - */ - txn = env->me_txn0; - goto renew; - } - if ((txn = calloc(1, size)) == NULL) { - DPRINTF(("calloc: %s", strerror(errno))); - return ENOMEM; - } -#ifdef MDB_VL32 - if (!parent) { - txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3)); - if (!txn->mt_rpages) { - free(txn); - return ENOMEM; - } - txn->mt_rpages[0].mid = 0; - txn->mt_rpcheck = MDB_TRPAGE_SIZE/2; - } -#endif - txn->mt_dbxs = env->me_dbxs; /* static */ - txn->mt_dbs = (MDB_db *) ((char *)txn + tsize); - txn->mt_dbflags = (unsigned char *)txn + size - env->me_maxdbs; - txn->mt_flags = flags; - txn->mt_env = env; - - if (parent) { - unsigned int i; - txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); - txn->mt_dbiseqs = parent->mt_dbiseqs; - txn->mt_u.dirty_list = malloc(sizeof(MDB_ID2)*MDB_IDL_UM_SIZE); - if (!txn->mt_u.dirty_list || - !(txn->mt_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX))) - { - free(txn->mt_u.dirty_list); - free(txn); - return ENOMEM; - } - txn->mt_txnid = parent->mt_txnid; - txn->mt_dirty_room = parent->mt_dirty_room; - txn->mt_u.dirty_list[0].mid = 0; - txn->mt_spill_pgs = NULL; - txn->mt_next_pgno = parent->mt_next_pgno; - parent->mt_flags |= MDB_TXN_HAS_CHILD; - parent->mt_child = txn; - txn->mt_parent = parent; - txn->mt_numdbs = parent->mt_numdbs; -#ifdef MDB_VL32 - txn->mt_rpages = parent->mt_rpages; -#endif - memcpy(txn->mt_dbs, parent->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); - /* Copy parent's mt_dbflags, but clear DB_NEW */ - for (i=0; imt_numdbs; i++) - txn->mt_dbflags[i] = parent->mt_dbflags[i] & ~DB_NEW; - rc = 0; - ntxn = (MDB_ntxn *)txn; - ntxn->mnt_pgstate = env->me_pgstate; /* save parent me_pghead & co */ - if (env->me_pghead) { - size = MDB_IDL_SIZEOF(env->me_pghead); - env->me_pghead = mdb_midl_alloc(env->me_pghead[0]); - if (env->me_pghead) - memcpy(env->me_pghead, ntxn->mnt_pgstate.mf_pghead, size); - else - rc = ENOMEM; - } - if (!rc) - rc = mdb_cursor_shadow(parent, txn); - if (rc) - mdb_txn_end(txn, MDB_END_FAIL_BEGINCHILD); - } else { /* MDB_RDONLY */ - txn->mt_dbiseqs = env->me_dbiseqs; -renew: - rc = mdb_txn_renew0(txn); - } - if (rc) { - if (txn != env->me_txn0) { -#ifdef MDB_VL32 - free(txn->mt_rpages); -#endif - free(txn); - } - } else { - txn->mt_flags |= flags; /* could not change txn=me_txn0 earlier */ - *ret = txn; - DPRINTF(("begin txn %"Yu"%c %p on mdbenv %p, root page %"Yu, - txn->mt_txnid, (flags & MDB_RDONLY) ? 'r' : 'w', - (void *) txn, (void *) env, txn->mt_dbs[MAIN_DBI].md_root)); - } - - return rc; -} - -MDB_env * -mdb_txn_env(MDB_txn *txn) -{ - if(!txn) return NULL; - return txn->mt_env; -} - -mdb_size_t -mdb_txn_id(MDB_txn *txn) -{ - if(!txn) return 0; - return txn->mt_txnid; -} - -/** Export or close DBI handles opened in this txn. */ -static void -mdb_dbis_update(MDB_txn *txn, int keep) -{ - int i; - MDB_dbi n = txn->mt_numdbs; - MDB_env *env = txn->mt_env; - unsigned char *tdbflags = txn->mt_dbflags; - - for (i = n; --i >= CORE_DBS;) { - if (tdbflags[i] & DB_NEW) { - if (keep) { - env->me_dbflags[i] = txn->mt_dbs[i].md_flags | MDB_VALID; - } else { - char *ptr = env->me_dbxs[i].md_name.mv_data; - if (ptr) { - env->me_dbxs[i].md_name.mv_data = NULL; - env->me_dbxs[i].md_name.mv_size = 0; - env->me_dbflags[i] = 0; - env->me_dbiseqs[i]++; - free(ptr); - } - } - } - } - if (keep && env->me_numdbs < n) - env->me_numdbs = n; -} - -/** End a transaction, except successful commit of a nested transaction. - * May be called twice for readonly txns: First reset it, then abort. - * @param[in] txn the transaction handle to end - * @param[in] mode why and how to end the transaction - */ -static void -mdb_txn_end(MDB_txn *txn, unsigned mode) -{ - MDB_env *env = txn->mt_env; -#if MDB_DEBUG - static const char *const names[] = MDB_END_NAMES; -#endif - - /* Export or close DBI handles opened in this txn */ - mdb_dbis_update(txn, mode & MDB_END_UPDATE); - - DPRINTF(("%s txn %"Yu"%c %p on mdbenv %p, root page %"Yu, - names[mode & MDB_END_OPMASK], - txn->mt_txnid, (txn->mt_flags & MDB_TXN_RDONLY) ? 'r' : 'w', - (void *) txn, (void *)env, txn->mt_dbs[MAIN_DBI].md_root)); - - if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { - if (txn->mt_u.reader) { - txn->mt_u.reader->mr_txnid = (txnid_t)-1; - if (!(env->me_flags & MDB_NOTLS)) { - txn->mt_u.reader = NULL; /* txn does not own reader */ - } else if (mode & MDB_END_SLOT) { - txn->mt_u.reader->mr_pid = 0; - txn->mt_u.reader = NULL; - } /* else txn owns the slot until it does MDB_END_SLOT */ - } - txn->mt_numdbs = 0; /* prevent further DBI activity */ - txn->mt_flags |= MDB_TXN_FINISHED; - - } else if (!F_ISSET(txn->mt_flags, MDB_TXN_FINISHED)) { - pgno_t *pghead = env->me_pghead; - - if (!(mode & MDB_END_UPDATE)) /* !(already closed cursors) */ - mdb_cursors_close(txn, 0); - if (!(env->me_flags & MDB_WRITEMAP)) { - mdb_dlist_free(txn); - } - - txn->mt_numdbs = 0; - txn->mt_flags = MDB_TXN_FINISHED; - - if (!txn->mt_parent) { - mdb_midl_shrink(&txn->mt_free_pgs); - env->me_free_pgs = txn->mt_free_pgs; - /* me_pgstate: */ - env->me_pghead = NULL; - env->me_pglast = 0; - - env->me_txn = NULL; - mode = 0; /* txn == env->me_txn0, do not free() it */ - - /* The writer mutex was locked in mdb_txn_begin. */ - if (env->me_txns) - UNLOCK_MUTEX(env->me_wmutex); - } else { - txn->mt_parent->mt_child = NULL; - txn->mt_parent->mt_flags &= ~MDB_TXN_HAS_CHILD; - env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate; - mdb_midl_free(txn->mt_free_pgs); - mdb_midl_free(txn->mt_spill_pgs); - free(txn->mt_u.dirty_list); - } - - mdb_midl_free(pghead); - } -#ifdef MDB_VL32 - if (!txn->mt_parent) { - MDB_ID3L el = env->me_rpages, tl = txn->mt_rpages; - unsigned i, x, n = tl[0].mid; - pthread_mutex_lock(&env->me_rpmutex); - for (i = 1; i <= n; i++) { - if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) { - /* tmp overflow pages that we didn't share in env */ - munmap(tl[i].mptr, tl[i].mcnt * env->me_psize); - } else { - x = mdb_mid3l_search(el, tl[i].mid); - if (tl[i].mptr == el[x].mptr) { - el[x].mref--; - } else { - /* another tmp overflow page */ - munmap(tl[i].mptr, tl[i].mcnt * env->me_psize); - } - } - } - pthread_mutex_unlock(&env->me_rpmutex); - tl[0].mid = 0; - if (mode & MDB_END_FREE) - free(tl); - } -#endif - if (mode & MDB_END_FREE) - free(txn); -} - -void -mdb_txn_reset(MDB_txn *txn) -{ - if (txn == NULL) - return; - - /* This call is only valid for read-only txns */ - if (!(txn->mt_flags & MDB_TXN_RDONLY)) - return; - - mdb_txn_end(txn, MDB_END_RESET); -} - -void -mdb_txn_abort(MDB_txn *txn) -{ - if (txn == NULL) - return; - - if (txn->mt_child) - mdb_txn_abort(txn->mt_child); - - mdb_txn_end(txn, MDB_END_ABORT|MDB_END_SLOT|MDB_END_FREE); -} - -/** Save the freelist as of this transaction to the freeDB. - * This changes the freelist. Keep trying until it stabilizes. - * - * When (MDB_DEVEL) & 2, the changes do not affect #mdb_page_alloc(), - * it then uses the transaction's original snapshot of the freeDB. - */ -static int -mdb_freelist_save(MDB_txn *txn) -{ - /* env->me_pghead[] can grow and shrink during this call. - * env->me_pglast and txn->mt_free_pgs[] can only grow. - * Page numbers cannot disappear from txn->mt_free_pgs[]. - */ - MDB_cursor mc; - MDB_env *env = txn->mt_env; - int rc, maxfree_1pg = env->me_maxfree_1pg, more = 1; - txnid_t pglast = 0, head_id = 0; - pgno_t freecnt = 0, *free_pgs, *mop; - ssize_t head_room = 0, total_room = 0, mop_len, clean_limit; - - mdb_cursor_init(&mc, txn, FREE_DBI, NULL); - - if (env->me_pghead) { - /* Make sure first page of freeDB is touched and on freelist */ - rc = mdb_page_search(&mc, NULL, MDB_PS_FIRST|MDB_PS_MODIFY); - if (rc && rc != MDB_NOTFOUND) - return rc; - } - - if (!env->me_pghead && txn->mt_loose_pgs) { - /* Put loose page numbers in mt_free_pgs, since - * we may be unable to return them to me_pghead. - */ - MDB_page *mp = txn->mt_loose_pgs; - MDB_ID2 *dl = txn->mt_u.dirty_list; - unsigned x; - if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) - return rc; - for (; mp; mp = NEXT_LOOSE_PAGE(mp)) { - mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); - /* must also remove from dirty list */ - if (txn->mt_flags & MDB_TXN_WRITEMAP) { - for (x=1; x<=dl[0].mid; x++) - if (dl[x].mid == mp->mp_pgno) - break; - mdb_tassert(txn, x <= dl[0].mid); - } else { - x = mdb_mid2l_search(dl, mp->mp_pgno); - mdb_tassert(txn, dl[x].mid == mp->mp_pgno); - mdb_dpage_free(env, mp); - } - dl[x].mptr = NULL; - } - { - /* squash freed slots out of the dirty list */ - unsigned y; - for (y=1; dl[y].mptr && y <= dl[0].mid; y++); - if (y <= dl[0].mid) { - for(x=y, y++;;) { - while (!dl[y].mptr && y <= dl[0].mid) y++; - if (y > dl[0].mid) break; - dl[x++] = dl[y++]; - } - dl[0].mid = x-1; - } else { - /* all slots freed */ - dl[0].mid = 0; - } - } - txn->mt_loose_pgs = NULL; - txn->mt_loose_count = 0; - } - - /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ - clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP)) - ? SSIZE_MAX : maxfree_1pg; - - for (;;) { - /* Come back here after each Put() in case freelist changed */ - MDB_val key, data; - pgno_t *pgs; - ssize_t j; - - /* If using records from freeDB which we have not yet - * deleted, delete them and any we reserved for me_pghead. - */ - while (pglast < env->me_pglast) { - rc = mdb_cursor_first(&mc, &key, NULL); - if (rc) - return rc; - pglast = head_id = *(txnid_t *)key.mv_data; - total_room = head_room = 0; - mdb_tassert(txn, pglast <= env->me_pglast); - rc = mdb_cursor_del(&mc, 0); - if (rc) - return rc; - } - - /* Save the IDL of pages freed by this txn, to a single record */ - if (freecnt < txn->mt_free_pgs[0]) { - if (!freecnt) { - /* Make sure last page of freeDB is touched and on freelist */ - rc = mdb_page_search(&mc, NULL, MDB_PS_LAST|MDB_PS_MODIFY); - if (rc && rc != MDB_NOTFOUND) - return rc; - } - free_pgs = txn->mt_free_pgs; - /* Write to last page of freeDB */ - key.mv_size = sizeof(txn->mt_txnid); - key.mv_data = &txn->mt_txnid; - do { - freecnt = free_pgs[0]; - data.mv_size = MDB_IDL_SIZEOF(free_pgs); - rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); - if (rc) - return rc; - /* Retry if mt_free_pgs[] grew during the Put() */ - free_pgs = txn->mt_free_pgs; - } while (freecnt < free_pgs[0]); - mdb_midl_sort(free_pgs); - memcpy(data.mv_data, free_pgs, data.mv_size); -#if (MDB_DEBUG) > 1 - { - unsigned int i = free_pgs[0]; - DPRINTF(("IDL write txn %"Yu" root %"Yu" num %u", - txn->mt_txnid, txn->mt_dbs[FREE_DBI].md_root, i)); - for (; i; i--) - DPRINTF(("IDL %"Yu, free_pgs[i])); - } -#endif - continue; - } - - mop = env->me_pghead; - mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; - - /* Reserve records for me_pghead[]. Split it if multi-page, - * to avoid searching freeDB for a page range. Use keys in - * range [1,me_pglast]: Smaller than txnid of oldest reader. - */ - if (total_room >= mop_len) { - if (total_room == mop_len || --more < 0) - break; - } else if (head_room >= maxfree_1pg && head_id > 1) { - /* Keep current record (overflow page), add a new one */ - head_id--; - head_room = 0; - } - /* (Re)write {key = head_id, IDL length = head_room} */ - total_room -= head_room; - head_room = mop_len - total_room; - if (head_room > maxfree_1pg && head_id > 1) { - /* Overflow multi-page for part of me_pghead */ - head_room /= head_id; /* amortize page sizes */ - head_room += maxfree_1pg - head_room % (maxfree_1pg + 1); - } else if (head_room < 0) { - /* Rare case, not bothering to delete this record */ - head_room = 0; - } - key.mv_size = sizeof(head_id); - key.mv_data = &head_id; - data.mv_size = (head_room + 1) * sizeof(pgno_t); - rc = mdb_cursor_put(&mc, &key, &data, MDB_RESERVE); - if (rc) - return rc; - /* IDL is initially empty, zero out at least the length */ - pgs = (pgno_t *)data.mv_data; - j = head_room > clean_limit ? head_room : 0; - do { - pgs[j] = 0; - } while (--j >= 0); - total_room += head_room; - } - - /* Return loose page numbers to me_pghead, though usually none are - * left at this point. The pages themselves remain in dirty_list. - */ - if (txn->mt_loose_pgs) { - MDB_page *mp = txn->mt_loose_pgs; - unsigned count = txn->mt_loose_count; - MDB_IDL loose; - /* Room for loose pages + temp IDL with same */ - if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) - return rc; - mop = env->me_pghead; - loose = mop + MDB_IDL_ALLOCLEN(mop) - count; - for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) - loose[ ++count ] = mp->mp_pgno; - loose[0] = count; - mdb_midl_sort(loose); - mdb_midl_xmerge(mop, loose); - txn->mt_loose_pgs = NULL; - txn->mt_loose_count = 0; - mop_len = mop[0]; - } - - /* Fill in the reserved me_pghead records */ - rc = MDB_SUCCESS; - if (mop_len) { - MDB_val key, data; - - mop += mop_len; - rc = mdb_cursor_first(&mc, &key, &data); - for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { - txnid_t id = *(txnid_t *)key.mv_data; - ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; - MDB_ID save; - - mdb_tassert(txn, len >= 0 && id <= env->me_pglast); - key.mv_data = &id; - if (len > mop_len) { - len = mop_len; - data.mv_size = (len + 1) * sizeof(MDB_ID); - } - data.mv_data = mop -= len; - save = mop[0]; - mop[0] = len; - rc = mdb_cursor_put(&mc, &key, &data, MDB_CURRENT); - mop[0] = save; - if (rc || !(mop_len -= len)) - break; - } - } - return rc; -} - -/** Flush (some) dirty pages to the map, after clearing their dirty flag. - * @param[in] txn the transaction that's being committed - * @param[in] keep number of initial pages in dirty_list to keep dirty. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_flush(MDB_txn *txn, int keep) -{ - MDB_env *env = txn->mt_env; - MDB_ID2L dl = txn->mt_u.dirty_list; - unsigned psize = env->me_psize, j; - int i, pagecount = dl[0].mid, rc; - size_t size = 0; - off_t pos = 0; - pgno_t pgno = 0; - MDB_page *dp = NULL; -#ifdef _WIN32 - OVERLAPPED ov; -#else - struct iovec iov[MDB_COMMIT_PAGES]; - ssize_t wsize = 0, wres; - off_t wpos = 0, next_pos = 1; /* impossible pos, so pos != next_pos */ - int n = 0; -#endif - - j = i = keep; - - if (env->me_flags & MDB_WRITEMAP) { - /* Clear dirty flags */ - while (++i <= pagecount) { - dp = dl[i].mptr; - /* Don't flush this page yet */ - if (dp->mp_flags & (P_LOOSE|P_KEEP)) { - dp->mp_flags &= ~P_KEEP; - dl[++j] = dl[i]; - continue; - } - dp->mp_flags &= ~P_DIRTY; - } - goto done; - } - - /* Write the pages */ - for (;;) { - if (++i <= pagecount) { - dp = dl[i].mptr; - /* Don't flush this page yet */ - if (dp->mp_flags & (P_LOOSE|P_KEEP)) { - dp->mp_flags &= ~P_KEEP; - dl[i].mid = 0; - continue; - } - pgno = dl[i].mid; - /* clear dirty flag */ - dp->mp_flags &= ~P_DIRTY; - pos = pgno * psize; - size = psize; - if (IS_OVERFLOW(dp)) size *= dp->mp_pages; - } -#ifdef _WIN32 - else break; - - /* Windows actually supports scatter/gather I/O, but only on - * unbuffered file handles. Since we're relying on the OS page - * cache for all our data, that's self-defeating. So we just - * write pages one at a time. We use the ov structure to set - * the write offset, to at least save the overhead of a Seek - * system call. - */ - DPRINTF(("committing page %"Yu, pgno)); - memset(&ov, 0, sizeof(ov)); - ov.Offset = pos & 0xffffffff; - ov.OffsetHigh = pos >> 16 >> 16; - if (!WriteFile(env->me_fd, dp, size, NULL, &ov)) { - rc = ErrCode(); - DPRINTF(("WriteFile: %d", rc)); - return rc; - } -#else - /* Write up to MDB_COMMIT_PAGES dirty pages at a time. */ - if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) { - if (n) { -retry_write: - /* Write previous page(s) */ -#ifdef MDB_USE_PWRITEV - wres = pwritev(env->me_fd, iov, n, wpos); -#else - if (n == 1) { - wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos); - } else { -retry_seek: - if (lseek(env->me_fd, wpos, SEEK_SET) == -1) { - rc = ErrCode(); - if (rc == EINTR) - goto retry_seek; - DPRINTF(("lseek: %s", strerror(rc))); - return rc; - } - wres = writev(env->me_fd, iov, n); - } -#endif - if (wres != wsize) { - if (wres < 0) { - rc = ErrCode(); - if (rc == EINTR) - goto retry_write; - DPRINTF(("Write error: %s", strerror(rc))); - } else { - rc = EIO; /* TODO: Use which error code? */ - DPUTS("short write, filesystem full?"); - } - return rc; - } - n = 0; - } - if (i > pagecount) - break; - wpos = pos; - wsize = 0; - } - DPRINTF(("committing page %"Yu, pgno)); - next_pos = pos + size; - iov[n].iov_len = size; - iov[n].iov_base = (char *)dp; - wsize += size; - n++; -#endif /* _WIN32 */ - } -#ifdef MDB_VL32 - if (pgno > txn->mt_last_pgno) - txn->mt_last_pgno = pgno; -#endif - - /* MIPS has cache coherency issues, this is a no-op everywhere else - * Note: for any size >= on-chip cache size, entire on-chip cache is - * flushed. - */ - CACHEFLUSH(env->me_map, txn->mt_next_pgno * env->me_psize, DCACHE); - - for (i = keep; ++i <= pagecount; ) { - dp = dl[i].mptr; - /* This is a page we skipped above */ - if (!dl[i].mid) { - dl[++j] = dl[i]; - dl[j].mid = dp->mp_pgno; - continue; - } - mdb_dpage_free(env, dp); - } - -done: - i--; - txn->mt_dirty_room += i - j; - dl[0].mid = j; - return MDB_SUCCESS; -} - -static int ESECT mdb_env_share_locks(MDB_env *env, int *excl); - -int -mdb_txn_commit(MDB_txn *txn) -{ - int rc; - unsigned int i, end_mode; - MDB_env *env; - - if (txn == NULL) - return EINVAL; - - /* mdb_txn_end() mode for a commit which writes nothing */ - end_mode = MDB_END_EMPTY_COMMIT|MDB_END_UPDATE|MDB_END_SLOT|MDB_END_FREE; - - if (txn->mt_child) { - rc = mdb_txn_commit(txn->mt_child); - if (rc) - goto fail; - } - - env = txn->mt_env; - - if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) { - goto done; - } - - if (txn->mt_flags & (MDB_TXN_FINISHED|MDB_TXN_ERROR)) { - DPUTS("txn has failed/finished, can't commit"); - if (txn->mt_parent) - txn->mt_parent->mt_flags |= MDB_TXN_ERROR; - rc = MDB_BAD_TXN; - goto fail; - } - - if (txn->mt_parent) { - MDB_txn *parent = txn->mt_parent; - MDB_page **lp; - MDB_ID2L dst, src; - MDB_IDL pspill; - unsigned x, y, len, ps_len; - - /* Append our free list to parent's */ - rc = mdb_midl_append_list(&parent->mt_free_pgs, txn->mt_free_pgs); - if (rc) - goto fail; - mdb_midl_free(txn->mt_free_pgs); - /* Failures after this must either undo the changes - * to the parent or set MDB_TXN_ERROR in the parent. - */ - - parent->mt_next_pgno = txn->mt_next_pgno; - parent->mt_flags = txn->mt_flags; - - /* Merge our cursors into parent's and close them */ - mdb_cursors_close(txn, 1); - - /* Update parent's DB table. */ - memcpy(parent->mt_dbs, txn->mt_dbs, txn->mt_numdbs * sizeof(MDB_db)); - parent->mt_numdbs = txn->mt_numdbs; - parent->mt_dbflags[FREE_DBI] = txn->mt_dbflags[FREE_DBI]; - parent->mt_dbflags[MAIN_DBI] = txn->mt_dbflags[MAIN_DBI]; - for (i=CORE_DBS; imt_numdbs; i++) { - /* preserve parent's DB_NEW status */ - x = parent->mt_dbflags[i] & DB_NEW; - parent->mt_dbflags[i] = txn->mt_dbflags[i] | x; - } - - dst = parent->mt_u.dirty_list; - src = txn->mt_u.dirty_list; - /* Remove anything in our dirty list from parent's spill list */ - if ((pspill = parent->mt_spill_pgs) && (ps_len = pspill[0])) { - x = y = ps_len; - pspill[0] = (pgno_t)-1; - /* Mark our dirty pages as deleted in parent spill list */ - for (i=0, len=src[0].mid; ++i <= len; ) { - MDB_ID pn = src[i].mid << 1; - while (pn > pspill[x]) - x--; - if (pn == pspill[x]) { - pspill[x] = 1; - y = --x; - } - } - /* Squash deleted pagenums if we deleted any */ - for (x=y; ++x <= ps_len; ) - if (!(pspill[x] & 1)) - pspill[++y] = pspill[x]; - pspill[0] = y; - } - - /* Remove anything in our spill list from parent's dirty list */ - if (txn->mt_spill_pgs && txn->mt_spill_pgs[0]) { - for (i=1; i<=txn->mt_spill_pgs[0]; i++) { - MDB_ID pn = txn->mt_spill_pgs[i]; - if (pn & 1) - continue; /* deleted spillpg */ - pn >>= 1; - y = mdb_mid2l_search(dst, pn); - if (y <= dst[0].mid && dst[y].mid == pn) { - free(dst[y].mptr); - while (y < dst[0].mid) { - dst[y] = dst[y+1]; - y++; - } - dst[0].mid--; - } - } - } - - /* Find len = length of merging our dirty list with parent's */ - x = dst[0].mid; - dst[0].mid = 0; /* simplify loops */ - if (parent->mt_parent) { - len = x + src[0].mid; - y = mdb_mid2l_search(src, dst[x].mid + 1) - 1; - for (i = x; y && i; y--) { - pgno_t yp = src[y].mid; - while (yp < dst[i].mid) - i--; - if (yp == dst[i].mid) { - i--; - len--; - } - } - } else { /* Simplify the above for single-ancestor case */ - len = MDB_IDL_UM_MAX - txn->mt_dirty_room; - } - /* Merge our dirty list with parent's */ - y = src[0].mid; - for (i = len; y; dst[i--] = src[y--]) { - pgno_t yp = src[y].mid; - while (yp < dst[x].mid) - dst[i--] = dst[x--]; - if (yp == dst[x].mid) - free(dst[x--].mptr); - } - mdb_tassert(txn, i == x); - dst[0].mid = len; - free(txn->mt_u.dirty_list); - parent->mt_dirty_room = txn->mt_dirty_room; - if (txn->mt_spill_pgs) { - if (parent->mt_spill_pgs) { - /* TODO: Prevent failure here, so parent does not fail */ - rc = mdb_midl_append_list(&parent->mt_spill_pgs, txn->mt_spill_pgs); - if (rc) - parent->mt_flags |= MDB_TXN_ERROR; - mdb_midl_free(txn->mt_spill_pgs); - mdb_midl_sort(parent->mt_spill_pgs); - } else { - parent->mt_spill_pgs = txn->mt_spill_pgs; - } - } - - /* Append our loose page list to parent's */ - for (lp = &parent->mt_loose_pgs; *lp; lp = &NEXT_LOOSE_PAGE(*lp)) - ; - *lp = txn->mt_loose_pgs; - parent->mt_loose_count += txn->mt_loose_count; - - parent->mt_child = NULL; - mdb_midl_free(((MDB_ntxn *)txn)->mnt_pgstate.mf_pghead); - free(txn); - return rc; - } - - if (txn != env->me_txn) { - DPUTS("attempt to commit unknown transaction"); - rc = EINVAL; - goto fail; - } - - mdb_cursors_close(txn, 0); - - if (!txn->mt_u.dirty_list[0].mid && - !(txn->mt_flags & (MDB_TXN_DIRTY|MDB_TXN_SPILLS))) - goto done; - - DPRINTF(("committing txn %"Yu" %p on mdbenv %p, root page %"Yu, - txn->mt_txnid, (void*)txn, (void*)env, txn->mt_dbs[MAIN_DBI].md_root)); - - /* Update DB root pointers */ - if (txn->mt_numdbs > CORE_DBS) { - MDB_cursor mc; - MDB_dbi i; - MDB_val data; - data.mv_size = sizeof(MDB_db); - - mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); - for (i = CORE_DBS; i < txn->mt_numdbs; i++) { - if (txn->mt_dbflags[i] & DB_DIRTY) { - if (TXN_DBI_CHANGED(txn, i)) { - rc = MDB_BAD_DBI; - goto fail; - } - data.mv_data = &txn->mt_dbs[i]; - rc = mdb_cursor_put(&mc, &txn->mt_dbxs[i].md_name, &data, - F_SUBDATA); - if (rc) - goto fail; - } - } - } - - rc = mdb_freelist_save(txn); - if (rc) - goto fail; - - mdb_midl_free(env->me_pghead); - env->me_pghead = NULL; - mdb_midl_shrink(&txn->mt_free_pgs); - -#if (MDB_DEBUG) > 2 - mdb_audit(txn); -#endif - - if ((rc = mdb_page_flush(txn, 0))) - goto fail; - if (!F_ISSET(txn->mt_flags, MDB_TXN_NOSYNC) && - (rc = mdb_env_sync0(env, 0, txn->mt_next_pgno))) - goto fail; - if ((rc = mdb_env_write_meta(txn))) - goto fail; - end_mode = MDB_END_COMMITTED|MDB_END_UPDATE; - if (env->me_flags & MDB_PREVSNAPSHOT) { - if (!(env->me_flags & MDB_NOLOCK)) { - int excl; - rc = mdb_env_share_locks(env, &excl); - if (rc) - goto fail; - } - env->me_flags ^= MDB_PREVSNAPSHOT; - } - -done: - mdb_txn_end(txn, end_mode); - return MDB_SUCCESS; - -fail: - mdb_txn_abort(txn); - return rc; -} - -/** Read the environment parameters of a DB environment before - * mapping it into memory. - * @param[in] env the environment handle - * @param[in] prev whether to read the backup meta page - * @param[out] meta address of where to store the meta information - * @return 0 on success, non-zero on failure. - */ -static int ESECT -mdb_env_read_header(MDB_env *env, int prev, MDB_meta *meta) -{ - MDB_metabuf pbuf; - MDB_page *p; - MDB_meta *m; - int i, rc, off; - enum { Size = sizeof(pbuf) }; - - /* We don't know the page size yet, so use a minimum value. - * Read both meta pages so we can use the latest one. - */ - - for (i=off=0; imm_psize) { -#ifdef _WIN32 - DWORD len; - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); - ov.Offset = off; - rc = ReadFile(env->me_fd, &pbuf, Size, &len, &ov) ? (int)len : -1; - if (rc == -1 && ErrCode() == ERROR_HANDLE_EOF) - rc = 0; -#else - rc = pread(env->me_fd, &pbuf, Size, off); -#endif - if (rc != Size) { - if (rc == 0 && off == 0) - return ENOENT; - rc = rc < 0 ? (int) ErrCode() : MDB_INVALID; - DPRINTF(("read: %s", mdb_strerror(rc))); - return rc; - } - - p = (MDB_page *)&pbuf; - - if (!F_ISSET(p->mp_flags, P_META)) { - DPRINTF(("page %"Yu" not a meta page", p->mp_pgno)); - return MDB_INVALID; - } - - m = METADATA(p); - if (m->mm_magic != MDB_MAGIC) { - DPUTS("meta has invalid magic"); - return MDB_INVALID; - } - - if (m->mm_version != MDB_DATA_VERSION) { - DPRINTF(("database is version %u, expected version %u", - m->mm_version, MDB_DATA_VERSION)); - return MDB_VERSION_MISMATCH; - } - - if (off == 0 || (prev ? m->mm_txnid < meta->mm_txnid : m->mm_txnid > meta->mm_txnid)) - *meta = *m; - } - return 0; -} - -/** Fill in most of the zeroed #MDB_meta for an empty database environment */ -static void ESECT -mdb_env_init_meta0(MDB_env *env, MDB_meta *meta) -{ - meta->mm_magic = MDB_MAGIC; - meta->mm_version = MDB_DATA_VERSION; - meta->mm_mapsize = env->me_mapsize; - meta->mm_psize = env->me_psize; - meta->mm_last_pg = NUM_METAS-1; - meta->mm_flags = env->me_flags & 0xffff; - meta->mm_flags |= MDB_INTEGERKEY; /* this is mm_dbs[FREE_DBI].md_flags */ - meta->mm_dbs[FREE_DBI].md_root = P_INVALID; - meta->mm_dbs[MAIN_DBI].md_root = P_INVALID; -} - -/** Write the environment parameters of a freshly created DB environment. - * @param[in] env the environment handle - * @param[in] meta the #MDB_meta to write - * @return 0 on success, non-zero on failure. - */ -static int ESECT -mdb_env_init_meta(MDB_env *env, MDB_meta *meta) -{ - MDB_page *p, *q; - int rc; - unsigned int psize; -#ifdef _WIN32 - DWORD len; - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); -#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ - ov.Offset = pos; \ - rc = WriteFile(fd, ptr, size, &len, &ov); } while(0) -#else - int len; -#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \ - len = pwrite(fd, ptr, size, pos); \ - if (len == -1 && ErrCode() == EINTR) continue; \ - rc = (len >= 0); break; } while(1) -#endif - - DPUTS("writing new meta page"); - - psize = env->me_psize; - - p = calloc(NUM_METAS, psize); - if (!p) - return ENOMEM; - p->mp_pgno = 0; - p->mp_flags = P_META; - *(MDB_meta *)METADATA(p) = *meta; - - q = (MDB_page *)((char *)p + psize); - q->mp_pgno = 1; - q->mp_flags = P_META; - *(MDB_meta *)METADATA(q) = *meta; - - DO_PWRITE(rc, env->me_fd, p, psize * NUM_METAS, len, 0); - if (!rc) - rc = ErrCode(); - else if ((unsigned) len == psize * NUM_METAS) - rc = MDB_SUCCESS; - else - rc = ENOSPC; - free(p); - return rc; -} - -/** Update the environment info to commit a transaction. - * @param[in] txn the transaction that's being committed - * @return 0 on success, non-zero on failure. - */ -static int -mdb_env_write_meta(MDB_txn *txn) -{ - MDB_env *env; - MDB_meta meta, metab, *mp; - unsigned flags; - mdb_size_t mapsize; - off_t off; - int rc, len, toggle; - char *ptr; - HANDLE mfd; -#ifdef _WIN32 - OVERLAPPED ov; -#else - int r2; -#endif - - toggle = txn->mt_txnid & 1; - DPRINTF(("writing meta page %d for root page %"Yu, - toggle, txn->mt_dbs[MAIN_DBI].md_root)); - - env = txn->mt_env; - flags = txn->mt_flags | env->me_flags; - mp = env->me_metas[toggle]; - mapsize = env->me_metas[toggle ^ 1]->mm_mapsize; - /* Persist any increases of mapsize config */ - if (mapsize < env->me_mapsize) - mapsize = env->me_mapsize; - - if (flags & MDB_WRITEMAP) { - mp->mm_mapsize = mapsize; - mp->mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI]; - mp->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; - mp->mm_last_pg = txn->mt_next_pgno - 1; -#if (__GNUC__ * 100 + __GNUC_MINOR__ >= 404) && /* TODO: portability */ \ - !(defined(__i386__) || defined(__x86_64__)) - /* LY: issue a memory barrier, if not x86. ITS#7969 */ - __sync_synchronize(); -#endif - mp->mm_txnid = txn->mt_txnid; - if (!(flags & (MDB_NOMETASYNC|MDB_NOSYNC))) { - unsigned meta_size = env->me_psize; - rc = (env->me_flags & MDB_MAPASYNC) ? MS_ASYNC : MS_SYNC; - ptr = (char *)mp - PAGEHDRSZ; -#ifndef _WIN32 /* POSIX msync() requires ptr = start of OS page */ - r2 = (ptr - env->me_map) & (env->me_os_psize - 1); - ptr -= r2; - meta_size += r2; -#endif - if (MDB_MSYNC(ptr, meta_size, rc)) { - rc = ErrCode(); - goto fail; - } - } - goto done; - } - metab.mm_txnid = mp->mm_txnid; - metab.mm_last_pg = mp->mm_last_pg; - - meta.mm_mapsize = mapsize; - meta.mm_dbs[FREE_DBI] = txn->mt_dbs[FREE_DBI]; - meta.mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; - meta.mm_last_pg = txn->mt_next_pgno - 1; - meta.mm_txnid = txn->mt_txnid; - - off = offsetof(MDB_meta, mm_mapsize); - ptr = (char *)&meta + off; - len = sizeof(MDB_meta) - off; - off += (char *)mp - env->me_map; - - /* Write to the SYNC fd unless MDB_NOSYNC/MDB_NOMETASYNC. - * (me_mfd goes to the same file as me_fd, but writing to it - * also syncs to disk. Avoids a separate fdatasync() call.) - */ - mfd = (flags & (MDB_NOSYNC|MDB_NOMETASYNC)) ? env->me_fd : env->me_mfd; -#ifdef _WIN32 - { - memset(&ov, 0, sizeof(ov)); - ov.Offset = off; - if (!WriteFile(mfd, ptr, len, (DWORD *)&rc, &ov)) - rc = -1; - } -#else -retry_write: - rc = pwrite(mfd, ptr, len, off); -#endif - if (rc != len) { - rc = rc < 0 ? ErrCode() : EIO; -#ifndef _WIN32 - if (rc == EINTR) - goto retry_write; -#endif - DPUTS("write failed, disk error?"); - /* On a failure, the pagecache still contains the new data. - * Write some old data back, to prevent it from being used. - * Use the non-SYNC fd; we know it will fail anyway. - */ - meta.mm_last_pg = metab.mm_last_pg; - meta.mm_txnid = metab.mm_txnid; -#ifdef _WIN32 - memset(&ov, 0, sizeof(ov)); - ov.Offset = off; - WriteFile(env->me_fd, ptr, len, NULL, &ov); -#else - r2 = pwrite(env->me_fd, ptr, len, off); - (void)r2; /* Silence warnings. We don't care about pwrite's return value */ -#endif -fail: - env->me_flags |= MDB_FATAL_ERROR; - return rc; - } - /* MIPS has cache coherency issues, this is a no-op everywhere else */ - CACHEFLUSH(env->me_map + off, len, DCACHE); -done: - /* Memory ordering issues are irrelevant; since the entire writer - * is wrapped by wmutex, all of these changes will become visible - * after the wmutex is unlocked. Since the DB is multi-version, - * readers will get consistent data regardless of how fresh or - * how stale their view of these values is. - */ - if (env->me_txns) - env->me_txns->mti_txnid = txn->mt_txnid; - - return MDB_SUCCESS; -} - -/** Check both meta pages to see which one is newer. - * @param[in] env the environment handle - * @return newest #MDB_meta. - */ -static MDB_meta * -mdb_env_pick_meta(const MDB_env *env) -{ - MDB_meta *const *metas = env->me_metas; - return metas[ (metas[0]->mm_txnid < metas[1]->mm_txnid) ^ - ((env->me_flags & MDB_PREVSNAPSHOT) != 0) ]; -} - -int ESECT -mdb_env_create(MDB_env **env) -{ - MDB_env *e; - - e = calloc(1, sizeof(MDB_env)); - if (!e) - return ENOMEM; - - e->me_maxreaders = DEFAULT_READERS; - e->me_maxdbs = e->me_numdbs = CORE_DBS; - e->me_fd = INVALID_HANDLE_VALUE; - e->me_lfd = INVALID_HANDLE_VALUE; - e->me_mfd = INVALID_HANDLE_VALUE; -#ifdef MDB_USE_POSIX_SEM - e->me_rmutex = SEM_FAILED; - e->me_wmutex = SEM_FAILED; -#elif defined MDB_USE_SYSV_SEM - e->me_rmutex->semid = -1; - e->me_wmutex->semid = -1; -#endif - e->me_pid = getpid(); - GET_PAGESIZE(e->me_os_psize); - VGMEMP_CREATE(e,0,0); - *env = e; - return MDB_SUCCESS; -} - -#ifdef _WIN32 -/** @brief Map a result from an NTAPI call to WIN32. */ -static DWORD -mdb_nt2win32(NTSTATUS st) -{ - OVERLAPPED o = {0}; - DWORD br; - o.Internal = st; - GetOverlappedResult(NULL, &o, &br, FALSE); - return GetLastError(); -} -#endif - -static int ESECT -mdb_env_map(MDB_env *env, void *addr) -{ - MDB_page *p; - unsigned int flags = env->me_flags; -#ifdef _WIN32 - int rc; - int access = SECTION_MAP_READ; - HANDLE mh; - void *map; - SIZE_T msize; - ULONG pageprot = PAGE_READONLY, secprot, alloctype; - - if (flags & MDB_WRITEMAP) { - access |= SECTION_MAP_WRITE; - pageprot = PAGE_READWRITE; - } - if (flags & MDB_RDONLY) { - secprot = PAGE_READONLY; - msize = 0; - alloctype = 0; - } else { - secprot = PAGE_READWRITE; - msize = env->me_mapsize; - alloctype = MEM_RESERVE; - } - - rc = NtCreateSection(&mh, access, NULL, NULL, secprot, SEC_RESERVE, env->me_fd); - if (rc) - return mdb_nt2win32(rc); - map = addr; -#ifdef MDB_VL32 - msize = NUM_METAS * env->me_psize; -#endif - rc = NtMapViewOfSection(mh, GetCurrentProcess(), &map, 0, 0, NULL, &msize, ViewUnmap, alloctype, pageprot); -#ifdef MDB_VL32 - env->me_fmh = mh; -#else - NtClose(mh); -#endif - if (rc) - return mdb_nt2win32(rc); - env->me_map = map; -#else -#ifdef MDB_VL32 - (void) flags; - env->me_map = mmap(addr, NUM_METAS * env->me_psize, PROT_READ, MAP_SHARED, - env->me_fd, 0); - if (env->me_map == MAP_FAILED) { - env->me_map = NULL; - return ErrCode(); - } -#else - int prot = PROT_READ; - if (flags & MDB_WRITEMAP) { - prot |= PROT_WRITE; - if (ftruncate(env->me_fd, env->me_mapsize) < 0) - return ErrCode(); - } - env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED, - env->me_fd, 0); - if (env->me_map == MAP_FAILED) { - env->me_map = NULL; - return ErrCode(); - } - - if (flags & MDB_NORDAHEAD) { - /* Turn off readahead. It's harmful when the DB is larger than RAM. */ -#ifdef MADV_RANDOM - madvise(env->me_map, env->me_mapsize, MADV_RANDOM); -#else -#ifdef POSIX_MADV_RANDOM - posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM); -#endif /* POSIX_MADV_RANDOM */ -#endif /* MADV_RANDOM */ - } -#endif /* _WIN32 */ - - /* Can happen because the address argument to mmap() is just a - * hint. mmap() can pick another, e.g. if the range is in use. - * The MAP_FIXED flag would prevent that, but then mmap could - * instead unmap existing pages to make room for the new map. - */ - if (addr && env->me_map != addr) - return EBUSY; /* TODO: Make a new MDB_* error code? */ -#endif - - p = (MDB_page *)env->me_map; - env->me_metas[0] = METADATA(p); - env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize); - - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_mapsize(MDB_env *env, mdb_size_t size) -{ - /* If env is already open, caller is responsible for making - * sure there are no active txns. - */ - if (env->me_map) { - MDB_meta *meta; -#ifndef MDB_VL32 - void *old; - int rc; -#endif - if (env->me_txn) - return EINVAL; - meta = mdb_env_pick_meta(env); - if (!size) - size = meta->mm_mapsize; - { - /* Silently round up to minimum if the size is too small */ - mdb_size_t minsize = (meta->mm_last_pg + 1) * env->me_psize; - if (size < minsize) - size = minsize; - } -#ifndef MDB_VL32 - /* For MDB_VL32 this bit is a noop since we dynamically remap - * chunks of the DB anyway. - */ - munmap(env->me_map, env->me_mapsize); - env->me_mapsize = size; - old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL; - rc = mdb_env_map(env, old); - if (rc) - return rc; -#endif /* !MDB_VL32 */ - } - env->me_mapsize = size; - if (env->me_psize) - env->me_maxpg = env->me_mapsize / env->me_psize; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_maxdbs(MDB_env *env, MDB_dbi dbs) -{ - if (env->me_map) - return EINVAL; - env->me_maxdbs = dbs + CORE_DBS; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_maxreaders(MDB_env *env, unsigned int readers) -{ - if (env->me_map || readers < 1) - return EINVAL; - env->me_maxreaders = readers; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers) -{ - if (!env || !readers) - return EINVAL; - *readers = env->me_maxreaders; - return MDB_SUCCESS; -} - -static int ESECT -mdb_fsize(HANDLE fd, mdb_size_t *size) -{ -#ifdef _WIN32 - LARGE_INTEGER fsize; - - if (!GetFileSizeEx(fd, &fsize)) - return ErrCode(); - - *size = fsize.QuadPart; -#else - struct stat st; - - if (fstat(fd, &st)) - return ErrCode(); - - *size = st.st_size; -#endif - return MDB_SUCCESS; -} - - -#ifdef _WIN32 -typedef wchar_t mdb_nchar_t; -# define MDB_NAME(str) L##str -# define mdb_name_cpy wcscpy -#else -/** Character type for file names: char on Unix, wchar_t on Windows */ -typedef char mdb_nchar_t; -# define MDB_NAME(str) str /**< #mdb_nchar_t[] string literal */ -# define mdb_name_cpy strcpy /**< Copy name (#mdb_nchar_t string) */ -#endif - -/** Filename - string of #mdb_nchar_t[] */ -typedef struct MDB_name { - int mn_len; /**< Length */ - int mn_alloced; /**< True if #mn_val was malloced */ - mdb_nchar_t *mn_val; /**< Contents */ -} MDB_name; - -/** Filename suffixes [datafile,lockfile][without,with MDB_NOSUBDIR] */ -static const mdb_nchar_t *const mdb_suffixes[2][2] = { - { MDB_NAME("/data.mdb"), MDB_NAME("") }, - { MDB_NAME("/lock.mdb"), MDB_NAME("-lock") } -}; - -#define MDB_SUFFLEN 9 /**< Max string length in #mdb_suffixes[] */ - -/** Set up filename + scratch area for filename suffix, for opening files. - * It should be freed with #mdb_fname_destroy(). - * On Windows, paths are converted from char *UTF-8 to wchar_t *UTF-16. - * - * @param[in] path Pathname for #mdb_env_open(). - * @param[in] envflags Whether a subdir and/or lockfile will be used. - * @param[out] fname Resulting filename, with room for a suffix if necessary. - */ -static int ESECT -mdb_fname_init(const char *path, unsigned envflags, MDB_name *fname) -{ - int no_suffix = F_ISSET(envflags, MDB_NOSUBDIR|MDB_NOLOCK); - fname->mn_alloced = 0; -#ifdef _WIN32 - return utf8_to_utf16(path, fname, no_suffix ? 0 : MDB_SUFFLEN); -#else - fname->mn_len = strlen(path); - if (no_suffix) - fname->mn_val = (char *) path; - else if ((fname->mn_val = malloc(fname->mn_len + MDB_SUFFLEN+1)) != NULL) { - fname->mn_alloced = 1; - strcpy(fname->mn_val, path); - } - else - return ENOMEM; - return MDB_SUCCESS; -#endif -} - -/** Destroy \b fname from #mdb_fname_init() */ -#define mdb_fname_destroy(fname) \ - do { if ((fname).mn_alloced) free((fname).mn_val); } while (0) - -#ifdef O_CLOEXEC /* POSIX.1-2008: Set FD_CLOEXEC atomically at open() */ -# define MDB_CLOEXEC O_CLOEXEC -#else -# define MDB_CLOEXEC 0 -#endif - -/** File type, access mode etc. for #mdb_fopen() */ -enum mdb_fopen_type { -#ifdef _WIN32 - MDB_O_RDONLY, MDB_O_RDWR, MDB_O_META, MDB_O_COPY, MDB_O_LOCKS -#else - /* A comment in mdb_fopen() explains some O_* flag choices. */ - MDB_O_RDONLY= O_RDONLY, /**< for RDONLY me_fd */ - MDB_O_RDWR = O_RDWR |O_CREAT, /**< for me_fd */ - MDB_O_META = O_WRONLY|MDB_DSYNC |MDB_CLOEXEC, /**< for me_mfd */ - MDB_O_COPY = O_WRONLY|O_CREAT|O_EXCL|MDB_CLOEXEC, /**< for #mdb_env_copy() */ - /** Bitmask for open() flags in enum #mdb_fopen_type. The other bits - * distinguish otherwise-equal MDB_O_* constants from each other. - */ - MDB_O_MASK = MDB_O_RDWR|MDB_CLOEXEC | MDB_O_RDONLY|MDB_O_META|MDB_O_COPY, - MDB_O_LOCKS = MDB_O_RDWR|MDB_CLOEXEC | ((MDB_O_MASK+1) & ~MDB_O_MASK) /**< for me_lfd */ -#endif -}; - -/** Open an LMDB file. - * @param[in] env The LMDB environment. - * @param[in,out] fname Path from from #mdb_fname_init(). A suffix is - * appended if necessary to create the filename, without changing mn_len. - * @param[in] which Determines file type, access mode, etc. - * @param[in] mode The Unix permissions for the file, if we create it. - * @param[out] res Resulting file handle. - * @return 0 on success, non-zero on failure. - */ -static int ESECT -mdb_fopen(const MDB_env *env, MDB_name *fname, - enum mdb_fopen_type which, mdb_mode_t mode, - HANDLE *res) -{ - int rc = MDB_SUCCESS; - HANDLE fd; -#ifdef _WIN32 - DWORD acc, share, disp, attrs; -#else - int flags; -#endif - - if (fname->mn_alloced) /* modifiable copy */ - mdb_name_cpy(fname->mn_val + fname->mn_len, - mdb_suffixes[which==MDB_O_LOCKS][F_ISSET(env->me_flags, MDB_NOSUBDIR)]); - - /* The directory must already exist. Usually the file need not. - * MDB_O_META requires the file because we already created it using - * MDB_O_RDWR. MDB_O_COPY must not overwrite an existing file. - * - * With MDB_O_COPY we do not want the OS to cache the writes, since - * the source data is already in the OS cache. - * - * The lockfile needs FD_CLOEXEC (close file descriptor on exec*()) - * to avoid the flock() issues noted under Caveats in lmdb.h. - * Also set it for other filehandles which the user cannot get at - * and close himself, which he may need after fork(). I.e. all but - * me_fd, which programs do use via mdb_env_get_fd(). - */ - -#ifdef _WIN32 - acc = GENERIC_READ|GENERIC_WRITE; - share = FILE_SHARE_READ|FILE_SHARE_WRITE; - disp = OPEN_ALWAYS; - attrs = FILE_ATTRIBUTE_NORMAL; - switch (which) { - case MDB_O_RDONLY: /* read-only datafile */ - acc = GENERIC_READ; - disp = OPEN_EXISTING; - break; - case MDB_O_META: /* for writing metapages */ - acc = GENERIC_WRITE; - disp = OPEN_EXISTING; - attrs = FILE_ATTRIBUTE_NORMAL|FILE_FLAG_WRITE_THROUGH; - break; - case MDB_O_COPY: /* mdb_env_copy() & co */ - acc = GENERIC_WRITE; - share = 0; - disp = CREATE_NEW; - attrs = FILE_FLAG_NO_BUFFERING|FILE_FLAG_WRITE_THROUGH; - break; - default: break; /* silence gcc -Wswitch (not all enum values handled) */ - } - fd = CreateFileW(fname->mn_val, acc, share, NULL, disp, attrs, NULL); -#else - fd = open(fname->mn_val, which & MDB_O_MASK, mode); -#endif - - if (fd == INVALID_HANDLE_VALUE) - rc = ErrCode(); -#ifndef _WIN32 - else { - if (which != MDB_O_RDONLY && which != MDB_O_RDWR) { - /* Set CLOEXEC if we could not pass it to open() */ - if (!MDB_CLOEXEC && (flags = fcntl(fd, F_GETFD)) != -1) - (void) fcntl(fd, F_SETFD, flags | FD_CLOEXEC); - } - if (which == MDB_O_COPY && env->me_psize >= env->me_os_psize) { - /* This may require buffer alignment. There is no portable - * way to ask how much, so we require OS pagesize alignment. - */ -# ifdef F_NOCACHE /* __APPLE__ */ - (void) fcntl(fd, F_NOCACHE, 1); -# elif defined O_DIRECT - /* open(...O_DIRECT...) would break on filesystems without - * O_DIRECT support (ITS#7682). Try to set it here instead. - */ - if ((flags = fcntl(fd, F_GETFL)) != -1) - (void) fcntl(fd, F_SETFL, flags | O_DIRECT); -# endif - } - } -#endif /* !_WIN32 */ - - *res = fd; - return rc; -} - - -#ifdef BROKEN_FDATASYNC -#include -#include -#endif - -/** Further setup required for opening an LMDB environment - */ -static int ESECT -mdb_env_open2(MDB_env *env, int prev) -{ - unsigned int flags = env->me_flags; - int i, newenv = 0, rc; - MDB_meta meta; - -#ifdef _WIN32 - /* See if we should use QueryLimited */ - rc = GetVersion(); - if ((rc & 0xff) > 5) - env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION; - else - env->me_pidquery = PROCESS_QUERY_INFORMATION; - /* Grab functions we need from NTDLL */ - if (!NtCreateSection) { - HMODULE h = GetModuleHandleW(L"NTDLL.DLL"); - if (!h) - return MDB_PROBLEM; - NtClose = (NtCloseFunc *)GetProcAddress(h, "NtClose"); - if (!NtClose) - return MDB_PROBLEM; - NtMapViewOfSection = (NtMapViewOfSectionFunc *)GetProcAddress(h, "NtMapViewOfSection"); - if (!NtMapViewOfSection) - return MDB_PROBLEM; - NtCreateSection = (NtCreateSectionFunc *)GetProcAddress(h, "NtCreateSection"); - if (!NtCreateSection) - return MDB_PROBLEM; - } -#endif /* _WIN32 */ - -#ifdef BROKEN_FDATASYNC - /* ext3/ext4 fdatasync is broken on some older Linux kernels. - * https://lkml.org/lkml/2012/9/3/83 - * Kernels after 3.6-rc6 are known good. - * https://lkml.org/lkml/2012/9/10/556 - * See if the DB is on ext3/ext4, then check for new enough kernel - * Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known - * to be patched. - */ - { - struct statfs st; - fstatfs(env->me_fd, &st); - while (st.f_type == 0xEF53) { - struct utsname uts; - int i; - uname(&uts); - if (uts.release[0] < '3') { - if (!strncmp(uts.release, "2.6.32.", 7)) { - i = atoi(uts.release+7); - if (i >= 60) - break; /* 2.6.32.60 and newer is OK */ - } else if (!strncmp(uts.release, "2.6.34.", 7)) { - i = atoi(uts.release+7); - if (i >= 15) - break; /* 2.6.34.15 and newer is OK */ - } - } else if (uts.release[0] == '3') { - i = atoi(uts.release+2); - if (i > 5) - break; /* 3.6 and newer is OK */ - if (i == 5) { - i = atoi(uts.release+4); - if (i >= 4) - break; /* 3.5.4 and newer is OK */ - } else if (i == 2) { - i = atoi(uts.release+4); - if (i >= 30) - break; /* 3.2.30 and newer is OK */ - } - } else { /* 4.x and newer is OK */ - break; - } - env->me_flags |= MDB_FSYNCONLY; - break; - } - } -#endif - - if ((i = mdb_env_read_header(env, prev, &meta)) != 0) { - if (i != ENOENT) - return i; - DPUTS("new mdbenv"); - newenv = 1; - env->me_psize = env->me_os_psize; - if (env->me_psize > MAX_PAGESIZE) - env->me_psize = MAX_PAGESIZE; - memset(&meta, 0, sizeof(meta)); - mdb_env_init_meta0(env, &meta); - meta.mm_mapsize = DEFAULT_MAPSIZE; - } else { - env->me_psize = meta.mm_psize; - } - - /* Was a mapsize configured? */ - if (!env->me_mapsize) { - env->me_mapsize = meta.mm_mapsize; - } - { - /* Make sure mapsize >= committed data size. Even when using - * mm_mapsize, which could be broken in old files (ITS#7789). - */ - mdb_size_t minsize = (meta.mm_last_pg + 1) * meta.mm_psize; - if (env->me_mapsize < minsize) - env->me_mapsize = minsize; - } - meta.mm_mapsize = env->me_mapsize; - - if (newenv && !(flags & MDB_FIXEDMAP)) { - /* mdb_env_map() may grow the datafile. Write the metapages - * first, so the file will be valid if initialization fails. - * Except with FIXEDMAP, since we do not yet know mm_address. - * We could fill in mm_address later, but then a different - * program might end up doing that - one with a memory layout - * and map address which does not suit the main program. - */ - rc = mdb_env_init_meta(env, &meta); - if (rc) - return rc; - newenv = 0; - } -#ifdef _WIN32 - /* For FIXEDMAP, make sure the file is non-empty before we attempt to map it */ - if (newenv) { - char dummy = 0; - DWORD len; - rc = WriteFile(env->me_fd, &dummy, 1, &len, NULL); - if (!rc) { - rc = ErrCode(); - return rc; - } - } -#endif - - rc = mdb_env_map(env, (flags & MDB_FIXEDMAP) ? meta.mm_address : NULL); - if (rc) - return rc; - - if (newenv) { - if (flags & MDB_FIXEDMAP) - meta.mm_address = env->me_map; - i = mdb_env_init_meta(env, &meta); - if (i != MDB_SUCCESS) { - return i; - } - } - - env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1; - env->me_nodemax = (((env->me_psize - PAGEHDRSZ) / MDB_MINKEYS) & -2) - - sizeof(indx_t); -#if !(MDB_MAXKEYSIZE) - env->me_maxkey = env->me_nodemax - (NODESIZE + sizeof(MDB_db)); -#endif - env->me_maxpg = env->me_mapsize / env->me_psize; - - if (env->me_txns) - env->me_txns->mti_txnid = meta.mm_txnid; - -#if MDB_DEBUG - { - MDB_meta *meta = mdb_env_pick_meta(env); - MDB_db *db = &meta->mm_dbs[MAIN_DBI]; - - DPRINTF(("opened database version %u, pagesize %u", - meta->mm_version, env->me_psize)); - DPRINTF(("using meta page %d", (int) (meta->mm_txnid & 1))); - DPRINTF(("depth: %u", db->md_depth)); - DPRINTF(("entries: %"Yu, db->md_entries)); - DPRINTF(("branch pages: %"Yu, db->md_branch_pages)); - DPRINTF(("leaf pages: %"Yu, db->md_leaf_pages)); - DPRINTF(("overflow pages: %"Yu, db->md_overflow_pages)); - DPRINTF(("root: %"Yu, db->md_root)); - } -#endif - - return MDB_SUCCESS; -} - - -/** Release a reader thread's slot in the reader lock table. - * This function is called automatically when a thread exits. - * @param[in] ptr This points to the slot in the reader lock table. - */ -static void -mdb_env_reader_dest(void *ptr) -{ - MDB_reader *reader = ptr; - -#ifndef _WIN32 - if (reader->mr_pid == getpid()) /* catch pthread_exit() in child process */ -#endif - /* We omit the mutex, so do this atomically (i.e. skip mr_txnid) */ - reader->mr_pid = 0; -} - -#ifdef _WIN32 -/** Junk for arranging thread-specific callbacks on Windows. This is - * necessarily platform and compiler-specific. Windows supports up - * to 1088 keys. Let's assume nobody opens more than 64 environments - * in a single process, for now. They can override this if needed. - */ -#ifndef MAX_TLS_KEYS -#define MAX_TLS_KEYS 64 -#endif -static pthread_key_t mdb_tls_keys[MAX_TLS_KEYS]; -static int mdb_tls_nkeys; - -static void NTAPI mdb_tls_callback(PVOID module, DWORD reason, PVOID ptr) -{ - int i; - switch(reason) { - case DLL_PROCESS_ATTACH: break; - case DLL_THREAD_ATTACH: break; - case DLL_THREAD_DETACH: - for (i=0; ime_lfd, 0, 0, 1, 0, &ov)) { - rc = ErrCode(); - } else { - UnlockFile(env->me_lfd, 0, 0, 1, 0); - *excl = 0; - } - } -#else - { - struct flock lock_info; - /* The shared lock replaces the existing lock */ - memset((void *)&lock_info, 0, sizeof(lock_info)); - lock_info.l_type = F_RDLCK; - lock_info.l_whence = SEEK_SET; - lock_info.l_start = 0; - lock_info.l_len = 1; - while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && - (rc = ErrCode()) == EINTR) ; - *excl = rc ? -1 : 0; /* error may mean we lost the lock */ - } -#endif - - return rc; -} - -/** Try to get exclusive lock, otherwise shared. - * Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive. - */ -static int ESECT -mdb_env_excl_lock(MDB_env *env, int *excl) -{ - int rc = 0; -#ifdef _WIN32 - if (LockFile(env->me_lfd, 0, 0, 1, 0)) { - *excl = 1; - } else { - OVERLAPPED ov; - memset(&ov, 0, sizeof(ov)); - if (LockFileEx(env->me_lfd, 0, 0, 1, 0, &ov)) { - *excl = 0; - } else { - rc = ErrCode(); - } - } -#else - struct flock lock_info; - memset((void *)&lock_info, 0, sizeof(lock_info)); - lock_info.l_type = F_WRLCK; - lock_info.l_whence = SEEK_SET; - lock_info.l_start = 0; - lock_info.l_len = 1; - while ((rc = fcntl(env->me_lfd, F_SETLK, &lock_info)) && - (rc = ErrCode()) == EINTR) ; - if (!rc) { - *excl = 1; - } else -# ifndef MDB_USE_POSIX_MUTEX - if (*excl < 0) /* always true when MDB_USE_POSIX_MUTEX */ -# endif - { - lock_info.l_type = F_RDLCK; - while ((rc = fcntl(env->me_lfd, F_SETLKW, &lock_info)) && - (rc = ErrCode()) == EINTR) ; - if (rc == 0) - *excl = 0; - } -#endif - return rc; -} - -#ifdef MDB_USE_HASH -/* - * hash_64 - 64 bit Fowler/Noll/Vo-0 FNV-1a hash code - * - * @(#) $Revision: 5.1 $ - * @(#) $Id: hash_64a.c,v 5.1 2009/06/30 09:01:38 chongo Exp $ - * @(#) $Source: /usr/local/src/cmd/fnv/RCS/hash_64a.c,v $ - * - * http://www.isthe.com/chongo/tech/comp/fnv/index.html - * - *** - * - * Please do not copyright this code. This code is in the public domain. - * - * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, - * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO - * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR - * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF - * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR - * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - * PERFORMANCE OF THIS SOFTWARE. - * - * By: - * chongo /\oo/\ - * http://www.isthe.com/chongo/ - * - * Share and Enjoy! :-) - */ - -/** perform a 64 bit Fowler/Noll/Vo FNV-1a hash on a buffer - * @param[in] val value to hash - * @param[in] len length of value - * @return 64 bit hash - */ -static mdb_hash_t -mdb_hash(const void *val, size_t len) -{ - const unsigned char *s = (const unsigned char *) val, *end = s + len; - mdb_hash_t hval = 0xcbf29ce484222325ULL; - /* - * FNV-1a hash each octet of the buffer - */ - while (s < end) { - hval = (hval ^ *s++) * 0x100000001b3ULL; - } - /* return our new hash value */ - return hval; -} - -/** Hash the string and output the encoded hash. - * This uses modified RFC1924 Ascii85 encoding to accommodate systems with - * very short name limits. We don't care about the encoding being reversible, - * we just want to preserve as many bits of the input as possible in a - * small printable string. - * @param[in] str string to hash - * @param[out] encbuf an array of 11 chars to hold the hash - */ -static const char mdb_a85[]= "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~"; - -static void ESECT -mdb_pack85(unsigned long long l, char *out) -{ - int i; - - for (i=0; i<10 && l; i++) { - *out++ = mdb_a85[l % 85]; - l /= 85; - } - *out = '\0'; -} - -/** Init #MDB_env.me_mutexname[] except the char which #MUTEXNAME() will set. - * Changes to this code must be reflected in #MDB_LOCK_FORMAT. - */ -static void ESECT -mdb_env_mname_init(MDB_env *env) -{ - char *nm = env->me_mutexname; - strcpy(nm, MUTEXNAME_PREFIX); - mdb_pack85(env->me_txns->mti_mutexid, nm + sizeof(MUTEXNAME_PREFIX)); -} - -/** Return env->me_mutexname after filling in ch ('r'/'w') for convenience */ -#define MUTEXNAME(env, ch) ( \ - (void) ((env)->me_mutexname[sizeof(MUTEXNAME_PREFIX)-1] = (ch)), \ - (env)->me_mutexname) - -#endif - -/** Open and/or initialize the lock region for the environment. - * @param[in] env The LMDB environment. - * @param[in] fname Filename + scratch area, from #mdb_fname_init(). - * @param[in] mode The Unix permissions for the file, if we create it. - * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive - * @return 0 on success, non-zero on failure. - */ -static int ESECT -mdb_env_setup_locks(MDB_env *env, MDB_name *fname, int mode, int *excl) -{ -#ifdef _WIN32 -# define MDB_ERRCODE_ROFS ERROR_WRITE_PROTECT -#else -# define MDB_ERRCODE_ROFS EROFS -#endif -#ifdef MDB_USE_SYSV_SEM - int semid; - union semun semu; -#endif - int rc; - off_t size, rsize; - - rc = mdb_fopen(env, fname, MDB_O_LOCKS, mode, &env->me_lfd); - if (rc) { - /* Omit lockfile if read-only env on read-only filesystem */ - if (rc == MDB_ERRCODE_ROFS && (env->me_flags & MDB_RDONLY)) { - return MDB_SUCCESS; - } - goto fail; - } - - if (!(env->me_flags & MDB_NOTLS)) { - rc = pthread_key_create(&env->me_txkey, mdb_env_reader_dest); - if (rc) - goto fail; - env->me_flags |= MDB_ENV_TXKEY; -#ifdef _WIN32 - /* Windows TLS callbacks need help finding their TLS info. */ - if (mdb_tls_nkeys >= MAX_TLS_KEYS) { - rc = MDB_TLS_FULL; - goto fail; - } - mdb_tls_keys[mdb_tls_nkeys++] = env->me_txkey; -#endif - } - - /* Try to get exclusive lock. If we succeed, then - * nobody is using the lock region and we should initialize it. - */ - if ((rc = mdb_env_excl_lock(env, excl))) goto fail; - -#ifdef _WIN32 - size = GetFileSize(env->me_lfd, NULL); -#else - size = lseek(env->me_lfd, 0, SEEK_END); - if (size == -1) goto fail_errno; -#endif - rsize = (env->me_maxreaders-1) * sizeof(MDB_reader) + sizeof(MDB_txninfo); - if (size < rsize && *excl > 0) { -#ifdef _WIN32 - if (SetFilePointer(env->me_lfd, rsize, NULL, FILE_BEGIN) != (DWORD)rsize - || !SetEndOfFile(env->me_lfd)) - goto fail_errno; -#else - if (ftruncate(env->me_lfd, rsize) != 0) goto fail_errno; -#endif - } else { - rsize = size; - size = rsize - sizeof(MDB_txninfo); - env->me_maxreaders = size/sizeof(MDB_reader) + 1; - } - { -#ifdef _WIN32 - HANDLE mh; - mh = CreateFileMapping(env->me_lfd, NULL, PAGE_READWRITE, - 0, 0, NULL); - if (!mh) goto fail_errno; - env->me_txns = MapViewOfFileEx(mh, FILE_MAP_WRITE, 0, 0, rsize, NULL); - CloseHandle(mh); - if (!env->me_txns) goto fail_errno; -#else - void *m = mmap(NULL, rsize, PROT_READ|PROT_WRITE, MAP_SHARED, - env->me_lfd, 0); - if (m == MAP_FAILED) goto fail_errno; - env->me_txns = m; -#endif - } - if (*excl > 0) { -#ifdef _WIN32 - BY_HANDLE_FILE_INFORMATION stbuf; - struct { - DWORD volume; - DWORD nhigh; - DWORD nlow; - } idbuf; - - if (!mdb_sec_inited) { - InitializeSecurityDescriptor(&mdb_null_sd, - SECURITY_DESCRIPTOR_REVISION); - SetSecurityDescriptorDacl(&mdb_null_sd, TRUE, 0, FALSE); - mdb_all_sa.nLength = sizeof(SECURITY_ATTRIBUTES); - mdb_all_sa.bInheritHandle = FALSE; - mdb_all_sa.lpSecurityDescriptor = &mdb_null_sd; - mdb_sec_inited = 1; - } - if (!GetFileInformationByHandle(env->me_lfd, &stbuf)) goto fail_errno; - idbuf.volume = stbuf.dwVolumeSerialNumber; - idbuf.nhigh = stbuf.nFileIndexHigh; - idbuf.nlow = stbuf.nFileIndexLow; - env->me_txns->mti_mutexid = mdb_hash(&idbuf, sizeof(idbuf)); - mdb_env_mname_init(env); - env->me_rmutex = CreateMutexA(&mdb_all_sa, FALSE, MUTEXNAME(env, 'r')); - if (!env->me_rmutex) goto fail_errno; - env->me_wmutex = CreateMutexA(&mdb_all_sa, FALSE, MUTEXNAME(env, 'w')); - if (!env->me_wmutex) goto fail_errno; -#elif defined(MDB_USE_POSIX_SEM) - struct stat stbuf; - struct { - dev_t dev; - ino_t ino; - } idbuf; - -#if defined(__NetBSD__) -#define MDB_SHORT_SEMNAMES 1 /* limited to 14 chars */ -#endif - if (fstat(env->me_lfd, &stbuf)) goto fail_errno; - memset(&idbuf, 0, sizeof(idbuf)); - idbuf.dev = stbuf.st_dev; - idbuf.ino = stbuf.st_ino; - env->me_txns->mti_mutexid = mdb_hash(&idbuf, sizeof(idbuf)) -#ifdef MDB_SHORT_SEMNAMES - /* Max 9 base85-digits. We truncate here instead of in - * mdb_env_mname_init() to keep the latter portable. - */ - % ((mdb_hash_t)85*85*85*85*85*85*85*85*85) -#endif - ; - mdb_env_mname_init(env); - /* Clean up after a previous run, if needed: Try to - * remove both semaphores before doing anything else. - */ - sem_unlink(MUTEXNAME(env, 'r')); - sem_unlink(MUTEXNAME(env, 'w')); - env->me_rmutex = sem_open(MUTEXNAME(env, 'r'), O_CREAT|O_EXCL, mode, 1); - if (env->me_rmutex == SEM_FAILED) goto fail_errno; - env->me_wmutex = sem_open(MUTEXNAME(env, 'w'), O_CREAT|O_EXCL, mode, 1); - if (env->me_wmutex == SEM_FAILED) goto fail_errno; -#elif defined(MDB_USE_SYSV_SEM) - unsigned short vals[2] = {1, 1}; - key_t key = ftok(fname->mn_val, 'M'); /* fname is lockfile path now */ - if (key == -1) - goto fail_errno; - semid = semget(key, 2, (mode & 0777) | IPC_CREAT); - if (semid < 0) - goto fail_errno; - semu.array = vals; - if (semctl(semid, 0, SETALL, semu) < 0) - goto fail_errno; - env->me_txns->mti_semid = semid; - env->me_txns->mti_rlocked = 0; - env->me_txns->mti_wlocked = 0; -#else /* MDB_USE_POSIX_MUTEX: */ - pthread_mutexattr_t mattr; - - /* Solaris needs this before initing a robust mutex. Otherwise - * it may skip the init and return EBUSY "seems someone already - * inited" or EINVAL "it was inited differently". - */ - memset(env->me_txns->mti_rmutex, 0, sizeof(*env->me_txns->mti_rmutex)); - memset(env->me_txns->mti_wmutex, 0, sizeof(*env->me_txns->mti_wmutex)); - - if ((rc = pthread_mutexattr_init(&mattr)) != 0) - goto fail; - rc = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED); -#ifdef MDB_ROBUST_SUPPORTED - if (!rc) rc = pthread_mutexattr_setrobust(&mattr, PTHREAD_MUTEX_ROBUST); -#endif - if (!rc) rc = pthread_mutex_init(env->me_txns->mti_rmutex, &mattr); - if (!rc) rc = pthread_mutex_init(env->me_txns->mti_wmutex, &mattr); - pthread_mutexattr_destroy(&mattr); - if (rc) - goto fail; -#endif /* _WIN32 || ... */ - - env->me_txns->mti_magic = MDB_MAGIC; - env->me_txns->mti_format = MDB_LOCK_FORMAT; - env->me_txns->mti_txnid = 0; - env->me_txns->mti_numreaders = 0; - - } else { -#ifdef MDB_USE_SYSV_SEM - struct semid_ds buf; -#endif - if (env->me_txns->mti_magic != MDB_MAGIC) { - DPUTS("lock region has invalid magic"); - rc = MDB_INVALID; - goto fail; - } - if (env->me_txns->mti_format != MDB_LOCK_FORMAT) { - DPRINTF(("lock region has format+version 0x%x, expected 0x%x", - env->me_txns->mti_format, MDB_LOCK_FORMAT)); - rc = MDB_VERSION_MISMATCH; - goto fail; - } - rc = ErrCode(); - if (rc && rc != EACCES && rc != EAGAIN) { - goto fail; - } -#ifdef _WIN32 - mdb_env_mname_init(env); - env->me_rmutex = OpenMutexA(SYNCHRONIZE, FALSE, MUTEXNAME(env, 'r')); - if (!env->me_rmutex) goto fail_errno; - env->me_wmutex = OpenMutexA(SYNCHRONIZE, FALSE, MUTEXNAME(env, 'w')); - if (!env->me_wmutex) goto fail_errno; -#elif defined(MDB_USE_POSIX_SEM) - mdb_env_mname_init(env); - env->me_rmutex = sem_open(MUTEXNAME(env, 'r'), 0); - if (env->me_rmutex == SEM_FAILED) goto fail_errno; - env->me_wmutex = sem_open(MUTEXNAME(env, 'w'), 0); - if (env->me_wmutex == SEM_FAILED) goto fail_errno; -#elif defined(MDB_USE_SYSV_SEM) - semid = env->me_txns->mti_semid; - semu.buf = &buf; - /* check for read access */ - if (semctl(semid, 0, IPC_STAT, semu) < 0) - goto fail_errno; - /* check for write access */ - if (semctl(semid, 0, IPC_SET, semu) < 0) - goto fail_errno; -#endif - } -#ifdef MDB_USE_SYSV_SEM - env->me_rmutex->semid = semid; - env->me_wmutex->semid = semid; - env->me_rmutex->semnum = 0; - env->me_wmutex->semnum = 1; - env->me_rmutex->locked = &env->me_txns->mti_rlocked; - env->me_wmutex->locked = &env->me_txns->mti_wlocked; -#endif - - return MDB_SUCCESS; - -fail_errno: - rc = ErrCode(); -fail: - return rc; -} - - /** Only a subset of the @ref mdb_env flags can be changed - * at runtime. Changing other flags requires closing the - * environment and re-opening it with the new flags. - */ -#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) -#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY| \ - MDB_WRITEMAP|MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD|MDB_PREVSNAPSHOT) - -#if VALID_FLAGS & PERSISTENT_FLAGS & (CHANGEABLE|CHANGELESS) -# error "Persistent DB flags & env flags overlap, but both go in mm_flags" -#endif - -int ESECT -mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode) -{ - int rc, excl = -1; - MDB_name fname; - - if (env->me_fd!=INVALID_HANDLE_VALUE || (flags & ~(CHANGEABLE|CHANGELESS))) - return EINVAL; - -#ifdef MDB_VL32 - if (flags & MDB_WRITEMAP) { - /* silently ignore WRITEMAP in 32 bit mode */ - flags ^= MDB_WRITEMAP; - } - if (flags & MDB_FIXEDMAP) { - /* cannot support FIXEDMAP */ - return EINVAL; - } -#endif - flags |= env->me_flags; - - rc = mdb_fname_init(path, flags, &fname); - if (rc) - return rc; - -#ifdef MDB_VL32 -#ifdef _WIN32 - env->me_rpmutex = CreateMutex(NULL, FALSE, NULL); - if (!env->me_rpmutex) { - rc = ErrCode(); - goto leave; - } -#else - rc = pthread_mutex_init(&env->me_rpmutex, NULL); - if (rc) - goto leave; -#endif -#endif - flags |= MDB_ENV_ACTIVE; /* tell mdb_env_close0() to clean up */ - - if (flags & MDB_RDONLY) { - /* silently ignore WRITEMAP when we're only getting read access */ - flags &= ~MDB_WRITEMAP; - } else { - if (!((env->me_free_pgs = mdb_midl_alloc(MDB_IDL_UM_MAX)) && - (env->me_dirty_list = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2))))) - rc = ENOMEM; - } - - env->me_flags = flags; - if (rc) - goto leave; - -#ifdef MDB_VL32 - { - env->me_rpages = malloc(MDB_ERPAGE_SIZE * sizeof(MDB_ID3)); - if (!env->me_rpages) { - rc = ENOMEM; - goto leave; - } - env->me_rpages[0].mid = 0; - env->me_rpcheck = MDB_ERPAGE_SIZE/2; - } -#endif - - env->me_path = strdup(path); - env->me_dbxs = calloc(env->me_maxdbs, sizeof(MDB_dbx)); - env->me_dbflags = calloc(env->me_maxdbs, sizeof(uint16_t)); - env->me_dbiseqs = calloc(env->me_maxdbs, sizeof(unsigned int)); - if (!(env->me_dbxs && env->me_path && env->me_dbflags && env->me_dbiseqs)) { - rc = ENOMEM; - goto leave; - } - env->me_dbxs[FREE_DBI].md_cmp = mdb_cmp_long; /* aligned MDB_INTEGERKEY */ - - /* For RDONLY, get lockfile after we know datafile exists */ - if (!(flags & (MDB_RDONLY|MDB_NOLOCK))) { - rc = mdb_env_setup_locks(env, &fname, mode, &excl); - if (rc) - goto leave; - if ((flags & MDB_PREVSNAPSHOT) && !excl) { - rc = EAGAIN; - goto leave; - } - } - - rc = mdb_fopen(env, &fname, - (flags & MDB_RDONLY) ? MDB_O_RDONLY : MDB_O_RDWR, - mode, &env->me_fd); - if (rc) - goto leave; - - if ((flags & (MDB_RDONLY|MDB_NOLOCK)) == MDB_RDONLY) { - rc = mdb_env_setup_locks(env, &fname, mode, &excl); - if (rc) - goto leave; - } - - if ((rc = mdb_env_open2(env, flags & MDB_PREVSNAPSHOT)) == MDB_SUCCESS) { - if (!(flags & (MDB_RDONLY|MDB_WRITEMAP))) { - /* Synchronous fd for meta writes. Needed even with - * MDB_NOSYNC/MDB_NOMETASYNC, in case these get reset. - */ - rc = mdb_fopen(env, &fname, MDB_O_META, mode, &env->me_mfd); - if (rc) - goto leave; - } - DPRINTF(("opened dbenv %p", (void *) env)); - if (excl > 0 && !(flags & MDB_PREVSNAPSHOT)) { - rc = mdb_env_share_locks(env, &excl); - if (rc) - goto leave; - } - if (!(flags & MDB_RDONLY)) { - MDB_txn *txn; - int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs * - (sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1); - if ((env->me_pbuf = calloc(1, env->me_psize)) && - (txn = calloc(1, size))) - { - txn->mt_dbs = (MDB_db *)((char *)txn + tsize); - txn->mt_cursors = (MDB_cursor **)(txn->mt_dbs + env->me_maxdbs); - txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs); - txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs); - txn->mt_env = env; -#ifdef MDB_VL32 - txn->mt_rpages = malloc(MDB_TRPAGE_SIZE * sizeof(MDB_ID3)); - if (!txn->mt_rpages) { - free(txn); - rc = ENOMEM; - goto leave; - } - txn->mt_rpages[0].mid = 0; - txn->mt_rpcheck = MDB_TRPAGE_SIZE/2; -#endif - txn->mt_dbxs = env->me_dbxs; - txn->mt_flags = MDB_TXN_FINISHED; - env->me_txn0 = txn; - } else { - rc = ENOMEM; - } - } - } - -leave: - if (rc) { - mdb_env_close0(env, excl); - } - mdb_fname_destroy(fname); - return rc; -} - -/** Destroy resources from mdb_env_open(), clear our readers & DBIs */ -static void ESECT -mdb_env_close0(MDB_env *env, int excl) -{ - int i; - - if (!(env->me_flags & MDB_ENV_ACTIVE)) - return; - - /* Doing this here since me_dbxs may not exist during mdb_env_close */ - if (env->me_dbxs) { - for (i = env->me_maxdbs; --i >= CORE_DBS; ) - free(env->me_dbxs[i].md_name.mv_data); - free(env->me_dbxs); - } - - free(env->me_pbuf); - free(env->me_dbiseqs); - free(env->me_dbflags); - free(env->me_path); - free(env->me_dirty_list); -#ifdef MDB_VL32 - if (env->me_txn0 && env->me_txn0->mt_rpages) - free(env->me_txn0->mt_rpages); - if (env->me_rpages) { - MDB_ID3L el = env->me_rpages; - unsigned int x; - for (x=1; x<=el[0].mid; x++) - munmap(el[x].mptr, el[x].mcnt * env->me_psize); - free(el); - } -#endif - free(env->me_txn0); - mdb_midl_free(env->me_free_pgs); - - if (env->me_flags & MDB_ENV_TXKEY) { - pthread_key_delete(env->me_txkey); -#ifdef _WIN32 - /* Delete our key from the global list */ - for (i=0; ime_txkey) { - mdb_tls_keys[i] = mdb_tls_keys[mdb_tls_nkeys-1]; - mdb_tls_nkeys--; - break; - } -#endif - } - - if (env->me_map) { -#ifdef MDB_VL32 - munmap(env->me_map, NUM_METAS*env->me_psize); -#else - munmap(env->me_map, env->me_mapsize); -#endif - } - if (env->me_mfd != INVALID_HANDLE_VALUE) - (void) close(env->me_mfd); - if (env->me_fd != INVALID_HANDLE_VALUE) - (void) close(env->me_fd); - if (env->me_txns) { - MDB_PID_T pid = getpid(); - /* Clearing readers is done in this function because - * me_txkey with its destructor must be disabled first. - * - * We skip the the reader mutex, so we touch only - * data owned by this process (me_close_readers and - * our readers), and clear each reader atomically. - */ - for (i = env->me_close_readers; --i >= 0; ) - if (env->me_txns->mti_readers[i].mr_pid == pid) - env->me_txns->mti_readers[i].mr_pid = 0; -#ifdef _WIN32 - if (env->me_rmutex) { - CloseHandle(env->me_rmutex); - if (env->me_wmutex) CloseHandle(env->me_wmutex); - } - /* Windows automatically destroys the mutexes when - * the last handle closes. - */ -#elif defined(MDB_USE_POSIX_SEM) - if (env->me_rmutex != SEM_FAILED) { - sem_close(env->me_rmutex); - if (env->me_wmutex != SEM_FAILED) - sem_close(env->me_wmutex); - /* If we have the filelock: If we are the - * only remaining user, clean up semaphores. - */ - if (excl == 0) - mdb_env_excl_lock(env, &excl); - if (excl > 0) { - sem_unlink(MUTEXNAME(env, 'r')); - sem_unlink(MUTEXNAME(env, 'w')); - } - } -#elif defined(MDB_USE_SYSV_SEM) - if (env->me_rmutex->semid != -1) { - /* If we have the filelock: If we are the - * only remaining user, clean up semaphores. - */ - if (excl == 0) - mdb_env_excl_lock(env, &excl); - if (excl > 0) - semctl(env->me_rmutex->semid, 0, IPC_RMID); - } -#endif - munmap((void *)env->me_txns, (env->me_maxreaders-1)*sizeof(MDB_reader)+sizeof(MDB_txninfo)); - } - if (env->me_lfd != INVALID_HANDLE_VALUE) { -#ifdef _WIN32 - if (excl >= 0) { - /* Unlock the lockfile. Windows would have unlocked it - * after closing anyway, but not necessarily at once. - */ - UnlockFile(env->me_lfd, 0, 0, 1, 0); - } -#endif - (void) close(env->me_lfd); - } -#ifdef MDB_VL32 -#ifdef _WIN32 - if (env->me_fmh) CloseHandle(env->me_fmh); - if (env->me_rpmutex) CloseHandle(env->me_rpmutex); -#else - pthread_mutex_destroy(&env->me_rpmutex); -#endif -#endif - - env->me_flags &= ~(MDB_ENV_ACTIVE|MDB_ENV_TXKEY); -} - -void ESECT -mdb_env_close(MDB_env *env) -{ - MDB_page *dp; - - if (env == NULL) - return; - - VGMEMP_DESTROY(env); - while ((dp = env->me_dpages) != NULL) { - VGMEMP_DEFINED(&dp->mp_next, sizeof(dp->mp_next)); - env->me_dpages = dp->mp_next; - free(dp); - } - - mdb_env_close0(env, 0); - free(env); -} - -/** Compare two items pointing at aligned #mdb_size_t's */ -static int -mdb_cmp_long(const MDB_val *a, const MDB_val *b) -{ - return (*(mdb_size_t *)a->mv_data < *(mdb_size_t *)b->mv_data) ? -1 : - *(mdb_size_t *)a->mv_data > *(mdb_size_t *)b->mv_data; -} - -/** Compare two items pointing at aligned unsigned int's. - * - * This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp, - * but #mdb_cmp_clong() is called instead if the data type is #mdb_size_t. - */ -static int -mdb_cmp_int(const MDB_val *a, const MDB_val *b) -{ - return (*(unsigned int *)a->mv_data < *(unsigned int *)b->mv_data) ? -1 : - *(unsigned int *)a->mv_data > *(unsigned int *)b->mv_data; -} - -/** Compare two items pointing at unsigned ints of unknown alignment. - * Nodes and keys are guaranteed to be 2-byte aligned. - */ -static int -mdb_cmp_cint(const MDB_val *a, const MDB_val *b) -{ -#if BYTE_ORDER == LITTLE_ENDIAN - unsigned short *u, *c; - int x; - - u = (unsigned short *) ((char *) a->mv_data + a->mv_size); - c = (unsigned short *) ((char *) b->mv_data + a->mv_size); - do { - x = *--u - *--c; - } while(!x && u > (unsigned short *)a->mv_data); - return x; -#else - unsigned short *u, *c, *end; - int x; - - end = (unsigned short *) ((char *) a->mv_data + a->mv_size); - u = (unsigned short *)a->mv_data; - c = (unsigned short *)b->mv_data; - do { - x = *u++ - *c++; - } while(!x && u < end); - return x; -#endif -} - -/** Compare two items lexically */ -static int -mdb_cmp_memn(const MDB_val *a, const MDB_val *b) -{ - int diff; - ssize_t len_diff; - unsigned int len; - - len = a->mv_size; - len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; - if (len_diff > 0) { - len = b->mv_size; - len_diff = 1; - } - - diff = memcmp(a->mv_data, b->mv_data, len); - return diff ? diff : len_diff<0 ? -1 : len_diff; -} - -/** Compare two items in reverse byte order */ -static int -mdb_cmp_memnr(const MDB_val *a, const MDB_val *b) -{ - const unsigned char *p1, *p2, *p1_lim; - ssize_t len_diff; - int diff; - - p1_lim = (const unsigned char *)a->mv_data; - p1 = (const unsigned char *)a->mv_data + a->mv_size; - p2 = (const unsigned char *)b->mv_data + b->mv_size; - - len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; - if (len_diff > 0) { - p1_lim += len_diff; - len_diff = 1; - } - - while (p1 > p1_lim) { - diff = *--p1 - *--p2; - if (diff) - return diff; - } - return len_diff<0 ? -1 : len_diff; -} - -/** Search for key within a page, using binary search. - * Returns the smallest entry larger or equal to the key. - * If exactp is non-null, stores whether the found entry was an exact match - * in *exactp (1 or 0). - * Updates the cursor index with the index of the found entry. - * If no entry larger or equal to the key is found, returns NULL. - */ -static MDB_node * -mdb_node_search(MDB_cursor *mc, MDB_val *key, int *exactp) -{ - unsigned int i = 0, nkeys; - int low, high; - int rc = 0; - MDB_page *mp = mc->mc_pg[mc->mc_top]; - MDB_node *node = NULL; - MDB_val nodekey; - MDB_cmp_func *cmp; - DKBUF; - - nkeys = NUMKEYS(mp); - - DPRINTF(("searching %u keys in %s %spage %"Yu, - nkeys, IS_LEAF(mp) ? "leaf" : "branch", IS_SUBP(mp) ? "sub-" : "", - mdb_dbg_pgno(mp))); - - low = IS_LEAF(mp) ? 0 : 1; - high = nkeys - 1; - cmp = mc->mc_dbx->md_cmp; - - /* Branch pages have no data, so if using integer keys, - * alignment is guaranteed. Use faster mdb_cmp_int. - */ - if (cmp == mdb_cmp_cint && IS_BRANCH(mp)) { - if (NODEPTR(mp, 1)->mn_ksize == sizeof(mdb_size_t)) - cmp = mdb_cmp_long; - else - cmp = mdb_cmp_int; - } - - if (IS_LEAF2(mp)) { - nodekey.mv_size = mc->mc_db->md_pad; - node = NODEPTR(mp, 0); /* fake */ - while (low <= high) { - i = (low + high) >> 1; - nodekey.mv_data = LEAF2KEY(mp, i, nodekey.mv_size); - rc = cmp(key, &nodekey); - DPRINTF(("found leaf index %u [%s], rc = %i", - i, DKEY(&nodekey), rc)); - if (rc == 0) - break; - if (rc > 0) - low = i + 1; - else - high = i - 1; - } - } else { - while (low <= high) { - i = (low + high) >> 1; - - node = NODEPTR(mp, i); - nodekey.mv_size = NODEKSZ(node); - nodekey.mv_data = NODEKEY(node); - - rc = cmp(key, &nodekey); -#if MDB_DEBUG - if (IS_LEAF(mp)) - DPRINTF(("found leaf index %u [%s], rc = %i", - i, DKEY(&nodekey), rc)); - else - DPRINTF(("found branch index %u [%s -> %"Yu"], rc = %i", - i, DKEY(&nodekey), NODEPGNO(node), rc)); -#endif - if (rc == 0) - break; - if (rc > 0) - low = i + 1; - else - high = i - 1; - } - } - - if (rc > 0) { /* Found entry is less than the key. */ - i++; /* Skip to get the smallest entry larger than key. */ - if (!IS_LEAF2(mp)) - node = NODEPTR(mp, i); - } - if (exactp) - *exactp = (rc == 0 && nkeys > 0); - /* store the key index */ - mc->mc_ki[mc->mc_top] = i; - if (i >= nkeys) - /* There is no entry larger or equal to the key. */ - return NULL; - - /* nodeptr is fake for LEAF2 */ - return node; -} - -#if 0 -static void -mdb_cursor_adjust(MDB_cursor *mc, func) -{ - MDB_cursor *m2; - - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2->mc_pg[m2->mc_top] == mc->mc_pg[mc->mc_top]) { - func(mc, m2); - } - } -} -#endif - -/** Pop a page off the top of the cursor's stack. */ -static void -mdb_cursor_pop(MDB_cursor *mc) -{ - if (mc->mc_snum) { - DPRINTF(("popping page %"Yu" off db %d cursor %p", - mc->mc_pg[mc->mc_top]->mp_pgno, DDBI(mc), (void *) mc)); - - mc->mc_snum--; - if (mc->mc_snum) { - mc->mc_top--; - } else { - mc->mc_flags &= ~C_INITIALIZED; - } - } -} - -/** Push a page onto the top of the cursor's stack. - * Set #MDB_TXN_ERROR on failure. - */ -static int -mdb_cursor_push(MDB_cursor *mc, MDB_page *mp) -{ - DPRINTF(("pushing page %"Yu" on db %d cursor %p", mp->mp_pgno, - DDBI(mc), (void *) mc)); - - if (mc->mc_snum >= CURSOR_STACK) { - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CURSOR_FULL; - } - - mc->mc_top = mc->mc_snum++; - mc->mc_pg[mc->mc_top] = mp; - mc->mc_ki[mc->mc_top] = 0; - - return MDB_SUCCESS; -} - -#ifdef MDB_VL32 -/** Map a read-only page. - * There are two levels of tracking in use, a per-txn list and a per-env list. - * ref'ing and unref'ing the per-txn list is faster since it requires no - * locking. Pages are cached in the per-env list for global reuse, and a lock - * is required. Pages are not immediately unmapped when their refcnt goes to - * zero; they hang around in case they will be reused again soon. - * - * When the per-txn list gets full, all pages with refcnt=0 are purged from the - * list and their refcnts in the per-env list are decremented. - * - * When the per-env list gets full, all pages with refcnt=0 are purged from the - * list and their pages are unmapped. - * - * @note "full" means the list has reached its respective rpcheck threshold. - * This threshold slowly raises if no pages could be purged on a given check, - * and returns to its original value when enough pages were purged. - * - * If purging doesn't free any slots, filling the per-txn list will return - * MDB_TXN_FULL, and filling the per-env list returns MDB_MAP_FULL. - * - * Reference tracking in a txn is imperfect, pages can linger with non-zero - * refcnt even without active references. It was deemed to be too invasive - * to add unrefs in every required location. However, all pages are unref'd - * at the end of the transaction. This guarantees that no stale references - * linger in the per-env list. - * - * Usually we map chunks of 16 pages at a time, but if an overflow page begins - * at the tail of the chunk we extend the chunk to include the entire overflow - * page. Unfortunately, pages can be turned into overflow pages after their - * chunk was already mapped. In that case we must remap the chunk if the - * overflow page is referenced. If the chunk's refcnt is 0 we can just remap - * it, otherwise we temporarily map a new chunk just for the overflow page. - * - * @note this chunk handling means we cannot guarantee that a data item - * returned from the DB will stay alive for the duration of the transaction: - * We unref pages as soon as a cursor moves away from the page - * A subsequent op may cause a purge, which may unmap any unref'd chunks - * The caller must copy the data if it must be used later in the same txn. - * - * Also - our reference counting revolves around cursors, but overflow pages - * aren't pointed to by a cursor's page stack. We have to remember them - * explicitly, in the added mc_ovpg field. A single cursor can only hold a - * reference to one overflow page at a time. - * - * @param[in] txn the transaction for this access. - * @param[in] pgno the page number for the page to retrieve. - * @param[out] ret address of a pointer where the page's address will be stored. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_rpage_get(MDB_txn *txn, pgno_t pg0, MDB_page **ret) -{ - MDB_env *env = txn->mt_env; - MDB_page *p; - MDB_ID3L tl = txn->mt_rpages; - MDB_ID3L el = env->me_rpages; - MDB_ID3 id3; - unsigned x, rem; - pgno_t pgno; - int rc, retries = 1; -#ifdef _WIN32 - LARGE_INTEGER off; - SIZE_T len; -#define SET_OFF(off,val) off.QuadPart = val -#define MAP(rc,env,addr,len,off) \ - addr = NULL; \ - rc = NtMapViewOfSection(env->me_fmh, GetCurrentProcess(), &addr, 0, \ - len, &off, &len, ViewUnmap, (env->me_flags & MDB_RDONLY) ? 0 : MEM_RESERVE, PAGE_READONLY); \ - if (rc) rc = mdb_nt2win32(rc) -#else - off_t off; - size_t len; -#define SET_OFF(off,val) off = val -#define MAP(rc,env,addr,len,off) \ - addr = mmap(NULL, len, PROT_READ, MAP_SHARED, env->me_fd, off); \ - rc = (addr == MAP_FAILED) ? errno : 0 -#endif - - /* remember the offset of the actual page number, so we can - * return the correct pointer at the end. - */ - rem = pg0 & (MDB_RPAGE_CHUNK-1); - pgno = pg0 ^ rem; - - id3.mid = 0; - x = mdb_mid3l_search(tl, pgno); - if (x <= tl[0].mid && tl[x].mid == pgno) { - if (x != tl[0].mid && tl[x+1].mid == pg0) - x++; - /* check for overflow size */ - p = (MDB_page *)((char *)tl[x].mptr + rem * env->me_psize); - if (IS_OVERFLOW(p) && p->mp_pages + rem > tl[x].mcnt) { - id3.mcnt = p->mp_pages + rem; - len = id3.mcnt * env->me_psize; - SET_OFF(off, pgno * env->me_psize); - MAP(rc, env, id3.mptr, len, off); - if (rc) - return rc; - /* check for local-only page */ - if (rem) { - mdb_tassert(txn, tl[x].mid != pg0); - /* hope there's room to insert this locally. - * setting mid here tells later code to just insert - * this id3 instead of searching for a match. - */ - id3.mid = pg0; - goto notlocal; - } else { - /* ignore the mapping we got from env, use new one */ - tl[x].mptr = id3.mptr; - tl[x].mcnt = id3.mcnt; - /* if no active ref, see if we can replace in env */ - if (!tl[x].mref) { - unsigned i; - pthread_mutex_lock(&env->me_rpmutex); - i = mdb_mid3l_search(el, tl[x].mid); - if (el[i].mref == 1) { - /* just us, replace it */ - munmap(el[i].mptr, el[i].mcnt * env->me_psize); - el[i].mptr = tl[x].mptr; - el[i].mcnt = tl[x].mcnt; - } else { - /* there are others, remove ourself */ - el[i].mref--; - } - pthread_mutex_unlock(&env->me_rpmutex); - } - } - } - id3.mptr = tl[x].mptr; - id3.mcnt = tl[x].mcnt; - tl[x].mref++; - goto ok; - } - -notlocal: - if (tl[0].mid >= MDB_TRPAGE_MAX - txn->mt_rpcheck) { - unsigned i, y; - /* purge unref'd pages from our list and unref in env */ - pthread_mutex_lock(&env->me_rpmutex); -retry: - y = 0; - for (i=1; i<=tl[0].mid; i++) { - if (!tl[i].mref) { - if (!y) y = i; - /* tmp overflow pages don't go to env */ - if (tl[i].mid & (MDB_RPAGE_CHUNK-1)) { - munmap(tl[i].mptr, tl[i].mcnt * env->me_psize); - continue; - } - x = mdb_mid3l_search(el, tl[i].mid); - el[x].mref--; - } - } - pthread_mutex_unlock(&env->me_rpmutex); - if (!y) { - /* we didn't find any unref'd chunks. - * if we're out of room, fail. - */ - if (tl[0].mid >= MDB_TRPAGE_MAX) - return MDB_TXN_FULL; - /* otherwise, raise threshold for next time around - * and let this go. - */ - txn->mt_rpcheck /= 2; - } else { - /* we found some unused; consolidate the list */ - for (i=y+1; i<= tl[0].mid; i++) - if (tl[i].mref) - tl[y++] = tl[i]; - tl[0].mid = y-1; - /* decrease the check threshold toward its original value */ - if (!txn->mt_rpcheck) - txn->mt_rpcheck = 1; - while (txn->mt_rpcheck < tl[0].mid && txn->mt_rpcheck < MDB_TRPAGE_SIZE/2) - txn->mt_rpcheck *= 2; - } - } - if (tl[0].mid < MDB_TRPAGE_SIZE) { - id3.mref = 1; - if (id3.mid) - goto found; - /* don't map past last written page in read-only envs */ - if ((env->me_flags & MDB_RDONLY) && pgno + MDB_RPAGE_CHUNK-1 > txn->mt_last_pgno) - id3.mcnt = txn->mt_last_pgno + 1 - pgno; - else - id3.mcnt = MDB_RPAGE_CHUNK; - len = id3.mcnt * env->me_psize; - id3.mid = pgno; - - /* search for page in env */ - pthread_mutex_lock(&env->me_rpmutex); - x = mdb_mid3l_search(el, pgno); - if (x <= el[0].mid && el[x].mid == pgno) { - id3.mptr = el[x].mptr; - id3.mcnt = el[x].mcnt; - /* check for overflow size */ - p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize); - if (IS_OVERFLOW(p) && p->mp_pages + rem > id3.mcnt) { - id3.mcnt = p->mp_pages + rem; - len = id3.mcnt * env->me_psize; - SET_OFF(off, pgno * env->me_psize); - MAP(rc, env, id3.mptr, len, off); - if (rc) - goto fail; - if (!el[x].mref) { - munmap(el[x].mptr, env->me_psize * el[x].mcnt); - el[x].mptr = id3.mptr; - el[x].mcnt = id3.mcnt; - } else { - id3.mid = pg0; - pthread_mutex_unlock(&env->me_rpmutex); - goto found; - } - } - el[x].mref++; - pthread_mutex_unlock(&env->me_rpmutex); - goto found; - } - if (el[0].mid >= MDB_ERPAGE_MAX - env->me_rpcheck) { - /* purge unref'd pages */ - unsigned i, y = 0; - for (i=1; i<=el[0].mid; i++) { - if (!el[i].mref) { - if (!y) y = i; - munmap(el[i].mptr, env->me_psize * el[i].mcnt); - } - } - if (!y) { - if (retries) { - /* see if we can unref some local pages */ - retries--; - id3.mid = 0; - goto retry; - } - if (el[0].mid >= MDB_ERPAGE_MAX) { - pthread_mutex_unlock(&env->me_rpmutex); - return MDB_MAP_FULL; - } - env->me_rpcheck /= 2; - } else { - for (i=y+1; i<= el[0].mid; i++) - if (el[i].mref) - el[y++] = el[i]; - el[0].mid = y-1; - if (!env->me_rpcheck) - env->me_rpcheck = 1; - while (env->me_rpcheck < el[0].mid && env->me_rpcheck < MDB_ERPAGE_SIZE/2) - env->me_rpcheck *= 2; - } - } - SET_OFF(off, pgno * env->me_psize); - MAP(rc, env, id3.mptr, len, off); - if (rc) { -fail: - pthread_mutex_unlock(&env->me_rpmutex); - return rc; - } - /* check for overflow size */ - p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize); - if (IS_OVERFLOW(p) && p->mp_pages + rem > id3.mcnt) { - id3.mcnt = p->mp_pages + rem; - munmap(id3.mptr, len); - len = id3.mcnt * env->me_psize; - MAP(rc, env, id3.mptr, len, off); - if (rc) - goto fail; - } - mdb_mid3l_insert(el, &id3); - pthread_mutex_unlock(&env->me_rpmutex); -found: - mdb_mid3l_insert(tl, &id3); - } else { - return MDB_TXN_FULL; - } -ok: - p = (MDB_page *)((char *)id3.mptr + rem * env->me_psize); -#if MDB_DEBUG /* we don't need this check any more */ - if (IS_OVERFLOW(p)) { - mdb_tassert(txn, p->mp_pages + rem <= id3.mcnt); - } -#endif - *ret = p; - return MDB_SUCCESS; -} -#endif - -/** Find the address of the page corresponding to a given page number. - * Set #MDB_TXN_ERROR on failure. - * @param[in] mc the cursor accessing the page. - * @param[in] pgno the page number for the page to retrieve. - * @param[out] ret address of a pointer where the page's address will be stored. - * @param[out] lvl dirty_list inheritance level of found page. 1=current txn, 0=mapped page. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_get(MDB_cursor *mc, pgno_t pgno, MDB_page **ret, int *lvl) -{ - MDB_txn *txn = mc->mc_txn; - MDB_page *p = NULL; - int level; - - if (! (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP))) { - MDB_txn *tx2 = txn; - level = 1; - do { - MDB_ID2L dl = tx2->mt_u.dirty_list; - unsigned x; - /* Spilled pages were dirtied in this txn and flushed - * because the dirty list got full. Bring this page - * back in from the map (but don't unspill it here, - * leave that unless page_touch happens again). - */ - if (tx2->mt_spill_pgs) { - MDB_ID pn = pgno << 1; - x = mdb_midl_search(tx2->mt_spill_pgs, pn); - if (x <= tx2->mt_spill_pgs[0] && tx2->mt_spill_pgs[x] == pn) { - goto mapped; - } - } - if (dl[0].mid) { - unsigned x = mdb_mid2l_search(dl, pgno); - if (x <= dl[0].mid && dl[x].mid == pgno) { - p = dl[x].mptr; - goto done; - } - } - level++; - } while ((tx2 = tx2->mt_parent) != NULL); - } - - if (pgno >= txn->mt_next_pgno) { - DPRINTF(("page %"Yu" not found", pgno)); - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PAGE_NOTFOUND; - } - - level = 0; - -mapped: - { -#ifdef MDB_VL32 - int rc = mdb_rpage_get(txn, pgno, &p); - if (rc) { - txn->mt_flags |= MDB_TXN_ERROR; - return rc; - } -#else - MDB_env *env = txn->mt_env; - p = (MDB_page *)(env->me_map + env->me_psize * pgno); -#endif - } - -done: - *ret = p; - if (lvl) - *lvl = level; - return MDB_SUCCESS; -} - -/** Finish #mdb_page_search() / #mdb_page_search_lowest(). - * The cursor is at the root page, set up the rest of it. - */ -static int -mdb_page_search_root(MDB_cursor *mc, MDB_val *key, int flags) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top]; - int rc; - DKBUF; - - while (IS_BRANCH(mp)) { - MDB_node *node; - indx_t i; - - DPRINTF(("branch page %"Yu" has %u keys", mp->mp_pgno, NUMKEYS(mp))); - /* Don't assert on branch pages in the FreeDB. We can get here - * while in the process of rebalancing a FreeDB branch page; we must - * let that proceed. ITS#8336 - */ - mdb_cassert(mc, !mc->mc_dbi || NUMKEYS(mp) > 1); - DPRINTF(("found index 0 to page %"Yu, NODEPGNO(NODEPTR(mp, 0)))); - - if (flags & (MDB_PS_FIRST|MDB_PS_LAST)) { - i = 0; - if (flags & MDB_PS_LAST) { - i = NUMKEYS(mp) - 1; - /* if already init'd, see if we're already in right place */ - if (mc->mc_flags & C_INITIALIZED) { - if (mc->mc_ki[mc->mc_top] == i) { - mc->mc_top = mc->mc_snum++; - mp = mc->mc_pg[mc->mc_top]; - goto ready; - } - } - } - } else { - int exact; - node = mdb_node_search(mc, key, &exact); - if (node == NULL) - i = NUMKEYS(mp) - 1; - else { - i = mc->mc_ki[mc->mc_top]; - if (!exact) { - mdb_cassert(mc, i > 0); - i--; - } - } - DPRINTF(("following index %u for key [%s]", i, DKEY(key))); - } - - mdb_cassert(mc, i < NUMKEYS(mp)); - node = NODEPTR(mp, i); - - if ((rc = mdb_page_get(mc, NODEPGNO(node), &mp, NULL)) != 0) - return rc; - - mc->mc_ki[mc->mc_top] = i; - if ((rc = mdb_cursor_push(mc, mp))) - return rc; - -ready: - if (flags & MDB_PS_MODIFY) { - if ((rc = mdb_page_touch(mc)) != 0) - return rc; - mp = mc->mc_pg[mc->mc_top]; - } - } - - if (!IS_LEAF(mp)) { - DPRINTF(("internal error, index points to a %02X page!?", - mp->mp_flags)); - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return MDB_CORRUPTED; - } - - DPRINTF(("found leaf page %"Yu" for key [%s]", mp->mp_pgno, - key ? DKEY(key) : "null")); - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - - return MDB_SUCCESS; -} - -/** Search for the lowest key under the current branch page. - * This just bypasses a NUMKEYS check in the current page - * before calling mdb_page_search_root(), because the callers - * are all in situations where the current page is known to - * be underfilled. - */ -static int -mdb_page_search_lowest(MDB_cursor *mc) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top]; - MDB_node *node = NODEPTR(mp, 0); - int rc; - - if ((rc = mdb_page_get(mc, NODEPGNO(node), &mp, NULL)) != 0) - return rc; - - mc->mc_ki[mc->mc_top] = 0; - if ((rc = mdb_cursor_push(mc, mp))) - return rc; - return mdb_page_search_root(mc, NULL, MDB_PS_FIRST); -} - -/** Search for the page a given key should be in. - * Push it and its parent pages on the cursor stack. - * @param[in,out] mc the cursor for this operation. - * @param[in] key the key to search for, or NULL for first/last page. - * @param[in] flags If MDB_PS_MODIFY is set, visited pages in the DB - * are touched (updated with new page numbers). - * If MDB_PS_FIRST or MDB_PS_LAST is set, find first or last leaf. - * This is used by #mdb_cursor_first() and #mdb_cursor_last(). - * If MDB_PS_ROOTONLY set, just fetch root node, no further lookups. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_search(MDB_cursor *mc, MDB_val *key, int flags) -{ - int rc; - pgno_t root; - - /* Make sure the txn is still viable, then find the root from - * the txn's db table and set it as the root of the cursor's stack. - */ - if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) { - DPUTS("transaction may not be used now"); - return MDB_BAD_TXN; - } else { - /* Make sure we're using an up-to-date root */ - if (*mc->mc_dbflag & DB_STALE) { - MDB_cursor mc2; - if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) - return MDB_BAD_DBI; - mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, NULL); - rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, 0); - if (rc) - return rc; - { - MDB_val data; - int exact = 0; - uint16_t flags; - MDB_node *leaf = mdb_node_search(&mc2, - &mc->mc_dbx->md_name, &exact); - if (!exact) - return MDB_NOTFOUND; - if ((leaf->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) - return MDB_INCOMPATIBLE; /* not a named DB */ - rc = mdb_node_read(&mc2, leaf, &data); - if (rc) - return rc; - memcpy(&flags, ((char *) data.mv_data + offsetof(MDB_db, md_flags)), - sizeof(uint16_t)); - /* The txn may not know this DBI, or another process may - * have dropped and recreated the DB with other flags. - */ - if ((mc->mc_db->md_flags & PERSISTENT_FLAGS) != flags) - return MDB_INCOMPATIBLE; - memcpy(mc->mc_db, data.mv_data, sizeof(MDB_db)); - } - *mc->mc_dbflag &= ~DB_STALE; - } - root = mc->mc_db->md_root; - - if (root == P_INVALID) { /* Tree is empty. */ - DPUTS("tree is empty"); - return MDB_NOTFOUND; - } - } - - mdb_cassert(mc, root > 1); - if (!mc->mc_pg[0] || mc->mc_pg[0]->mp_pgno != root) { -#ifdef MDB_VL32 - if (mc->mc_pg[0]) - MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[0]); -#endif - if ((rc = mdb_page_get(mc, root, &mc->mc_pg[0], NULL)) != 0) - return rc; - } - -#ifdef MDB_VL32 - { - int i; - for (i=1; imc_snum; i++) - MDB_PAGE_UNREF(mc->mc_txn, mc->mc_pg[i]); - } -#endif - mc->mc_snum = 1; - mc->mc_top = 0; - - DPRINTF(("db %d root page %"Yu" has flags 0x%X", - DDBI(mc), root, mc->mc_pg[0]->mp_flags)); - - if (flags & MDB_PS_MODIFY) { - if ((rc = mdb_page_touch(mc))) - return rc; - } - - if (flags & MDB_PS_ROOTONLY) - return MDB_SUCCESS; - - return mdb_page_search_root(mc, key, flags); -} - -static int -mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) -{ - MDB_txn *txn = mc->mc_txn; - pgno_t pg = mp->mp_pgno; - unsigned x = 0, ovpages = mp->mp_pages; - MDB_env *env = txn->mt_env; - MDB_IDL sl = txn->mt_spill_pgs; - MDB_ID pn = pg << 1; - int rc; - - DPRINTF(("free ov page %"Yu" (%d)", pg, ovpages)); - /* If the page is dirty or on the spill list we just acquired it, - * so we should give it back to our current free list, if any. - * Otherwise put it onto the list of pages we freed in this txn. - * - * Won't create me_pghead: me_pglast must be inited along with it. - * Unsupported in nested txns: They would need to hide the page - * range in ancestor txns' dirty and spilled lists. - */ - if (env->me_pghead && - !txn->mt_parent && - ((mp->mp_flags & P_DIRTY) || - (sl && (x = mdb_midl_search(sl, pn)) <= sl[0] && sl[x] == pn))) - { - unsigned i, j; - pgno_t *mop; - MDB_ID2 *dl, ix, iy; - rc = mdb_midl_need(&env->me_pghead, ovpages); - if (rc) - return rc; - if (!(mp->mp_flags & P_DIRTY)) { - /* This page is no longer spilled */ - if (x == sl[0]) - sl[0]--; - else - sl[x] |= 1; - goto release; - } - /* Remove from dirty list */ - dl = txn->mt_u.dirty_list; - x = dl[0].mid--; - for (ix = dl[x]; ix.mptr != mp; ix = iy) { - if (x > 1) { - x--; - iy = dl[x]; - dl[x] = ix; - } else { - mdb_cassert(mc, x > 1); - j = ++(dl[0].mid); - dl[j] = ix; /* Unsorted. OK when MDB_TXN_ERROR. */ - txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PROBLEM; - } - } - txn->mt_dirty_room++; - if (!(env->me_flags & MDB_WRITEMAP)) - mdb_dpage_free(env, mp); -release: - /* Insert in me_pghead */ - mop = env->me_pghead; - j = mop[0] + ovpages; - for (i = mop[0]; i && mop[i] < pg; i--) - mop[j--] = mop[i]; - while (j>i) - mop[j--] = pg++; - mop[0] += ovpages; - } else { - rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages); - if (rc) - return rc; - } -#ifdef MDB_VL32 - if (mc->mc_ovpg == mp) - mc->mc_ovpg = NULL; -#endif - mc->mc_db->md_overflow_pages -= ovpages; - return 0; -} - -/** Return the data associated with a given node. - * @param[in] mc The cursor for this operation. - * @param[in] leaf The node being read. - * @param[out] data Updated to point to the node's data. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_node_read(MDB_cursor *mc, MDB_node *leaf, MDB_val *data) -{ - MDB_page *omp; /* overflow page */ - pgno_t pgno; - int rc; - - if (MC_OVPG(mc)) { - MDB_PAGE_UNREF(mc->mc_txn, MC_OVPG(mc)); - MC_SET_OVPG(mc, NULL); - } - if (!F_ISSET(leaf->mn_flags, F_BIGDATA)) { - data->mv_size = NODEDSZ(leaf); - data->mv_data = NODEDATA(leaf); - return MDB_SUCCESS; - } - - /* Read overflow data. - */ - data->mv_size = NODEDSZ(leaf); - memcpy(&pgno, NODEDATA(leaf), sizeof(pgno)); - if ((rc = mdb_page_get(mc, pgno, &omp, NULL)) != 0) { - DPRINTF(("read overflow page %"Yu" failed", pgno)); - return rc; - } - data->mv_data = METADATA(omp); - MC_SET_OVPG(mc, omp); - - return MDB_SUCCESS; -} - -int -mdb_get(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data) -{ - MDB_cursor mc; - MDB_xcursor mx; - int exact = 0, rc; - DKBUF; - - DPRINTF(("===> get db %u key [%s]", dbi, DKEY(key))); - - if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_BLOCKED) - return MDB_BAD_TXN; - - mdb_cursor_init(&mc, txn, dbi, &mx); - rc = mdb_cursor_set(&mc, key, data, MDB_SET, &exact); - /* unref all the pages when MDB_VL32 - caller must copy the data - * before doing anything else - */ - MDB_CURSOR_UNREF(&mc, 1); - return rc; -} - -/** Find a sibling for a page. - * Replaces the page at the top of the cursor's stack with the - * specified sibling, if one exists. - * @param[in] mc The cursor for this operation. - * @param[in] move_right Non-zero if the right sibling is requested, - * otherwise the left sibling. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_cursor_sibling(MDB_cursor *mc, int move_right) -{ - int rc; - MDB_node *indx; - MDB_page *mp; -#ifdef MDB_VL32 - MDB_page *op; -#endif - - if (mc->mc_snum < 2) { - return MDB_NOTFOUND; /* root has no siblings */ - } - -#ifdef MDB_VL32 - op = mc->mc_pg[mc->mc_top]; -#endif - mdb_cursor_pop(mc); - DPRINTF(("parent page is page %"Yu", index %u", - mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top])); - - if (move_right ? (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mc->mc_pg[mc->mc_top])) - : (mc->mc_ki[mc->mc_top] == 0)) { - DPRINTF(("no more keys left, moving to %s sibling", - move_right ? "right" : "left")); - if ((rc = mdb_cursor_sibling(mc, move_right)) != MDB_SUCCESS) { - /* undo cursor_pop before returning */ - mc->mc_top++; - mc->mc_snum++; - return rc; - } - } else { - if (move_right) - mc->mc_ki[mc->mc_top]++; - else - mc->mc_ki[mc->mc_top]--; - DPRINTF(("just moving to %s index key %u", - move_right ? "right" : "left", mc->mc_ki[mc->mc_top])); - } - mdb_cassert(mc, IS_BRANCH(mc->mc_pg[mc->mc_top])); - - MDB_PAGE_UNREF(mc->mc_txn, op); - - indx = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if ((rc = mdb_page_get(mc, NODEPGNO(indx), &mp, NULL)) != 0) { - /* mc will be inconsistent if caller does mc_snum++ as above */ - mc->mc_flags &= ~(C_INITIALIZED|C_EOF); - return rc; - } - - mdb_cursor_push(mc, mp); - if (!move_right) - mc->mc_ki[mc->mc_top] = NUMKEYS(mp)-1; - - return MDB_SUCCESS; -} - -/** Move the cursor to the next data item. */ -static int -mdb_cursor_next(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) -{ - MDB_page *mp; - MDB_node *leaf; - int rc; - - if ((mc->mc_flags & C_DEL && op == MDB_NEXT_DUP)) - return MDB_NOTFOUND; - - if (!(mc->mc_flags & C_INITIALIZED)) - return mdb_cursor_first(mc, key, data); - - mp = mc->mc_pg[mc->mc_top]; - - if (mc->mc_flags & C_EOF) { - if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mp)-1) - return MDB_NOTFOUND; - mc->mc_flags ^= C_EOF; - } - - if (mc->mc_db->md_flags & MDB_DUPSORT) { - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (op == MDB_NEXT || op == MDB_NEXT_DUP) { - rc = mdb_cursor_next(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_NEXT); - if (op != MDB_NEXT || rc != MDB_NOTFOUND) { - if (rc == MDB_SUCCESS) - MDB_GET_KEY(leaf, key); - return rc; - } - } - else { - MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); - } - } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - if (op == MDB_NEXT_DUP) - return MDB_NOTFOUND; - } - } - - DPRINTF(("cursor_next: top page is %"Yu" in cursor %p", - mdb_dbg_pgno(mp), (void *) mc)); - if (mc->mc_flags & C_DEL) { - mc->mc_flags ^= C_DEL; - goto skip; - } - - if (mc->mc_ki[mc->mc_top] + 1u >= NUMKEYS(mp)) { - DPUTS("=====> move to next sibling page"); - if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { - mc->mc_flags |= C_EOF; - return rc; - } - mp = mc->mc_pg[mc->mc_top]; - DPRINTF(("next page is %"Yu", key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); - } else - mc->mc_ki[mc->mc_top]++; - -skip: - DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u", - mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); - - if (IS_LEAF2(mp)) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - return MDB_SUCCESS; - } - - mdb_cassert(mc, IS_LEAF(mp)); - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - } - if (data) { - if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) - return rc; - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc != MDB_SUCCESS) - return rc; - } - } - - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -/** Move the cursor to the previous data item. */ -static int -mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op) -{ - MDB_page *mp; - MDB_node *leaf; - int rc; - - if (!(mc->mc_flags & C_INITIALIZED)) { - rc = mdb_cursor_last(mc, key, data); - if (rc) - return rc; - mc->mc_ki[mc->mc_top]++; - } - - mp = mc->mc_pg[mc->mc_top]; - - if (mc->mc_db->md_flags & MDB_DUPSORT) { - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (op == MDB_PREV || op == MDB_PREV_DUP) { - rc = mdb_cursor_prev(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_PREV); - if (op != MDB_PREV || rc != MDB_NOTFOUND) { - if (rc == MDB_SUCCESS) { - MDB_GET_KEY(leaf, key); - mc->mc_flags &= ~C_EOF; - } - return rc; - } - } - else { - MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); - } - } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - if (op == MDB_PREV_DUP) - return MDB_NOTFOUND; - } - } - - DPRINTF(("cursor_prev: top page is %"Yu" in cursor %p", - mdb_dbg_pgno(mp), (void *) mc)); - - mc->mc_flags &= ~(C_EOF|C_DEL); - - if (mc->mc_ki[mc->mc_top] == 0) { - DPUTS("=====> move to prev sibling page"); - if ((rc = mdb_cursor_sibling(mc, 0)) != MDB_SUCCESS) { - return rc; - } - mp = mc->mc_pg[mc->mc_top]; - mc->mc_ki[mc->mc_top] = NUMKEYS(mp) - 1; - DPRINTF(("prev page is %"Yu", key index %u", mp->mp_pgno, mc->mc_ki[mc->mc_top])); - } else - mc->mc_ki[mc->mc_top]--; - - DPRINTF(("==> cursor points to page %"Yu" with %u keys, key index %u", - mdb_dbg_pgno(mp), NUMKEYS(mp), mc->mc_ki[mc->mc_top])); - - if (IS_LEAF2(mp)) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - return MDB_SUCCESS; - } - - mdb_cassert(mc, IS_LEAF(mp)); - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - } - if (data) { - if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) - return rc; - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc != MDB_SUCCESS) - return rc; - } - } - - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -/** Set the cursor on a specific data item. */ -static int -mdb_cursor_set(MDB_cursor *mc, MDB_val *key, MDB_val *data, - MDB_cursor_op op, int *exactp) -{ - int rc; - MDB_page *mp; - MDB_node *leaf = NULL; - DKBUF; - - if (key->mv_size == 0) - return MDB_BAD_VALSIZE; - - if (mc->mc_xcursor) { - MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - } - - /* See if we're already on the right page */ - if (mc->mc_flags & C_INITIALIZED) { - MDB_val nodekey; - - mp = mc->mc_pg[mc->mc_top]; - if (!NUMKEYS(mp)) { - mc->mc_ki[mc->mc_top] = 0; - return MDB_NOTFOUND; - } - if (mp->mp_flags & P_LEAF2) { - nodekey.mv_size = mc->mc_db->md_pad; - nodekey.mv_data = LEAF2KEY(mp, 0, nodekey.mv_size); - } else { - leaf = NODEPTR(mp, 0); - MDB_GET_KEY2(leaf, nodekey); - } - rc = mc->mc_dbx->md_cmp(key, &nodekey); - if (rc == 0) { - /* Probably happens rarely, but first node on the page - * was the one we wanted. - */ - mc->mc_ki[mc->mc_top] = 0; - if (exactp) - *exactp = 1; - goto set1; - } - if (rc > 0) { - unsigned int i; - unsigned int nkeys = NUMKEYS(mp); - if (nkeys > 1) { - if (mp->mp_flags & P_LEAF2) { - nodekey.mv_data = LEAF2KEY(mp, - nkeys-1, nodekey.mv_size); - } else { - leaf = NODEPTR(mp, nkeys-1); - MDB_GET_KEY2(leaf, nodekey); - } - rc = mc->mc_dbx->md_cmp(key, &nodekey); - if (rc == 0) { - /* last node was the one we wanted */ - mc->mc_ki[mc->mc_top] = nkeys-1; - if (exactp) - *exactp = 1; - goto set1; - } - if (rc < 0) { - if (mc->mc_ki[mc->mc_top] < NUMKEYS(mp)) { - /* This is definitely the right page, skip search_page */ - if (mp->mp_flags & P_LEAF2) { - nodekey.mv_data = LEAF2KEY(mp, - mc->mc_ki[mc->mc_top], nodekey.mv_size); - } else { - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - MDB_GET_KEY2(leaf, nodekey); - } - rc = mc->mc_dbx->md_cmp(key, &nodekey); - if (rc == 0) { - /* current node was the one we wanted */ - if (exactp) - *exactp = 1; - goto set1; - } - } - rc = 0; - mc->mc_flags &= ~C_EOF; - goto set2; - } - } - /* If any parents have right-sibs, search. - * Otherwise, there's nothing further. - */ - for (i=0; imc_top; i++) - if (mc->mc_ki[i] < - NUMKEYS(mc->mc_pg[i])-1) - break; - if (i == mc->mc_top) { - /* There are no other pages */ - mc->mc_ki[mc->mc_top] = nkeys; - return MDB_NOTFOUND; - } - } - if (!mc->mc_top) { - /* There are no other pages */ - mc->mc_ki[mc->mc_top] = 0; - if (op == MDB_SET_RANGE && !exactp) { - rc = 0; - goto set1; - } else - return MDB_NOTFOUND; - } - } else { - mc->mc_pg[0] = 0; - } - - rc = mdb_page_search(mc, key, 0); - if (rc != MDB_SUCCESS) - return rc; - - mp = mc->mc_pg[mc->mc_top]; - mdb_cassert(mc, IS_LEAF(mp)); - -set2: - leaf = mdb_node_search(mc, key, exactp); - if (exactp != NULL && !*exactp) { - /* MDB_SET specified and not an exact match. */ - return MDB_NOTFOUND; - } - - if (leaf == NULL) { - DPUTS("===> inexact leaf not found, goto sibling"); - if ((rc = mdb_cursor_sibling(mc, 1)) != MDB_SUCCESS) { - mc->mc_flags |= C_EOF; - return rc; /* no entries matched */ - } - mp = mc->mc_pg[mc->mc_top]; - mdb_cassert(mc, IS_LEAF(mp)); - leaf = NODEPTR(mp, 0); - } - -set1: - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - - if (IS_LEAF2(mp)) { - if (op == MDB_SET_RANGE || op == MDB_SET_KEY) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - } - return MDB_SUCCESS; - } - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - } - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (op == MDB_SET || op == MDB_SET_KEY || op == MDB_SET_RANGE) { - rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - } else { - int ex2, *ex2p; - if (op == MDB_GET_BOTH) { - ex2p = &ex2; - ex2 = 0; - } else { - ex2p = NULL; - } - rc = mdb_cursor_set(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_SET_RANGE, ex2p); - if (rc != MDB_SUCCESS) - return rc; - } - } else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) { - MDB_val olddata; - MDB_cmp_func *dcmp; - if ((rc = mdb_node_read(mc, leaf, &olddata)) != MDB_SUCCESS) - return rc; - dcmp = mc->mc_dbx->md_dcmp; - if (NEED_CMP_CLONG(dcmp, olddata.mv_size)) - dcmp = mdb_cmp_clong; - rc = dcmp(data, &olddata); - if (rc) { - if (op == MDB_GET_BOTH || rc > 0) - return MDB_NOTFOUND; - rc = 0; - } - *data = olddata; - - } else { - if (mc->mc_xcursor) - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) - return rc; - } - } - - /* The key already matches in all other cases */ - if (op == MDB_SET_RANGE || op == MDB_SET_KEY) - MDB_GET_KEY(leaf, key); - DPRINTF(("==> cursor placed on key [%s]", DKEY(key))); - - return rc; -} - -/** Move the cursor to the first item in the database. */ -static int -mdb_cursor_first(MDB_cursor *mc, MDB_val *key, MDB_val *data) -{ - int rc; - MDB_node *leaf; - - if (mc->mc_xcursor) { - MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - } - - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); - if (rc != MDB_SUCCESS) - return rc; - } - mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - - leaf = NODEPTR(mc->mc_pg[mc->mc_top], 0); - mc->mc_flags |= C_INITIALIZED; - mc->mc_flags &= ~C_EOF; - - mc->mc_ki[mc->mc_top] = 0; - - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], 0, key->mv_size); - return MDB_SUCCESS; - } - - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - rc = mdb_cursor_first(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc) - return rc; - } else { - if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) - return rc; - } - } - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -/** Move the cursor to the last item in the database. */ -static int -mdb_cursor_last(MDB_cursor *mc, MDB_val *key, MDB_val *data) -{ - int rc; - MDB_node *leaf; - - if (mc->mc_xcursor) { - MDB_CURSOR_UNREF(&mc->mc_xcursor->mx_cursor, 0); - mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - } - - if (!(mc->mc_flags & C_INITIALIZED) || mc->mc_top) { - rc = mdb_page_search(mc, NULL, MDB_PS_LAST); - if (rc != MDB_SUCCESS) - return rc; - } - mdb_cassert(mc, IS_LEAF(mc->mc_pg[mc->mc_top])); - - mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]) - 1; - mc->mc_flags |= C_INITIALIZED|C_EOF; - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], key->mv_size); - return MDB_SUCCESS; - } - - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - mdb_xcursor_init1(mc, leaf); - rc = mdb_cursor_last(&mc->mc_xcursor->mx_cursor, data, NULL); - if (rc) - return rc; - } else { - if ((rc = mdb_node_read(mc, leaf, data)) != MDB_SUCCESS) - return rc; - } - } - - MDB_GET_KEY(leaf, key); - return MDB_SUCCESS; -} - -int -mdb_cursor_get(MDB_cursor *mc, MDB_val *key, MDB_val *data, - MDB_cursor_op op) -{ - int rc; - int exact = 0; - int (*mfunc)(MDB_cursor *mc, MDB_val *key, MDB_val *data); - - if (mc == NULL) - return EINVAL; - - if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) - return MDB_BAD_TXN; - - switch (op) { - case MDB_GET_CURRENT: - if (!(mc->mc_flags & C_INITIALIZED)) { - rc = EINVAL; - } else { - MDB_page *mp = mc->mc_pg[mc->mc_top]; - int nkeys = NUMKEYS(mp); - if (!nkeys || mc->mc_ki[mc->mc_top] >= nkeys) { - mc->mc_ki[mc->mc_top] = nkeys; - rc = MDB_NOTFOUND; - break; - } - rc = MDB_SUCCESS; - if (IS_LEAF2(mp)) { - key->mv_size = mc->mc_db->md_pad; - key->mv_data = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], key->mv_size); - } else { - MDB_node *leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - MDB_GET_KEY(leaf, key); - if (data) { - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - rc = mdb_cursor_get(&mc->mc_xcursor->mx_cursor, data, NULL, MDB_GET_CURRENT); - } else { - rc = mdb_node_read(mc, leaf, data); - } - } - } - } - break; - case MDB_GET_BOTH: - case MDB_GET_BOTH_RANGE: - if (data == NULL) { - rc = EINVAL; - break; - } - if (mc->mc_xcursor == NULL) { - rc = MDB_INCOMPATIBLE; - break; - } - /* FALLTHRU */ - case MDB_SET: - case MDB_SET_KEY: - case MDB_SET_RANGE: - if (key == NULL) { - rc = EINVAL; - } else { - rc = mdb_cursor_set(mc, key, data, op, - op == MDB_SET_RANGE ? NULL : &exact); - } - break; - case MDB_GET_MULTIPLE: - if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { - rc = EINVAL; - break; - } - if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { - rc = MDB_INCOMPATIBLE; - break; - } - rc = MDB_SUCCESS; - if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) || - (mc->mc_xcursor->mx_cursor.mc_flags & C_EOF)) - break; - goto fetchm; - case MDB_NEXT_MULTIPLE: - if (data == NULL) { - rc = EINVAL; - break; - } - if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { - rc = MDB_INCOMPATIBLE; - break; - } - rc = mdb_cursor_next(mc, key, data, MDB_NEXT_DUP); - if (rc == MDB_SUCCESS) { - if (mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - MDB_cursor *mx; -fetchm: - mx = &mc->mc_xcursor->mx_cursor; - data->mv_size = NUMKEYS(mx->mc_pg[mx->mc_top]) * - mx->mc_db->md_pad; - data->mv_data = METADATA(mx->mc_pg[mx->mc_top]); - mx->mc_ki[mx->mc_top] = NUMKEYS(mx->mc_pg[mx->mc_top])-1; - } else { - rc = MDB_NOTFOUND; - } - } - break; - case MDB_PREV_MULTIPLE: - if (data == NULL) { - rc = EINVAL; - break; - } - if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { - rc = MDB_INCOMPATIBLE; - break; - } - if (!(mc->mc_flags & C_INITIALIZED)) - rc = mdb_cursor_last(mc, key, data); - else - rc = MDB_SUCCESS; - if (rc == MDB_SUCCESS) { - MDB_cursor *mx = &mc->mc_xcursor->mx_cursor; - if (mx->mc_flags & C_INITIALIZED) { - rc = mdb_cursor_sibling(mx, 0); - if (rc == MDB_SUCCESS) - goto fetchm; - } else { - rc = MDB_NOTFOUND; - } - } - break; - case MDB_NEXT: - case MDB_NEXT_DUP: - case MDB_NEXT_NODUP: - rc = mdb_cursor_next(mc, key, data, op); - break; - case MDB_PREV: - case MDB_PREV_DUP: - case MDB_PREV_NODUP: - rc = mdb_cursor_prev(mc, key, data, op); - break; - case MDB_FIRST: - rc = mdb_cursor_first(mc, key, data); - break; - case MDB_FIRST_DUP: - mfunc = mdb_cursor_first; - mmove: - if (data == NULL || !(mc->mc_flags & C_INITIALIZED)) { - rc = EINVAL; - break; - } - if (mc->mc_xcursor == NULL) { - rc = MDB_INCOMPATIBLE; - break; - } - if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) { - mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); - rc = MDB_NOTFOUND; - break; - } - { - MDB_node *leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { - MDB_GET_KEY(leaf, key); - rc = mdb_node_read(mc, leaf, data); - break; - } - } - if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) { - rc = EINVAL; - break; - } - rc = mfunc(&mc->mc_xcursor->mx_cursor, data, NULL); - break; - case MDB_LAST: - rc = mdb_cursor_last(mc, key, data); - break; - case MDB_LAST_DUP: - mfunc = mdb_cursor_last; - goto mmove; - default: - DPRINTF(("unhandled/unimplemented cursor operation %u", op)); - rc = EINVAL; - break; - } - - if (mc->mc_flags & C_DEL) - mc->mc_flags ^= C_DEL; - - return rc; -} - -/** Touch all the pages in the cursor stack. Set mc_top. - * Makes sure all the pages are writable, before attempting a write operation. - * @param[in] mc The cursor to operate on. - */ -static int -mdb_cursor_touch(MDB_cursor *mc) -{ - int rc = MDB_SUCCESS; - - if (mc->mc_dbi >= CORE_DBS && !(*mc->mc_dbflag & (DB_DIRTY|DB_DUPDATA))) { - /* Touch DB record of named DB */ - MDB_cursor mc2; - MDB_xcursor mcx; - if (TXN_DBI_CHANGED(mc->mc_txn, mc->mc_dbi)) - return MDB_BAD_DBI; - mdb_cursor_init(&mc2, mc->mc_txn, MAIN_DBI, &mcx); - rc = mdb_page_search(&mc2, &mc->mc_dbx->md_name, MDB_PS_MODIFY); - if (rc) - return rc; - *mc->mc_dbflag |= DB_DIRTY; - } - mc->mc_top = 0; - if (mc->mc_snum) { - do { - rc = mdb_page_touch(mc); - } while (!rc && ++(mc->mc_top) < mc->mc_snum); - mc->mc_top = mc->mc_snum-1; - } - return rc; -} - -/** Do not spill pages to disk if txn is getting full, may fail instead */ -#define MDB_NOSPILL 0x8000 - -int -mdb_cursor_put(MDB_cursor *mc, MDB_val *key, MDB_val *data, - unsigned int flags) -{ - MDB_env *env; - MDB_node *leaf = NULL; - MDB_page *fp, *mp, *sub_root = NULL; - uint16_t fp_flags; - MDB_val xdata, *rdata, dkey, olddata; - MDB_db dummy; - int do_sub = 0, insert_key, insert_data; - unsigned int mcount = 0, dcount = 0, nospill; - size_t nsize; - int rc, rc2; - unsigned int nflags; - DKBUF; - - if (mc == NULL || key == NULL) - return EINVAL; - - env = mc->mc_txn->mt_env; - - /* Check this first so counter will always be zero on any - * early failures. - */ - if (flags & MDB_MULTIPLE) { - dcount = data[1].mv_size; - data[1].mv_size = 0; - if (!F_ISSET(mc->mc_db->md_flags, MDB_DUPFIXED)) - return MDB_INCOMPATIBLE; - } - - nospill = flags & MDB_NOSPILL; - flags &= ~MDB_NOSPILL; - - if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) - return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; - - if (key->mv_size-1 >= ENV_MAXKEY(env)) - return MDB_BAD_VALSIZE; - -#if SIZE_MAX > MAXDATASIZE - if (data->mv_size > ((mc->mc_db->md_flags & MDB_DUPSORT) ? ENV_MAXKEY(env) : MAXDATASIZE)) - return MDB_BAD_VALSIZE; -#else - if ((mc->mc_db->md_flags & MDB_DUPSORT) && data->mv_size > ENV_MAXKEY(env)) - return MDB_BAD_VALSIZE; -#endif - - DPRINTF(("==> put db %d key [%s], size %"Z"u, data size %"Z"u", - DDBI(mc), DKEY(key), key ? key->mv_size : 0, data->mv_size)); - - dkey.mv_size = 0; - - if (flags & MDB_CURRENT) { - if (!(mc->mc_flags & C_INITIALIZED)) - return EINVAL; - rc = MDB_SUCCESS; - } else if (mc->mc_db->md_root == P_INVALID) { - /* new database, cursor has nothing to point to */ - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_flags &= ~C_INITIALIZED; - rc = MDB_NO_ROOT; - } else { - int exact = 0; - MDB_val d2; - if (flags & MDB_APPEND) { - MDB_val k2; - rc = mdb_cursor_last(mc, &k2, &d2); - if (rc == 0) { - rc = mc->mc_dbx->md_cmp(key, &k2); - if (rc > 0) { - rc = MDB_NOTFOUND; - mc->mc_ki[mc->mc_top]++; - } else { - /* new key is <= last key */ - rc = MDB_KEYEXIST; - } - } - } else { - rc = mdb_cursor_set(mc, key, &d2, MDB_SET, &exact); - } - if ((flags & MDB_NOOVERWRITE) && rc == 0) { - DPRINTF(("duplicate key [%s]", DKEY(key))); - *data = d2; - return MDB_KEYEXIST; - } - if (rc && rc != MDB_NOTFOUND) - return rc; - } - - if (mc->mc_flags & C_DEL) - mc->mc_flags ^= C_DEL; - - /* Cursor is positioned, check for room in the dirty list */ - if (!nospill) { - if (flags & MDB_MULTIPLE) { - rdata = &xdata; - xdata.mv_size = data->mv_size * dcount; - } else { - rdata = data; - } - if ((rc2 = mdb_page_spill(mc, key, rdata))) - return rc2; - } - - if (rc == MDB_NO_ROOT) { - MDB_page *np; - /* new database, write a root leaf page */ - DPUTS("allocating new root leaf page"); - if ((rc2 = mdb_page_new(mc, P_LEAF, 1, &np))) { - return rc2; - } - mdb_cursor_push(mc, np); - mc->mc_db->md_root = np->mp_pgno; - mc->mc_db->md_depth++; - *mc->mc_dbflag |= DB_DIRTY; - if ((mc->mc_db->md_flags & (MDB_DUPSORT|MDB_DUPFIXED)) - == MDB_DUPFIXED) - np->mp_flags |= P_LEAF2; - mc->mc_flags |= C_INITIALIZED; - } else { - /* make sure all cursor pages are writable */ - rc2 = mdb_cursor_touch(mc); - if (rc2) - return rc2; - } - - insert_key = insert_data = rc; - if (insert_key) { - /* The key does not exist */ - DPRINTF(("inserting key at index %i", mc->mc_ki[mc->mc_top])); - if ((mc->mc_db->md_flags & MDB_DUPSORT) && - LEAFSIZE(key, data) > env->me_nodemax) - { - /* Too big for a node, insert in sub-DB. Set up an empty - * "old sub-page" for prep_subDB to expand to a full page. - */ - fp_flags = P_LEAF|P_DIRTY; - fp = env->me_pbuf; - fp->mp_pad = data->mv_size; /* used if MDB_DUPFIXED */ - fp->mp_lower = fp->mp_upper = (PAGEHDRSZ-PAGEBASE); - olddata.mv_size = PAGEHDRSZ; - goto prep_subDB; - } - } else { - /* there's only a key anyway, so this is a no-op */ - if (IS_LEAF2(mc->mc_pg[mc->mc_top])) { - char *ptr; - unsigned int ksize = mc->mc_db->md_pad; - if (key->mv_size != ksize) - return MDB_BAD_VALSIZE; - ptr = LEAF2KEY(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top], ksize); - memcpy(ptr, key->mv_data, ksize); -fix_parent: - /* if overwriting slot 0 of leaf, need to - * update branch key if there is a parent page - */ - if (mc->mc_top && !mc->mc_ki[mc->mc_top]) { - unsigned short dtop = 1; - mc->mc_top--; - /* slot 0 is always an empty key, find real slot */ - while (mc->mc_top && !mc->mc_ki[mc->mc_top]) { - mc->mc_top--; - dtop++; - } - if (mc->mc_ki[mc->mc_top]) - rc2 = mdb_update_key(mc, key); - else - rc2 = MDB_SUCCESS; - mc->mc_top += dtop; - if (rc2) - return rc2; - } - return MDB_SUCCESS; - } - -more: - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - olddata.mv_size = NODEDSZ(leaf); - olddata.mv_data = NODEDATA(leaf); - - /* DB has dups? */ - if (F_ISSET(mc->mc_db->md_flags, MDB_DUPSORT)) { - /* Prepare (sub-)page/sub-DB to accept the new item, - * if needed. fp: old sub-page or a header faking - * it. mp: new (sub-)page. offset: growth in page - * size. xdata: node data with new page or DB. - */ - unsigned i, offset = 0; - mp = fp = xdata.mv_data = env->me_pbuf; - mp->mp_pgno = mc->mc_pg[mc->mc_top]->mp_pgno; - - /* Was a single item before, must convert now */ - if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { - MDB_cmp_func *dcmp; - /* Just overwrite the current item */ - if (flags == MDB_CURRENT) - goto current; - dcmp = mc->mc_dbx->md_dcmp; - if (NEED_CMP_CLONG(dcmp, olddata.mv_size)) - dcmp = mdb_cmp_clong; - /* does data match? */ - if (!dcmp(data, &olddata)) { - if (flags & (MDB_NODUPDATA|MDB_APPENDDUP)) - return MDB_KEYEXIST; - /* overwrite it */ - goto current; - } - - /* Back up original data item */ - dkey.mv_size = olddata.mv_size; - dkey.mv_data = memcpy(fp+1, olddata.mv_data, olddata.mv_size); - - /* Make sub-page header for the dup items, with dummy body */ - fp->mp_flags = P_LEAF|P_DIRTY|P_SUBP; - fp->mp_lower = (PAGEHDRSZ-PAGEBASE); - xdata.mv_size = PAGEHDRSZ + dkey.mv_size + data->mv_size; - if (mc->mc_db->md_flags & MDB_DUPFIXED) { - fp->mp_flags |= P_LEAF2; - fp->mp_pad = data->mv_size; - xdata.mv_size += 2 * data->mv_size; /* leave space for 2 more */ - } else { - xdata.mv_size += 2 * (sizeof(indx_t) + NODESIZE) + - (dkey.mv_size & 1) + (data->mv_size & 1); - } - fp->mp_upper = xdata.mv_size - PAGEBASE; - olddata.mv_size = xdata.mv_size; /* pretend olddata is fp */ - } else if (leaf->mn_flags & F_SUBDATA) { - /* Data is on sub-DB, just store it */ - flags |= F_DUPDATA|F_SUBDATA; - goto put_sub; - } else { - /* Data is on sub-page */ - fp = olddata.mv_data; - switch (flags) { - default: - if (!(mc->mc_db->md_flags & MDB_DUPFIXED)) { - offset = EVEN(NODESIZE + sizeof(indx_t) + - data->mv_size); - break; - } - offset = fp->mp_pad; - if (SIZELEFT(fp) < offset) { - offset *= 4; /* space for 4 more */ - break; - } - /* FALLTHRU: Big enough MDB_DUPFIXED sub-page */ - case MDB_CURRENT: - fp->mp_flags |= P_DIRTY; - COPY_PGNO(fp->mp_pgno, mp->mp_pgno); - mc->mc_xcursor->mx_cursor.mc_pg[0] = fp; - flags |= F_DUPDATA; - goto put_sub; - } - xdata.mv_size = olddata.mv_size + offset; - } - - fp_flags = fp->mp_flags; - if (NODESIZE + NODEKSZ(leaf) + xdata.mv_size > env->me_nodemax) { - /* Too big for a sub-page, convert to sub-DB */ - fp_flags &= ~P_SUBP; -prep_subDB: - if (mc->mc_db->md_flags & MDB_DUPFIXED) { - fp_flags |= P_LEAF2; - dummy.md_pad = fp->mp_pad; - dummy.md_flags = MDB_DUPFIXED; - if (mc->mc_db->md_flags & MDB_INTEGERDUP) - dummy.md_flags |= MDB_INTEGERKEY; - } else { - dummy.md_pad = 0; - dummy.md_flags = 0; - } - dummy.md_depth = 1; - dummy.md_branch_pages = 0; - dummy.md_leaf_pages = 1; - dummy.md_overflow_pages = 0; - dummy.md_entries = NUMKEYS(fp); - xdata.mv_size = sizeof(MDB_db); - xdata.mv_data = &dummy; - if ((rc = mdb_page_alloc(mc, 1, &mp))) - return rc; - offset = env->me_psize - olddata.mv_size; - flags |= F_DUPDATA|F_SUBDATA; - dummy.md_root = mp->mp_pgno; - sub_root = mp; - } - if (mp != fp) { - mp->mp_flags = fp_flags | P_DIRTY; - mp->mp_pad = fp->mp_pad; - mp->mp_lower = fp->mp_lower; - mp->mp_upper = fp->mp_upper + offset; - if (fp_flags & P_LEAF2) { - memcpy(METADATA(mp), METADATA(fp), NUMKEYS(fp) * fp->mp_pad); - } else { - memcpy((char *)mp + mp->mp_upper + PAGEBASE, (char *)fp + fp->mp_upper + PAGEBASE, - olddata.mv_size - fp->mp_upper - PAGEBASE); - memcpy((char *)(&mp->mp_ptrs), (char *)(&fp->mp_ptrs), NUMKEYS(fp) * sizeof(mp->mp_ptrs[0])); - for (i=0; imp_ptrs[i] += offset; - } - } - - rdata = &xdata; - flags |= F_DUPDATA; - do_sub = 1; - if (!insert_key) - mdb_node_del(mc, 0); - goto new_sub; - } -current: - /* LMDB passes F_SUBDATA in 'flags' to write a DB record */ - if ((leaf->mn_flags ^ flags) & F_SUBDATA) - return MDB_INCOMPATIBLE; - /* overflow page overwrites need special handling */ - if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { - MDB_page *omp; - pgno_t pg; - int level, ovpages, dpages = OVPAGES(data->mv_size, env->me_psize); - - memcpy(&pg, olddata.mv_data, sizeof(pg)); - if ((rc2 = mdb_page_get(mc, pg, &omp, &level)) != 0) - return rc2; - ovpages = omp->mp_pages; - - /* Is the ov page large enough? */ - if (ovpages >= dpages) { - if (!(omp->mp_flags & P_DIRTY) && - (level || (env->me_flags & MDB_WRITEMAP))) - { - rc = mdb_page_unspill(mc->mc_txn, omp, &omp); - if (rc) - return rc; - level = 0; /* dirty in this txn or clean */ - } - /* Is it dirty? */ - if (omp->mp_flags & P_DIRTY) { - /* yes, overwrite it. Note in this case we don't - * bother to try shrinking the page if the new data - * is smaller than the overflow threshold. - */ - if (level > 1) { - /* It is writable only in a parent txn */ - size_t sz = (size_t) env->me_psize * ovpages, off; - MDB_page *np = mdb_page_malloc(mc->mc_txn, ovpages); - MDB_ID2 id2; - if (!np) - return ENOMEM; - id2.mid = pg; - id2.mptr = np; - /* Note - this page is already counted in parent's dirty_room */ - rc2 = mdb_mid2l_insert(mc->mc_txn->mt_u.dirty_list, &id2); - mdb_cassert(mc, rc2 == 0); - /* Currently we make the page look as with put() in the - * parent txn, in case the user peeks at MDB_RESERVEd - * or unused parts. Some users treat ovpages specially. - */ - if (!(flags & MDB_RESERVE)) { - /* Skip the part where LMDB will put *data. - * Copy end of page, adjusting alignment so - * compiler may copy words instead of bytes. - */ - off = (PAGEHDRSZ + data->mv_size) & -sizeof(size_t); - memcpy((size_t *)((char *)np + off), - (size_t *)((char *)omp + off), sz - off); - sz = PAGEHDRSZ; - } - memcpy(np, omp, sz); /* Copy beginning of page */ - omp = np; - } - SETDSZ(leaf, data->mv_size); - if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = METADATA(omp); - else - memcpy(METADATA(omp), data->mv_data, data->mv_size); - return MDB_SUCCESS; - } - } - if ((rc2 = mdb_ovpage_free(mc, omp)) != MDB_SUCCESS) - return rc2; - } else if (data->mv_size == olddata.mv_size) { - /* same size, just replace it. Note that we could - * also reuse this node if the new data is smaller, - * but instead we opt to shrink the node in that case. - */ - if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = olddata.mv_data; - else if (!(mc->mc_flags & C_SUB)) - memcpy(olddata.mv_data, data->mv_data, data->mv_size); - else { - memcpy(NODEKEY(leaf), key->mv_data, key->mv_size); - goto fix_parent; - } - return MDB_SUCCESS; - } - mdb_node_del(mc, 0); - } - - rdata = data; - -new_sub: - nflags = flags & NODE_ADD_FLAGS; - nsize = IS_LEAF2(mc->mc_pg[mc->mc_top]) ? key->mv_size : mdb_leaf_size(env, key, rdata); - if (SIZELEFT(mc->mc_pg[mc->mc_top]) < nsize) { - if (( flags & (F_DUPDATA|F_SUBDATA)) == F_DUPDATA ) - nflags &= ~MDB_APPEND; /* sub-page may need room to grow */ - if (!insert_key) - nflags |= MDB_SPLIT_REPLACE; - rc = mdb_page_split(mc, key, rdata, P_INVALID, nflags); - } else { - /* There is room already in this leaf page. */ - rc = mdb_node_add(mc, mc->mc_ki[mc->mc_top], key, rdata, 0, nflags); - if (rc == 0) { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - unsigned i = mc->mc_top; - MDB_page *mp = mc->mc_pg[i]; - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == mc || m3->mc_snum < mc->mc_snum || m3->mc_pg[i] != mp) continue; - if (m3->mc_ki[i] >= mc->mc_ki[i] && insert_key) { - m3->mc_ki[i]++; - } - XCURSOR_REFRESH(m3, i, mp); - } - } - } - - if (rc == MDB_SUCCESS) { - /* Now store the actual data in the child DB. Note that we're - * storing the user data in the keys field, so there are strict - * size limits on dupdata. The actual data fields of the child - * DB are all zero size. - */ - if (do_sub) { - int xflags, new_dupdata; - mdb_size_t ecount; -put_sub: - xdata.mv_size = 0; - xdata.mv_data = ""; - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (flags == MDB_CURRENT) { - xflags = MDB_CURRENT|MDB_NOSPILL; - } else { - mdb_xcursor_init1(mc, leaf); - xflags = (flags & MDB_NODUPDATA) ? - MDB_NOOVERWRITE|MDB_NOSPILL : MDB_NOSPILL; - } - if (sub_root) - mc->mc_xcursor->mx_cursor.mc_pg[0] = sub_root; - new_dupdata = (int)dkey.mv_size; - /* converted, write the original data first */ - if (dkey.mv_size) { - rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, &dkey, &xdata, xflags); - if (rc) - goto bad_sub; - /* we've done our job */ - dkey.mv_size = 0; - } - if (!(leaf->mn_flags & F_SUBDATA) || sub_root) { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2; - MDB_xcursor *mx = mc->mc_xcursor; - unsigned i = mc->mc_top; - MDB_page *mp = mc->mc_pg[i]; - - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; - if (!(m2->mc_flags & C_INITIALIZED)) continue; - if (m2->mc_pg[i] == mp) { - if (m2->mc_ki[i] == mc->mc_ki[i]) { - mdb_xcursor_init2(m2, mx, new_dupdata); - } else if (!insert_key) { - XCURSOR_REFRESH(m2, i, mp); - } - } - } - } - ecount = mc->mc_xcursor->mx_db.md_entries; - if (flags & MDB_APPENDDUP) - xflags |= MDB_APPEND; - rc = mdb_cursor_put(&mc->mc_xcursor->mx_cursor, data, &xdata, xflags); - if (flags & F_SUBDATA) { - void *db = NODEDATA(leaf); - memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); - } - insert_data = mc->mc_xcursor->mx_db.md_entries - ecount; - } - /* Increment count unless we just replaced an existing item. */ - if (insert_data) - mc->mc_db->md_entries++; - if (insert_key) { - /* Invalidate txn if we created an empty sub-DB */ - if (rc) - goto bad_sub; - /* If we succeeded and the key didn't exist before, - * make sure the cursor is marked valid. - */ - mc->mc_flags |= C_INITIALIZED; - } - if (flags & MDB_MULTIPLE) { - if (!rc) { - mcount++; - /* let caller know how many succeeded, if any */ - data[1].mv_size = mcount; - if (mcount < dcount) { - data[0].mv_data = (char *)data[0].mv_data + data[0].mv_size; - insert_key = insert_data = 0; - goto more; - } - } - } - return rc; -bad_sub: - if (rc == MDB_KEYEXIST) /* should not happen, we deleted that item */ - rc = MDB_PROBLEM; - } - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_cursor_del(MDB_cursor *mc, unsigned int flags) -{ - MDB_node *leaf; - MDB_page *mp; - int rc; - - if (mc->mc_txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) - return (mc->mc_txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; - - if (!(mc->mc_flags & C_INITIALIZED)) - return EINVAL; - - if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) - return MDB_NOTFOUND; - - if (!(flags & MDB_NOSPILL) && (rc = mdb_page_spill(mc, NULL, NULL))) - return rc; - - rc = mdb_cursor_touch(mc); - if (rc) - return rc; - - mp = mc->mc_pg[mc->mc_top]; - if (IS_LEAF2(mp)) - goto del_key; - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - - if (F_ISSET(leaf->mn_flags, F_DUPDATA)) { - if (flags & MDB_NODUPDATA) { - /* mdb_cursor_del0() will subtract the final entry */ - mc->mc_db->md_entries -= mc->mc_xcursor->mx_db.md_entries - 1; - mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; - } else { - if (!F_ISSET(leaf->mn_flags, F_SUBDATA)) { - mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - } - rc = mdb_cursor_del(&mc->mc_xcursor->mx_cursor, MDB_NOSPILL); - if (rc) - return rc; - /* If sub-DB still has entries, we're done */ - if (mc->mc_xcursor->mx_db.md_entries) { - if (leaf->mn_flags & F_SUBDATA) { - /* update subDB info */ - void *db = NODEDATA(leaf); - memcpy(db, &mc->mc_xcursor->mx_db, sizeof(MDB_db)); - } else { - MDB_cursor *m2; - /* shrink fake page */ - mdb_node_shrink(mp, mc->mc_ki[mc->mc_top]); - leaf = NODEPTR(mp, mc->mc_ki[mc->mc_top]); - mc->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(leaf); - /* fix other sub-DB cursors pointed at fake pages on this page */ - for (m2 = mc->mc_txn->mt_cursors[mc->mc_dbi]; m2; m2=m2->mc_next) { - if (m2 == mc || m2->mc_snum < mc->mc_snum) continue; - if (!(m2->mc_flags & C_INITIALIZED)) continue; - if (m2->mc_pg[mc->mc_top] == mp) { - XCURSOR_REFRESH(m2, mc->mc_top, mp); - } - } - } - mc->mc_db->md_entries--; - return rc; - } else { - mc->mc_xcursor->mx_cursor.mc_flags &= ~C_INITIALIZED; - } - /* otherwise fall thru and delete the sub-DB */ - } - - if (leaf->mn_flags & F_SUBDATA) { - /* add all the child DB's pages to the free list */ - rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); - if (rc) - goto fail; - } - } - /* LMDB passes F_SUBDATA in 'flags' to delete a DB record */ - else if ((leaf->mn_flags ^ flags) & F_SUBDATA) { - rc = MDB_INCOMPATIBLE; - goto fail; - } - - /* add overflow pages to free list */ - if (F_ISSET(leaf->mn_flags, F_BIGDATA)) { - MDB_page *omp; - pgno_t pg; - - memcpy(&pg, NODEDATA(leaf), sizeof(pg)); - if ((rc = mdb_page_get(mc, pg, &omp, NULL)) || - (rc = mdb_ovpage_free(mc, omp))) - goto fail; - } - -del_key: - return mdb_cursor_del0(mc); - -fail: - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -/** Allocate and initialize new pages for a database. - * Set #MDB_TXN_ERROR on failure. - * @param[in] mc a cursor on the database being added to. - * @param[in] flags flags defining what type of page is being allocated. - * @param[in] num the number of pages to allocate. This is usually 1, - * unless allocating overflow pages for a large record. - * @param[out] mp Address of a page, or NULL on failure. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp) -{ - MDB_page *np; - int rc; - - if ((rc = mdb_page_alloc(mc, num, &np))) - return rc; - DPRINTF(("allocated new mpage %"Yu", page size %u", - np->mp_pgno, mc->mc_txn->mt_env->me_psize)); - np->mp_flags = flags | P_DIRTY; - np->mp_lower = (PAGEHDRSZ-PAGEBASE); - np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE; - - if (IS_BRANCH(np)) - mc->mc_db->md_branch_pages++; - else if (IS_LEAF(np)) - mc->mc_db->md_leaf_pages++; - else if (IS_OVERFLOW(np)) { - mc->mc_db->md_overflow_pages += num; - np->mp_pages = num; - } - *mp = np; - - return 0; -} - -/** Calculate the size of a leaf node. - * The size depends on the environment's page size; if a data item - * is too large it will be put onto an overflow page and the node - * size will only include the key and not the data. Sizes are always - * rounded up to an even number of bytes, to guarantee 2-byte alignment - * of the #MDB_node headers. - * @param[in] env The environment handle. - * @param[in] key The key for the node. - * @param[in] data The data for the node. - * @return The number of bytes needed to store the node. - */ -static size_t -mdb_leaf_size(MDB_env *env, MDB_val *key, MDB_val *data) -{ - size_t sz; - - sz = LEAFSIZE(key, data); - if (sz > env->me_nodemax) { - /* put on overflow page */ - sz -= data->mv_size - sizeof(pgno_t); - } - - return EVEN(sz + sizeof(indx_t)); -} - -/** Calculate the size of a branch node. - * The size should depend on the environment's page size but since - * we currently don't support spilling large keys onto overflow - * pages, it's simply the size of the #MDB_node header plus the - * size of the key. Sizes are always rounded up to an even number - * of bytes, to guarantee 2-byte alignment of the #MDB_node headers. - * @param[in] env The environment handle. - * @param[in] key The key for the node. - * @return The number of bytes needed to store the node. - */ -static size_t -mdb_branch_size(MDB_env *env, MDB_val *key) -{ - size_t sz; - - sz = INDXSIZE(key); - if (sz > env->me_nodemax) { - /* put on overflow page */ - /* not implemented */ - /* sz -= key->size - sizeof(pgno_t); */ - } - - return sz + sizeof(indx_t); -} - -/** Add a node to the page pointed to by the cursor. - * Set #MDB_TXN_ERROR on failure. - * @param[in] mc The cursor for this operation. - * @param[in] indx The index on the page where the new node should be added. - * @param[in] key The key for the new node. - * @param[in] data The data for the new node, if any. - * @param[in] pgno The page number, if adding a branch node. - * @param[in] flags Flags for the node. - * @return 0 on success, non-zero on failure. Possible errors are: - *
    - *
  • ENOMEM - failed to allocate overflow pages for the node. - *
  • MDB_PAGE_FULL - there is insufficient room in the page. This error - * should never happen since all callers already calculate the - * page's free space before calling this function. - *
- */ -static int -mdb_node_add(MDB_cursor *mc, indx_t indx, - MDB_val *key, MDB_val *data, pgno_t pgno, unsigned int flags) -{ - unsigned int i; - size_t node_size = NODESIZE; - ssize_t room; - indx_t ofs; - MDB_node *node; - MDB_page *mp = mc->mc_pg[mc->mc_top]; - MDB_page *ofp = NULL; /* overflow page */ - void *ndata; - DKBUF; - - mdb_cassert(mc, mp->mp_upper >= mp->mp_lower); - - DPRINTF(("add to %s %spage %"Yu" index %i, data size %"Z"u key size %"Z"u [%s]", - IS_LEAF(mp) ? "leaf" : "branch", - IS_SUBP(mp) ? "sub-" : "", - mdb_dbg_pgno(mp), indx, data ? data->mv_size : 0, - key ? key->mv_size : 0, key ? DKEY(key) : "null")); - - if (IS_LEAF2(mp)) { - /* Move higher keys up one slot. */ - int ksize = mc->mc_db->md_pad, dif; - char *ptr = LEAF2KEY(mp, indx, ksize); - dif = NUMKEYS(mp) - indx; - if (dif > 0) - memmove(ptr+ksize, ptr, dif*ksize); - /* insert new key */ - memcpy(ptr, key->mv_data, ksize); - - /* Just using these for counting */ - mp->mp_lower += sizeof(indx_t); - mp->mp_upper -= ksize - sizeof(indx_t); - return MDB_SUCCESS; - } - - room = (ssize_t)SIZELEFT(mp) - (ssize_t)sizeof(indx_t); - if (key != NULL) - node_size += key->mv_size; - if (IS_LEAF(mp)) { - mdb_cassert(mc, key && data); - if (F_ISSET(flags, F_BIGDATA)) { - /* Data already on overflow page. */ - node_size += sizeof(pgno_t); - } else if (node_size + data->mv_size > mc->mc_txn->mt_env->me_nodemax) { - int ovpages = OVPAGES(data->mv_size, mc->mc_txn->mt_env->me_psize); - int rc; - /* Put data on overflow page. */ - DPRINTF(("data size is %"Z"u, node would be %"Z"u, put data on overflow page", - data->mv_size, node_size+data->mv_size)); - node_size = EVEN(node_size + sizeof(pgno_t)); - if ((ssize_t)node_size > room) - goto full; - if ((rc = mdb_page_new(mc, P_OVERFLOW, ovpages, &ofp))) - return rc; - DPRINTF(("allocated overflow page %"Yu, ofp->mp_pgno)); - flags |= F_BIGDATA; - goto update; - } else { - node_size += data->mv_size; - } - } - node_size = EVEN(node_size); - if ((ssize_t)node_size > room) - goto full; - -update: - /* Move higher pointers up one slot. */ - for (i = NUMKEYS(mp); i > indx; i--) - mp->mp_ptrs[i] = mp->mp_ptrs[i - 1]; - - /* Adjust free space offsets. */ - ofs = mp->mp_upper - node_size; - mdb_cassert(mc, ofs >= mp->mp_lower + sizeof(indx_t)); - mp->mp_ptrs[indx] = ofs; - mp->mp_upper = ofs; - mp->mp_lower += sizeof(indx_t); - - /* Write the node data. */ - node = NODEPTR(mp, indx); - node->mn_ksize = (key == NULL) ? 0 : key->mv_size; - node->mn_flags = flags; - if (IS_LEAF(mp)) - SETDSZ(node,data->mv_size); - else - SETPGNO(node,pgno); - - if (key) - memcpy(NODEKEY(node), key->mv_data, key->mv_size); - - if (IS_LEAF(mp)) { - ndata = NODEDATA(node); - if (ofp == NULL) { - if (F_ISSET(flags, F_BIGDATA)) - memcpy(ndata, data->mv_data, sizeof(pgno_t)); - else if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = ndata; - else - memcpy(ndata, data->mv_data, data->mv_size); - } else { - memcpy(ndata, &ofp->mp_pgno, sizeof(pgno_t)); - ndata = METADATA(ofp); - if (F_ISSET(flags, MDB_RESERVE)) - data->mv_data = ndata; - else - memcpy(ndata, data->mv_data, data->mv_size); - } - } - - return MDB_SUCCESS; - -full: - DPRINTF(("not enough room in page %"Yu", got %u ptrs", - mdb_dbg_pgno(mp), NUMKEYS(mp))); - DPRINTF(("upper-lower = %u - %u = %"Z"d", mp->mp_upper,mp->mp_lower,room)); - DPRINTF(("node size = %"Z"u", node_size)); - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return MDB_PAGE_FULL; -} - -/** Delete the specified node from a page. - * @param[in] mc Cursor pointing to the node to delete. - * @param[in] ksize The size of a node. Only used if the page is - * part of a #MDB_DUPFIXED database. - */ -static void -mdb_node_del(MDB_cursor *mc, int ksize) -{ - MDB_page *mp = mc->mc_pg[mc->mc_top]; - indx_t indx = mc->mc_ki[mc->mc_top]; - unsigned int sz; - indx_t i, j, numkeys, ptr; - MDB_node *node; - char *base; - - DPRINTF(("delete node %u on %s page %"Yu, indx, - IS_LEAF(mp) ? "leaf" : "branch", mdb_dbg_pgno(mp))); - numkeys = NUMKEYS(mp); - mdb_cassert(mc, indx < numkeys); - - if (IS_LEAF2(mp)) { - int x = numkeys - 1 - indx; - base = LEAF2KEY(mp, indx, ksize); - if (x) - memmove(base, base + ksize, x * ksize); - mp->mp_lower -= sizeof(indx_t); - mp->mp_upper += ksize - sizeof(indx_t); - return; - } - - node = NODEPTR(mp, indx); - sz = NODESIZE + node->mn_ksize; - if (IS_LEAF(mp)) { - if (F_ISSET(node->mn_flags, F_BIGDATA)) - sz += sizeof(pgno_t); - else - sz += NODEDSZ(node); - } - sz = EVEN(sz); - - ptr = mp->mp_ptrs[indx]; - for (i = j = 0; i < numkeys; i++) { - if (i != indx) { - mp->mp_ptrs[j] = mp->mp_ptrs[i]; - if (mp->mp_ptrs[i] < ptr) - mp->mp_ptrs[j] += sz; - j++; - } - } - - base = (char *)mp + mp->mp_upper + PAGEBASE; - memmove(base + sz, base, ptr - mp->mp_upper); - - mp->mp_lower -= sizeof(indx_t); - mp->mp_upper += sz; -} - -/** Compact the main page after deleting a node on a subpage. - * @param[in] mp The main page to operate on. - * @param[in] indx The index of the subpage on the main page. - */ -static void -mdb_node_shrink(MDB_page *mp, indx_t indx) -{ - MDB_node *node; - MDB_page *sp, *xp; - char *base; - indx_t delta, nsize, len, ptr; - int i; - - node = NODEPTR(mp, indx); - sp = (MDB_page *)NODEDATA(node); - delta = SIZELEFT(sp); - nsize = NODEDSZ(node) - delta; - - /* Prepare to shift upward, set len = length(subpage part to shift) */ - if (IS_LEAF2(sp)) { - len = nsize; - if (nsize & 1) - return; /* do not make the node uneven-sized */ - } else { - xp = (MDB_page *)((char *)sp + delta); /* destination subpage */ - for (i = NUMKEYS(sp); --i >= 0; ) - xp->mp_ptrs[i] = sp->mp_ptrs[i] - delta; - len = PAGEHDRSZ; - } - sp->mp_upper = sp->mp_lower; - COPY_PGNO(sp->mp_pgno, mp->mp_pgno); - SETDSZ(node, nsize); - - /* Shift upward */ - base = (char *)mp + mp->mp_upper + PAGEBASE; - memmove(base + delta, base, (char *)sp + len - base); - - ptr = mp->mp_ptrs[indx]; - for (i = NUMKEYS(mp); --i >= 0; ) { - if (mp->mp_ptrs[i] <= ptr) - mp->mp_ptrs[i] += delta; - } - mp->mp_upper += delta; -} - -/** Initial setup of a sorted-dups cursor. - * Sorted duplicates are implemented as a sub-database for the given key. - * The duplicate data items are actually keys of the sub-database. - * Operations on the duplicate data items are performed using a sub-cursor - * initialized when the sub-database is first accessed. This function does - * the preliminary setup of the sub-cursor, filling in the fields that - * depend only on the parent DB. - * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. - */ -static void -mdb_xcursor_init0(MDB_cursor *mc) -{ - MDB_xcursor *mx = mc->mc_xcursor; - - mx->mx_cursor.mc_xcursor = NULL; - mx->mx_cursor.mc_txn = mc->mc_txn; - mx->mx_cursor.mc_db = &mx->mx_db; - mx->mx_cursor.mc_dbx = &mx->mx_dbx; - mx->mx_cursor.mc_dbi = mc->mc_dbi; - mx->mx_cursor.mc_dbflag = &mx->mx_dbflag; - mx->mx_cursor.mc_snum = 0; - mx->mx_cursor.mc_top = 0; - MC_SET_OVPG(&mx->mx_cursor, NULL); - mx->mx_cursor.mc_flags = C_SUB | (mc->mc_flags & (C_ORIG_RDONLY|C_WRITEMAP)); - mx->mx_dbx.md_name.mv_size = 0; - mx->mx_dbx.md_name.mv_data = NULL; - mx->mx_dbx.md_cmp = mc->mc_dbx->md_dcmp; - mx->mx_dbx.md_dcmp = NULL; - mx->mx_dbx.md_rel = mc->mc_dbx->md_rel; -} - -/** Final setup of a sorted-dups cursor. - * Sets up the fields that depend on the data from the main cursor. - * @param[in] mc The main cursor whose sorted-dups cursor is to be initialized. - * @param[in] node The data containing the #MDB_db record for the - * sorted-dup database. - */ -static void -mdb_xcursor_init1(MDB_cursor *mc, MDB_node *node) -{ - MDB_xcursor *mx = mc->mc_xcursor; - - mx->mx_cursor.mc_flags &= C_SUB|C_ORIG_RDONLY|C_WRITEMAP; - if (node->mn_flags & F_SUBDATA) { - memcpy(&mx->mx_db, NODEDATA(node), sizeof(MDB_db)); - mx->mx_cursor.mc_pg[0] = 0; - mx->mx_cursor.mc_snum = 0; - mx->mx_cursor.mc_top = 0; - } else { - MDB_page *fp = NODEDATA(node); - mx->mx_db.md_pad = 0; - mx->mx_db.md_flags = 0; - mx->mx_db.md_depth = 1; - mx->mx_db.md_branch_pages = 0; - mx->mx_db.md_leaf_pages = 1; - mx->mx_db.md_overflow_pages = 0; - mx->mx_db.md_entries = NUMKEYS(fp); - COPY_PGNO(mx->mx_db.md_root, fp->mp_pgno); - mx->mx_cursor.mc_snum = 1; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags |= C_INITIALIZED; - mx->mx_cursor.mc_pg[0] = fp; - mx->mx_cursor.mc_ki[0] = 0; - if (mc->mc_db->md_flags & MDB_DUPFIXED) { - mx->mx_db.md_flags = MDB_DUPFIXED; - mx->mx_db.md_pad = fp->mp_pad; - if (mc->mc_db->md_flags & MDB_INTEGERDUP) - mx->mx_db.md_flags |= MDB_INTEGERKEY; - } - } - DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi, - mx->mx_db.md_root)); - mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA; - if (NEED_CMP_CLONG(mx->mx_dbx.md_cmp, mx->mx_db.md_pad)) - mx->mx_dbx.md_cmp = mdb_cmp_clong; -} - - -/** Fixup a sorted-dups cursor due to underlying update. - * Sets up some fields that depend on the data from the main cursor. - * Almost the same as init1, but skips initialization steps if the - * xcursor had already been used. - * @param[in] mc The main cursor whose sorted-dups cursor is to be fixed up. - * @param[in] src_mx The xcursor of an up-to-date cursor. - * @param[in] new_dupdata True if converting from a non-#F_DUPDATA item. - */ -static void -mdb_xcursor_init2(MDB_cursor *mc, MDB_xcursor *src_mx, int new_dupdata) -{ - MDB_xcursor *mx = mc->mc_xcursor; - - if (new_dupdata) { - mx->mx_cursor.mc_snum = 1; - mx->mx_cursor.mc_top = 0; - mx->mx_cursor.mc_flags |= C_INITIALIZED; - mx->mx_cursor.mc_ki[0] = 0; - mx->mx_dbflag = DB_VALID|DB_USRVALID|DB_DUPDATA; -#if UINT_MAX < MDB_SIZE_MAX /* matches mdb_xcursor_init1:NEED_CMP_CLONG() */ - mx->mx_dbx.md_cmp = src_mx->mx_dbx.md_cmp; -#endif - } else if (!(mx->mx_cursor.mc_flags & C_INITIALIZED)) { - return; - } - mx->mx_db = src_mx->mx_db; - mx->mx_cursor.mc_pg[0] = src_mx->mx_cursor.mc_pg[0]; - DPRINTF(("Sub-db -%u root page %"Yu, mx->mx_cursor.mc_dbi, - mx->mx_db.md_root)); -} - -/** Initialize a cursor for a given transaction and database. */ -static void -mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx) -{ - mc->mc_next = NULL; - mc->mc_backup = NULL; - mc->mc_dbi = dbi; - mc->mc_txn = txn; - mc->mc_db = &txn->mt_dbs[dbi]; - mc->mc_dbx = &txn->mt_dbxs[dbi]; - mc->mc_dbflag = &txn->mt_dbflags[dbi]; - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_pg[0] = 0; - mc->mc_ki[0] = 0; - MC_SET_OVPG(mc, NULL); - mc->mc_flags = txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP); - if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) { - mdb_tassert(txn, mx != NULL); - mc->mc_xcursor = mx; - mdb_xcursor_init0(mc); - } else { - mc->mc_xcursor = NULL; - } - if (*mc->mc_dbflag & DB_STALE) { - mdb_page_search(mc, NULL, MDB_PS_ROOTONLY); - } -} - -int -mdb_cursor_open(MDB_txn *txn, MDB_dbi dbi, MDB_cursor **ret) -{ - MDB_cursor *mc; - size_t size = sizeof(MDB_cursor); - - if (!ret || !TXN_DBI_EXIST(txn, dbi, DB_VALID)) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_BLOCKED) - return MDB_BAD_TXN; - - if (dbi == FREE_DBI && !F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) - return EINVAL; - - if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) - size += sizeof(MDB_xcursor); - - if ((mc = malloc(size)) != NULL) { - mdb_cursor_init(mc, txn, dbi, (MDB_xcursor *)(mc + 1)); - if (txn->mt_cursors) { - mc->mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = mc; - mc->mc_flags |= C_UNTRACK; - } - } else { - return ENOMEM; - } - - *ret = mc; - - return MDB_SUCCESS; -} - -int -mdb_cursor_renew(MDB_txn *txn, MDB_cursor *mc) -{ - if (!mc || !TXN_DBI_EXIST(txn, mc->mc_dbi, DB_VALID)) - return EINVAL; - - if ((mc->mc_flags & C_UNTRACK) || txn->mt_cursors) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_BLOCKED) - return MDB_BAD_TXN; - - mdb_cursor_init(mc, txn, mc->mc_dbi, mc->mc_xcursor); - return MDB_SUCCESS; -} - -/* Return the count of duplicate data items for the current key */ -int -mdb_cursor_count(MDB_cursor *mc, mdb_size_t *countp) -{ - MDB_node *leaf; - - if (mc == NULL || countp == NULL) - return EINVAL; - - if (mc->mc_xcursor == NULL) - return MDB_INCOMPATIBLE; - - if (mc->mc_txn->mt_flags & MDB_TXN_BLOCKED) - return MDB_BAD_TXN; - - if (!(mc->mc_flags & C_INITIALIZED)) - return EINVAL; - - if (!mc->mc_snum) - return MDB_NOTFOUND; - - if (mc->mc_flags & C_EOF) { - if (mc->mc_ki[mc->mc_top] >= NUMKEYS(mc->mc_pg[mc->mc_top])) - return MDB_NOTFOUND; - mc->mc_flags ^= C_EOF; - } - - leaf = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) { - *countp = 1; - } else { - if (!(mc->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED)) - return EINVAL; - - *countp = mc->mc_xcursor->mx_db.md_entries; - } - return MDB_SUCCESS; -} - -void -mdb_cursor_close(MDB_cursor *mc) -{ - if (mc) { - MDB_CURSOR_UNREF(mc, 0); - } - if (mc && !mc->mc_backup) { - /* Remove from txn, if tracked. - * A read-only txn (!C_UNTRACK) may have been freed already, - * so do not peek inside it. Only write txns track cursors. - */ - if ((mc->mc_flags & C_UNTRACK) && mc->mc_txn->mt_cursors) { - MDB_cursor **prev = &mc->mc_txn->mt_cursors[mc->mc_dbi]; - while (*prev && *prev != mc) prev = &(*prev)->mc_next; - if (*prev == mc) - *prev = mc->mc_next; - } - free(mc); - } -} - -MDB_txn * -mdb_cursor_txn(MDB_cursor *mc) -{ - if (!mc) return NULL; - return mc->mc_txn; -} - -MDB_dbi -mdb_cursor_dbi(MDB_cursor *mc) -{ - return mc->mc_dbi; -} - -/** Replace the key for a branch node with a new key. - * Set #MDB_TXN_ERROR on failure. - * @param[in] mc Cursor pointing to the node to operate on. - * @param[in] key The new key to use. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_update_key(MDB_cursor *mc, MDB_val *key) -{ - MDB_page *mp; - MDB_node *node; - char *base; - size_t len; - int delta, ksize, oksize; - indx_t ptr, i, numkeys, indx; - DKBUF; - - indx = mc->mc_ki[mc->mc_top]; - mp = mc->mc_pg[mc->mc_top]; - node = NODEPTR(mp, indx); - ptr = mp->mp_ptrs[indx]; -#if MDB_DEBUG - { - MDB_val k2; - char kbuf2[DKBUF_MAXKEYSIZE*2+1]; - k2.mv_data = NODEKEY(node); - k2.mv_size = node->mn_ksize; - DPRINTF(("update key %u (ofs %u) [%s] to [%s] on page %"Yu, - indx, ptr, - mdb_dkey(&k2, kbuf2), - DKEY(key), - mp->mp_pgno)); - } -#endif - - /* Sizes must be 2-byte aligned. */ - ksize = EVEN(key->mv_size); - oksize = EVEN(node->mn_ksize); - delta = ksize - oksize; - - /* Shift node contents if EVEN(key length) changed. */ - if (delta) { - if (delta > 0 && SIZELEFT(mp) < delta) { - pgno_t pgno; - /* not enough space left, do a delete and split */ - DPRINTF(("Not enough room, delta = %d, splitting...", delta)); - pgno = NODEPGNO(node); - mdb_node_del(mc, 0); - return mdb_page_split(mc, key, NULL, pgno, MDB_SPLIT_REPLACE); - } - - numkeys = NUMKEYS(mp); - for (i = 0; i < numkeys; i++) { - if (mp->mp_ptrs[i] <= ptr) - mp->mp_ptrs[i] -= delta; - } - - base = (char *)mp + mp->mp_upper + PAGEBASE; - len = ptr - mp->mp_upper + NODESIZE; - memmove(base - delta, base, len); - mp->mp_upper -= delta; - - node = NODEPTR(mp, indx); - } - - /* But even if no shift was needed, update ksize */ - if (node->mn_ksize != key->mv_size) - node->mn_ksize = key->mv_size; - - if (key->mv_size) - memcpy(NODEKEY(node), key->mv_data, key->mv_size); - - return MDB_SUCCESS; -} - -static void -mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst); - -/** Perform \b act while tracking temporary cursor \b mn */ -#define WITH_CURSOR_TRACKING(mn, act) do { \ - MDB_cursor dummy, *tracked, **tp = &(mn).mc_txn->mt_cursors[mn.mc_dbi]; \ - if ((mn).mc_flags & C_SUB) { \ - dummy.mc_flags = C_INITIALIZED; \ - dummy.mc_xcursor = (MDB_xcursor *)&(mn); \ - tracked = &dummy; \ - } else { \ - tracked = &(mn); \ - } \ - tracked->mc_next = *tp; \ - *tp = tracked; \ - { act; } \ - *tp = tracked->mc_next; \ -} while (0) - -/** Move a node from csrc to cdst. - */ -static int -mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst, int fromleft) -{ - MDB_node *srcnode; - MDB_val key, data; - pgno_t srcpg; - MDB_cursor mn; - int rc; - unsigned short flags; - - DKBUF; - - /* Mark src and dst as dirty. */ - if ((rc = mdb_page_touch(csrc)) || - (rc = mdb_page_touch(cdst))) - return rc; - - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_size = csrc->mc_db->md_pad; - key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top], key.mv_size); - data.mv_size = 0; - data.mv_data = NULL; - srcpg = 0; - flags = 0; - } else { - srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], csrc->mc_ki[csrc->mc_top]); - mdb_cassert(csrc, !((size_t)srcnode & 1)); - srcpg = NODEPGNO(srcnode); - flags = srcnode->mn_flags; - if (csrc->mc_ki[csrc->mc_top] == 0 && IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { - unsigned int snum = csrc->mc_snum; - MDB_node *s2; - /* must find the lowest key below src */ - rc = mdb_page_search_lowest(csrc); - if (rc) - return rc; - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_size = csrc->mc_db->md_pad; - key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); - } else { - s2 = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); - key.mv_size = NODEKSZ(s2); - key.mv_data = NODEKEY(s2); - } - csrc->mc_snum = snum--; - csrc->mc_top = snum; - } else { - key.mv_size = NODEKSZ(srcnode); - key.mv_data = NODEKEY(srcnode); - } - data.mv_size = NODEDSZ(srcnode); - data.mv_data = NODEDATA(srcnode); - } - mn.mc_xcursor = NULL; - if (IS_BRANCH(cdst->mc_pg[cdst->mc_top]) && cdst->mc_ki[cdst->mc_top] == 0) { - unsigned int snum = cdst->mc_snum; - MDB_node *s2; - MDB_val bkey; - /* must find the lowest key below dst */ - mdb_cursor_copy(cdst, &mn); - rc = mdb_page_search_lowest(&mn); - if (rc) - return rc; - if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { - bkey.mv_size = mn.mc_db->md_pad; - bkey.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, bkey.mv_size); - } else { - s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); - bkey.mv_size = NODEKSZ(s2); - bkey.mv_data = NODEKEY(s2); - } - mn.mc_snum = snum--; - mn.mc_top = snum; - mn.mc_ki[snum] = 0; - rc = mdb_update_key(&mn, &bkey); - if (rc) - return rc; - } - - DPRINTF(("moving %s node %u [%s] on page %"Yu" to node %u on page %"Yu, - IS_LEAF(csrc->mc_pg[csrc->mc_top]) ? "leaf" : "branch", - csrc->mc_ki[csrc->mc_top], - DKEY(&key), - csrc->mc_pg[csrc->mc_top]->mp_pgno, - cdst->mc_ki[cdst->mc_top], cdst->mc_pg[cdst->mc_top]->mp_pgno)); - - /* Add the node to the destination page. - */ - rc = mdb_node_add(cdst, cdst->mc_ki[cdst->mc_top], &key, &data, srcpg, flags); - if (rc != MDB_SUCCESS) - return rc; - - /* Delete the node from the source page. - */ - mdb_node_del(csrc, key.mv_size); - - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = csrc->mc_dbi; - MDB_page *mpd, *mps; - - mps = csrc->mc_pg[csrc->mc_top]; - /* If we're adding on the left, bump others up */ - if (fromleft) { - mpd = cdst->mc_pg[csrc->mc_top]; - for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (csrc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top) - continue; - if (m3 != cdst && - m3->mc_pg[csrc->mc_top] == mpd && - m3->mc_ki[csrc->mc_top] >= cdst->mc_ki[csrc->mc_top]) { - m3->mc_ki[csrc->mc_top]++; - } - if (m3 !=csrc && - m3->mc_pg[csrc->mc_top] == mps && - m3->mc_ki[csrc->mc_top] == csrc->mc_ki[csrc->mc_top]) { - m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; - m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; - m3->mc_ki[csrc->mc_top-1]++; - } - if (IS_LEAF(mps)) - XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]); - } - } else - /* Adding on the right, bump others down */ - { - for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (csrc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == csrc) continue; - if (!(m3->mc_flags & C_INITIALIZED) || m3->mc_top < csrc->mc_top) - continue; - if (m3->mc_pg[csrc->mc_top] == mps) { - if (!m3->mc_ki[csrc->mc_top]) { - m3->mc_pg[csrc->mc_top] = cdst->mc_pg[cdst->mc_top]; - m3->mc_ki[csrc->mc_top] = cdst->mc_ki[cdst->mc_top]; - m3->mc_ki[csrc->mc_top-1]--; - } else { - m3->mc_ki[csrc->mc_top]--; - } - if (IS_LEAF(mps)) - XCURSOR_REFRESH(m3, csrc->mc_top, m3->mc_pg[csrc->mc_top]); - } - } - } - } - - /* Update the parent separators. - */ - if (csrc->mc_ki[csrc->mc_top] == 0) { - if (csrc->mc_ki[csrc->mc_top-1] != 0) { - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_data = LEAF2KEY(csrc->mc_pg[csrc->mc_top], 0, key.mv_size); - } else { - srcnode = NODEPTR(csrc->mc_pg[csrc->mc_top], 0); - key.mv_size = NODEKSZ(srcnode); - key.mv_data = NODEKEY(srcnode); - } - DPRINTF(("update separator for source page %"Yu" to [%s]", - csrc->mc_pg[csrc->mc_top]->mp_pgno, DKEY(&key))); - mdb_cursor_copy(csrc, &mn); - mn.mc_snum--; - mn.mc_top--; - /* We want mdb_rebalance to find mn when doing fixups */ - WITH_CURSOR_TRACKING(mn, - rc = mdb_update_key(&mn, &key)); - if (rc) - return rc; - } - if (IS_BRANCH(csrc->mc_pg[csrc->mc_top])) { - MDB_val nullkey; - indx_t ix = csrc->mc_ki[csrc->mc_top]; - nullkey.mv_size = 0; - csrc->mc_ki[csrc->mc_top] = 0; - rc = mdb_update_key(csrc, &nullkey); - csrc->mc_ki[csrc->mc_top] = ix; - mdb_cassert(csrc, rc == MDB_SUCCESS); - } - } - - if (cdst->mc_ki[cdst->mc_top] == 0) { - if (cdst->mc_ki[cdst->mc_top-1] != 0) { - if (IS_LEAF2(csrc->mc_pg[csrc->mc_top])) { - key.mv_data = LEAF2KEY(cdst->mc_pg[cdst->mc_top], 0, key.mv_size); - } else { - srcnode = NODEPTR(cdst->mc_pg[cdst->mc_top], 0); - key.mv_size = NODEKSZ(srcnode); - key.mv_data = NODEKEY(srcnode); - } - DPRINTF(("update separator for destination page %"Yu" to [%s]", - cdst->mc_pg[cdst->mc_top]->mp_pgno, DKEY(&key))); - mdb_cursor_copy(cdst, &mn); - mn.mc_snum--; - mn.mc_top--; - /* We want mdb_rebalance to find mn when doing fixups */ - WITH_CURSOR_TRACKING(mn, - rc = mdb_update_key(&mn, &key)); - if (rc) - return rc; - } - if (IS_BRANCH(cdst->mc_pg[cdst->mc_top])) { - MDB_val nullkey; - indx_t ix = cdst->mc_ki[cdst->mc_top]; - nullkey.mv_size = 0; - cdst->mc_ki[cdst->mc_top] = 0; - rc = mdb_update_key(cdst, &nullkey); - cdst->mc_ki[cdst->mc_top] = ix; - mdb_cassert(cdst, rc == MDB_SUCCESS); - } - } - - return MDB_SUCCESS; -} - -/** Merge one page into another. - * The nodes from the page pointed to by \b csrc will - * be copied to the page pointed to by \b cdst and then - * the \b csrc page will be freed. - * @param[in] csrc Cursor pointing to the source page. - * @param[in] cdst Cursor pointing to the destination page. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_merge(MDB_cursor *csrc, MDB_cursor *cdst) -{ - MDB_page *psrc, *pdst; - MDB_node *srcnode; - MDB_val key, data; - unsigned nkeys; - int rc; - indx_t i, j; - - psrc = csrc->mc_pg[csrc->mc_top]; - pdst = cdst->mc_pg[cdst->mc_top]; - - DPRINTF(("merging page %"Yu" into %"Yu, psrc->mp_pgno, pdst->mp_pgno)); - - mdb_cassert(csrc, csrc->mc_snum > 1); /* can't merge root page */ - mdb_cassert(csrc, cdst->mc_snum > 1); - - /* Mark dst as dirty. */ - if ((rc = mdb_page_touch(cdst))) - return rc; - - /* get dst page again now that we've touched it. */ - pdst = cdst->mc_pg[cdst->mc_top]; - - /* Move all nodes from src to dst. - */ - j = nkeys = NUMKEYS(pdst); - if (IS_LEAF2(psrc)) { - key.mv_size = csrc->mc_db->md_pad; - key.mv_data = METADATA(psrc); - for (i = 0; i < NUMKEYS(psrc); i++, j++) { - rc = mdb_node_add(cdst, j, &key, NULL, 0, 0); - if (rc != MDB_SUCCESS) - return rc; - key.mv_data = (char *)key.mv_data + key.mv_size; - } - } else { - for (i = 0; i < NUMKEYS(psrc); i++, j++) { - srcnode = NODEPTR(psrc, i); - if (i == 0 && IS_BRANCH(psrc)) { - MDB_cursor mn; - MDB_node *s2; - mdb_cursor_copy(csrc, &mn); - mn.mc_xcursor = NULL; - /* must find the lowest key below src */ - rc = mdb_page_search_lowest(&mn); - if (rc) - return rc; - if (IS_LEAF2(mn.mc_pg[mn.mc_top])) { - key.mv_size = mn.mc_db->md_pad; - key.mv_data = LEAF2KEY(mn.mc_pg[mn.mc_top], 0, key.mv_size); - } else { - s2 = NODEPTR(mn.mc_pg[mn.mc_top], 0); - key.mv_size = NODEKSZ(s2); - key.mv_data = NODEKEY(s2); - } - } else { - key.mv_size = srcnode->mn_ksize; - key.mv_data = NODEKEY(srcnode); - } - - data.mv_size = NODEDSZ(srcnode); - data.mv_data = NODEDATA(srcnode); - rc = mdb_node_add(cdst, j, &key, &data, NODEPGNO(srcnode), srcnode->mn_flags); - if (rc != MDB_SUCCESS) - return rc; - } - } - - DPRINTF(("dst page %"Yu" now has %u keys (%.1f%% filled)", - pdst->mp_pgno, NUMKEYS(pdst), - (float)PAGEFILL(cdst->mc_txn->mt_env, pdst) / 10)); - - /* Unlink the src page from parent and add to free list. - */ - csrc->mc_top--; - mdb_node_del(csrc, 0); - if (csrc->mc_ki[csrc->mc_top] == 0) { - key.mv_size = 0; - rc = mdb_update_key(csrc, &key); - if (rc) { - csrc->mc_top++; - return rc; - } - } - csrc->mc_top++; - - psrc = csrc->mc_pg[csrc->mc_top]; - /* If not operating on FreeDB, allow this page to be reused - * in this txn. Otherwise just add to free list. - */ - rc = mdb_page_loose(csrc, psrc); - if (rc) - return rc; - if (IS_LEAF(psrc)) - csrc->mc_db->md_leaf_pages--; - else - csrc->mc_db->md_branch_pages--; - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = csrc->mc_dbi; - unsigned int top = csrc->mc_top; - - for (m2 = csrc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (csrc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == csrc) continue; - if (m3->mc_snum < csrc->mc_snum) continue; - if (m3->mc_pg[top] == psrc) { - m3->mc_pg[top] = pdst; - m3->mc_ki[top] += nkeys; - m3->mc_ki[top-1] = cdst->mc_ki[top-1]; - } else if (m3->mc_pg[top-1] == csrc->mc_pg[top-1] && - m3->mc_ki[top-1] > csrc->mc_ki[top-1]) { - m3->mc_ki[top-1]--; - } - if (IS_LEAF(psrc)) - XCURSOR_REFRESH(m3, top, m3->mc_pg[top]); - } - } - { - unsigned int snum = cdst->mc_snum; - uint16_t depth = cdst->mc_db->md_depth; - mdb_cursor_pop(cdst); - rc = mdb_rebalance(cdst); - /* Did the tree height change? */ - if (depth != cdst->mc_db->md_depth) - snum += cdst->mc_db->md_depth - depth; - cdst->mc_snum = snum; - cdst->mc_top = snum-1; - } - return rc; -} - -/** Copy the contents of a cursor. - * @param[in] csrc The cursor to copy from. - * @param[out] cdst The cursor to copy to. - */ -static void -mdb_cursor_copy(const MDB_cursor *csrc, MDB_cursor *cdst) -{ - unsigned int i; - - cdst->mc_txn = csrc->mc_txn; - cdst->mc_dbi = csrc->mc_dbi; - cdst->mc_db = csrc->mc_db; - cdst->mc_dbx = csrc->mc_dbx; - cdst->mc_snum = csrc->mc_snum; - cdst->mc_top = csrc->mc_top; - cdst->mc_flags = csrc->mc_flags; - MC_SET_OVPG(cdst, MC_OVPG(csrc)); - - for (i=0; imc_snum; i++) { - cdst->mc_pg[i] = csrc->mc_pg[i]; - cdst->mc_ki[i] = csrc->mc_ki[i]; - } -} - -/** Rebalance the tree after a delete operation. - * @param[in] mc Cursor pointing to the page where rebalancing - * should begin. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_rebalance(MDB_cursor *mc) -{ - MDB_node *node; - int rc, fromleft; - unsigned int ptop, minkeys, thresh; - MDB_cursor mn; - indx_t oldki; - - if (IS_BRANCH(mc->mc_pg[mc->mc_top])) { - minkeys = 2; - thresh = 1; - } else { - minkeys = 1; - thresh = FILL_THRESHOLD; - } - DPRINTF(("rebalancing %s page %"Yu" (has %u keys, %.1f%% full)", - IS_LEAF(mc->mc_pg[mc->mc_top]) ? "leaf" : "branch", - mdb_dbg_pgno(mc->mc_pg[mc->mc_top]), NUMKEYS(mc->mc_pg[mc->mc_top]), - (float)PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) / 10)); - - if (PAGEFILL(mc->mc_txn->mt_env, mc->mc_pg[mc->mc_top]) >= thresh && - NUMKEYS(mc->mc_pg[mc->mc_top]) >= minkeys) { - DPRINTF(("no need to rebalance page %"Yu", above fill threshold", - mdb_dbg_pgno(mc->mc_pg[mc->mc_top]))); - return MDB_SUCCESS; - } - - if (mc->mc_snum < 2) { - MDB_page *mp = mc->mc_pg[0]; - if (IS_SUBP(mp)) { - DPUTS("Can't rebalance a subpage, ignoring"); - return MDB_SUCCESS; - } - if (NUMKEYS(mp) == 0) { - DPUTS("tree is completely empty"); - mc->mc_db->md_root = P_INVALID; - mc->mc_db->md_depth = 0; - mc->mc_db->md_leaf_pages = 0; - rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); - if (rc) - return rc; - /* Adjust cursors pointing to mp */ - mc->mc_snum = 0; - mc->mc_top = 0; - mc->mc_flags &= ~C_INITIALIZED; - { - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (!(m3->mc_flags & C_INITIALIZED) || (m3->mc_snum < mc->mc_snum)) - continue; - if (m3->mc_pg[0] == mp) { - m3->mc_snum = 0; - m3->mc_top = 0; - m3->mc_flags &= ~C_INITIALIZED; - } - } - } - } else if (IS_BRANCH(mp) && NUMKEYS(mp) == 1) { - int i; - DPUTS("collapsing root page!"); - rc = mdb_midl_append(&mc->mc_txn->mt_free_pgs, mp->mp_pgno); - if (rc) - return rc; - mc->mc_db->md_root = NODEPGNO(NODEPTR(mp, 0)); - rc = mdb_page_get(mc, mc->mc_db->md_root, &mc->mc_pg[0], NULL); - if (rc) - return rc; - mc->mc_db->md_depth--; - mc->mc_db->md_branch_pages--; - mc->mc_ki[0] = mc->mc_ki[1]; - for (i = 1; imc_db->md_depth; i++) { - mc->mc_pg[i] = mc->mc_pg[i+1]; - mc->mc_ki[i] = mc->mc_ki[i+1]; - } - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == mc) continue; - if (!(m3->mc_flags & C_INITIALIZED)) - continue; - if (m3->mc_pg[0] == mp) { - for (i=0; imc_db->md_depth; i++) { - m3->mc_pg[i] = m3->mc_pg[i+1]; - m3->mc_ki[i] = m3->mc_ki[i+1]; - } - m3->mc_snum--; - m3->mc_top--; - } - } - } - } else - DPUTS("root page doesn't need rebalancing"); - return MDB_SUCCESS; - } - - /* The parent (branch page) must have at least 2 pointers, - * otherwise the tree is invalid. - */ - ptop = mc->mc_top-1; - mdb_cassert(mc, NUMKEYS(mc->mc_pg[ptop]) > 1); - - /* Leaf page fill factor is below the threshold. - * Try to move keys from left or right neighbor, or - * merge with a neighbor page. - */ - - /* Find neighbors. - */ - mdb_cursor_copy(mc, &mn); - mn.mc_xcursor = NULL; - - oldki = mc->mc_ki[mc->mc_top]; - if (mc->mc_ki[ptop] == 0) { - /* We're the leftmost leaf in our parent. - */ - DPUTS("reading right neighbor"); - mn.mc_ki[ptop]++; - node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); - rc = mdb_page_get(mc, NODEPGNO(node), &mn.mc_pg[mn.mc_top], NULL); - if (rc) - return rc; - mn.mc_ki[mn.mc_top] = 0; - mc->mc_ki[mc->mc_top] = NUMKEYS(mc->mc_pg[mc->mc_top]); - fromleft = 0; - } else { - /* There is at least one neighbor to the left. - */ - DPUTS("reading left neighbor"); - mn.mc_ki[ptop]--; - node = NODEPTR(mc->mc_pg[ptop], mn.mc_ki[ptop]); - rc = mdb_page_get(mc, NODEPGNO(node), &mn.mc_pg[mn.mc_top], NULL); - if (rc) - return rc; - mn.mc_ki[mn.mc_top] = NUMKEYS(mn.mc_pg[mn.mc_top]) - 1; - mc->mc_ki[mc->mc_top] = 0; - fromleft = 1; - } - - DPRINTF(("found neighbor page %"Yu" (%u keys, %.1f%% full)", - mn.mc_pg[mn.mc_top]->mp_pgno, NUMKEYS(mn.mc_pg[mn.mc_top]), - (float)PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) / 10)); - - /* If the neighbor page is above threshold and has enough keys, - * move one key from it. Otherwise we should try to merge them. - * (A branch page must never have less than 2 keys.) - */ - if (PAGEFILL(mc->mc_txn->mt_env, mn.mc_pg[mn.mc_top]) >= thresh && NUMKEYS(mn.mc_pg[mn.mc_top]) > minkeys) { - rc = mdb_node_move(&mn, mc, fromleft); - if (fromleft) { - /* if we inserted on left, bump position up */ - oldki++; - } - } else { - if (!fromleft) { - rc = mdb_page_merge(&mn, mc); - } else { - oldki += NUMKEYS(mn.mc_pg[mn.mc_top]); - mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1; - /* We want mdb_rebalance to find mn when doing fixups */ - WITH_CURSOR_TRACKING(mn, - rc = mdb_page_merge(mc, &mn)); - mdb_cursor_copy(&mn, mc); - } - mc->mc_flags &= ~C_EOF; - } - mc->mc_ki[mc->mc_top] = oldki; - return rc; -} - -/** Complete a delete operation started by #mdb_cursor_del(). */ -static int -mdb_cursor_del0(MDB_cursor *mc) -{ - int rc; - MDB_page *mp; - indx_t ki; - unsigned int nkeys; - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - - ki = mc->mc_ki[mc->mc_top]; - mp = mc->mc_pg[mc->mc_top]; - mdb_node_del(mc, mc->mc_db->md_pad); - mc->mc_db->md_entries--; - { - /* Adjust other cursors pointing to mp */ - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) - continue; - if (m3 == mc || m3->mc_snum < mc->mc_snum) - continue; - if (m3->mc_pg[mc->mc_top] == mp) { - if (m3->mc_ki[mc->mc_top] == ki) { - m3->mc_flags |= C_DEL; - if (mc->mc_db->md_flags & MDB_DUPSORT) { - /* Sub-cursor referred into dataset which is gone */ - m3->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF); - } - continue; - } else if (m3->mc_ki[mc->mc_top] > ki) { - m3->mc_ki[mc->mc_top]--; - } - XCURSOR_REFRESH(m3, mc->mc_top, mp); - } - } - } - rc = mdb_rebalance(mc); - - if (rc == MDB_SUCCESS) { - /* DB is totally empty now, just bail out. - * Other cursors adjustments were already done - * by mdb_rebalance and aren't needed here. - */ - if (!mc->mc_snum) - return rc; - - mp = mc->mc_pg[mc->mc_top]; - nkeys = NUMKEYS(mp); - - /* Adjust other cursors pointing to mp */ - for (m2 = mc->mc_txn->mt_cursors[dbi]; !rc && m2; m2=m2->mc_next) { - m3 = (mc->mc_flags & C_SUB) ? &m2->mc_xcursor->mx_cursor : m2; - if (! (m2->mc_flags & m3->mc_flags & C_INITIALIZED)) - continue; - if (m3->mc_snum < mc->mc_snum) - continue; - if (m3->mc_pg[mc->mc_top] == mp) { - /* if m3 points past last node in page, find next sibling */ - if (m3->mc_ki[mc->mc_top] >= mc->mc_ki[mc->mc_top]) { - if (m3->mc_ki[mc->mc_top] >= nkeys) { - rc = mdb_cursor_sibling(m3, 1); - if (rc == MDB_NOTFOUND) { - m3->mc_flags |= C_EOF; - rc = MDB_SUCCESS; - continue; - } - } - if (mc->mc_db->md_flags & MDB_DUPSORT) { - MDB_node *node = NODEPTR(m3->mc_pg[m3->mc_top], m3->mc_ki[m3->mc_top]); - /* If this node has dupdata, it may need to be reinited - * because its data has moved. - * If the xcursor was not initd it must be reinited. - * Else if node points to a subDB, nothing is needed. - * Else (xcursor was initd, not a subDB) needs mc_pg[0] reset. - */ - if (node->mn_flags & F_DUPDATA) { - if (m3->mc_xcursor->mx_cursor.mc_flags & C_INITIALIZED) { - if (!(node->mn_flags & F_SUBDATA)) - m3->mc_xcursor->mx_cursor.mc_pg[0] = NODEDATA(node); - } else { - mdb_xcursor_init1(m3, node); - m3->mc_xcursor->mx_cursor.mc_flags |= C_DEL; - } - } - } - } - } - } - mc->mc_flags |= C_DEL; - } - - if (rc) - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_del(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data) -{ - if (!key || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) - return EINVAL; - - if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) - return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; - - if (!F_ISSET(txn->mt_dbs[dbi].md_flags, MDB_DUPSORT)) { - /* must ignore any data */ - data = NULL; - } - - return mdb_del0(txn, dbi, key, data, 0); -} - -static int -mdb_del0(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data, unsigned flags) -{ - MDB_cursor mc; - MDB_xcursor mx; - MDB_cursor_op op; - MDB_val rdata, *xdata; - int rc, exact = 0; - DKBUF; - - DPRINTF(("====> delete db %u key [%s]", dbi, DKEY(key))); - - mdb_cursor_init(&mc, txn, dbi, &mx); - - if (data) { - op = MDB_GET_BOTH; - rdata = *data; - xdata = &rdata; - } else { - op = MDB_SET; - xdata = NULL; - flags |= MDB_NODUPDATA; - } - rc = mdb_cursor_set(&mc, key, xdata, op, &exact); - if (rc == 0) { - /* let mdb_page_split know about this cursor if needed: - * delete will trigger a rebalance; if it needs to move - * a node from one page to another, it will have to - * update the parent's separator key(s). If the new sepkey - * is larger than the current one, the parent page may - * run out of space, triggering a split. We need this - * cursor to be consistent until the end of the rebalance. - */ - mc.mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = &mc; - rc = mdb_cursor_del(&mc, flags); - txn->mt_cursors[dbi] = mc.mc_next; - } - return rc; -} - -/** Split a page and insert a new node. - * Set #MDB_TXN_ERROR on failure. - * @param[in,out] mc Cursor pointing to the page and desired insertion index. - * The cursor will be updated to point to the actual page and index where - * the node got inserted after the split. - * @param[in] newkey The key for the newly inserted node. - * @param[in] newdata The data for the newly inserted node. - * @param[in] newpgno The page number, if the new node is a branch node. - * @param[in] nflags The #NODE_ADD_FLAGS for the new node. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_page_split(MDB_cursor *mc, MDB_val *newkey, MDB_val *newdata, pgno_t newpgno, - unsigned int nflags) -{ - unsigned int flags; - int rc = MDB_SUCCESS, new_root = 0, did_split = 0; - indx_t newindx; - pgno_t pgno = 0; - int i, j, split_indx, nkeys, pmax; - MDB_env *env = mc->mc_txn->mt_env; - MDB_node *node; - MDB_val sepkey, rkey, xdata, *rdata = &xdata; - MDB_page *copy = NULL; - MDB_page *mp, *rp, *pp; - int ptop; - MDB_cursor mn; - DKBUF; - - mp = mc->mc_pg[mc->mc_top]; - newindx = mc->mc_ki[mc->mc_top]; - nkeys = NUMKEYS(mp); - - DPRINTF(("-----> splitting %s page %"Yu" and adding [%s] at index %i/%i", - IS_LEAF(mp) ? "leaf" : "branch", mp->mp_pgno, - DKEY(newkey), mc->mc_ki[mc->mc_top], nkeys)); - - /* Create a right sibling. */ - if ((rc = mdb_page_new(mc, mp->mp_flags, 1, &rp))) - return rc; - rp->mp_pad = mp->mp_pad; - DPRINTF(("new right sibling: page %"Yu, rp->mp_pgno)); - - /* Usually when splitting the root page, the cursor - * height is 1. But when called from mdb_update_key, - * the cursor height may be greater because it walks - * up the stack while finding the branch slot to update. - */ - if (mc->mc_top < 1) { - if ((rc = mdb_page_new(mc, P_BRANCH, 1, &pp))) - goto done; - /* shift current top to make room for new parent */ - for (i=mc->mc_snum; i>0; i--) { - mc->mc_pg[i] = mc->mc_pg[i-1]; - mc->mc_ki[i] = mc->mc_ki[i-1]; - } - mc->mc_pg[0] = pp; - mc->mc_ki[0] = 0; - mc->mc_db->md_root = pp->mp_pgno; - DPRINTF(("root split! new root = %"Yu, pp->mp_pgno)); - new_root = mc->mc_db->md_depth++; - - /* Add left (implicit) pointer. */ - if ((rc = mdb_node_add(mc, 0, NULL, NULL, mp->mp_pgno, 0)) != MDB_SUCCESS) { - /* undo the pre-push */ - mc->mc_pg[0] = mc->mc_pg[1]; - mc->mc_ki[0] = mc->mc_ki[1]; - mc->mc_db->md_root = mp->mp_pgno; - mc->mc_db->md_depth--; - goto done; - } - mc->mc_snum++; - mc->mc_top++; - ptop = 0; - } else { - ptop = mc->mc_top-1; - DPRINTF(("parent branch page is %"Yu, mc->mc_pg[ptop]->mp_pgno)); - } - - mdb_cursor_copy(mc, &mn); - mn.mc_xcursor = NULL; - mn.mc_pg[mn.mc_top] = rp; - mn.mc_ki[ptop] = mc->mc_ki[ptop]+1; - - if (nflags & MDB_APPEND) { - mn.mc_ki[mn.mc_top] = 0; - sepkey = *newkey; - split_indx = newindx; - nkeys = 0; - } else { - - split_indx = (nkeys+1) / 2; - - if (IS_LEAF2(rp)) { - char *split, *ins; - int x; - unsigned int lsize, rsize, ksize; - /* Move half of the keys to the right sibling */ - x = mc->mc_ki[mc->mc_top] - split_indx; - ksize = mc->mc_db->md_pad; - split = LEAF2KEY(mp, split_indx, ksize); - rsize = (nkeys - split_indx) * ksize; - lsize = (nkeys - split_indx) * sizeof(indx_t); - mp->mp_lower -= lsize; - rp->mp_lower += lsize; - mp->mp_upper += rsize - lsize; - rp->mp_upper -= rsize - lsize; - sepkey.mv_size = ksize; - if (newindx == split_indx) { - sepkey.mv_data = newkey->mv_data; - } else { - sepkey.mv_data = split; - } - if (x<0) { - ins = LEAF2KEY(mp, mc->mc_ki[mc->mc_top], ksize); - memcpy(rp->mp_ptrs, split, rsize); - sepkey.mv_data = rp->mp_ptrs; - memmove(ins+ksize, ins, (split_indx - mc->mc_ki[mc->mc_top]) * ksize); - memcpy(ins, newkey->mv_data, ksize); - mp->mp_lower += sizeof(indx_t); - mp->mp_upper -= ksize - sizeof(indx_t); - } else { - if (x) - memcpy(rp->mp_ptrs, split, x * ksize); - ins = LEAF2KEY(rp, x, ksize); - memcpy(ins, newkey->mv_data, ksize); - memcpy(ins+ksize, split + x * ksize, rsize - x * ksize); - rp->mp_lower += sizeof(indx_t); - rp->mp_upper -= ksize - sizeof(indx_t); - mc->mc_ki[mc->mc_top] = x; - } - } else { - int psize, nsize, k; - /* Maximum free space in an empty page */ - pmax = env->me_psize - PAGEHDRSZ; - if (IS_LEAF(mp)) - nsize = mdb_leaf_size(env, newkey, newdata); - else - nsize = mdb_branch_size(env, newkey); - nsize = EVEN(nsize); - - /* grab a page to hold a temporary copy */ - copy = mdb_page_malloc(mc->mc_txn, 1); - if (copy == NULL) { - rc = ENOMEM; - goto done; - } - copy->mp_pgno = mp->mp_pgno; - copy->mp_flags = mp->mp_flags; - copy->mp_lower = (PAGEHDRSZ-PAGEBASE); - copy->mp_upper = env->me_psize - PAGEBASE; - - /* prepare to insert */ - for (i=0, j=0; imp_ptrs[j++] = 0; - } - copy->mp_ptrs[j++] = mp->mp_ptrs[i]; - } - - /* When items are relatively large the split point needs - * to be checked, because being off-by-one will make the - * difference between success or failure in mdb_node_add. - * - * It's also relevant if a page happens to be laid out - * such that one half of its nodes are all "small" and - * the other half of its nodes are "large." If the new - * item is also "large" and falls on the half with - * "large" nodes, it also may not fit. - * - * As a final tweak, if the new item goes on the last - * spot on the page (and thus, onto the new page), bias - * the split so the new page is emptier than the old page. - * This yields better packing during sequential inserts. - */ - if (nkeys < 32 || nsize > pmax/16 || newindx >= nkeys) { - /* Find split point */ - psize = 0; - if (newindx <= split_indx || newindx >= nkeys) { - i = 0; j = 1; - k = newindx >= nkeys ? nkeys : split_indx+1+IS_LEAF(mp); - } else { - i = nkeys; j = -1; - k = split_indx-1; - } - for (; i!=k; i+=j) { - if (i == newindx) { - psize += nsize; - node = NULL; - } else { - node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); - psize += NODESIZE + NODEKSZ(node) + sizeof(indx_t); - if (IS_LEAF(mp)) { - if (F_ISSET(node->mn_flags, F_BIGDATA)) - psize += sizeof(pgno_t); - else - psize += NODEDSZ(node); - } - psize = EVEN(psize); - } - if (psize > pmax || i == k-j) { - split_indx = i + (j<0); - break; - } - } - } - if (split_indx == newindx) { - sepkey.mv_size = newkey->mv_size; - sepkey.mv_data = newkey->mv_data; - } else { - node = (MDB_node *)((char *)mp + copy->mp_ptrs[split_indx] + PAGEBASE); - sepkey.mv_size = node->mn_ksize; - sepkey.mv_data = NODEKEY(node); - } - } - } - - DPRINTF(("separator is %d [%s]", split_indx, DKEY(&sepkey))); - - /* Copy separator key to the parent. - */ - if (SIZELEFT(mn.mc_pg[ptop]) < mdb_branch_size(env, &sepkey)) { - int snum = mc->mc_snum; - mn.mc_snum--; - mn.mc_top--; - did_split = 1; - /* We want other splits to find mn when doing fixups */ - WITH_CURSOR_TRACKING(mn, - rc = mdb_page_split(&mn, &sepkey, NULL, rp->mp_pgno, 0)); - if (rc) - goto done; - - /* root split? */ - if (mc->mc_snum > snum) { - ptop++; - } - /* Right page might now have changed parent. - * Check if left page also changed parent. - */ - if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && - mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { - for (i=0; imc_pg[i] = mn.mc_pg[i]; - mc->mc_ki[i] = mn.mc_ki[i]; - } - mc->mc_pg[ptop] = mn.mc_pg[ptop]; - if (mn.mc_ki[ptop]) { - mc->mc_ki[ptop] = mn.mc_ki[ptop] - 1; - } else { - /* find right page's left sibling */ - mc->mc_ki[ptop] = mn.mc_ki[ptop]; - rc = mdb_cursor_sibling(mc, 0); - } - } - } else { - mn.mc_top--; - rc = mdb_node_add(&mn, mn.mc_ki[ptop], &sepkey, NULL, rp->mp_pgno, 0); - mn.mc_top++; - } - if (rc != MDB_SUCCESS) { - if (rc == MDB_NOTFOUND) /* improper mdb_cursor_sibling() result */ - rc = MDB_PROBLEM; - goto done; - } - if (nflags & MDB_APPEND) { - mc->mc_pg[mc->mc_top] = rp; - mc->mc_ki[mc->mc_top] = 0; - rc = mdb_node_add(mc, 0, newkey, newdata, newpgno, nflags); - if (rc) - goto done; - for (i=0; imc_top; i++) - mc->mc_ki[i] = mn.mc_ki[i]; - } else if (!IS_LEAF2(mp)) { - /* Move nodes */ - mc->mc_pg[mc->mc_top] = rp; - i = split_indx; - j = 0; - do { - if (i == newindx) { - rkey.mv_data = newkey->mv_data; - rkey.mv_size = newkey->mv_size; - if (IS_LEAF(mp)) { - rdata = newdata; - } else - pgno = newpgno; - flags = nflags; - /* Update index for the new key. */ - mc->mc_ki[mc->mc_top] = j; - } else { - node = (MDB_node *)((char *)mp + copy->mp_ptrs[i] + PAGEBASE); - rkey.mv_data = NODEKEY(node); - rkey.mv_size = node->mn_ksize; - if (IS_LEAF(mp)) { - xdata.mv_data = NODEDATA(node); - xdata.mv_size = NODEDSZ(node); - rdata = &xdata; - } else - pgno = NODEPGNO(node); - flags = node->mn_flags; - } - - if (!IS_LEAF(mp) && j == 0) { - /* First branch index doesn't need key data. */ - rkey.mv_size = 0; - } - - rc = mdb_node_add(mc, j, &rkey, rdata, pgno, flags); - if (rc) - goto done; - if (i == nkeys) { - i = 0; - j = 0; - mc->mc_pg[mc->mc_top] = copy; - } else { - i++; - j++; - } - } while (i != split_indx); - - nkeys = NUMKEYS(copy); - for (i=0; imp_ptrs[i] = copy->mp_ptrs[i]; - mp->mp_lower = copy->mp_lower; - mp->mp_upper = copy->mp_upper; - memcpy(NODEPTR(mp, nkeys-1), NODEPTR(copy, nkeys-1), - env->me_psize - copy->mp_upper - PAGEBASE); - - /* reset back to original page */ - if (newindx < split_indx) { - mc->mc_pg[mc->mc_top] = mp; - } else { - mc->mc_pg[mc->mc_top] = rp; - mc->mc_ki[ptop]++; - /* Make sure mc_ki is still valid. - */ - if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && - mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { - for (i=0; i<=ptop; i++) { - mc->mc_pg[i] = mn.mc_pg[i]; - mc->mc_ki[i] = mn.mc_ki[i]; - } - } - } - if (nflags & MDB_RESERVE) { - node = NODEPTR(mc->mc_pg[mc->mc_top], mc->mc_ki[mc->mc_top]); - if (!(node->mn_flags & F_BIGDATA)) - newdata->mv_data = NODEDATA(node); - } - } else { - if (newindx >= split_indx) { - mc->mc_pg[mc->mc_top] = rp; - mc->mc_ki[ptop]++; - /* Make sure mc_ki is still valid. - */ - if (mn.mc_pg[ptop] != mc->mc_pg[ptop] && - mc->mc_ki[ptop] >= NUMKEYS(mc->mc_pg[ptop])) { - for (i=0; i<=ptop; i++) { - mc->mc_pg[i] = mn.mc_pg[i]; - mc->mc_ki[i] = mn.mc_ki[i]; - } - } - } - } - - { - /* Adjust other cursors pointing to mp */ - MDB_cursor *m2, *m3; - MDB_dbi dbi = mc->mc_dbi; - nkeys = NUMKEYS(mp); - - for (m2 = mc->mc_txn->mt_cursors[dbi]; m2; m2=m2->mc_next) { - if (mc->mc_flags & C_SUB) - m3 = &m2->mc_xcursor->mx_cursor; - else - m3 = m2; - if (m3 == mc) - continue; - if (!(m2->mc_flags & m3->mc_flags & C_INITIALIZED)) - continue; - if (new_root) { - int k; - /* sub cursors may be on different DB */ - if (m3->mc_pg[0] != mp) - continue; - /* root split */ - for (k=new_root; k>=0; k--) { - m3->mc_ki[k+1] = m3->mc_ki[k]; - m3->mc_pg[k+1] = m3->mc_pg[k]; - } - if (m3->mc_ki[0] >= nkeys) { - m3->mc_ki[0] = 1; - } else { - m3->mc_ki[0] = 0; - } - m3->mc_pg[0] = mc->mc_pg[0]; - m3->mc_snum++; - m3->mc_top++; - } - if (m3->mc_top >= mc->mc_top && m3->mc_pg[mc->mc_top] == mp) { - if (m3->mc_ki[mc->mc_top] >= newindx && !(nflags & MDB_SPLIT_REPLACE)) - m3->mc_ki[mc->mc_top]++; - if (m3->mc_ki[mc->mc_top] >= nkeys) { - m3->mc_pg[mc->mc_top] = rp; - m3->mc_ki[mc->mc_top] -= nkeys; - for (i=0; imc_top; i++) { - m3->mc_ki[i] = mn.mc_ki[i]; - m3->mc_pg[i] = mn.mc_pg[i]; - } - } - } else if (!did_split && m3->mc_top >= ptop && m3->mc_pg[ptop] == mc->mc_pg[ptop] && - m3->mc_ki[ptop] >= mc->mc_ki[ptop]) { - m3->mc_ki[ptop]++; - } - if (IS_LEAF(mp)) - XCURSOR_REFRESH(m3, mc->mc_top, m3->mc_pg[mc->mc_top]); - } - } - DPRINTF(("mp left: %d, rp left: %d", SIZELEFT(mp), SIZELEFT(rp))); - -done: - if (copy) /* tmp page */ - mdb_page_free(env, copy); - if (rc) - mc->mc_txn->mt_flags |= MDB_TXN_ERROR; - return rc; -} - -int -mdb_put(MDB_txn *txn, MDB_dbi dbi, - MDB_val *key, MDB_val *data, unsigned int flags) -{ - MDB_cursor mc; - MDB_xcursor mx; - int rc; - - if (!key || !data || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) - return EINVAL; - - if (flags & ~(MDB_NOOVERWRITE|MDB_NODUPDATA|MDB_RESERVE|MDB_APPEND|MDB_APPENDDUP)) - return EINVAL; - - if (txn->mt_flags & (MDB_TXN_RDONLY|MDB_TXN_BLOCKED)) - return (txn->mt_flags & MDB_TXN_RDONLY) ? EACCES : MDB_BAD_TXN; - - mdb_cursor_init(&mc, txn, dbi, &mx); - mc.mc_next = txn->mt_cursors[dbi]; - txn->mt_cursors[dbi] = &mc; - rc = mdb_cursor_put(&mc, key, data, flags); - txn->mt_cursors[dbi] = mc.mc_next; - return rc; -} - -#ifndef MDB_WBUF -#define MDB_WBUF (1024*1024) -#endif -#define MDB_EOF 0x10 /**< #mdb_env_copyfd1() is done reading */ - - /** State needed for a double-buffering compacting copy. */ -typedef struct mdb_copy { - MDB_env *mc_env; - MDB_txn *mc_txn; - pthread_mutex_t mc_mutex; - pthread_cond_t mc_cond; /**< Condition variable for #mc_new */ - char *mc_wbuf[2]; - char *mc_over[2]; - int mc_wlen[2]; - int mc_olen[2]; - pgno_t mc_next_pgno; - HANDLE mc_fd; - int mc_toggle; /**< Buffer number in provider */ - int mc_new; /**< (0-2 buffers to write) | (#MDB_EOF at end) */ - /** Error code. Never cleared if set. Both threads can set nonzero - * to fail the copy. Not mutex-protected, LMDB expects atomic int. - */ - volatile int mc_error; -} mdb_copy; - - /** Dedicated writer thread for compacting copy. */ -static THREAD_RET ESECT CALL_CONV -mdb_env_copythr(void *arg) -{ - mdb_copy *my = arg; - char *ptr; - int toggle = 0, wsize, rc; -#ifdef _WIN32 - DWORD len; -#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) -#else - int len; -#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) -#ifdef SIGPIPE - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGPIPE); - if ((rc = pthread_sigmask(SIG_BLOCK, &set, NULL)) != 0) - my->mc_error = rc; -#endif -#endif - - pthread_mutex_lock(&my->mc_mutex); - for(;;) { - while (!my->mc_new) - pthread_cond_wait(&my->mc_cond, &my->mc_mutex); - if (my->mc_new == 0 + MDB_EOF) /* 0 buffers, just EOF */ - break; - wsize = my->mc_wlen[toggle]; - ptr = my->mc_wbuf[toggle]; -again: - rc = MDB_SUCCESS; - while (wsize > 0 && !my->mc_error) { - DO_WRITE(rc, my->mc_fd, ptr, wsize, len); - if (!rc) { - rc = ErrCode(); -#if defined(SIGPIPE) && !defined(_WIN32) - if (rc == EPIPE) { - /* Collect the pending SIGPIPE, otherwise at least OS X - * gives it to the process on thread-exit (ITS#8504). - */ - int tmp; - sigwait(&set, &tmp); - } -#endif - break; - } else if (len > 0) { - rc = MDB_SUCCESS; - ptr += len; - wsize -= len; - continue; - } else { - rc = EIO; - break; - } - } - if (rc) { - my->mc_error = rc; - } - /* If there's an overflow page tail, write it too */ - if (my->mc_olen[toggle]) { - wsize = my->mc_olen[toggle]; - ptr = my->mc_over[toggle]; - my->mc_olen[toggle] = 0; - goto again; - } - my->mc_wlen[toggle] = 0; - toggle ^= 1; - /* Return the empty buffer to provider */ - my->mc_new--; - pthread_cond_signal(&my->mc_cond); - } - pthread_mutex_unlock(&my->mc_mutex); - return (THREAD_RET)0; -#undef DO_WRITE -} - - /** Give buffer and/or #MDB_EOF to writer thread, await unused buffer. - * - * @param[in] my control structure. - * @param[in] adjust (1 to hand off 1 buffer) | (MDB_EOF when ending). - */ -static int ESECT -mdb_env_cthr_toggle(mdb_copy *my, int adjust) -{ - pthread_mutex_lock(&my->mc_mutex); - my->mc_new += adjust; - pthread_cond_signal(&my->mc_cond); - while (my->mc_new & 2) /* both buffers in use */ - pthread_cond_wait(&my->mc_cond, &my->mc_mutex); - pthread_mutex_unlock(&my->mc_mutex); - - my->mc_toggle ^= (adjust & 1); - /* Both threads reset mc_wlen, to be safe from threading errors */ - my->mc_wlen[my->mc_toggle] = 0; - return my->mc_error; -} - - /** Depth-first tree traversal for compacting copy. - * @param[in] my control structure. - * @param[in,out] pg database root. - * @param[in] flags includes #F_DUPDATA if it is a sorted-duplicate sub-DB. - */ -static int ESECT -mdb_env_cwalk(mdb_copy *my, pgno_t *pg, int flags) -{ - MDB_cursor mc = {0}; - MDB_node *ni; - MDB_page *mo, *mp, *leaf; - char *buf, *ptr; - int rc, toggle; - unsigned int i; - - /* Empty DB, nothing to do */ - if (*pg == P_INVALID) - return MDB_SUCCESS; - - mc.mc_snum = 1; - mc.mc_txn = my->mc_txn; - mc.mc_flags = my->mc_txn->mt_flags & (C_ORIG_RDONLY|C_WRITEMAP); - - rc = mdb_page_get(&mc, *pg, &mc.mc_pg[0], NULL); - if (rc) - return rc; - rc = mdb_page_search_root(&mc, NULL, MDB_PS_FIRST); - if (rc) - return rc; - - /* Make cursor pages writable */ - buf = ptr = malloc(my->mc_env->me_psize * mc.mc_snum); - if (buf == NULL) - return ENOMEM; - - for (i=0; imc_env->me_psize); - mc.mc_pg[i] = (MDB_page *)ptr; - ptr += my->mc_env->me_psize; - } - - /* This is writable space for a leaf page. Usually not needed. */ - leaf = (MDB_page *)ptr; - - toggle = my->mc_toggle; - while (mc.mc_snum > 0) { - unsigned n; - mp = mc.mc_pg[mc.mc_top]; - n = NUMKEYS(mp); - - if (IS_LEAF(mp)) { - if (!IS_LEAF2(mp) && !(flags & F_DUPDATA)) { - for (i=0; imn_flags & F_BIGDATA) { - MDB_page *omp; - pgno_t pg; - - /* Need writable leaf */ - if (mp != leaf) { - mc.mc_pg[mc.mc_top] = leaf; - mdb_page_copy(leaf, mp, my->mc_env->me_psize); - mp = leaf; - ni = NODEPTR(mp, i); - } - - memcpy(&pg, NODEDATA(ni), sizeof(pg)); - memcpy(NODEDATA(ni), &my->mc_next_pgno, sizeof(pgno_t)); - rc = mdb_page_get(&mc, pg, &omp, NULL); - if (rc) - goto done; - if (my->mc_wlen[toggle] >= MDB_WBUF) { - rc = mdb_env_cthr_toggle(my, 1); - if (rc) - goto done; - toggle = my->mc_toggle; - } - mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); - memcpy(mo, omp, my->mc_env->me_psize); - mo->mp_pgno = my->mc_next_pgno; - my->mc_next_pgno += omp->mp_pages; - my->mc_wlen[toggle] += my->mc_env->me_psize; - if (omp->mp_pages > 1) { - my->mc_olen[toggle] = my->mc_env->me_psize * (omp->mp_pages - 1); - my->mc_over[toggle] = (char *)omp + my->mc_env->me_psize; - rc = mdb_env_cthr_toggle(my, 1); - if (rc) - goto done; - toggle = my->mc_toggle; - } - } else if (ni->mn_flags & F_SUBDATA) { - MDB_db db; - - /* Need writable leaf */ - if (mp != leaf) { - mc.mc_pg[mc.mc_top] = leaf; - mdb_page_copy(leaf, mp, my->mc_env->me_psize); - mp = leaf; - ni = NODEPTR(mp, i); - } - - memcpy(&db, NODEDATA(ni), sizeof(db)); - my->mc_toggle = toggle; - rc = mdb_env_cwalk(my, &db.md_root, ni->mn_flags & F_DUPDATA); - if (rc) - goto done; - toggle = my->mc_toggle; - memcpy(NODEDATA(ni), &db, sizeof(db)); - } - } - } - } else { - mc.mc_ki[mc.mc_top]++; - if (mc.mc_ki[mc.mc_top] < n) { - pgno_t pg; -again: - ni = NODEPTR(mp, mc.mc_ki[mc.mc_top]); - pg = NODEPGNO(ni); - rc = mdb_page_get(&mc, pg, &mp, NULL); - if (rc) - goto done; - mc.mc_top++; - mc.mc_snum++; - mc.mc_ki[mc.mc_top] = 0; - if (IS_BRANCH(mp)) { - /* Whenever we advance to a sibling branch page, - * we must proceed all the way down to its first leaf. - */ - mdb_page_copy(mc.mc_pg[mc.mc_top], mp, my->mc_env->me_psize); - goto again; - } else - mc.mc_pg[mc.mc_top] = mp; - continue; - } - } - if (my->mc_wlen[toggle] >= MDB_WBUF) { - rc = mdb_env_cthr_toggle(my, 1); - if (rc) - goto done; - toggle = my->mc_toggle; - } - mo = (MDB_page *)(my->mc_wbuf[toggle] + my->mc_wlen[toggle]); - mdb_page_copy(mo, mp, my->mc_env->me_psize); - mo->mp_pgno = my->mc_next_pgno++; - my->mc_wlen[toggle] += my->mc_env->me_psize; - if (mc.mc_top) { - /* Update parent if there is one */ - ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]); - SETPGNO(ni, mo->mp_pgno); - mdb_cursor_pop(&mc); - } else { - /* Otherwise we're done */ - *pg = mo->mp_pgno; - break; - } - } -done: - free(buf); - return rc; -} - - /** Copy environment with compaction. */ -static int ESECT -mdb_env_copyfd1(MDB_env *env, HANDLE fd) -{ - MDB_meta *mm; - MDB_page *mp; - mdb_copy my = {0}; - MDB_txn *txn = NULL; - pthread_t thr; - pgno_t root, new_root; - int rc = MDB_SUCCESS; - -#ifdef _WIN32 - if (!(my.mc_mutex = CreateMutex(NULL, FALSE, NULL)) || - !(my.mc_cond = CreateEvent(NULL, FALSE, FALSE, NULL))) { - rc = ErrCode(); - goto done; - } - my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_os_psize); - if (my.mc_wbuf[0] == NULL) { - /* _aligned_malloc() sets errno, but we use Windows error codes */ - rc = ERROR_NOT_ENOUGH_MEMORY; - goto done; - } -#else - if ((rc = pthread_mutex_init(&my.mc_mutex, NULL)) != 0) - return rc; - if ((rc = pthread_cond_init(&my.mc_cond, NULL)) != 0) - goto done2; -#ifdef HAVE_MEMALIGN - my.mc_wbuf[0] = memalign(env->me_os_psize, MDB_WBUF*2); - if (my.mc_wbuf[0] == NULL) { - rc = errno; - goto done; - } -#else - { - void *p; - if ((rc = posix_memalign(&p, env->me_os_psize, MDB_WBUF*2)) != 0) - goto done; - my.mc_wbuf[0] = p; - } -#endif -#endif - memset(my.mc_wbuf[0], 0, MDB_WBUF*2); - my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; - my.mc_next_pgno = NUM_METAS; - my.mc_env = env; - my.mc_fd = fd; - rc = THREAD_CREATE(thr, mdb_env_copythr, &my); - if (rc) - goto done; - - rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); - if (rc) - goto finish; - - mp = (MDB_page *)my.mc_wbuf[0]; - memset(mp, 0, NUM_METAS * env->me_psize); - mp->mp_pgno = 0; - mp->mp_flags = P_META; - mm = (MDB_meta *)METADATA(mp); - mdb_env_init_meta0(env, mm); - mm->mm_address = env->me_metas[0]->mm_address; - - mp = (MDB_page *)(my.mc_wbuf[0] + env->me_psize); - mp->mp_pgno = 1; - mp->mp_flags = P_META; - *(MDB_meta *)METADATA(mp) = *mm; - mm = (MDB_meta *)METADATA(mp); - - /* Set metapage 1 with current main DB */ - root = new_root = txn->mt_dbs[MAIN_DBI].md_root; - if (root != P_INVALID) { - /* Count free pages + freeDB pages. Subtract from last_pg - * to find the new last_pg, which also becomes the new root. - */ - MDB_ID freecount = 0; - MDB_cursor mc; - MDB_val key, data; - mdb_cursor_init(&mc, txn, FREE_DBI, NULL); - while ((rc = mdb_cursor_get(&mc, &key, &data, MDB_NEXT)) == 0) - freecount += *(MDB_ID *)data.mv_data; - if (rc != MDB_NOTFOUND) - goto finish; - freecount += txn->mt_dbs[FREE_DBI].md_branch_pages + - txn->mt_dbs[FREE_DBI].md_leaf_pages + - txn->mt_dbs[FREE_DBI].md_overflow_pages; - - new_root = txn->mt_next_pgno - 1 - freecount; - mm->mm_last_pg = new_root; - mm->mm_dbs[MAIN_DBI] = txn->mt_dbs[MAIN_DBI]; - mm->mm_dbs[MAIN_DBI].md_root = new_root; - } else { - /* When the DB is empty, handle it specially to - * fix any breakage like page leaks from ITS#8174. - */ - mm->mm_dbs[MAIN_DBI].md_flags = txn->mt_dbs[MAIN_DBI].md_flags; - } - if (root != P_INVALID || mm->mm_dbs[MAIN_DBI].md_flags) { - mm->mm_txnid = 1; /* use metapage 1 */ - } - - my.mc_wlen[0] = env->me_psize * NUM_METAS; - my.mc_txn = txn; - rc = mdb_env_cwalk(&my, &root, 0); - if (rc == MDB_SUCCESS && root != new_root) { - rc = MDB_INCOMPATIBLE; /* page leak or corrupt DB */ - } - -finish: - if (rc) - my.mc_error = rc; - mdb_env_cthr_toggle(&my, 1 | MDB_EOF); - rc = THREAD_FINISH(thr); - mdb_txn_abort(txn); - -done: -#ifdef _WIN32 - if (my.mc_wbuf[0]) _aligned_free(my.mc_wbuf[0]); - if (my.mc_cond) CloseHandle(my.mc_cond); - if (my.mc_mutex) CloseHandle(my.mc_mutex); -#else - free(my.mc_wbuf[0]); - pthread_cond_destroy(&my.mc_cond); -done2: - pthread_mutex_destroy(&my.mc_mutex); -#endif - return rc ? rc : my.mc_error; -} - - /** Copy environment as-is. */ -static int ESECT -mdb_env_copyfd0(MDB_env *env, HANDLE fd) -{ - MDB_txn *txn = NULL; - mdb_mutexref_t wmutex = NULL; - int rc; - mdb_size_t wsize, w3; - char *ptr; -#ifdef _WIN32 - DWORD len, w2; -#define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) -#else - ssize_t len; - size_t w2; -#define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) -#endif - - /* Do the lock/unlock of the reader mutex before starting the - * write txn. Otherwise other read txns could block writers. - */ - rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); - if (rc) - return rc; - - if (env->me_txns) { - /* We must start the actual read txn after blocking writers */ - mdb_txn_end(txn, MDB_END_RESET_TMP); - - /* Temporarily block writers until we snapshot the meta pages */ - wmutex = env->me_wmutex; - if (LOCK_MUTEX(rc, env, wmutex)) - goto leave; - - rc = mdb_txn_renew0(txn); - if (rc) { - UNLOCK_MUTEX(wmutex); - goto leave; - } - } - - wsize = env->me_psize * NUM_METAS; - ptr = env->me_map; - w2 = wsize; - while (w2 > 0) { - DO_WRITE(rc, fd, ptr, w2, len); - if (!rc) { - rc = ErrCode(); - break; - } else if (len > 0) { - rc = MDB_SUCCESS; - ptr += len; - w2 -= len; - continue; - } else { - /* Non-blocking or async handles are not supported */ - rc = EIO; - break; - } - } - if (wmutex) - UNLOCK_MUTEX(wmutex); - - if (rc) - goto leave; - - w3 = txn->mt_next_pgno * env->me_psize; - { - mdb_size_t fsize = 0; - if ((rc = mdb_fsize(env->me_fd, &fsize))) - goto leave; - if (w3 > fsize) - w3 = fsize; - } - wsize = w3 - wsize; - while (wsize > 0) { - if (wsize > MAX_WRITE) - w2 = MAX_WRITE; - else - w2 = wsize; - DO_WRITE(rc, fd, ptr, w2, len); - if (!rc) { - rc = ErrCode(); - break; - } else if (len > 0) { - rc = MDB_SUCCESS; - ptr += len; - wsize -= len; - continue; - } else { - rc = EIO; - break; - } - } - -leave: - mdb_txn_abort(txn); - return rc; -} - -int ESECT -mdb_env_copyfd2(MDB_env *env, HANDLE fd, unsigned int flags) -{ - if (flags & MDB_CP_COMPACT) - return mdb_env_copyfd1(env, fd); - else - return mdb_env_copyfd0(env, fd); -} - -int ESECT -mdb_env_copyfd(MDB_env *env, HANDLE fd) -{ - return mdb_env_copyfd2(env, fd, 0); -} - -int ESECT -mdb_env_copy2(MDB_env *env, const char *path, unsigned int flags) -{ - int rc; - MDB_name fname; - HANDLE newfd = INVALID_HANDLE_VALUE; - - rc = mdb_fname_init(path, env->me_flags | MDB_NOLOCK, &fname); - if (rc == MDB_SUCCESS) { - rc = mdb_fopen(env, &fname, MDB_O_COPY, 0666, &newfd); - mdb_fname_destroy(fname); - } - if (rc == MDB_SUCCESS) { - rc = mdb_env_copyfd2(env, newfd, flags); - if (close(newfd) < 0 && rc == MDB_SUCCESS) - rc = ErrCode(); - } - return rc; -} - -int ESECT -mdb_env_copy(MDB_env *env, const char *path) -{ - return mdb_env_copy2(env, path, 0); -} - -int ESECT -mdb_env_set_flags(MDB_env *env, unsigned int flag, int onoff) -{ - if (flag & ~CHANGEABLE) - return EINVAL; - if (onoff) - env->me_flags |= flag; - else - env->me_flags &= ~flag; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_flags(MDB_env *env, unsigned int *arg) -{ - if (!env || !arg) - return EINVAL; - - *arg = env->me_flags & (CHANGEABLE|CHANGELESS); - return MDB_SUCCESS; -} - -int ESECT -mdb_env_set_userctx(MDB_env *env, void *ctx) -{ - if (!env) - return EINVAL; - env->me_userctx = ctx; - return MDB_SUCCESS; -} - -void * ESECT -mdb_env_get_userctx(MDB_env *env) -{ - return env ? env->me_userctx : NULL; -} - -int ESECT -mdb_env_set_assert(MDB_env *env, MDB_assert_func *func) -{ - if (!env) - return EINVAL; -#ifndef NDEBUG - env->me_assert_func = func; -#endif - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_path(MDB_env *env, const char **arg) -{ - if (!env || !arg) - return EINVAL; - - *arg = env->me_path; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_fd(MDB_env *env, mdb_filehandle_t *arg) -{ - if (!env || !arg) - return EINVAL; - - *arg = env->me_fd; - return MDB_SUCCESS; -} - -/** Common code for #mdb_stat() and #mdb_env_stat(). - * @param[in] env the environment to operate in. - * @param[in] db the #MDB_db record containing the stats to return. - * @param[out] arg the address of an #MDB_stat structure to receive the stats. - * @return 0, this function always succeeds. - */ -static int ESECT -mdb_stat0(MDB_env *env, MDB_db *db, MDB_stat *arg) -{ - arg->ms_psize = env->me_psize; - arg->ms_depth = db->md_depth; - arg->ms_branch_pages = db->md_branch_pages; - arg->ms_leaf_pages = db->md_leaf_pages; - arg->ms_overflow_pages = db->md_overflow_pages; - arg->ms_entries = db->md_entries; - - return MDB_SUCCESS; -} - -int ESECT -mdb_env_stat(MDB_env *env, MDB_stat *arg) -{ - MDB_meta *meta; - - if (env == NULL || arg == NULL) - return EINVAL; - - meta = mdb_env_pick_meta(env); - - return mdb_stat0(env, &meta->mm_dbs[MAIN_DBI], arg); -} - -int ESECT -mdb_env_info(MDB_env *env, MDB_envinfo *arg) -{ - MDB_meta *meta; - - if (env == NULL || arg == NULL) - return EINVAL; - - meta = mdb_env_pick_meta(env); - arg->me_mapaddr = meta->mm_address; - arg->me_last_pgno = meta->mm_last_pg; - arg->me_last_txnid = meta->mm_txnid; - - arg->me_mapsize = env->me_mapsize; - arg->me_maxreaders = env->me_maxreaders; - arg->me_numreaders = env->me_txns ? env->me_txns->mti_numreaders : 0; - return MDB_SUCCESS; -} - -/** Set the default comparison functions for a database. - * Called immediately after a database is opened to set the defaults. - * The user can then override them with #mdb_set_compare() or - * #mdb_set_dupsort(). - * @param[in] txn A transaction handle returned by #mdb_txn_begin() - * @param[in] dbi A database handle returned by #mdb_dbi_open() - */ -static void -mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi) -{ - uint16_t f = txn->mt_dbs[dbi].md_flags; - - txn->mt_dbxs[dbi].md_cmp = - (f & MDB_REVERSEKEY) ? mdb_cmp_memnr : - (f & MDB_INTEGERKEY) ? mdb_cmp_cint : mdb_cmp_memn; - - txn->mt_dbxs[dbi].md_dcmp = - !(f & MDB_DUPSORT) ? 0 : - ((f & MDB_INTEGERDUP) - ? ((f & MDB_DUPFIXED) ? mdb_cmp_int : mdb_cmp_cint) - : ((f & MDB_REVERSEDUP) ? mdb_cmp_memnr : mdb_cmp_memn)); -} - -int mdb_dbi_open(MDB_txn *txn, const char *name, unsigned int flags, MDB_dbi *dbi) -{ - MDB_val key, data; - MDB_dbi i; - MDB_cursor mc; - MDB_db dummy; - int rc, dbflag, exact; - unsigned int unused = 0, seq; - char *namedup; - size_t len; - - if (flags & ~VALID_FLAGS) - return EINVAL; - if (txn->mt_flags & MDB_TXN_BLOCKED) - return MDB_BAD_TXN; - - /* main DB? */ - if (!name) { - *dbi = MAIN_DBI; - if (flags & PERSISTENT_FLAGS) { - uint16_t f2 = flags & PERSISTENT_FLAGS; - /* make sure flag changes get committed */ - if ((txn->mt_dbs[MAIN_DBI].md_flags | f2) != txn->mt_dbs[MAIN_DBI].md_flags) { - txn->mt_dbs[MAIN_DBI].md_flags |= f2; - txn->mt_flags |= MDB_TXN_DIRTY; - } - } - mdb_default_cmp(txn, MAIN_DBI); - return MDB_SUCCESS; - } - - if (txn->mt_dbxs[MAIN_DBI].md_cmp == NULL) { - mdb_default_cmp(txn, MAIN_DBI); - } - - /* Is the DB already open? */ - len = strlen(name); - for (i=CORE_DBS; imt_numdbs; i++) { - if (!txn->mt_dbxs[i].md_name.mv_size) { - /* Remember this free slot */ - if (!unused) unused = i; - continue; - } - if (len == txn->mt_dbxs[i].md_name.mv_size && - !strncmp(name, txn->mt_dbxs[i].md_name.mv_data, len)) { - *dbi = i; - return MDB_SUCCESS; - } - } - - /* If no free slot and max hit, fail */ - if (!unused && txn->mt_numdbs >= txn->mt_env->me_maxdbs) - return MDB_DBS_FULL; - - /* Cannot mix named databases with some mainDB flags */ - if (txn->mt_dbs[MAIN_DBI].md_flags & (MDB_DUPSORT|MDB_INTEGERKEY)) - return (flags & MDB_CREATE) ? MDB_INCOMPATIBLE : MDB_NOTFOUND; - - /* Find the DB info */ - dbflag = DB_NEW|DB_VALID|DB_USRVALID; - exact = 0; - key.mv_size = len; - key.mv_data = (void *)name; - mdb_cursor_init(&mc, txn, MAIN_DBI, NULL); - rc = mdb_cursor_set(&mc, &key, &data, MDB_SET, &exact); - if (rc == MDB_SUCCESS) { - /* make sure this is actually a DB */ - MDB_node *node = NODEPTR(mc.mc_pg[mc.mc_top], mc.mc_ki[mc.mc_top]); - if ((node->mn_flags & (F_DUPDATA|F_SUBDATA)) != F_SUBDATA) - return MDB_INCOMPATIBLE; - } else { - if (rc != MDB_NOTFOUND || !(flags & MDB_CREATE)) - return rc; - if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) - return EACCES; - } - - /* Done here so we cannot fail after creating a new DB */ - if ((namedup = strdup(name)) == NULL) - return ENOMEM; - - if (rc) { - /* MDB_NOTFOUND and MDB_CREATE: Create new DB */ - data.mv_size = sizeof(MDB_db); - data.mv_data = &dummy; - memset(&dummy, 0, sizeof(dummy)); - dummy.md_root = P_INVALID; - dummy.md_flags = flags & PERSISTENT_FLAGS; - WITH_CURSOR_TRACKING(mc, - rc = mdb_cursor_put(&mc, &key, &data, F_SUBDATA)); - dbflag |= DB_DIRTY; - } - - if (rc) { - free(namedup); - } else { - /* Got info, register DBI in this txn */ - unsigned int slot = unused ? unused : txn->mt_numdbs; - txn->mt_dbxs[slot].md_name.mv_data = namedup; - txn->mt_dbxs[slot].md_name.mv_size = len; - txn->mt_dbxs[slot].md_rel = NULL; - txn->mt_dbflags[slot] = dbflag; - /* txn-> and env-> are the same in read txns, use - * tmp variable to avoid undefined assignment - */ - seq = ++txn->mt_env->me_dbiseqs[slot]; - txn->mt_dbiseqs[slot] = seq; - - memcpy(&txn->mt_dbs[slot], data.mv_data, sizeof(MDB_db)); - *dbi = slot; - mdb_default_cmp(txn, slot); - if (!unused) { - txn->mt_numdbs++; - } - } - - return rc; -} - -int ESECT -mdb_stat(MDB_txn *txn, MDB_dbi dbi, MDB_stat *arg) -{ - if (!arg || !TXN_DBI_EXIST(txn, dbi, DB_VALID)) - return EINVAL; - - if (txn->mt_flags & MDB_TXN_BLOCKED) - return MDB_BAD_TXN; - - if (txn->mt_dbflags[dbi] & DB_STALE) { - MDB_cursor mc; - MDB_xcursor mx; - /* Stale, must read the DB's root. cursor_init does it for us. */ - mdb_cursor_init(&mc, txn, dbi, &mx); - } - return mdb_stat0(txn->mt_env, &txn->mt_dbs[dbi], arg); -} - -void mdb_dbi_close(MDB_env *env, MDB_dbi dbi) -{ - char *ptr; - if (dbi < CORE_DBS || dbi >= env->me_maxdbs) - return; - ptr = env->me_dbxs[dbi].md_name.mv_data; - /* If there was no name, this was already closed */ - if (ptr) { - env->me_dbxs[dbi].md_name.mv_data = NULL; - env->me_dbxs[dbi].md_name.mv_size = 0; - env->me_dbflags[dbi] = 0; - env->me_dbiseqs[dbi]++; - free(ptr); - } -} - -int mdb_dbi_flags(MDB_txn *txn, MDB_dbi dbi, unsigned int *flags) -{ - /* We could return the flags for the FREE_DBI too but what's the point? */ - if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) - return EINVAL; - *flags = txn->mt_dbs[dbi].md_flags & PERSISTENT_FLAGS; - return MDB_SUCCESS; -} - -/** Add all the DB's pages to the free list. - * @param[in] mc Cursor on the DB to free. - * @param[in] subs non-Zero to check for sub-DBs in this DB. - * @return 0 on success, non-zero on failure. - */ -static int -mdb_drop0(MDB_cursor *mc, int subs) -{ - int rc; - - rc = mdb_page_search(mc, NULL, MDB_PS_FIRST); - if (rc == MDB_SUCCESS) { - MDB_txn *txn = mc->mc_txn; - MDB_node *ni; - MDB_cursor mx; - unsigned int i; - - /* DUPSORT sub-DBs have no ovpages/DBs. Omit scanning leaves. - * This also avoids any P_LEAF2 pages, which have no nodes. - * Also if the DB doesn't have sub-DBs and has no overflow - * pages, omit scanning leaves. - */ - if ((mc->mc_flags & C_SUB) || - (!subs && !mc->mc_db->md_overflow_pages)) - mdb_cursor_pop(mc); - - mdb_cursor_copy(mc, &mx); -#ifdef MDB_VL32 - /* bump refcount for mx's pages */ - for (i=0; imc_snum; i++) - mdb_page_get(&mx, mc->mc_pg[i]->mp_pgno, &mx.mc_pg[i], NULL); -#endif - while (mc->mc_snum > 0) { - MDB_page *mp = mc->mc_pg[mc->mc_top]; - unsigned n = NUMKEYS(mp); - if (IS_LEAF(mp)) { - for (i=0; imn_flags & F_BIGDATA) { - MDB_page *omp; - pgno_t pg; - memcpy(&pg, NODEDATA(ni), sizeof(pg)); - rc = mdb_page_get(mc, pg, &omp, NULL); - if (rc != 0) - goto done; - mdb_cassert(mc, IS_OVERFLOW(omp)); - rc = mdb_midl_append_range(&txn->mt_free_pgs, - pg, omp->mp_pages); - if (rc) - goto done; - mc->mc_db->md_overflow_pages -= omp->mp_pages; - if (!mc->mc_db->md_overflow_pages && !subs) - break; - } else if (subs && (ni->mn_flags & F_SUBDATA)) { - mdb_xcursor_init1(mc, ni); - rc = mdb_drop0(&mc->mc_xcursor->mx_cursor, 0); - if (rc) - goto done; - } - } - if (!subs && !mc->mc_db->md_overflow_pages) - goto pop; - } else { - if ((rc = mdb_midl_need(&txn->mt_free_pgs, n)) != 0) - goto done; - for (i=0; imt_free_pgs, pg); - } - } - if (!mc->mc_top) - break; - mc->mc_ki[mc->mc_top] = i; - rc = mdb_cursor_sibling(mc, 1); - if (rc) { - if (rc != MDB_NOTFOUND) - goto done; - /* no more siblings, go back to beginning - * of previous level. - */ -pop: - mdb_cursor_pop(mc); - mc->mc_ki[0] = 0; - for (i=1; imc_snum; i++) { - mc->mc_ki[i] = 0; - mc->mc_pg[i] = mx.mc_pg[i]; - } - } - } - /* free it */ - rc = mdb_midl_append(&txn->mt_free_pgs, mc->mc_db->md_root); -done: - if (rc) - txn->mt_flags |= MDB_TXN_ERROR; - /* drop refcount for mx's pages */ - MDB_CURSOR_UNREF(&mx, 0); - } else if (rc == MDB_NOTFOUND) { - rc = MDB_SUCCESS; - } - mc->mc_flags &= ~C_INITIALIZED; - return rc; -} - -int mdb_drop(MDB_txn *txn, MDB_dbi dbi, int del) -{ - MDB_cursor *mc, *m2; - int rc; - - if ((unsigned)del > 1 || !TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) - return EINVAL; - - if (F_ISSET(txn->mt_flags, MDB_TXN_RDONLY)) - return EACCES; - - if (TXN_DBI_CHANGED(txn, dbi)) - return MDB_BAD_DBI; - - rc = mdb_cursor_open(txn, dbi, &mc); - if (rc) - return rc; - - rc = mdb_drop0(mc, mc->mc_db->md_flags & MDB_DUPSORT); - /* Invalidate the dropped DB's cursors */ - for (m2 = txn->mt_cursors[dbi]; m2; m2 = m2->mc_next) - m2->mc_flags &= ~(C_INITIALIZED|C_EOF); - if (rc) - goto leave; - - /* Can't delete the main DB */ - if (del && dbi >= CORE_DBS) { - rc = mdb_del0(txn, MAIN_DBI, &mc->mc_dbx->md_name, NULL, F_SUBDATA); - if (!rc) { - txn->mt_dbflags[dbi] = DB_STALE; - mdb_dbi_close(txn->mt_env, dbi); - } else { - txn->mt_flags |= MDB_TXN_ERROR; - } - } else { - /* reset the DB record, mark it dirty */ - txn->mt_dbflags[dbi] |= DB_DIRTY; - txn->mt_dbs[dbi].md_depth = 0; - txn->mt_dbs[dbi].md_branch_pages = 0; - txn->mt_dbs[dbi].md_leaf_pages = 0; - txn->mt_dbs[dbi].md_overflow_pages = 0; - txn->mt_dbs[dbi].md_entries = 0; - txn->mt_dbs[dbi].md_root = P_INVALID; - - txn->mt_flags |= MDB_TXN_DIRTY; - } -leave: - mdb_cursor_close(mc); - return rc; -} - -int mdb_set_compare(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) -{ - if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) - return EINVAL; - - txn->mt_dbxs[dbi].md_cmp = cmp; - return MDB_SUCCESS; -} - -int mdb_set_dupsort(MDB_txn *txn, MDB_dbi dbi, MDB_cmp_func *cmp) -{ - if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) - return EINVAL; - - txn->mt_dbxs[dbi].md_dcmp = cmp; - return MDB_SUCCESS; -} - -int mdb_set_relfunc(MDB_txn *txn, MDB_dbi dbi, MDB_rel_func *rel) -{ - if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) - return EINVAL; - - txn->mt_dbxs[dbi].md_rel = rel; - return MDB_SUCCESS; -} - -int mdb_set_relctx(MDB_txn *txn, MDB_dbi dbi, void *ctx) -{ - if (!TXN_DBI_EXIST(txn, dbi, DB_USRVALID)) - return EINVAL; - - txn->mt_dbxs[dbi].md_relctx = ctx; - return MDB_SUCCESS; -} - -int ESECT -mdb_env_get_maxkeysize(MDB_env *env) -{ - return ENV_MAXKEY(env); -} - -int ESECT -mdb_reader_list(MDB_env *env, MDB_msg_func *func, void *ctx) -{ - unsigned int i, rdrs; - MDB_reader *mr; - char buf[64]; - int rc = 0, first = 1; - - if (!env || !func) - return -1; - if (!env->me_txns) { - return func("(no reader locks)\n", ctx); - } - rdrs = env->me_txns->mti_numreaders; - mr = env->me_txns->mti_readers; - for (i=0; i> 1; - cursor = base + pivot + 1; - val = pid - ids[cursor]; - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - /* found, so it's a duplicate */ - return -1; - } - } - - if( val > 0 ) { - ++cursor; - } - ids[0]++; - for (n = ids[0]; n > cursor; n--) - ids[n] = ids[n-1]; - ids[n] = pid; - return 0; -} - -int ESECT -mdb_reader_check(MDB_env *env, int *dead) -{ - if (!env) - return EINVAL; - if (dead) - *dead = 0; - return env->me_txns ? mdb_reader_check0(env, 0, dead) : MDB_SUCCESS; -} - -/** As #mdb_reader_check(). \b rlocked is set if caller locked #me_rmutex. */ -static int ESECT -mdb_reader_check0(MDB_env *env, int rlocked, int *dead) -{ - mdb_mutexref_t rmutex = rlocked ? NULL : env->me_rmutex; - unsigned int i, j, rdrs; - MDB_reader *mr; - MDB_PID_T *pids, pid; - int rc = MDB_SUCCESS, count = 0; - - rdrs = env->me_txns->mti_numreaders; - pids = malloc((rdrs+1) * sizeof(MDB_PID_T)); - if (!pids) - return ENOMEM; - pids[0] = 0; - mr = env->me_txns->mti_readers; - for (i=0; ime_pid) { - if (mdb_pid_insert(pids, pid) == 0) { - if (!mdb_reader_pid(env, Pidcheck, pid)) { - /* Stale reader found */ - j = i; - if (rmutex) { - if ((rc = LOCK_MUTEX0(rmutex)) != 0) { - if ((rc = mdb_mutex_failed(env, rmutex, rc))) - break; - rdrs = 0; /* the above checked all readers */ - } else { - /* Recheck, a new process may have reused pid */ - if (mdb_reader_pid(env, Pidcheck, pid)) - j = rdrs; - } - } - for (; jme_rmutex); - if (!rlocked) { - /* Keep mti_txnid updated, otherwise next writer can - * overwrite data which latest meta page refers to. - */ - meta = mdb_env_pick_meta(env); - env->me_txns->mti_txnid = meta->mm_txnid; - /* env is hosed if the dead thread was ours */ - if (env->me_txn) { - env->me_flags |= MDB_FATAL_ERROR; - env->me_txn = NULL; - rc = MDB_PANIC; - } - } - DPRINTF(("%cmutex owner died, %s", (rlocked ? 'r' : 'w'), - (rc ? "this process' env is hosed" : "recovering"))); - rc2 = mdb_reader_check0(env, rlocked, NULL); - if (rc2 == 0) - rc2 = mdb_mutex_consistent(mutex); - if (rc || (rc = rc2)) { - DPRINTF(("LOCK_MUTEX recovery failed, %s", mdb_strerror(rc))); - UNLOCK_MUTEX(mutex); - } - } else { -#ifdef _WIN32 - rc = ErrCode(); -#endif - DPRINTF(("LOCK_MUTEX failed, %s", mdb_strerror(rc))); - } - - return rc; -} -#endif /* MDB_ROBUST_SUPPORTED */ - -#if defined(_WIN32) -/** Convert \b src to new wchar_t[] string with room for \b xtra extra chars */ -static int ESECT -utf8_to_utf16(const char *src, MDB_name *dst, int xtra) -{ - int rc, need = 0; - wchar_t *result = NULL; - for (;;) { /* malloc result, then fill it in */ - need = MultiByteToWideChar(CP_UTF8, 0, src, -1, result, need); - if (!need) { - rc = ErrCode(); - free(result); - return rc; - } - if (!result) { - result = malloc(sizeof(wchar_t) * (need + xtra)); - if (!result) - return ENOMEM; - continue; - } - dst->mn_alloced = 1; - dst->mn_len = need - 1; - dst->mn_val = result; - return MDB_SUCCESS; - } -} -#endif /* defined(_WIN32) */ -/** @} */ diff --git a/pkg/hs/lmdb-static/cbits/midl.c b/pkg/hs/lmdb-static/cbits/midl.c deleted file mode 100644 index ee89822e39..0000000000 --- a/pkg/hs/lmdb-static/cbits/midl.c +++ /dev/null @@ -1,421 +0,0 @@ -/** @file midl.c - * @brief ldap bdb back-end ID List functions */ -/* $OpenLDAP$ */ -/* This work is part of OpenLDAP Software . - * - * Copyright 2000-2019 The OpenLDAP Foundation. - * Portions Copyright 2001-2018 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -#include -#include -#include -#include -#include -#include "midl.h" - -/** @defgroup internal LMDB Internals - * @{ - */ -/** @defgroup idls ID List Management - * @{ - */ -#define CMP(x,y) ( (x) < (y) ? -1 : (x) > (y) ) - -unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) -{ - /* - * binary search of id in ids - * if found, returns position of id - * if not found, returns first position greater than id - */ - unsigned base = 0; - unsigned cursor = 1; - int val = 0; - unsigned n = ids[0]; - - while( 0 < n ) { - unsigned pivot = n >> 1; - cursor = base + pivot + 1; - val = CMP( ids[cursor], id ); - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - return cursor; - } - } - - if( val > 0 ) { - ++cursor; - } - return cursor; -} - -#if 0 /* superseded by append/sort */ -int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) -{ - unsigned x, i; - - x = mdb_midl_search( ids, id ); - assert( x > 0 ); - - if( x < 1 ) { - /* internal error */ - return -2; - } - - if ( x <= ids[0] && ids[x] == id ) { - /* duplicate */ - assert(0); - return -1; - } - - if ( ++ids[0] >= MDB_IDL_DB_MAX ) { - /* no room */ - --ids[0]; - return -2; - - } else { - /* insert id */ - for (i=ids[0]; i>x; i--) - ids[i] = ids[i-1]; - ids[x] = id; - } - - return 0; -} -#endif - -MDB_IDL mdb_midl_alloc(int num) -{ - MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); - if (ids) { - *ids++ = num; - *ids = 0; - } - return ids; -} - -void mdb_midl_free(MDB_IDL ids) -{ - if (ids) - free(ids-1); -} - -void mdb_midl_shrink( MDB_IDL *idp ) -{ - MDB_IDL ids = *idp; - if (*(--ids) > MDB_IDL_UM_MAX && - (ids = realloc(ids, (MDB_IDL_UM_MAX+2) * sizeof(MDB_ID)))) - { - *ids++ = MDB_IDL_UM_MAX; - *idp = ids; - } -} - -static int mdb_midl_grow( MDB_IDL *idp, int num ) -{ - MDB_IDL idn = *idp-1; - /* grow it */ - idn = realloc(idn, (*idn + num + 2) * sizeof(MDB_ID)); - if (!idn) - return ENOMEM; - *idn++ += num; - *idp = idn; - return 0; -} - -int mdb_midl_need( MDB_IDL *idp, unsigned num ) -{ - MDB_IDL ids = *idp; - num += ids[0]; - if (num > ids[-1]) { - num = (num + num/4 + (256 + 2)) & -256; - if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) - return ENOMEM; - *ids++ = num - 2; - *idp = ids; - } - return 0; -} - -int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) -{ - MDB_IDL ids = *idp; - /* Too big? */ - if (ids[0] >= ids[-1]) { - if (mdb_midl_grow(idp, MDB_IDL_UM_MAX)) - return ENOMEM; - ids = *idp; - } - ids[0]++; - ids[ids[0]] = id; - return 0; -} - -int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) -{ - MDB_IDL ids = *idp; - /* Too big? */ - if (ids[0] + app[0] >= ids[-1]) { - if (mdb_midl_grow(idp, app[0])) - return ENOMEM; - ids = *idp; - } - memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); - ids[0] += app[0]; - return 0; -} - -int mdb_midl_append_range( MDB_IDL *idp, MDB_ID id, unsigned n ) -{ - MDB_ID *ids = *idp, len = ids[0]; - /* Too big? */ - if (len + n > ids[-1]) { - if (mdb_midl_grow(idp, n | MDB_IDL_UM_MAX)) - return ENOMEM; - ids = *idp; - } - ids[0] = len + n; - ids += len; - while (n) - ids[n--] = id++; - return 0; -} - -void mdb_midl_xmerge( MDB_IDL idl, MDB_IDL merge ) -{ - MDB_ID old_id, merge_id, i = merge[0], j = idl[0], k = i+j, total = k; - idl[0] = (MDB_ID)-1; /* delimiter for idl scan below */ - old_id = idl[j]; - while (i) { - merge_id = merge[i--]; - for (; old_id < merge_id; old_id = idl[--j]) - idl[k--] = old_id; - idl[k--] = merge_id; - } - idl[0] = total; -} - -/* Quicksort + Insertion sort for small arrays */ - -#define SMALL 8 -#define MIDL_SWAP(a,b) { itmp=(a); (a)=(b); (b)=itmp; } - -void -mdb_midl_sort( MDB_IDL ids ) -{ - /* Max possible depth of int-indexed tree * 2 items/level */ - int istack[sizeof(int)*CHAR_BIT * 2]; - int i,j,k,l,ir,jstack; - MDB_ID a, itmp; - - ir = (int)ids[0]; - l = 1; - jstack = 0; - for(;;) { - if (ir - l < SMALL) { /* Insertion sort */ - for (j=l+1;j<=ir;j++) { - a = ids[j]; - for (i=j-1;i>=1;i--) { - if (ids[i] >= a) break; - ids[i+1] = ids[i]; - } - ids[i+1] = a; - } - if (jstack == 0) break; - ir = istack[jstack--]; - l = istack[jstack--]; - } else { - k = (l + ir) >> 1; /* Choose median of left, center, right */ - MIDL_SWAP(ids[k], ids[l+1]); - if (ids[l] < ids[ir]) { - MIDL_SWAP(ids[l], ids[ir]); - } - if (ids[l+1] < ids[ir]) { - MIDL_SWAP(ids[l+1], ids[ir]); - } - if (ids[l] < ids[l+1]) { - MIDL_SWAP(ids[l], ids[l+1]); - } - i = l+1; - j = ir; - a = ids[l+1]; - for(;;) { - do i++; while(ids[i] > a); - do j--; while(ids[j] < a); - if (j < i) break; - MIDL_SWAP(ids[i],ids[j]); - } - ids[l+1] = ids[j]; - ids[j] = a; - jstack += 2; - if (ir-i+1 >= j-l) { - istack[jstack] = ir; - istack[jstack-1] = i; - ir = j-1; - } else { - istack[jstack] = j-1; - istack[jstack-1] = l; - l = i; - } - } - } -} - -unsigned mdb_mid2l_search( MDB_ID2L ids, MDB_ID id ) -{ - /* - * binary search of id in ids - * if found, returns position of id - * if not found, returns first position greater than id - */ - unsigned base = 0; - unsigned cursor = 1; - int val = 0; - unsigned n = (unsigned)ids[0].mid; - - while( 0 < n ) { - unsigned pivot = n >> 1; - cursor = base + pivot + 1; - val = CMP( id, ids[cursor].mid ); - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - return cursor; - } - } - - if( val > 0 ) { - ++cursor; - } - return cursor; -} - -int mdb_mid2l_insert( MDB_ID2L ids, MDB_ID2 *id ) -{ - unsigned x, i; - - x = mdb_mid2l_search( ids, id->mid ); - - if( x < 1 ) { - /* internal error */ - return -2; - } - - if ( x <= ids[0].mid && ids[x].mid == id->mid ) { - /* duplicate */ - return -1; - } - - if ( ids[0].mid >= MDB_IDL_UM_MAX ) { - /* too big */ - return -2; - - } else { - /* insert id */ - ids[0].mid++; - for (i=(unsigned)ids[0].mid; i>x; i--) - ids[i] = ids[i-1]; - ids[x] = *id; - } - - return 0; -} - -int mdb_mid2l_append( MDB_ID2L ids, MDB_ID2 *id ) -{ - /* Too big? */ - if (ids[0].mid >= MDB_IDL_UM_MAX) { - return -2; - } - ids[0].mid++; - ids[ids[0].mid] = *id; - return 0; -} - -#ifdef MDB_VL32 -unsigned mdb_mid3l_search( MDB_ID3L ids, MDB_ID id ) -{ - /* - * binary search of id in ids - * if found, returns position of id - * if not found, returns first position greater than id - */ - unsigned base = 0; - unsigned cursor = 1; - int val = 0; - unsigned n = (unsigned)ids[0].mid; - - while( 0 < n ) { - unsigned pivot = n >> 1; - cursor = base + pivot + 1; - val = CMP( id, ids[cursor].mid ); - - if( val < 0 ) { - n = pivot; - - } else if ( val > 0 ) { - base = cursor; - n -= pivot + 1; - - } else { - return cursor; - } - } - - if( val > 0 ) { - ++cursor; - } - return cursor; -} - -int mdb_mid3l_insert( MDB_ID3L ids, MDB_ID3 *id ) -{ - unsigned x, i; - - x = mdb_mid3l_search( ids, id->mid ); - - if( x < 1 ) { - /* internal error */ - return -2; - } - - if ( x <= ids[0].mid && ids[x].mid == id->mid ) { - /* duplicate */ - return -1; - } - - /* insert id */ - ids[0].mid++; - for (i=(unsigned)ids[0].mid; i>x; i--) - ids[i] = ids[i-1]; - ids[x] = *id; - - return 0; -} -#endif /* MDB_VL32 */ - -/** @} */ -/** @} */ diff --git a/pkg/hs/lmdb-static/cbits/midl.h b/pkg/hs/lmdb-static/cbits/midl.h deleted file mode 100644 index 0f826771a1..0000000000 --- a/pkg/hs/lmdb-static/cbits/midl.h +++ /dev/null @@ -1,200 +0,0 @@ -/** @file midl.h - * @brief LMDB ID List header file. - * - * This file was originally part of back-bdb but has been - * modified for use in libmdb. Most of the macros defined - * in this file are unused, just left over from the original. - * - * This file is only used internally in libmdb and its definitions - * are not exposed publicly. - */ -/* $OpenLDAP$ */ -/* This work is part of OpenLDAP Software . - * - * Copyright 2000-2019 The OpenLDAP Foundation. - * Portions Copyright 2001-2019 Howard Chu, Symas Corp. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted only as authorized by the OpenLDAP - * Public License. - * - * A copy of this license is available in the file LICENSE in the - * top-level directory of the distribution or, alternatively, at - * . - */ - -#ifndef _MDB_MIDL_H_ -#define _MDB_MIDL_H_ - -#include "lmdb.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/** @defgroup internal LMDB Internals - * @{ - */ - -/** @defgroup idls ID List Management - * @{ - */ - /** A generic unsigned ID number. These were entryIDs in back-bdb. - * Preferably it should have the same size as a pointer. - */ -typedef mdb_size_t MDB_ID; - - /** An IDL is an ID List, a sorted array of IDs. The first - * element of the array is a counter for how many actual - * IDs are in the list. In the original back-bdb code, IDLs are - * sorted in ascending order. For libmdb IDLs are sorted in - * descending order. - */ -typedef MDB_ID *MDB_IDL; - -/* IDL sizes - likely should be even bigger - * limiting factors: sizeof(ID), thread stack size - */ -#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */ -#define MDB_IDL_DB_SIZE (1< - -import Prelude - -import Foreign -import Foreign.C -import Control.Monad -import Control.Exception -import Control.Concurrent -import qualified Data.Array.Unboxed as A -import qualified Data.List as L -import Data.Typeable ---import System.IO (FilePath) -import Data.Function (on) -import Data.Maybe (isNothing) -import Data.IORef - -#let alignment t = "%lu", (unsigned long)offsetof(struct {char x__; t (y__); }, y__) - --- FFI --- 'safe': higher overhead, thread juggling, allows callbacks into Haskell --- 'unsafe': lower overhead, reduced concurrency, no callbacks into Haskell -foreign import ccall unsafe "lmdb.h mdb_version" _mdb_version :: Ptr CInt -> Ptr CInt -> Ptr CInt -> IO CString -foreign import ccall unsafe "lmdb.h mdb_strerror" _mdb_strerror :: CInt -> CString - -foreign import ccall "lmdb.h mdb_env_create" _mdb_env_create :: Ptr (Ptr MDB_env) -> IO CInt -foreign import ccall "lmdb.h mdb_env_open" _mdb_env_open :: Ptr MDB_env -> CString -> CUInt -> MDB_mode_t -> IO CInt -foreign import ccall "lmdb.h mdb_env_copy" _mdb_env_copy :: Ptr MDB_env -> CString -> IO CInt -foreign import ccall "lmdb.h mdb_env_stat" _mdb_env_stat :: Ptr MDB_env -> Ptr MDB_stat -> IO CInt -foreign import ccall "lmdb.h mdb_env_info" _mdb_env_info :: Ptr MDB_env -> Ptr MDB_envinfo -> IO CInt -foreign import ccall "lmdb.h mdb_env_sync" _mdb_env_sync :: Ptr MDB_env -> CInt -> IO CInt -foreign import ccall "lmdb.h mdb_env_close" _mdb_env_close :: Ptr MDB_env -> IO () -foreign import ccall "lmdb.h mdb_env_set_flags" _mdb_env_set_flags :: Ptr MDB_env -> CUInt -> CInt -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_env_get_flags" _mdb_env_get_flags :: Ptr MDB_env -> Ptr CUInt -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_env_get_path" _mdb_env_get_path :: Ptr MDB_env -> Ptr CString -> IO CInt -foreign import ccall "lmdb.h mdb_env_set_mapsize" _mdb_env_set_mapsize :: Ptr MDB_env -> CSize -> IO CInt -foreign import ccall "lmdb.h mdb_env_set_maxreaders" _mdb_env_set_maxreaders :: Ptr MDB_env -> CUInt -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_env_get_maxreaders" _mdb_env_get_maxreaders :: Ptr MDB_env -> Ptr CUInt -> IO CInt -foreign import ccall "lmdb.h mdb_env_set_maxdbs" _mdb_env_set_maxdbs :: Ptr MDB_env -> MDB_dbi_t -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_env_get_maxkeysize" _mdb_env_get_maxkeysize :: Ptr MDB_env -> IO CInt - -foreign import ccall "lmdb.h mdb_txn_begin" _mdb_txn_begin :: Ptr MDB_env -> Ptr MDB_txn -> CUInt -> Ptr (Ptr MDB_txn) -> IO CInt --- foreign import ccall "lmdb.h mdb_txn_env" _mdb_txn_env :: MDB_txn -> IO (Ptr MDB_env) -foreign import ccall "lmdb.h mdb_txn_commit" _mdb_txn_commit :: Ptr MDB_txn -> IO CInt -foreign import ccall "lmdb.h mdb_txn_abort" _mdb_txn_abort :: Ptr MDB_txn -> IO () - --- I'm hoping to get a patch adding the following function into LMDB: --- foreign import ccall "lmdb.h mdb_txn_id" _mdb_txn_id :: MDB_txn -> IO MDB_txnid_t - -foreign import ccall "lmdb.h mdb_dbi_open" _mdb_dbi_open :: Ptr MDB_txn -> CString -> CUInt -> Ptr MDB_dbi_t -> IO CInt -foreign import ccall "lmdb.h mdb_stat" _mdb_stat :: Ptr MDB_txn -> MDB_dbi_t -> Ptr MDB_stat -> IO CInt -foreign import ccall "lmdb.h mdb_dbi_flags" _mdb_dbi_flags :: Ptr MDB_txn -> MDB_dbi_t -> Ptr CUInt -> IO CInt -foreign import ccall "lmdb.h mdb_dbi_close" _mdb_dbi_close :: Ptr MDB_env -> MDB_dbi_t -> IO () -foreign import ccall "lmdb.h mdb_drop" _mdb_drop :: Ptr MDB_txn -> MDB_dbi_t -> CInt -> IO CInt - --- comparisons may only be configured for a 'safe' MDB_dbi. -foreign import ccall "lmdb.h mdb_set_compare" _mdb_set_compare :: Ptr MDB_txn -> MDB_dbi -> FunPtr MDB_cmp_func -> IO CInt -foreign import ccall "lmdb.h mdb_set_dupsort" _mdb_set_dupsort :: Ptr MDB_txn -> MDB_dbi -> FunPtr MDB_cmp_func -> IO CInt - -foreign import ccall safe "lmdb.h mdb_cmp" _mdb_cmp :: Ptr MDB_txn -> MDB_dbi -> Ptr MDB_val -> Ptr MDB_val -> IO CInt -foreign import ccall safe "lmdb.h mdb_dcmp" _mdb_dcmp :: Ptr MDB_txn -> MDB_dbi -> Ptr MDB_val -> Ptr MDB_val -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_cmp" _mdb_cmp' :: Ptr MDB_txn -> MDB_dbi' -> Ptr MDB_val -> Ptr MDB_val -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_dcmp" _mdb_dcmp' :: Ptr MDB_txn -> MDB_dbi' -> Ptr MDB_val -> Ptr MDB_val -> IO CInt - -foreign import ccall safe "lmdb.h mdb_get" _mdb_get :: Ptr MDB_txn -> MDB_dbi -> Ptr MDB_val -> Ptr MDB_val -> IO CInt -foreign import ccall safe "lmdb.h mdb_put" _mdb_put :: Ptr MDB_txn -> MDB_dbi -> Ptr MDB_val -> Ptr MDB_val -> MDB_WriteFlags -> IO CInt -foreign import ccall safe "lmdb.h mdb_del" _mdb_del :: Ptr MDB_txn -> MDB_dbi -> Ptr MDB_val -> Ptr MDB_val -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_get" _mdb_get' :: Ptr MDB_txn -> MDB_dbi' -> Ptr MDB_val -> Ptr MDB_val -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_put" _mdb_put' :: Ptr MDB_txn -> MDB_dbi' -> Ptr MDB_val -> Ptr MDB_val -> MDB_WriteFlags -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_del" _mdb_del' :: Ptr MDB_txn -> MDB_dbi' -> Ptr MDB_val -> Ptr MDB_val -> IO CInt - --- I dislike LMDB's cursor interface: one 'get' function with 18 special cases. --- Seems like it should be 18 functions. -foreign import ccall safe "lmdb.h mdb_cursor_open" _mdb_cursor_open :: Ptr MDB_txn -> MDB_dbi -> Ptr (Ptr MDB_cursor) -> IO CInt -foreign import ccall safe "lmdb.h mdb_cursor_close" _mdb_cursor_close :: Ptr MDB_cursor -> IO () -foreign import ccall safe "lmdb.h mdb_cursor_get" _mdb_cursor_get :: Ptr MDB_cursor -> Ptr MDB_val -> Ptr MDB_val -> (#type MDB_cursor_op) -> IO CInt -foreign import ccall safe "lmdb.h mdb_cursor_put" _mdb_cursor_put :: Ptr MDB_cursor -> Ptr MDB_val -> Ptr MDB_val -> MDB_WriteFlags -> IO CInt -foreign import ccall safe "lmdb.h mdb_cursor_del" _mdb_cursor_del :: Ptr MDB_cursor -> MDB_WriteFlags -> IO CInt -foreign import ccall safe "lmdb.h mdb_cursor_count" _mdb_cursor_count :: Ptr MDB_cursor -> Ptr CSize -> IO CInt --- foreign import ccall safe "lmdb.h mdb_cursor_txn" _mdb_cursor_txn :: Ptr MDB_cursor -> IO (Ptr MDB_txn) --- foreign import ccall safe "lmdb.h mdb_cursor_dbi" _mdb_cursor_dbi :: Ptr MDB_cursor -> IO MDB_dbi - -foreign import ccall unsafe "lmdb.h mdb_cursor_open" _mdb_cursor_open' :: Ptr MDB_txn -> MDB_dbi' -> Ptr (Ptr MDB_cursor') -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_cursor_close" _mdb_cursor_close' :: Ptr MDB_cursor' -> IO () -foreign import ccall unsafe "lmdb.h mdb_cursor_get" _mdb_cursor_get' :: Ptr MDB_cursor' -> Ptr MDB_val -> Ptr MDB_val -> (#type MDB_cursor_op) -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_cursor_put" _mdb_cursor_put' :: Ptr MDB_cursor' -> Ptr MDB_val -> Ptr MDB_val -> MDB_WriteFlags -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_cursor_del" _mdb_cursor_del' :: Ptr MDB_cursor' -> MDB_WriteFlags -> IO CInt -foreign import ccall unsafe "lmdb.h mdb_cursor_count" _mdb_cursor_count' :: Ptr MDB_cursor' -> Ptr CSize -> IO CInt --- foreign import ccall unsafe "lmdb.h mdb_cursor_txn" _mdb_cursor_txn' :: Ptr MDB_cursor -> IO (Ptr MDB_txn) --- foreign import ccall unsafe "lmdb.h mdb_cursor_dbi" _mdb_cursor_dbi' :: Ptr MDB_cursor -> IO MDB_dbi - -foreign import ccall unsafe "lmdb.h mdb_txn_reset" _mdb_txn_reset :: Ptr MDB_txn -> IO () -foreign import ccall "lmdb.h mdb_txn_renew" _mdb_txn_renew :: Ptr MDB_txn -> IO CInt - -foreign import ccall "lmdb.h mdb_reader_list" _mdb_reader_list :: Ptr MDB_env -> FunPtr MDB_msg_func -> Ptr () -> IO CInt -foreign import ccall "lmdb.h mdb_reader_check" _mdb_reader_check :: Ptr MDB_env -> Ptr CInt -> IO CInt - --- | User-defined comparison functions for keys. -type MDB_cmp_func = Ptr MDB_val -> Ptr MDB_val -> IO CInt -foreign import ccall "wrapper" wrapCmpFn :: MDB_cmp_func -> IO (FunPtr MDB_cmp_func) - --- callback function for reader list (used internally to this binding) -type MDB_msg_func = CString -> Ptr () -> IO CInt -foreign import ccall "wrapper" wrapMsgFunc :: MDB_msg_func -> IO (FunPtr MDB_msg_func) - - --- Haskell seems to have difficulty inferring the `Ptr CInt` from --- the _mdb_version call. (This seriously annoys me.) -_peekCInt :: Ptr CInt -> IO CInt -_peekCInt = peek - -_peekCUInt :: Ptr CUInt -> IO CUInt -_peekCUInt = peek - --- | Version information for LMDB. Two potentially different versions --- can be obtained: lmdb_version returns the version at the time of --- binding (via C preprocessor macros) and lmdb_dyn_version returns a --- version for the bound library. --- --- These bindings to Haskell will refuse to open the database when --- the dynamic version of LMDB is different in the major or minor --- fields. -data LMDB_Version = LMDB_Version - { v_major :: {-# UNPACK #-} !Int - , v_minor :: {-# UNPACK #-} !Int - , v_patch :: {-# UNPACK #-} !Int - , v_text :: !String - } deriving (Eq, Ord, Show) - --- | Version of LMDB when the Haskell-LMDB binding was compiled. -lmdb_version :: LMDB_Version -lmdb_version = LMDB_Version - { v_major = #const MDB_VERSION_MAJOR - , v_minor = #const MDB_VERSION_MINOR - , v_patch = #const MDB_VERSION_PATCH - , v_text = #const_str MDB_VERSION_STRING - } - - --- | Version of LMDB linked to the current Haskell process. -lmdb_dyn_version :: IO LMDB_Version -lmdb_dyn_version = - let szInt = sizeOf (undefined :: CInt) in - allocaBytes (3 * szInt) $ \ pMajor -> do - let pMinor = pMajor `plusPtr` szInt - let pPatch = pMinor `plusPtr` szInt - cvText <- _mdb_version pMajor pMinor pPatch - vMajor <- fromIntegral <$> _peekCInt pMajor - vMinor <- fromIntegral <$> _peekCInt pMinor - vPatch <- fromIntegral <$> _peekCInt pPatch - vText <- peekCString cvText - return $! LMDB_Version - { v_major = vMajor - , v_minor = vMinor - , v_patch = vPatch - , v_text = vText - } - --- | LMDB_Error is the exception type thrown in case a function from --- the LMDB API does not return successfully. Clients should be --- prepared to catch exceptions from any LMDB operation. -data LMDB_Error = LMDB_Error - { e_context :: String - , e_description :: String - , e_code :: Either Int MDB_ErrCode - } deriving (Eq, Ord, Show, Typeable) -instance Exception LMDB_Error - --- | Opaque structure for LMDB environment. --- --- The environment additionally contains an MVar to enforce at most --- one lightweight Haskell thread is writing at a time. This is --- necessary so long as LMDB uses a long-lived mutex for writes, as --- in v0.9.10. --- -data MDB_env = MDB_env - { _env_ptr :: {-# UNPACK #-} !(Ptr MDB_env) -- opaque pointer to LMDB object - , _env_wlock :: {-# UNPACK #-} !(MVar ThreadId) -- write lock - } - --- | Opaque structure for LMDB transaction. -data MDB_txn = MDB_txn - { _txn_ptr :: {-# UNPACK #-} !(Ptr MDB_txn) - , _txn_env :: !MDB_env -- environment that owns this transaction. - , _txn_rw :: !Bool -- is this a read-write transaction? (vs read-only) - , _txn_p :: !(Maybe MDB_txn) -- parent transaction, if any - } - --- | Identifier for a transaction. -newtype MDB_txnid = MDB_txnid { _txnid :: MDB_txnid_t } deriving (Ord, Eq, Show) - --- | Handle for a database in the environment. -newtype MDB_dbi = MDB_dbi { _dbi :: MDB_dbi_t } - --- | Opaque structure for LMDB cursor. -data MDB_cursor = MDB_cursor - { _crs_ptr :: {-# UNPACK #-} !(Ptr MDB_cursor) - , _crs_dbi :: {-# UNPACK #-} !MDB_dbi - , _crs_txn :: !MDB_txn - } - --- | Handle for a database in the environment. --- --- This variation is associated with 'unsafe' FFI calls, with reduced --- overhead but no user-defined comparisons. I expect most code using --- LMDB could use this variation. -newtype MDB_dbi' = MDB_dbi' { _dbi' :: MDB_dbi_t } - --- | Opaque structure for a cursor on an MDB_dbi' object. Cursors --- in this case also use the 'unsafe' FFI calls. -data MDB_cursor' = MDB_cursor' - { _crs_ptr' :: {-# UNPACK #-} !(Ptr MDB_cursor') - , _crs_dbi' :: {-# UNPACK #-} !MDB_dbi' - , _crs_txn' :: !MDB_txn - } - -type MDB_mode_t = #type mdb_mode_t -type MDB_dbi_t = #type MDB_dbi -type MDB_txnid_t = CSize -- typedef not currently exposed - --- | A value stored in the database. Be cautious; committing the --- transaction that obtained a value should also invalidate it; --- avoid capturing MDB_val in a lazy value. A safe interface --- similar to STRef could be provided by another module. -data MDB_val = MDB_val - { mv_size :: {-# UNPACK #-} !CSize - , mv_data :: {-# UNPACK #-} !(Ptr Word8) - } - -data MDB_stat = MDB_stat - { ms_psize :: {-# UNPACK #-} !CUInt - , ms_depth :: {-# UNPACK #-} !CUInt - , ms_branch_pages :: {-# UNPACK #-} !CSize - , ms_leaf_pages :: {-# UNPACK #-} !CSize - , ms_overflow_pages :: {-# UNPACK #-} !CSize - , ms_entries :: {-# UNPACK #-} !CSize - } deriving (Eq, Ord, Show) - -data MDB_envinfo = MDB_envinfo - { me_mapaddr :: {-# UNPACK #-} !(Ptr ()) - , me_mapsize :: {-# UNPACK #-} !CSize - , me_last_pgno :: {-# UNPACK #-} !CSize - , me_last_txnid :: {-# UNPACK #-} !MDB_txnid - , me_maxreaders :: {-# UNPACK #-} !CUInt - , me_numreaders :: {-# UNPACK #-} !CUInt - } deriving (Eq, Ord, Show) - - --- | Environment flags from lmdb.h --- --- Note: MDB_NOTLS is implicit and enforced for this binding. -data MDB_EnvFlag - = MDB_FIXEDMAP - | MDB_NOSUBDIR - | MDB_NOSYNC - | MDB_RDONLY - | MDB_NOMETASYNC - | MDB_WRITEMAP - | MDB_MAPASYNC - -- | MDB_NOTLS - | MDB_NOLOCK - | MDB_NORDAHEAD - | MDB_NOMEMINIT - deriving (Eq, Ord, Bounded, A.Ix, Show) - -envFlags :: [(MDB_EnvFlag, Int)] -envFlags = - [(MDB_FIXEDMAP, #const MDB_FIXEDMAP) - ,(MDB_NOSUBDIR, #const MDB_NOSUBDIR) - ,(MDB_NOSYNC, #const MDB_NOSYNC) - ,(MDB_RDONLY, #const MDB_RDONLY) - ,(MDB_NOMETASYNC, #const MDB_NOMETASYNC) - ,(MDB_WRITEMAP, #const MDB_WRITEMAP) - ,(MDB_MAPASYNC, #const MDB_MAPASYNC) - -- ,(MDB_NOTLS, #const MDB_NOTLS) - ,(MDB_NOLOCK, #const MDB_NOLOCK) - ,(MDB_NORDAHEAD, #const MDB_NORDAHEAD) - ,(MDB_NOMEMINIT, #const MDB_NOMEMINIT) - ] - -envFlagsArray :: A.UArray MDB_EnvFlag Int -envFlagsArray = A.accumArray (.|.) 0 (minBound, maxBound) envFlags - -compileEnvFlags :: [MDB_EnvFlag] -> CUInt -compileEnvFlags = fromIntegral . L.foldl' (.|.) 0 . fmap ((A.!) envFlagsArray) - -decompileBitFlags :: [(a,Int)] -> Int -> [a] -decompileBitFlags optFlags n = fmap fst $ L.filter fullMatch optFlags where - fullMatch (_,f) = (f == (n .&. f)) - -decompileEnvFlags :: CUInt -> [MDB_EnvFlag] -decompileEnvFlags = decompileBitFlags envFlags . fromIntegral - - -data MDB_DbFlag - = MDB_REVERSEKEY - | MDB_DUPSORT - | MDB_INTEGERKEY - | MDB_DUPFIXED - | MDB_INTEGERDUP - | MDB_REVERSEDUP - | MDB_CREATE - deriving (Eq, Ord, Bounded, A.Ix, Show) - -dbFlags :: [(MDB_DbFlag, Int)] -dbFlags = - [(MDB_REVERSEKEY, #const MDB_REVERSEKEY) - ,(MDB_DUPSORT, #const MDB_DUPSORT) - ,(MDB_INTEGERKEY, #const MDB_INTEGERKEY) - ,(MDB_DUPFIXED, #const MDB_DUPFIXED) - ,(MDB_INTEGERDUP, #const MDB_INTEGERDUP) - ,(MDB_REVERSEDUP, #const MDB_REVERSEDUP) - ,(MDB_CREATE, #const MDB_CREATE) - ] - -dbFlagsArray :: A.UArray MDB_DbFlag Int -dbFlagsArray = A.accumArray (.|.) 0 (minBound,maxBound) dbFlags - -compileDBFlags :: [MDB_DbFlag] -> CUInt -compileDBFlags = fromIntegral . L.foldl' (.|.) 0 . fmap ((A.!) dbFlagsArray) - -decompileDBFlags :: CUInt -> [MDB_DbFlag] -decompileDBFlags = decompileBitFlags dbFlags . fromIntegral - -data MDB_WriteFlag - = MDB_NOOVERWRITE - | MDB_NODUPDATA - | MDB_CURRENT - -- | MDB_RESERVE -- (needs dedicated function) - | MDB_APPEND - | MDB_APPENDDUP - -- | MDB_MULTIPLE -- (needs special handling) - deriving (Eq, Ord, Bounded, A.Ix, Show) - - -writeFlags :: [(MDB_WriteFlag, Int)] -writeFlags = - [(MDB_NOOVERWRITE, #const MDB_NOOVERWRITE) - ,(MDB_NODUPDATA, #const MDB_NODUPDATA) - ,(MDB_CURRENT, #const MDB_CURRENT) - -- ,(MDB_RESERVE, #const MDB_RESERVE) - ,(MDB_APPEND, #const MDB_APPEND) - ,(MDB_APPENDDUP, #const MDB_APPENDDUP) - -- ,(MDB_MULTIPLE, #const MDB_MULTIPLE) - ] - -writeFlagsArray :: A.UArray MDB_WriteFlag Int -writeFlagsArray = A.accumArray (.|.) 0 (minBound,maxBound) writeFlags - --- | compiled write flags, corresponding to a [WriteFlag] list. Used --- because writes are frequent enough that we want to avoid building --- from a list on a per-write basis. -newtype MDB_WriteFlags = MDB_WriteFlags CUInt - --- | compile a list of write flags. -compileWriteFlags :: [MDB_WriteFlag] -> MDB_WriteFlags -compileWriteFlags = MDB_WriteFlags . L.foldl' addWF 0 where - addWF n wf = n .|. fromIntegral (writeFlagsArray A.! wf) - -data MDB_cursor_op - = MDB_FIRST - | MDB_FIRST_DUP - | MDB_GET_BOTH - | MDB_GET_BOTH_RANGE - | MDB_GET_CURRENT - | MDB_GET_MULTIPLE - | MDB_LAST - | MDB_LAST_DUP - | MDB_NEXT - | MDB_NEXT_DUP - | MDB_NEXT_MULTIPLE - | MDB_NEXT_NODUP - | MDB_PREV - | MDB_PREV_DUP - | MDB_PREV_NODUP - | MDB_SET - | MDB_SET_KEY - | MDB_SET_RANGE - deriving (Eq, Ord, Bounded, A.Ix, Show) - -cursorOps :: [(MDB_cursor_op, Int)] -cursorOps = - [(MDB_FIRST, #const MDB_FIRST) - ,(MDB_FIRST_DUP, #const MDB_FIRST_DUP) - ,(MDB_GET_BOTH, #const MDB_GET_BOTH) - ,(MDB_GET_BOTH_RANGE, #const MDB_GET_BOTH_RANGE) - ,(MDB_GET_CURRENT, #const MDB_GET_CURRENT) - ,(MDB_GET_MULTIPLE, #const MDB_GET_MULTIPLE) - ,(MDB_LAST, #const MDB_LAST) - ,(MDB_LAST_DUP, #const MDB_LAST_DUP) - ,(MDB_NEXT, #const MDB_NEXT) - ,(MDB_NEXT_DUP, #const MDB_NEXT_DUP) - ,(MDB_NEXT_MULTIPLE, #const MDB_NEXT_MULTIPLE) - ,(MDB_NEXT_NODUP, #const MDB_NEXT_NODUP) - ,(MDB_PREV, #const MDB_PREV) - ,(MDB_PREV_DUP, #const MDB_PREV_DUP) - ,(MDB_PREV_NODUP, #const MDB_PREV_NODUP) - ,(MDB_SET, #const MDB_SET) - ,(MDB_SET_KEY, #const MDB_SET_KEY) - ,(MDB_SET_RANGE, #const MDB_SET_RANGE) - ] - -cursorOpsArray :: A.UArray MDB_cursor_op Int -cursorOpsArray = A.accumArray (flip const) minBound (minBound,maxBound) cursorOps - -cursorOp :: MDB_cursor_op -> (#type MDB_cursor_op) -cursorOp = fromIntegral . (A.!) cursorOpsArray - --- | Error codes from MDB. Note, however, that this API for MDB will mostly --- use exceptions for any non-successful return codes. This is mostly included --- because I feel the binding would be incomplete otherwise. --- --- (The MDB_SUCCESS return value is excluded.) -data MDB_ErrCode - = MDB_KEYEXIST - | MDB_NOTFOUND - | MDB_PAGE_NOTFOUND - | MDB_CORRUPTED - | MDB_PANIC - | MDB_VERSION_MISMATCH - | MDB_INVALID - | MDB_MAP_FULL - | MDB_DBS_FULL - | MDB_READERS_FULL - | MDB_TLS_FULL - | MDB_TXN_FULL - | MDB_CURSOR_FULL - | MDB_PAGE_FULL - | MDB_MAP_RESIZED - | MDB_INCOMPATIBLE - | MDB_BAD_RSLOT - | MDB_BAD_TXN - | MDB_BAD_VALSIZE - deriving (Eq, Ord, Bounded, A.Ix, Show) - -errCodes :: [(MDB_ErrCode, Int)] -errCodes = - [(MDB_KEYEXIST, #const MDB_KEYEXIST) - ,(MDB_NOTFOUND, #const MDB_NOTFOUND) - ,(MDB_PAGE_NOTFOUND, #const MDB_PAGE_NOTFOUND) - ,(MDB_CORRUPTED, #const MDB_CORRUPTED) - ,(MDB_PANIC, #const MDB_PANIC) - ,(MDB_VERSION_MISMATCH, #const MDB_VERSION_MISMATCH) - ,(MDB_INVALID, #const MDB_INVALID) - ,(MDB_MAP_FULL, #const MDB_MAP_FULL) - ,(MDB_DBS_FULL, #const MDB_DBS_FULL) - ,(MDB_READERS_FULL, #const MDB_READERS_FULL) - ,(MDB_TLS_FULL, #const MDB_TLS_FULL) - ,(MDB_TXN_FULL, #const MDB_TXN_FULL) - ,(MDB_CURSOR_FULL, #const MDB_CURSOR_FULL) - ,(MDB_PAGE_FULL, #const MDB_PAGE_FULL) - ,(MDB_MAP_RESIZED, #const MDB_MAP_RESIZED) - ,(MDB_INCOMPATIBLE, #const MDB_INCOMPATIBLE) - ,(MDB_BAD_RSLOT, #const MDB_BAD_RSLOT) - ,(MDB_BAD_TXN, #const MDB_BAD_TXN) - ,(MDB_BAD_VALSIZE, #const MDB_BAD_VALSIZE) - ] - -_numToErrVal :: Int -> Either Int MDB_ErrCode -_numToErrVal code = - case L.find ((== code) . snd) errCodes of - Nothing -> Left code - Just (ec,_) -> Right ec - -_throwLMDBErrNum :: String -> CInt -> IO noReturn -_throwLMDBErrNum context errNum = do - desc <- peekCString (_mdb_strerror errNum) - throwIO $! LMDB_Error - { e_context = context - , e_description = desc - , e_code = _numToErrVal (fromIntegral errNum) - } -{-# NOINLINE _throwLMDBErrNum #-} - --- | Allocate an environment object. This doesn't open the environment. --- --- After creation, but before opening, please use: --- --- mdb_env_set_mapsize --- mdb_env_set_maxreaders --- mdb_env_set_maxdbs --- --- Then, just after opening, you should create a transaction to open --- all the databases (MDB_dbi and MDB_dbi' values) your application --- will use. --- --- The typical use case would then involve keeping all these open --- until your application either shuts down or crashes. --- --- In addition to normal LMDB errors, this operation may throw an --- MDB_VERSION_MISMATCH if the Haskell LMDB bindings don't match --- the dynamic version. If this happens, you'll need to rebuild the --- lmdb Haskell package. -mdb_env_create :: IO MDB_env -mdb_env_create = alloca $ \ ppEnv -> - lmdb_validate_version_match >> - _mdb_env_create ppEnv >>= \ rc -> - if (0 /= rc) then _throwLMDBErrNum "mdb_env_create" rc else - MDB_env <$> peek ppEnv <*> newEmptyMVar - - -lmdb_validate_version_match :: IO () -lmdb_validate_version_match = - let vStat = lmdb_version in - lmdb_dyn_version >>= \ vDyn -> - unless (versionMatch vStat vDyn) $ - throwIO $! LMDB_Error - { e_context = "lmdb_validate_version_match" - , e_description = "Haskell bindings: " ++ show vStat - ++ "\tDynamic library: " ++ show vDyn - , e_code = Right MDB_VERSION_MISMATCH - } - --- this match function is a bit relaxed, e.g. it will accept --- LMDB 0.9.10 with 0.9.8, so long as the first two numbers --- match. -versionMatch :: LMDB_Version -> LMDB_Version -> Bool -versionMatch vA vB = matchMajor && matchMinor where - matchMajor = ((==) `on` v_major) vA vB - matchMinor = ((==) `on` v_minor) vA vB - --- | open or build a database in the filesystem. The named directory --- must already exist and be writeable. Before opening, be sure to --- at least apply `mdb_env_set_mapsize`. --- --- After opening the environment, you should open the databases: --- --- Create the environment. --- Open a transaction. --- Open all DBI handles the app will need. --- Commit the transaction. --- Use those DBI handles in subsequent transactions --- -mdb_env_open :: MDB_env -> FilePath -> [MDB_EnvFlag] -> IO () -mdb_env_open env fp flags = - let iFlags = (#const MDB_NOTLS) .|. (compileEnvFlags flags) in - let unix_mode = (6 * 64 + 6 * 8) in -- mode 0660, read-write for user+group - withCString fp $ \ cfp -> - _mdb_env_open (_env_ptr env) cfp iFlags unix_mode >>= \ rc -> - unless (0 == rc) $ - _throwLMDBErrNum "mdb_env_open" rc - --- | Copy the environment to an empty (but existing) directory. --- --- Note: the LMDB copy operation temporarily grabs the writer mutex. --- Unfortunately, this greatly complicates the binding to Haskell. --- This interface, mdb_env_copy, conservatively blocks all writers --- in the same process for the entire duration of copy. --- --- Recommendation: Don't use this function in the same process with --- writers. Consider use of the `mdb_copy` command line utility if --- you need hot copies. --- -mdb_env_copy :: MDB_env -> FilePath -> IO () -mdb_env_copy env fp = - runInBoundThread $ - bracket_ (_lockEnv env) (_unlockEnv env) $ - withCString fp $ \ cfp -> - _mdb_env_copy (_env_ptr env) cfp >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_env_copy" rc) - - --- | obtain statistics for environment -mdb_env_stat :: MDB_env -> IO MDB_stat -mdb_env_stat env = - alloca $ \ pStats -> - _mdb_env_stat (_env_ptr env) pStats >>= \ rc -> - if (0 == rc) then peek pStats else - _throwLMDBErrNum "mdb_env_stat" rc - --- | obtain ad-hoc information about the environment. -mdb_env_info :: MDB_env -> IO MDB_envinfo -mdb_env_info env = - alloca $ \ pInfo -> - _mdb_env_info (_env_ptr env) pInfo >>= \ rc -> - if (0 == rc) then peek pInfo else - _throwLMDBErrNum "mdb_env_info" rc - --- | Initiate synchronization of environment with disk. However, if --- the MDB_NOSYNC or MDB_MAPASYNC flags are active, this won't wait --- for the operation to finish. Cf. mdb_env_sync_flush. -mdb_env_sync :: MDB_env -> IO () -mdb_env_sync env = - _mdb_env_sync (_env_ptr env) 0 >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_env_sync" rc) - --- | Force buffered writes to disk before returning. -mdb_env_sync_flush :: MDB_env -> IO () -mdb_env_sync_flush env = - _mdb_env_sync (_env_ptr env) 1 >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_env_sync_flush" rc) - --- | Close the environment. The MDB_env object should not be used by --- any operations during or after closing. -mdb_env_close :: MDB_env -> IO () -mdb_env_close env = _lockEnv env >> _mdb_env_close (_env_ptr env) - --- | Set flags for the environment. -mdb_env_set_flags :: MDB_env -> [MDB_EnvFlag] -> IO () -mdb_env_set_flags env flags = - _mdb_env_set_flags (_env_ptr env) (compileEnvFlags flags) 1 >>= \ rc -> - unless (0 == rc) $ _throwLMDBErrNum "mdb_env_set_flags" rc - --- | Unset flags for the environment. -mdb_env_unset_flags :: MDB_env -> [MDB_EnvFlag] -> IO () -mdb_env_unset_flags env flags = - _mdb_env_set_flags (_env_ptr env) (compileEnvFlags flags) 0 >>= \ rc -> - unless (0 == rc) $ _throwLMDBErrNum "mdb_env_unset_flags" rc - --- | View the current set of flags for the environment. -mdb_env_get_flags :: MDB_env -> IO [MDB_EnvFlag] -mdb_env_get_flags env = decompileEnvFlags <$> _mdb_env_get_flags_u env - -_mdb_env_get_flags_u :: MDB_env -> IO CUInt -_mdb_env_get_flags_u env = alloca $ \ pFlags -> - _mdb_env_get_flags (_env_ptr env) pFlags >>= \ rc -> - if (0 == rc) then peek pFlags else - _throwLMDBErrNum "mdb_env_get_flags" rc - --- | Obtain filesystem path for this environment. -mdb_env_get_path :: MDB_env -> IO FilePath -mdb_env_get_path env = alloca $ \ pPathStr -> - _mdb_env_get_path (_env_ptr env) pPathStr >>= \ rc -> - if (0 == rc) then peekCString =<< peek pPathStr else - _throwLMDBErrNum "mdb_env_get_path" rc - --- | Set the memory map size, in bytes, for this environment. This --- determines the maximum size for the environment and databases, --- but typically only a small fraction of the database is in memory --- at any given moment. --- --- It is not a problem to set this to a very large number, hundreds --- of gigabytes or even terabytes, assuming a sufficiently large --- address space. It should be set to a multiple of page size. --- --- The default map size is 1MB, intentionally set low to force --- developers to select something larger. -mdb_env_set_mapsize :: MDB_env -> Int -> IO () -mdb_env_set_mapsize env nBytes = - _mdb_env_set_mapsize (_env_ptr env) (fromIntegral nBytes) >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_env_set_mapsize" rc) - --- | Set the maximum number of concurrent readers. -mdb_env_set_maxreaders :: MDB_env -> Int -> IO () -mdb_env_set_maxreaders env nReaders = - _mdb_env_set_maxreaders (_env_ptr env) (fromIntegral nReaders) >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_env_set_maxreaders" rc) - --- | Get the maximum number of concurrent readers. -mdb_env_get_maxreaders :: MDB_env -> IO Int -mdb_env_get_maxreaders env = alloca $ \ pCount -> - _mdb_env_get_maxreaders (_env_ptr env) pCount >>= \ rc -> - if (0 == rc) then fromIntegral <$> _peekCUInt pCount else - _throwLMDBErrNum "mdb_env_get_maxreaders" rc - --- | Set the maximum number of named databases. LMDB is designed to --- support a small handful of databases. -mdb_env_set_maxdbs :: MDB_env -> Int -> IO () -mdb_env_set_maxdbs env nDBs = - _mdb_env_set_maxdbs (_env_ptr env) (fromIntegral nDBs) >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_env_set_maxdbs" rc) - --- | Key sizes in LMDB are determined by a compile-time constant, --- defaulting to 511 bytes. This function returns the maximum. -mdb_env_get_maxkeysize :: MDB_env -> IO Int -mdb_env_get_maxkeysize env = fromIntegral <$> _mdb_env_get_maxkeysize (_env_ptr env) - --- | Check for stale readers, and return number of stale readers cleared. -mdb_reader_check :: MDB_env -> IO Int -mdb_reader_check env = - alloca $ \ pCount -> - _mdb_reader_check (_env_ptr env) pCount >>= \ rc -> - if (0 == rc) then fromIntegral <$> _peekCInt pCount else - _throwLMDBErrNum "mdb_reader_check" rc - --- | Dump entries from reader lock table (for human consumption) -mdb_reader_list :: MDB_env -> IO String -mdb_reader_list env = - newIORef [] >>= \ rf -> - let onMsg cs _ = - peekCString cs >>= \ msg -> - modifyIORef rf (msg:) >> - return 0 - in - withMsgFunc onMsg $ \ pMsgFunc -> - _mdb_reader_list (_env_ptr env) pMsgFunc nullPtr >>= \ rc -> - let toMsg = L.foldl (flip (++)) [] in - if (0 == rc) then toMsg <$> readIORef rf else - _throwLMDBErrNum "mdb_reader_list" rc - -withMsgFunc :: MDB_msg_func -> (FunPtr MDB_msg_func -> IO a) -> IO a -withMsgFunc f = bracket (wrapMsgFunc f) freeHaskellFunPtr - --- | Begin a new transaction, possibly read-only, with a possible parent. --- --- mdb_txn_begin env parent bReadOnly --- --- NOTE: Unless your MDB_env was created with MDB_NOLOCK, it is necessary --- that read-write transactions be created and completed in one Haskell --- 'bound' thread, e.g. via forkOS or runInBoundThread. The bound threads --- are necessary because LMDB uses OS-level mutexes which track the thread --- ID of their owning thread. --- --- This LMDB adapter includes its own MVar mutex to prevent more than one --- Haskell-level thread from trying to write at the same time. --- --- The hierarchical transactions are useful for read-write transactions. --- They allow trying something out then aborting if it doesn't work. But --- only one child should be active at a time, all in the same OS thread. --- -mdb_txn_begin :: MDB_env -> Maybe MDB_txn -> Bool -> IO MDB_txn -mdb_txn_begin env parent bReadOnly = mask_ $ - let bWriteTxn = not bReadOnly in - let bLockForWrite = bWriteTxn && isNothing parent in - - -- allow only one toplevel write operation at a time. - when bLockForWrite (_lockEnv env) >> - let pEnv = _env_ptr env in - let pParent = maybe nullPtr _txn_ptr parent in - let iFlags = if bReadOnly then (#const MDB_RDONLY) else 0 in - let onFailure rc = - when bLockForWrite (_unlockEnv env) >> - _throwLMDBErrNum "mdb_txn_begin" rc - in - alloca $ \ ppChildTxn -> - _mdb_txn_begin pEnv pParent iFlags ppChildTxn >>= \ rc -> - if (0 /= rc) then onFailure rc else - peek ppChildTxn >>= \ pChildTxn -> - return $! MDB_txn { _txn_ptr = pChildTxn - , _txn_env = env - , _txn_rw = bWriteTxn - , _txn_p = parent - } - --- Haskell-level writer lock for the environment. --- --- This is necessary because otherwise multiple Haskell threads on --- the same OS thread can acquire the same lock (in windows) or --- deadlock (in posix systems). LMDB doesn't really support M:N --- writer threads. --- --- Note: this will also enforce that the caller is operating in a --- bound thread. So we can only _lockEnv from a bound thread, and --- we can only _unlockEnv from the same thread. -_lockEnv, _unlockEnv :: MDB_env -> IO () -_lockErr, _unlockErr :: LMDB_Error -_lockErr = LMDB_Error - { e_context = "locking LMDB for write in Haskell layer" - , e_description = "must lock from a 'bound' thread!" - , e_code = Right MDB_PANIC - } -_unlockErr = LMDB_Error - { e_context = "unlock Haskell layer LMDB after write" - , e_description = "calling thread does not own the lock!" - , e_code = Right MDB_PANIC - } - -_lockEnv env = do - safeThread <- isCurrentThreadBound <||> isUnlockedEnv - unless safeThread (throwIO _lockErr) - tid <- myThreadId - putMVar (_env_wlock env) tid - where - isUnlockedEnv = hasFlag (#const MDB_NOLOCK) <$> _mdb_env_get_flags_u env - hasFlag f fs = (f == (f .&. fs)) - getA <||> getB = getA >>= \ a -> if a then return True else getB - -_unlockEnv env = - myThreadId >>= \ self -> - let m = (_env_wlock env) in - mask_ $ - takeMVar m >>= \ owner -> - unless (self == owner) $ - putMVar m owner >> -- oops! - throwIO _unlockErr - --- compute whether this transaction should hold the write lock. --- If it does, unlock it. Otherwise, return. -_unlockTxn :: MDB_txn -> IO () -_unlockTxn txn = - let bHasLock = _txn_rw txn && isNothing (_txn_p txn) in - when bHasLock (_unlockEnv (_txn_env txn)) - --- | Access environment for a transaction. -mdb_txn_env :: MDB_txn -> MDB_env -mdb_txn_env = _txn_env - --- | Commit a transaction. Don't use the transaction after this. -mdb_txn_commit :: MDB_txn -> IO () -mdb_txn_commit txn = mask_ $ - _mdb_txn_commit (_txn_ptr txn) >>= \ rc -> - _unlockTxn txn >> - unless (0 == rc) (_throwLMDBErrNum "mdb_txn_commit" rc) - --- | Abort a transaction. Don't use the transaction after this. -mdb_txn_abort :: MDB_txn -> IO () -mdb_txn_abort txn = mask_ $ - _mdb_txn_abort (_txn_ptr txn) >> - _unlockTxn txn - --- | Abort a read-only transaction, but don't destroy it. --- Keep it available for mdb_txn_renew. -mdb_txn_reset :: MDB_txn -> IO () -mdb_txn_reset txn = _mdb_txn_reset (_txn_ptr txn) - --- | Renew a read-only transaction that was previously _reset. -mdb_txn_renew :: MDB_txn -> IO () -mdb_txn_renew txn = - _mdb_txn_renew (_txn_ptr txn) >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_txn_renew" rc) - - -{- - --- I'm hoping to get a patch adding the following function into the main LMDB: --- foreign import ccall "lmdb.h mdb_txn_id" _mdb_txn_id :: MDB_txn -> IO MDB_txnid_t - --} - --- | Open a database that supports user-defined comparisons, but --- has slightly more FFI overhead for reads and writes. --- --- LMDB supports a small set of named databases, plus one 'main' --- database using the null argument for the database name. -mdb_dbi_open :: MDB_txn -> Maybe String -> [MDB_DbFlag] -> IO MDB_dbi -mdb_dbi_open txn dbName flags = MDB_dbi <$> mdb_dbi_open_t txn dbName flags - --- | database statistics -mdb_stat :: MDB_txn -> MDB_dbi -> IO MDB_stat -mdb_stat txn = mdb_stat_t txn . _dbi - --- | review flags from database -mdb_dbi_flags :: MDB_txn -> MDB_dbi -> IO [MDB_DbFlag] -mdb_dbi_flags txn = mdb_dbi_flags_t txn . _dbi - --- | close the database handle. --- --- Note: the normal use-case for LMDB is to open all the database --- handles up front, then hold onto them until the application is --- closed or crashed. In that case, you don't need to bother with --- closing database handles. -mdb_dbi_close :: MDB_env -> MDB_dbi -> IO () -mdb_dbi_close env = mdb_dbi_close_t env . _dbi - --- | remove the database and close the handle; don't use MDB_dbi --- after this -mdb_drop :: MDB_txn -> MDB_dbi -> IO () -mdb_drop txn = mdb_drop_t txn . _dbi - --- | clear contents of database, reset to empty -mdb_clear :: MDB_txn -> MDB_dbi -> IO () -mdb_clear txn = mdb_clear_t txn . _dbi - -mdb_dbi_open' :: MDB_txn -> Maybe String -> [MDB_DbFlag] -> IO MDB_dbi' -mdb_dbi_open' txn dbName flags = MDB_dbi' <$> mdb_dbi_open_t txn dbName flags - -mdb_stat' :: MDB_txn -> MDB_dbi' -> IO MDB_stat -mdb_stat' txn = mdb_stat_t txn . _dbi' - -mdb_dbi_flags' :: MDB_txn -> MDB_dbi' -> IO [MDB_DbFlag] -mdb_dbi_flags' txn = mdb_dbi_flags_t txn . _dbi' - -mdb_dbi_close' :: MDB_env -> MDB_dbi' -> IO () -mdb_dbi_close' txn = mdb_dbi_close_t txn . _dbi' - -mdb_drop' :: MDB_txn -> MDB_dbi' -> IO () -mdb_drop' txn = mdb_drop_t txn . _dbi' - -mdb_clear' :: MDB_txn -> MDB_dbi' -> IO () -mdb_clear' txn = mdb_clear_t txn . _dbi' - --- | use a nullable CString -withCStringMaybe :: Maybe String -> (CString -> IO a) -> IO a -withCStringMaybe Nothing f = f nullPtr -withCStringMaybe (Just s) f = withCString s f - -mdb_dbi_open_t :: MDB_txn -> Maybe String -> [MDB_DbFlag] -> IO MDB_dbi_t -mdb_dbi_open_t txn dbName flags = -- use string name - let cdbFlags = compileDBFlags flags in - alloca $ \ pDBI -> - withCStringMaybe dbName $ \ cdbName -> - _mdb_dbi_open (_txn_ptr txn) cdbName cdbFlags pDBI >>= \ rc -> - if (0 == rc) then peek pDBI else - _throwLMDBErrNum "mdb_dbi_open" rc - -mdb_stat_t :: MDB_txn -> MDB_dbi_t -> IO MDB_stat -mdb_stat_t txn dbi = - alloca $ \ pStat -> - _mdb_stat (_txn_ptr txn) dbi pStat >>= \ rc -> - if (0 == rc) then peek pStat else - _throwLMDBErrNum "mdb_stat" rc - -mdb_dbi_flags_t :: MDB_txn -> MDB_dbi_t -> IO [MDB_DbFlag] -mdb_dbi_flags_t txn dbi = - alloca $ \ pFlags -> - _mdb_dbi_flags (_txn_ptr txn) dbi pFlags >>= \ rc -> - if (0 == rc) then decompileDBFlags <$> peek pFlags else - _throwLMDBErrNum "mdb_dbi_flags" rc - -mdb_dbi_close_t :: MDB_env -> MDB_dbi_t -> IO () -mdb_dbi_close_t env dbi = _mdb_dbi_close (_env_ptr env) dbi - -mdb_drop_t :: MDB_txn -> MDB_dbi_t -> IO () -mdb_drop_t txn dbi = - _mdb_drop (_txn_ptr txn) dbi 1 >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_drop" rc) - -mdb_clear_t :: MDB_txn -> MDB_dbi_t -> IO () -mdb_clear_t txn dbi = - _mdb_drop (_txn_ptr txn) dbi 0 >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_clear" rc) - --- | Set a user-defined key comparison function for a database. -mdb_set_compare :: MDB_txn -> MDB_dbi -> FunPtr MDB_cmp_func -> IO () -mdb_set_compare txn dbi fcmp = - _mdb_set_compare (_txn_ptr txn) dbi fcmp >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_set_compare" rc) - --- | Set a user-defined data comparison operator for MDB_DUPSORT databases. -mdb_set_dupsort :: MDB_txn -> MDB_dbi -> FunPtr MDB_cmp_func -> IO () -mdb_set_dupsort txn dbi fcmp = - _mdb_set_dupsort (_txn_ptr txn) dbi fcmp >>= \ rc -> - unless (0 == rc) (_throwLMDBErrNum "mdb_set_dupsort" rc) - --- zero datum -zed :: MDB_val -zed = MDB_val 0 nullPtr - --- | Access a value by key. Returns Nothing if the key is not found. -mdb_get :: MDB_txn -> MDB_dbi -> MDB_val -> IO (Maybe MDB_val) -mdb_get txn dbi key = - withKVPtrs key zed $ \ pKey pVal -> - _mdb_get (_txn_ptr txn) dbi pKey pVal >>= \ rc -> - r_get rc pVal -{-# INLINE mdb_get #-} - -r_get :: CInt -> Ptr MDB_val -> IO (Maybe MDB_val) -r_get rc pVal = - if (0 == rc) then Just <$> peek pVal else - if ((#const MDB_NOTFOUND) == rc) then return Nothing else - _throwLMDBErrNum "mdb_get" rc -{-# INLINE r_get #-} - --- | utility function: prepare pointers suitable for mdb_cursor_get. -withKVPtrs :: MDB_val -> MDB_val -> (Ptr MDB_val -> Ptr MDB_val -> IO a) -> IO a -withKVPtrs k v fn = - allocaBytes (2 * sizeOf k) $ \ pK -> - let pV = pK `plusPtr` sizeOf k in - do poke pK k - poke pV v - fn pK pV -{-# INLINE withKVPtrs #-} - --- | variation on withKVPtrs with nullable value. -withKVOptPtrs :: MDB_val -> Maybe MDB_val -> (Ptr MDB_val -> Ptr MDB_val -> IO a) -> IO a -withKVOptPtrs k (Just v) fn = withKVPtrs k v fn -withKVOptPtrs k Nothing fn = alloca $ \ pK -> poke pK k >> fn pK nullPtr - - --- | Add a (key,value) pair to the database. --- --- Returns False on MDB_KEYEXIST, and True on MDB_SUCCESS. Any other --- return value from LMDB results in an exception. The MDB_KEYEXIST --- result can be returned only if certain write flags are enabled. -mdb_put :: MDB_WriteFlags -> MDB_txn -> MDB_dbi -> MDB_val -> MDB_val -> IO Bool -mdb_put wf txn dbi key val = - withKVPtrs key val $ \ pKey pVal -> - _mdb_put (_txn_ptr txn) dbi pKey pVal wf >>= \ rc -> - r_put rc -{-# INLINE mdb_put #-} - -r_put :: CInt -> IO Bool -r_put rc = - if (0 == rc) then return True else - if ((#const MDB_KEYEXIST) == rc) then return False else - _throwLMDBErrNum "mdb_put" rc -{-# INLINE r_put #-} - --- | Allocate space for data under a given key. This space must be --- filled before the write transaction commits. The idea here is to --- avoid an extra allocation. --- --- mdb_reserve flags txn dbi key byteCount --- --- Note: not safe to use with MDB_DUPSORT. --- Note: MDB_KEYEXIST will result in an exception here. -mdb_reserve :: MDB_WriteFlags -> MDB_txn -> MDB_dbi -> MDB_val -> Int -> IO MDB_val -mdb_reserve wf txn dbi key szBytes = - withKVPtrs key (reserveData szBytes) $ \ pKey pVal -> - _mdb_put (_txn_ptr txn) dbi pKey pVal (wfReserve wf) >>= \ rc -> - if (0 == rc) then peek pVal else - _throwLMDBErrNum "mdb_reserve" rc -{-# INLINE mdb_reserve #-} - -wfReserve :: MDB_WriteFlags -> MDB_WriteFlags -wfReserve (MDB_WriteFlags wf) = MDB_WriteFlags ((#const MDB_RESERVE) .|. wf) - -reserveData :: Int -> MDB_val -reserveData szBytes = MDB_val (fromIntegral szBytes) nullPtr - --- | Delete a given key, or a specific (key,value) pair in case of --- MDB_DUPSORT. This function will return False on a MDB_NOTFOUND --- result, and True on MDB_SUCCESS. --- --- Note: Ideally, LMDB would match the value even without MDB_DUPSORT. --- But it doesn't. Under the hood, the data is replaced by a null ptr --- if MDB_DUPSORT is not enabled (v0.9.10). -mdb_del :: MDB_txn -> MDB_dbi -> MDB_val -> Maybe MDB_val -> IO Bool -mdb_del txn dbi key mbVal = - withKVOptPtrs key mbVal $ \ pKey pVal -> - _mdb_del (_txn_ptr txn) dbi pKey pVal >>= \ rc -> - r_del rc -{-# INLINE mdb_del #-} - -r_del :: CInt -> IO Bool -r_del rc = - if (0 == rc) then return True else - if ((#const MDB_NOTFOUND) == rc) then return False else - _throwLMDBErrNum "mdb_del" rc -{-# INLINE r_del #-} - -mdb_get' :: MDB_txn -> MDB_dbi' -> MDB_val -> IO (Maybe MDB_val) -mdb_get' txn dbi key = - withKVPtrs key zed $ \ pKey pVal -> - _mdb_get' (_txn_ptr txn) dbi pKey pVal >>= \ rc -> - r_get rc pVal -{-# INLINE mdb_get' #-} - -mdb_put' :: MDB_WriteFlags -> MDB_txn -> MDB_dbi' -> MDB_val -> MDB_val -> IO Bool -mdb_put' wf txn dbi key val = - withKVPtrs key val $ \ pKey pVal -> - _mdb_put' (_txn_ptr txn) dbi pKey pVal wf >>= \ rc -> - r_put rc -{-# INLINE mdb_put' #-} - -mdb_reserve' :: MDB_WriteFlags -> MDB_txn -> MDB_dbi' -> MDB_val -> Int -> IO MDB_val -mdb_reserve' wf txn dbi key szBytes = - withKVPtrs key (reserveData szBytes) $ \ pKey pVal -> - _mdb_put' (_txn_ptr txn) dbi pKey pVal (wfReserve wf) >>= \ rc -> - if (0 == rc) then peek pVal else - _throwLMDBErrNum "mdb_reserve" rc -{-# INLINE mdb_reserve' #-} - -mdb_del' :: MDB_txn -> MDB_dbi' -> MDB_val -> Maybe MDB_val -> IO Bool -mdb_del' txn dbi key mbVal = - withKVOptPtrs key mbVal $ \ pKey pVal -> - _mdb_del' (_txn_ptr txn) dbi pKey pVal >>= \rc -> - r_del rc -{-# INLINE mdb_del' #-} - --- | compare two values as keys in a database -mdb_cmp :: MDB_txn -> MDB_dbi -> MDB_val -> MDB_val -> IO Ordering -mdb_cmp txn dbi a b = - withKVPtrs a b $ \ pA pB -> - _mdb_cmp (_txn_ptr txn) dbi pA pB >>= \ rc -> - return (compare rc 0) -{-# INLINE mdb_cmp #-} - --- | compare two values as data in an MDB_DUPSORT database -mdb_dcmp :: MDB_txn -> MDB_dbi -> MDB_val -> MDB_val -> IO Ordering -mdb_dcmp txn dbi a b = - withKVPtrs a b $ \ pA pB -> - _mdb_dcmp (_txn_ptr txn) dbi pA pB >>= \ rc -> - return (compare rc 0) -{-# INLINE mdb_dcmp #-} - -mdb_cmp' :: MDB_txn -> MDB_dbi' -> MDB_val -> MDB_val -> IO Ordering -mdb_cmp' txn dbi a b = - withKVPtrs a b $ \ pA pB -> - _mdb_cmp' (_txn_ptr txn) dbi pA pB >>= \ rc -> - return (compare rc 0) -{-# INLINE mdb_cmp' #-} - -mdb_dcmp' :: MDB_txn -> MDB_dbi' -> MDB_val -> MDB_val -> IO Ordering -mdb_dcmp' txn dbi a b = - withKVPtrs a b $ \ pA pB -> - _mdb_dcmp' (_txn_ptr txn) dbi pA pB >>= \ rc -> - return (compare rc 0) -{-# INLINE mdb_dcmp' #-} - --- | open a cursor for the database. -mdb_cursor_open :: MDB_txn -> MDB_dbi -> IO MDB_cursor -mdb_cursor_open txn dbi = - alloca $ \ ppCursor -> - _mdb_cursor_open (_txn_ptr txn) dbi ppCursor >>= \ rc -> - if (0 /= rc) then _throwLMDBErrNum "mdb_cursor_open" rc else - peek ppCursor >>= \ pCursor -> - return $! MDB_cursor - { _crs_ptr = pCursor - , _crs_dbi = dbi - , _crs_txn = txn - } - -mdb_cursor_open' :: MDB_txn -> MDB_dbi' -> IO MDB_cursor' -mdb_cursor_open' txn dbi = - alloca $ \ ppCursor -> - _mdb_cursor_open' (_txn_ptr txn) dbi ppCursor >>= \ rc -> - if (0 /= rc) then _throwLMDBErrNum "mdb_cursor_open" rc else - peek ppCursor >>= \ pCursor -> - return $! MDB_cursor' - { _crs_ptr' = pCursor - , _crs_dbi' = dbi - , _crs_txn' = txn - } - --- | Low-level mdb_cursor_get operation, with direct control of how --- pointers to values are allocated, whether an argument is a nullPtr, --- and so on. --- --- In this case, False is returned for MDB_NOTFOUND (in which case the --- cursor should not be moved), and True is returned for MDB_SUCCESS. --- Any other return value from LMDB will result in an exception. --- --- Depending on the MDB_cursor_op, additional values may be returned --- via the pointers. At the moment -mdb_cursor_get :: MDB_cursor_op -> MDB_cursor -> Ptr MDB_val -> Ptr MDB_val -> IO Bool -mdb_cursor_get op crs pKey pData = _mdb_cursor_get (_crs_ptr crs) pKey pData (cursorOp op) >>= r_cursor_get -{-# INLINE mdb_cursor_get #-} - -r_cursor_get :: CInt -> IO Bool -r_cursor_get rc = - if(0 == rc) then return True else - if((#const MDB_NOTFOUND) == rc) then return False else - _throwLMDBErrNum "mdb_cursor_get" rc -{-# INLINE r_cursor_get #-} - -mdb_cursor_get' :: MDB_cursor_op -> MDB_cursor' -> Ptr MDB_val -> Ptr MDB_val -> IO Bool -mdb_cursor_get' op crs pKey pData = _mdb_cursor_get' (_crs_ptr' crs) pKey pData (cursorOp op) >>= r_cursor_get -{-# INLINE mdb_cursor_get' #-} - --- | Low-level 'mdb_cursor_put' operation. --- --- As with mdb_put, this returns True on MDB_SUCCESS and False for MDB_KEYEXIST, --- and otherwise throws an exception. -mdb_cursor_put :: MDB_WriteFlags -> MDB_cursor -> MDB_val -> MDB_val -> IO Bool -mdb_cursor_put wf crs key val = - withKVPtrs key val $ \ pKey pVal -> - _mdb_cursor_put (_crs_ptr crs) pKey pVal wf >>= \ rc -> - r_cursor_put rc -{-# INLINE mdb_cursor_put #-} - -r_cursor_put :: CInt -> IO Bool -r_cursor_put rc = - if(0 == rc) then return True else - if((#const MDB_KEYEXIST) == rc) then return False else - _throwLMDBErrNum "mdb_cursor_put" rc -{-# INLINE r_cursor_put #-} - -mdb_cursor_put' :: MDB_WriteFlags -> MDB_cursor' -> MDB_val -> MDB_val -> IO Bool -mdb_cursor_put' wf crs key val = - withKVPtrs key val $ \ pKey pVal -> - _mdb_cursor_put' (_crs_ptr' crs) pKey pVal wf >>= \ rc -> - r_cursor_put rc -{-# INLINE mdb_cursor_put' #-} - --- | Delete the value at the cursor. -mdb_cursor_del :: MDB_WriteFlags -> MDB_cursor -> IO () -mdb_cursor_del wf crs = _mdb_cursor_del (_crs_ptr crs) wf >>= r_cursor_del -{-# INLINE mdb_cursor_del #-} - -r_cursor_del :: CInt -> IO () -r_cursor_del rc = unless (0 == rc) (_throwLMDBErrNum "mdb_cursor_del" rc) -{-# INLINE r_cursor_del #-} - -mdb_cursor_del' :: MDB_WriteFlags -> MDB_cursor' -> IO () -mdb_cursor_del' wf crs = _mdb_cursor_del' (_crs_ptr' crs) wf >>= r_cursor_del -{-# INLINE mdb_cursor_del' #-} - --- | Close a cursor. don't use after this. In general, cursors should --- be closed before their associated transaction is commited or aborted. -mdb_cursor_close :: MDB_cursor -> IO () -mdb_cursor_close crs = _mdb_cursor_close (_crs_ptr crs) - -mdb_cursor_close' :: MDB_cursor' -> IO () -mdb_cursor_close' crs = _mdb_cursor_close' (_crs_ptr' crs) - --- | Access transaction associated with a cursor. -mdb_cursor_txn :: MDB_cursor -> MDB_txn -mdb_cursor_txn = _crs_txn - -mdb_cursor_txn' :: MDB_cursor' -> MDB_txn -mdb_cursor_txn' = _crs_txn' - --- | Access the database associated with a cursor. -mdb_cursor_dbi :: MDB_cursor -> MDB_dbi -mdb_cursor_dbi = _crs_dbi - -mdb_cursor_dbi' :: MDB_cursor' -> MDB_dbi' -mdb_cursor_dbi' = _crs_dbi' - --- | count number of duplicate data items at cursor's current location. -mdb_cursor_count :: MDB_cursor -> IO Int -mdb_cursor_count crs = - alloca $ \ pCount -> - _mdb_cursor_count (_crs_ptr crs) pCount >>= \ rc -> - if (0 == rc) then fromIntegral <$> _peekSize pCount else - _throwLMDBErrNum "mdb_cursor_count" rc -{-# INLINE mdb_cursor_count #-} - -_peekSize :: Ptr CSize -> IO CSize -_peekSize = peek - -mdb_cursor_count' :: MDB_cursor' -> IO Int -mdb_cursor_count' crs = - alloca $ \ pCount -> - _mdb_cursor_count' (_crs_ptr' crs) pCount >>= \ rc -> - if (0 == rc) then fromIntegral <$> _peekSize pCount else - _throwLMDBErrNum "mdb_cursor_count" rc -{-# INLINE mdb_cursor_count' #-} - - --- for cursor get... --- I'm not really sure what I want to do here, not quite yet. --- maybe I should write a bunch of individual functions? - -{- -foreign import ccall safe "lmdb.h mdb_cursor_get" _mdb_cursor_get :: Ptr MDB_cursor -> Ptr MDB_val -> Ptr MDB_val -> (#type MDB_cursor_op) -> IO CInt --} - -{- -cmpBytesToCmpFn :: (ByteString -> ByteString -> Ord) -> CmpFn -cmpBytesToCmpFn cmp vL vR = do - lBytes <- valToVolatileByteString vL - rBytes <- valToVolatileByteString vR - return $! case cmp lBytes rBytes of - LT -> -1 - EQ -> 0 - GT -> 1 - --- | Create a user-defined comparison funcion over ByteStrings -wrapCmpBytes :: (ByteString -> ByteString -> Ord) -> MDB_cmp_func -wrapCmpBytes = as_MDB_cmp_func . cmpBytesToCmpFn - --- | Convert a value to a bytestring in O(1) time. Note, however, --- that this bytestring refers into a memory-mapped page in the --- database, which may be reused after the transaction that obtained --- the value is dropped. Developers must be careful to ensure the --- bytestring doesn't stick around in any lazy computations. --- --- Consider use of the safer, higher level API that will strongly --- associate a value with a particular transaction. -valToBytes :: MDB_val -> IO ByteString -valToBytes (MDB_val sz pd) = do - fpd <- newForeignPtr_ pd - return $! B.fromForeignPtr fpd 0 (fromIntegral sz) - --} - - -instance Storable MDB_val where - alignment _ = #{alignment MDB_val} - sizeOf _ = #{size MDB_val} - peek ptr = do - sz <- #{peek MDB_val, mv_size} ptr - pd <- #{peek MDB_val, mv_data} ptr - return $! MDB_val sz pd - poke ptr (MDB_val sz pd) = do - #{poke MDB_val, mv_size} ptr sz - #{poke MDB_val, mv_data} ptr pd - -instance Storable MDB_stat where - alignment _ = #{alignment MDB_stat} - sizeOf _ = #{size MDB_stat} - peek ptr = do - psize <- #{peek MDB_stat, ms_psize} ptr - depth <- #{peek MDB_stat, ms_depth} ptr - branch_pages <- #{peek MDB_stat, ms_branch_pages} ptr - leaf_pages <- #{peek MDB_stat, ms_leaf_pages} ptr - overflow_pages <- #{peek MDB_stat, ms_overflow_pages} ptr - entries <- #{peek MDB_stat, ms_entries} ptr - return $! MDB_stat - { ms_psize = psize - , ms_depth = depth - , ms_branch_pages = branch_pages - , ms_leaf_pages = leaf_pages - , ms_overflow_pages = overflow_pages - , ms_entries = entries - } - poke ptr val = do - #{poke MDB_stat, ms_psize} ptr (ms_psize val) - #{poke MDB_stat, ms_depth} ptr (ms_depth val) - #{poke MDB_stat, ms_branch_pages} ptr (ms_branch_pages val) - #{poke MDB_stat, ms_leaf_pages} ptr (ms_leaf_pages val) - #{poke MDB_stat, ms_overflow_pages} ptr (ms_overflow_pages val) - #{poke MDB_stat, ms_entries} ptr (ms_entries val) - -instance Storable MDB_envinfo where - alignment _ = #{alignment MDB_envinfo} - sizeOf _ = #{size MDB_envinfo} - peek ptr = do - mapaddr <- #{peek MDB_envinfo, me_mapaddr} ptr - mapsize <- #{peek MDB_envinfo, me_mapsize} ptr - last_pgno <- #{peek MDB_envinfo, me_last_pgno} ptr - last_txnid <- #{peek MDB_envinfo, me_last_txnid} ptr - maxreaders <- #{peek MDB_envinfo, me_maxreaders} ptr - numreaders <- #{peek MDB_envinfo, me_numreaders} ptr - return $! MDB_envinfo - { me_mapaddr = mapaddr - , me_mapsize = mapsize - , me_last_pgno = last_pgno - , me_last_txnid = MDB_txnid last_txnid - , me_maxreaders = maxreaders - , me_numreaders = numreaders - } - poke ptr val = do - #{poke MDB_envinfo, me_mapaddr} ptr (me_mapaddr val) - #{poke MDB_envinfo, me_mapsize} ptr (me_mapsize val) - #{poke MDB_envinfo, me_last_pgno} ptr (me_last_pgno val) - #{poke MDB_envinfo, me_last_txnid} ptr (_txnid $ me_last_txnid val) - #{poke MDB_envinfo, me_maxreaders} ptr (me_maxreaders val) - #{poke MDB_envinfo, me_numreaders} ptr (me_numreaders val) - - diff --git a/pkg/hs/lmdb-static/lmdb-static.cabal b/pkg/hs/lmdb-static/lmdb-static.cabal deleted file mode 100644 index d34c9d60c4..0000000000 --- a/pkg/hs/lmdb-static/lmdb-static.cabal +++ /dev/null @@ -1,89 +0,0 @@ -Name: lmdb-static -Version: 0.2.5 -Synopsis: Lightning MDB bindings -Category: Database -Description: - LMDB is a read-optimized Berkeley DB replacement developed by Symas - for the OpenLDAP project. LMDB has impressive performance characteristics - and a friendly BSD-style OpenLDAP license. See . - . - This library has Haskell bindings to the LMDB library. You must install - the lmdb development files before installing this library, - e.g. `sudo apt-get install liblmdb-dev` works for Ubuntu 14.04. - . - For now, only a low level interface is provided, and the author is moving - on to use LMDB rather than further develop its bindings. If a higher level - API is desired, please consider contributing, or develop a separate package. - -Author: David Barbour -Maintainer: dmbarbour@gmail.com -Homepage: http://github.com/dmbarbour/haskell-lmdb - -Package-Url: -Copyright: (c) 2014 by David Barbour -License: BSD2 -license-file: LICENSE -Stability: experimental -build-type: Simple -cabal-version: >= 1.16.0.3 - -Source-repository head - type: git - location: http://github.com/dmbarbour/haskell-lmdb.git - -Library - hs-Source-Dirs: hsrc_lib - default-language: Haskell2010 - Build-Depends: base (>= 4.6 && < 5), array - Build-Tools: hsc2hs - - Exposed-Modules: - Database.LMDB.Raw - - Include-dirs: cbits - Includes: lmdb.h midl.h - C-Sources: cbits/mdb.c cbits/midl.c - cc-options: -Wall -O2 -g -pthread -fPIC - ghc-options: -Wall -fprof-auto -fPIC - - default-extensions: ApplicativeDo - , BangPatterns - , BlockArguments - , DataKinds - , DefaultSignatures - , DeriveAnyClass - , DeriveDataTypeable - , DeriveFoldable - , DeriveGeneric - , DeriveTraversable - , DerivingStrategies - , EmptyCase - , EmptyDataDecls - , FlexibleContexts - , FlexibleInstances - , FunctionalDependencies - , GADTs - , GeneralizedNewtypeDeriving - , LambdaCase - , MagicHash - , MultiParamTypeClasses - , NamedFieldPuns - , NoImplicitPrelude - , NumericUnderscores - , OverloadedStrings - , PartialTypeSignatures - , PatternSynonyms - , QuasiQuotes - , Rank2Types - , RankNTypes - , RecordWildCards - , ScopedTypeVariables - , StandaloneDeriving - , TemplateHaskell - , TupleSections - , TypeApplications - , TypeFamilies - , TypeOperators - , UnboxedTuples - , UnicodeSyntax - , ViewPatterns diff --git a/pkg/hs/stack.yaml b/pkg/hs/stack.yaml index efd4d81f9b..0ce45ed38b 100644 --- a/pkg/hs/stack.yaml +++ b/pkg/hs/stack.yaml @@ -1,7 +1,6 @@ resolver: lts-16.15 packages: - - lmdb-static - natpmp-static - proto - racquire diff --git a/pkg/hs/urbit-eventlog-lmdb/package.yaml b/pkg/hs/urbit-eventlog-lmdb/package.yaml index ff00e0a721..398e69a8b2 100644 --- a/pkg/hs/urbit-eventlog-lmdb/package.yaml +++ b/pkg/hs/urbit-eventlog-lmdb/package.yaml @@ -20,7 +20,7 @@ dependencies: - rio - vector - bytestring - - lmdb-static + - lmdb - conduit - racquire - urbit-noun-core diff --git a/pkg/hs/urbit-king/lib/Urbit/Arvo/Common.hs b/pkg/hs/urbit-king/lib/Urbit/Arvo/Common.hs index cb30b01960..3168ba1e32 100644 --- a/pkg/hs/urbit-king/lib/Urbit/Arvo/Common.hs +++ b/pkg/hs/urbit-king/lib/Urbit/Arvo/Common.hs @@ -25,10 +25,10 @@ module Urbit.Arvo.Common import Urbit.Prelude import Control.Monad.Fail (fail) -import Data.Bits import Data.Serialize import qualified Network.HTTP.Types.Method as H +import qualified Network.Socket as N import qualified Urbit.Ob as Ob @@ -159,6 +159,19 @@ deriveNoun ''JsonNode -- Ames Destinations ------------------------------------------------- +serializeToNoun :: Serialize a => a -> Noun +serializeToNoun = A . bytesAtom . encode + +serializeParseNoun :: Serialize a => String -> Int -> Noun -> Parser a +serializeParseNoun desc len = named (pack desc) . \case + A (atomBytes -> bs) + -- Atoms lose leading 0s, but since lsb, these become trailing NULs + | length bs <= len -> case decode $ bs <> replicate (len - length bs) 0 of + Right aa -> pure aa + Left msg -> fail msg + | otherwise -> fail ("putative " <> desc <> " " <> show bs <> " too long") + C{} -> fail ("unexpected cell in " <> desc) + newtype Patp a = Patp { unPatp :: a } deriving newtype (Eq, Ord, Enum, Real, Integral, Num, ToNoun, FromNoun) @@ -167,17 +180,29 @@ newtype Port = Port { unPort :: Word16 } deriving newtype (Eq, Ord, Show, Enum, Real, Integral, Num, ToNoun, FromNoun) -- @if -newtype Ipv4 = Ipv4 { unIpv4 :: Word32 } - deriving newtype (Eq, Ord, Enum, Real, Integral, Num, ToNoun, FromNoun) +newtype Ipv4 = Ipv4 { unIpv4 :: N.HostAddress } + deriving newtype (Eq, Ord, Enum) + +instance Serialize Ipv4 where + get = (\a b c d -> Ipv4 $ N.tupleToHostAddress $ (d, c, b, a)) + <$> getWord8 <*> getWord8 <*> getWord8 <*> getWord8 + put (Ipv4 (N.hostAddressToTuple -> (a, b, c, d))) = for_ [d, c, b, a] putWord8 + +instance ToNoun Ipv4 where + toNoun = serializeToNoun + +instance FromNoun Ipv4 where + parseNoun = serializeParseNoun "Ipv4" 4 instance Show Ipv4 where - show (Ipv4 i) = - show ((shiftR i 24) .&. 0xff) ++ "." ++ - show ((shiftR i 16) .&. 0xff) ++ "." ++ - show ((shiftR i 8) .&. 0xff) ++ "." ++ - show (i .&. 0xff) + show (Ipv4 (N.hostAddressToTuple -> (a, b, c, d))) = + show a ++ "." ++ + show b ++ "." ++ + show c ++ "." ++ + show d -- @is +-- should probably use hostAddress6ToTuple here, but no one uses it right now newtype Ipv6 = Ipv6 { unIpv6 :: Word128 } deriving newtype (Eq, Ord, Show, Enum, Real, Integral, Num, ToNoun, FromNoun) @@ -190,21 +215,14 @@ data AmesAddress = AAIpv4 Ipv4 Port deriving (Eq, Ord, Show) instance Serialize AmesAddress where - get = AAIpv4 <$> (Ipv4 <$> getWord32le) <*> (Port <$> getWord16le) - put (AAIpv4 (Ipv4 ip) (Port port)) = putWord32le ip >> putWord16le port + get = AAIpv4 <$> get <*> (Port <$> getWord16le) + put (AAIpv4 ip (Port port)) = put ip >> putWord16le port instance FromNoun AmesAddress where - parseNoun = named "AmesAddress" . \case - A (atomBytes -> bs) - -- Atoms lose leading 0s, but since lsb, these become trailing NULs - | length bs <= 6 -> case decode $ bs <> replicate (6 - length bs) 0 of - Right aa -> pure aa - Left msg -> fail msg - | otherwise -> fail ("putative address " <> show bs <> " too long") - C{} -> fail "unexpected cell in ames address" + parseNoun = serializeParseNoun "AmesAddress" 6 instance ToNoun AmesAddress where - toNoun = A . bytesAtom . encode + toNoun = serializeToNoun type AmesDest = Each Galaxy AmesAddress diff --git a/pkg/hs/urbit-king/lib/Urbit/King/Main.hs b/pkg/hs/urbit-king/lib/Urbit/King/Main.hs index 1a394110b6..9dbb71822a 100644 --- a/pkg/hs/urbit-king/lib/Urbit/King/Main.hs +++ b/pkg/hs/urbit-king/lib/Urbit/King/Main.hs @@ -99,6 +99,7 @@ import qualified Data.Set as Set import qualified Data.Text as T import qualified Network.HTTP.Client as C import qualified System.Posix.Signals as Sys +import qualified System.Posix.Resource as Sys import qualified System.ProgressBar as PB import qualified System.Random as Sys import qualified Urbit.EventLog.LMDB as Log @@ -460,6 +461,13 @@ pillFrom = \case noun <- cueBS body & either throwIO pure fromNounErr noun & either (throwIO . uncurry ParseErr) pure +multiOnFatal :: HasKingEnv e => e -> IO () +multiOnFatal env = runRIO env $ do + (view stderrLogFuncL >>=) $ flip runRIO $ logError + ("Urbit is shutting down because of a problem with the HTTP server.\n" + <> "Please restart it at your leisure.") + view killKingActionL >>= atomically + newShip :: CLI.New -> CLI.Opts -> RIO KingEnv () newShip CLI.New{..} opts = do {- @@ -472,7 +480,8 @@ newShip CLI.New{..} opts = do "run ship" flow, and possibly sequence them from the outside if that's really needed. -} - multi <- multiEyre (MultiEyreConf Nothing Nothing True) + env <- ask + multi <- multiEyre (multiOnFatal env) (MultiEyreConf Nothing Nothing True) -- TODO: We hit the same problem as above: we need a host env to boot a ship -- because it may autostart the ship, so build an inactive port configuration. @@ -660,6 +669,7 @@ main = do hSetBuffering stdout NoBuffering setupSignalHandlers + setRLimits runKingEnv args log $ case args of CLI.CmdRun ko ships -> runShips ko ships @@ -693,6 +703,15 @@ main = do for_ [Sys.sigTERM, Sys.sigINT] $ \sig -> do Sys.installHandler sig (Sys.Catch onKillSig) Nothing + setRLimits = do + openFiles <- Sys.getResourceLimit Sys.ResourceOpenFiles + let soft = case Sys.hardLimit openFiles of + Sys.ResourceLimit lim -> Sys.ResourceLimit lim + Sys.ResourceLimitInfinity -> Sys.ResourceLimit 10240 -- macOS + Sys.ResourceLimitUnknown -> Sys.ResourceLimit 10240 + Sys.setResourceLimit Sys.ResourceOpenFiles + openFiles { Sys.softLimit = soft } + verboseLogging :: CLI.Cmd -> Bool verboseLogging = \case CLI.CmdRun ko ships -> any CLI.oVerbose (ships <&> \(_, o, _) -> o) @@ -716,7 +735,6 @@ main = do CLI.CmdRun ko _ -> CLI.LogStderr _ -> CLI.LogStderr - {- Runs a ship but restarts it if it crashes or shuts down on it's own. @@ -792,7 +810,8 @@ runShips CLI.Host {..} ships = do -- a king-wide option. } - multi <- multiEyre meConf + env <- ask + multi <- multiEyre (multiOnFatal env) meConf ports <- buildPortHandler hUseNatPmp diff --git a/pkg/hs/urbit-king/lib/Urbit/Vere/Ames/Packet.hs b/pkg/hs/urbit-king/lib/Urbit/Vere/Ames/Packet.hs index 8c90bd169e..d2ddcdfc42 100644 --- a/pkg/hs/urbit-king/lib/Urbit/Vere/Ames/Packet.hs +++ b/pkg/hs/urbit-king/lib/Urbit/Vere/Ames/Packet.hs @@ -80,10 +80,6 @@ data ShipClass muk :: ByteString -> Word20 muk bs = mugBS bs .&. (2 ^ 20 - 1) --- XX check this -getAmesAddress :: Get AmesAddress -getAmesAddress = AAIpv4 <$> (Ipv4 <$> getWord32le) <*> (Port <$> getWord16le) - putAmesAddress :: Putter AmesAddress putAmesAddress = \case AAIpv4 (Ipv4 ip) (Port port) -> putWord32le ip >> putWord16le port @@ -104,7 +100,7 @@ instance Serialize Packet where guard isAmes pktOrigin <- if isRelayed - then Just <$> getAmesAddress + then Just <$> get else pure Nothing -- body @@ -157,9 +153,10 @@ instance Serialize Packet where putWord32le head case pktOrigin of - Just o -> putAmesAddress o + Just o -> put o Nothing -> pure () putByteString body + where putShipGetRank s@(Ship (LargeKey p q)) = case () of _ | s < 2 ^ 16 -> (0, putWord16le $ fromIntegral s) -- lord diff --git a/pkg/hs/urbit-king/lib/Urbit/Vere/Ames/UDP.hs b/pkg/hs/urbit-king/lib/Urbit/Vere/Ames/UDP.hs index f38b1a868b..0189306325 100644 --- a/pkg/hs/urbit-king/lib/Urbit/Vere/Ames/UDP.hs +++ b/pkg/hs/urbit-king/lib/Urbit/Vere/Ames/UDP.hs @@ -4,8 +4,12 @@ 1. Opens a UDP socket and makes sure that it stays open. - If can't open the port, wait and try again repeatedly. - - If there is an error reading or writting from the open socket, - close it and open another. + - If there is an error reading to or writing from the open socket, + close it and open another, making sure, however, to reuse the + same port + NOTE: It's not clear what, if anything, closing and reopening + the socket does. We're keeping this behavior out of conservatism + until we understand it better. 2. Receives packets from the socket. @@ -158,7 +162,7 @@ realUdpServ -> HostAddress -> AmesStat -> RIO e UdpServ -realUdpServ por hos sat = do +realUdpServ startPort hos sat = do logInfo $ displayShow ("AMES", "UDP", "Starting real UDP server.") env <- ask @@ -202,23 +206,30 @@ realUdpServ por hos sat = do did <- atomically (tryWriteTBQueue qSend (a, b)) when (did == False) $ do logWarn "AMES: UDP: Dropping outbound packet because queue is full." + let opener por = do + logInfo $ displayShow $ ("AMES", "UDP", "Trying to open socket, port",) + por + sk <- forceBind por hos + sn <- io $ getSocketName sk + sp <- io $ socketPort sk + logInfo $ displayShow $ ("AMES", "UDP", "Got socket", sn, sp) - tOpen <- async $ forever $ do - sk <- forceBind por hos - sn <- io $ getSocketName sk + let waitForRelease = do + atomically (writeTVar vSock (Just sk)) + broken <- atomically (takeTMVar vFail) + logWarn "AMES: UDP: Closing broken socket." + io (close broken) - let waitForRelease = do - atomically (writeTVar vSock (Just sk)) - broken <- atomically (takeTMVar vFail) - logWarn "AMES: UDP: Closing broken socket." - io (close broken) + case sn of + (SockAddrInet boundPort _) -> + -- When we're on IPv4, maybe port forward at the NAT. + rwith (requestPortAccess $ fromIntegral boundPort) $ + \() -> waitForRelease + _ -> waitForRelease - case sn of - (SockAddrInet boundPort _) -> - -- When we're on IPv4, maybe port forward at the NAT. - rwith (requestPortAccess $ fromIntegral boundPort) $ - \() -> waitForRelease - _ -> waitForRelease + opener sp + + tOpen <- async $ opener startPort tSend <- async $ forever $ join $ atomically $ do (adr, byt) <- readTBQueue qSend diff --git a/pkg/hs/urbit-king/lib/Urbit/Vere/Clay.hs b/pkg/hs/urbit-king/lib/Urbit/Vere/Clay.hs index c29fced795..e76fdba2c9 100644 --- a/pkg/hs/urbit-king/lib/Urbit/Vere/Clay.hs +++ b/pkg/hs/urbit-king/lib/Urbit/Vere/Clay.hs @@ -37,12 +37,12 @@ textPlain = Path [(MkKnot "text"), (MkKnot "plain")] -- | Filter for dotfiles, tempfiles and backup files. validClaySyncPath :: FilePath -> Bool -validClaySyncPath fp = hasPeriod && notTildeFile && notDotHash && notDoubleHash +validClaySyncPath fp = hasPeriod && notTildeFile && notDotFile && notDoubleHash where fileName = takeFileName fp hasPeriod = elem '.' fileName notTildeFile = not $ "~" `isSuffixOf` fileName - notDotHash = not $ ".#" `isPrefixOf` fileName + notDotFile = not $ "." `isPrefixOf` fileName notDoubleHash = not $ ("#" `isPrefixOf` fileName) && ("#" `isSuffixOf` fileName) diff --git a/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre.hs b/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre.hs index bea9e1962b..b426009f18 100644 --- a/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre.hs +++ b/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre.hs @@ -11,7 +11,11 @@ where import Urbit.Prelude hiding (Builder) import Urbit.Arvo hiding (ServerId, reqUrl) -import Urbit.King.App (HasKingId(..), HasMultiEyreApi(..), HasPierEnv(..)) +import Urbit.King.App ( killKingActionL + , HasKingId(..) + , HasMultiEyreApi(..) + , HasPierEnv(..) + ) import Urbit.King.Config import Urbit.Vere.Eyre.Multi import Urbit.Vere.Eyre.PortsFile @@ -177,9 +181,10 @@ startServ -> HttpServerConf -> (EvErr -> STM ()) -> (Text -> RIO e ()) + -> IO () -> KingSubsite -> RIO e Serv -startServ who isFake conf plan stderr sub = do +startServ who isFake conf plan stderr onFatal sub = do logInfo (displayShow ("EYRE", "startServ")) multi <- view multiEyreApiL @@ -228,7 +233,7 @@ startServ who isFake conf plan stderr sub = do atomically (joinMultiEyre multi who mCre onReq onKilReq sub) logInfo $ displayShow ("EYRE", "Starting loopback server") - lop <- serv vLive $ ServConf + lop <- serv vLive onFatal $ ServConf { scHost = soHost (pttLop ptt) , scPort = soWhich (pttLop ptt) , scRedi = Nothing @@ -240,7 +245,7 @@ startServ who isFake conf plan stderr sub = do } logInfo $ displayShow ("EYRE", "Starting insecure server") - ins <- serv vLive $ ServConf + ins <- serv vLive onFatal $ ServConf { scHost = soHost (pttIns ptt) , scPort = soWhich (pttIns ptt) , scRedi = secRedi @@ -253,7 +258,7 @@ startServ who isFake conf plan stderr sub = do mSec <- for mTls $ \tls -> do logInfo "Starting secure server" - serv vLive $ ServConf + serv vLive onFatal $ ServConf { scHost = soHost (pttSec ptt) , scPort = soWhich (pttSec ptt) , scRedi = Nothing @@ -356,7 +361,11 @@ eyre env who plan isFake stderr sub = (initialEvents, runHttpServer) restart :: Drv -> HttpServerConf -> RIO e Serv restart (Drv var) conf = do logInfo "Restarting http server" - let startAct = startServ who isFake conf plan stderr sub + let onFatal = runRIO env $ do + -- XX instead maybe restart following logic under HSESetConfig below + stderr "A web server problem has occurred. Please restart your ship." + view killKingActionL >>= atomically + let startAct = startServ who isFake conf plan stderr onFatal sub res <- fromEither =<< restartService var startAct kill logInfo "Done restating http server" pure res diff --git a/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/KingSubsite.hs b/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/KingSubsite.hs index c75d78c0db..ae4d59d630 100644 --- a/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/KingSubsite.hs +++ b/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/KingSubsite.hs @@ -116,7 +116,7 @@ kingSubsite who scry stat func = do => Text -> RIO e (Maybe Bool) scryAuth cookie = - scryNow scry "ex" "" ["authenticated", "cookie", textAsTa cookie] + scryNow scry "ex" "" ["authenticated", "cookie", textAsT cookie] fourOhFourSubsite :: Ship -> KingSubsite fourOhFourSubsite who = KS $ \req respond -> diff --git a/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/Multi.hs b/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/Multi.hs index 7a3c27c12a..09d1f974e3 100644 --- a/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/Multi.hs +++ b/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/Multi.hs @@ -75,8 +75,8 @@ leaveMultiEyre MultiEyreApi {..} who = do modifyTVar' meaTlsC (deleteMap who) modifyTVar' meaSite (deleteMap who) -multiEyre :: HasLogFunc e => MultiEyreConf -> RIO e MultiEyreApi -multiEyre conf@MultiEyreConf {..} = do +multiEyre :: HasLogFunc e => IO () -> MultiEyreConf -> RIO e MultiEyreApi +multiEyre onFatal conf@MultiEyreConf {..} = do logInfo (displayShow ("EYRE", "MULTI", conf)) vLive <- io emptyLiveReqs >>= newTVarIO @@ -108,7 +108,7 @@ multiEyre conf@MultiEyreConf {..} = do mIns <- for mecHttpPort $ \por -> do logInfo (displayShow ("EYRE", "MULTI", "HTTP", por)) - serv vLive $ ServConf + serv vLive onFatal $ ServConf { scHost = host , scPort = SPChoices $ singleton $ fromIntegral por , scRedi = Nothing -- TODO @@ -121,7 +121,7 @@ multiEyre conf@MultiEyreConf {..} = do mSec <- for mecHttpsPort $ \por -> do logInfo (displayShow ("EYRE", "MULTI", "HTTPS", por)) - serv vLive $ ServConf + serv vLive onFatal $ ServConf { scHost = host , scPort = SPChoices $ singleton $ fromIntegral por , scRedi = Nothing diff --git a/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/Serv.hs b/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/Serv.hs index 052c11e0cc..2bd8a22645 100644 --- a/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/Serv.hs +++ b/pkg/hs/urbit-king/lib/Urbit/Vere/Eyre/Serv.hs @@ -35,6 +35,7 @@ import Urbit.Prelude hiding (Builder) import Data.Default (def) import Data.List.NonEmpty (NonEmpty((:|))) +import GHC.IO.Exception (IOException(..), IOErrorType(..)) import Network.TLS ( Credential , Credentials(..) , ServerHooks(..) @@ -254,19 +255,38 @@ startServer -> Net.Socket -> Maybe W.Port -> TVar E.LiveReqs + -> IO () -> RIO e () -startServer typ hos por sok red vLive = do +startServer typ hos por sok red vLive onFatal = do envir <- ask let host = case hos of SHLocalhost -> "127.0.0.1" SHAnyHostOk -> "*" + let handler r e = do + when (isFatal e) $ do + runRIO envir $ logError $ display $ msg r e + onFatal + when (W.defaultShouldDisplayException e) $ do + runRIO envir $ logWarn $ display $ msg r e + + isFatal e + | Just (IOError {ioe_type = ResourceExhausted}) <- fromException e + = True + | otherwise = False + + msg r e = case r of + Just r -> "eyre: failed request from " <> (tshow $ W.remoteHost r) + <> " for " <> (tshow $ W.rawPathInfo r) <> ": " <> tshow e + Nothing -> "eyre: server exception: " <> tshow e + let opts = W.defaultSettings & W.setHost host & W.setPort (fromIntegral por) - & W.setTimeout (5 * 60) + & W.setTimeout 30 + & W.setOnException handler -- TODO build Eyre.Site.app in pier, thread through here let runAppl who = E.app envir who vLive @@ -338,8 +358,9 @@ getFirstTlsConfig (MTC var) = do [] -> STM.retry x:_ -> pure (fst x) -realServ :: HasLogFunc e => TVar E.LiveReqs -> ServConf -> RIO e ServApi -realServ vLive conf@ServConf {..} = do +realServ :: HasLogFunc e + => TVar E.LiveReqs -> IO () -> ServConf -> RIO e ServApi +realServ vLive onFatal conf@ServConf {..} = do logInfo (displayShow ("EYRE", "SERV", "Running Real Server")) por <- newEmptyTMVarIO @@ -354,10 +375,10 @@ realServ vLive conf@ServConf {..} = do logInfo (displayShow ("EYRE", "SERV", "runServ")) rwith (forceOpenSocket scHost scPort) $ \(por, sok) -> do atomically (putTMVar vPort por) - startServer scType scHost por sok scRedi vLive + startServer scType scHost por sok scRedi vLive onFatal -serv :: HasLogFunc e => TVar E.LiveReqs -> ServConf -> RIO e ServApi -serv vLive conf = do +serv :: HasLogFunc e => TVar E.LiveReqs -> IO () -> ServConf -> RIO e ServApi +serv vLive onFatal conf = do if scFake conf then fakeServ conf - else realServ vLive conf + else realServ vLive onFatal conf diff --git a/pkg/hs/urbit-king/lib/Urbit/Vere/Http/Client.hs b/pkg/hs/urbit-king/lib/Urbit/Vere/Http/Client.hs index b3a24ddff9..88cddb505f 100644 --- a/pkg/hs/urbit-king/lib/Urbit/Vere/Http/Client.hs +++ b/pkg/hs/urbit-king/lib/Urbit/Vere/Http/Client.hs @@ -7,7 +7,7 @@ module Urbit.Vere.Http.Client where -import Urbit.Prelude hiding (Builder) +import Urbit.Prelude hiding (Builder, finally) import Urbit.Vere.Http import Urbit.Vere.Pier.Types @@ -16,6 +16,8 @@ import Urbit.King.App import Urbit.Arvo (BlipEv(..), Ev(..), HttpClientEf(..), HttpClientEv(..), HttpClientReq(..), HttpEvent(..), KingId, ResponseHeader(..)) +import RIO.Orphans () +import Control.Monad.Catch (finally) import qualified Data.Map.Strict as M import qualified Network.HTTP.Client as H @@ -126,7 +128,11 @@ client env plan = (initialEvents, runHttpClient) newReq :: HttpClientDrv -> ReqId -> HttpClientReq -> RIO e () newReq drv id req = do async <- runReq drv id req - atomically $ modifyTVar' (hcdLive drv) (insertMap id async) + -- If the async has somehow already completed, don't put it in the map, + -- because then it might never get out. + atomically $ pollSTM async >>= \case + Nothing -> modifyTVar' (hcdLive drv) (insertMap id async) + Just _ -> pure () -- The problem with the original http client code was that it was written -- to the idea of what the events "should have" been instead of what they @@ -137,7 +143,7 @@ client env plan = (initialEvents, runHttpClient) -- Continue (with File) event, and the Continue (completed) event are three -- separate things. runReq :: HttpClientDrv -> ReqId -> HttpClientReq -> RIO e (Async ()) - runReq HttpClientDrv{..} id req = async $ + runReq HttpClientDrv{..} id req = async $ flip finally aftr $ case cvtReq req of Nothing -> do logInfo $ displayShow ("(malformed http client request)", id, req) @@ -147,6 +153,11 @@ client env plan = (initialEvents, runHttpClient) withRunInIO $ \run -> H.withResponse r hcdManager $ \x -> run (exec x) where + -- Make sure to remove our entry from hcdLive after we're done so the + -- map doesn't grow without bound. + aftr :: RIO e () + aftr = atomically $ modifyTVar' hcdLive (deleteMap id) + recv :: H.BodyReader -> RIO e (Maybe ByteString) recv read = io $ read <&> \case chunk | null chunk -> Nothing | otherwise -> Just chunk diff --git a/pkg/hs/urbit-king/lib/Urbit/Vere/Serf/IPC.hs b/pkg/hs/urbit-king/lib/Urbit/Vere/Serf/IPC.hs index f977907a2b..67128821f4 100644 --- a/pkg/hs/urbit-king/lib/Urbit/Vere/Serf/IPC.hs +++ b/pkg/hs/urbit-king/lib/Urbit/Vere/Serf/IPC.hs @@ -522,10 +522,13 @@ run serf maxBatchSize getLastEvInLog onInput sendOn spin = topLoop que <- newTBMQueueIO 1 () <- atomically (writeTBMQueue que firstWorkErr) tWork <- async (processWork serf maxBatchSize que onWorkResp spin) - flip onException (cancel tWork) $ do + -- Avoid wrapping all subsequent runs of the event loop in an exception + -- handler which retains tWork. + nexSt <- flip onException (cancel tWork) $ do nexSt <- workLoop que wait tWork - nexSt + pure nexSt + nexSt workLoop :: TBMQueue EvErr -> IO (IO ()) workLoop que = atomically onInput >>= \case diff --git a/pkg/hs/urbit-king/package.yaml b/pkg/hs/urbit-king/package.yaml index 1493ceb49f..472b156058 100644 --- a/pkg/hs/urbit-king/package.yaml +++ b/pkg/hs/urbit-king/package.yaml @@ -1,5 +1,5 @@ name: urbit-king -version: 1.2 +version: 1.5 license: MIT license-file: LICENSE data-files: @@ -65,7 +65,6 @@ dependencies: - iproute - largeword - lens - - lmdb-static - lock-file - megaparsec - memory @@ -86,6 +85,7 @@ dependencies: - regex-tdfa - resourcet - rio + - rio-orphans - semigroups - smallcheck - stm diff --git a/pkg/hs/urbit-noun/lib/Urbit/Noun/Conversions.hs b/pkg/hs/urbit-noun/lib/Urbit/Noun/Conversions.hs index 13bc39419c..09dd4aa739 100644 --- a/pkg/hs/urbit-noun/lib/Urbit/Noun/Conversions.hs +++ b/pkg/hs/urbit-noun/lib/Urbit/Noun/Conversions.hs @@ -13,7 +13,7 @@ module Urbit.Noun.Conversions , Path(..), EvilPath(..), Ship(..) , Lenient(..), pathToFilePath, filePathToPath , showUD, tshowUD - , textAsTa + , textAsT ) where import ClassyPrelude hiding (hash) @@ -556,13 +556,13 @@ instance FromNoun Knot where else fail ("Non-ASCII chars in knot: " <> unpack txt) -- equivalent of (cury scot %t) -textAsTa :: Text -> Text -textAsTa = ("~~" <>) . concatMap \case +textAsT :: Text -> Text +textAsT = ("~~" <>) . concatMap \case ' ' -> "." '.' -> "~." '~' -> "~~" c -> - if C.isAlphaNum c || (c == '-') then + if (C.isAlphaNum c && not (C.isUpper c)) || (c == '-') then T.singleton c else if C.ord c < 0x10 then "~0" else "~" diff --git a/pkg/hs/urbit-noun/lib/Urbit/Noun/Time.hs b/pkg/hs/urbit-noun/lib/Urbit/Noun/Time.hs index 55122fcb34..1976c11a58 100644 --- a/pkg/hs/urbit-noun/lib/Urbit/Noun/Time.hs +++ b/pkg/hs/urbit-noun/lib/Urbit/Noun/Time.hs @@ -17,22 +17,22 @@ import Data.Time.Clock.System (systemToUTCTime, utcToSystemTime) import Data.Time.LocalTime (TimeOfDay(..), timeToTimeOfDay) import Data.Word (Word64) import Text.Printf (printf) -import Urbit.Noun (FromNoun, ToNoun) +import Urbit.Noun (deriveToNoun, FromNoun, ToNoun(..)) -- Types ----------------------------------------------------------------------- newtype Gap = Gap { _fractoSecs :: Integer } - deriving newtype (Eq, Ord, Show, Num, ToNoun, FromNoun) + deriving newtype (Eq, Ord, Show, Num, FromNoun) newtype Unix = Unix { _sinceUnixEpoch :: Gap } - deriving newtype (Eq, Ord, Show, ToNoun, FromNoun) + deriving newtype (Eq, Ord, Show, FromNoun) newtype Wen = Wen { _sinceUrbitEpoch :: Gap } - deriving newtype (Eq, Ord, Show, Num, ToNoun, FromNoun) + deriving newtype (Eq, Ord, Show, Num, FromNoun) newtype Date = MkDate { _dateWen :: Wen } - deriving newtype (Eq, Ord, Num, ToNoun, FromNoun) + deriving newtype (Eq, Ord, Num, FromNoun) -- Record Lenses --------------------------------------------------------------- @@ -45,6 +45,20 @@ makeLenses ''Date -- Instances ------------------------------------------------------------------- +instance ToNoun Gap where + toNoun (reducePrecision -> Gap fs) = toNoun fs + +-- | Produce a Gap with fewer digits after the binary point, more +-- appropriately capturing the precision our clock gives us. +reducePrecision :: Gap -> Gap +reducePrecision (Gap fs) = Gap (chop fs) + where + chop fs = shiftL (shiftR fs 32) 32 + +deriveToNoun ''Unix +deriveToNoun ''Wen +deriveToNoun ''Date + instance Show Date where show (MkDate wen) = if fs == 0 then printf "~%i.%u.%u..%02u.%02u.%02u" y m d h min s diff --git a/pkg/interface/.eslintrc.js b/pkg/interface/.eslintrc.js index fba1ccabcc..ad4f8c0c9c 100644 --- a/pkg/interface/.eslintrc.js +++ b/pkg/interface/.eslintrc.js @@ -1,3 +1,3 @@ module.exports = { - extends: "@urbit" -}; \ No newline at end of file + extends: '@urbit' +}; diff --git a/pkg/interface/CONTRIBUTING.md b/pkg/interface/CONTRIBUTING.md index 711e359d8f..f3bfbb5a0e 100644 --- a/pkg/interface/CONTRIBUTING.md +++ b/pkg/interface/CONTRIBUTING.md @@ -32,7 +32,7 @@ same (if [developing on a local development ship][local]). Then, from 'pkg/interface': ``` -npm install +npm ci npm run start ``` @@ -59,7 +59,7 @@ module.exports = { ``` The dev environment will attempt to match the subdomain against the keys of this -object, and if matched will proxy to the corresponding URL. For example, the +object, and if matched will proxy to the corresponding URL. For example, the above config will proxy `zod.localhost:9000` to `http://localhost:8080`, `bus.localhost:9000` to `http://localhost:8081` and so on and so forth. If no match is found, then it will fallback to the `URL` property. @@ -71,7 +71,7 @@ linter and for usage through the command, do the following: ```bash $ cd ./pkg/interface -$ npm install +$ npm ci $ npm run lint ``` diff --git a/pkg/interface/config/webpack.dev.js b/pkg/interface/config/webpack.dev.js index 66bf83ce9d..27c5e4e63a 100644 --- a/pkg/interface/config/webpack.dev.js +++ b/pkg/interface/config/webpack.dev.js @@ -6,7 +6,10 @@ const urbitrc = require('./urbitrc'); const fs = require('fs'); const util = require('util'); const _ = require('lodash'); -const exec = util.promisify(require('child_process').exec); +const { execSync } = require('child_process'); + +const GIT_DESC = execSync('git describe --always', { encoding: 'utf8' }).trim(); + function copyFile(src,dest) { return new Promise((res,rej) => @@ -64,12 +67,12 @@ if(urbitrc.URL) { return '/index.js' } }, - '/~landscape/js/serviceworker.js': { - target: 'http://localhost:9000', - pathRewrite: (req, path) => { - return '/serviceworker.js' - } - }, + // '/~landscape/js/serviceworker.js': { + // target: 'http://localhost:9000', + // pathRewrite: (req, path) => { + // return '/serviceworker.js' + // } + // }, '**': { changeOrigin: true, target: urbitrc.URL, @@ -85,7 +88,7 @@ module.exports = { mode: 'development', entry: { app: './src/index.js', - serviceworker: './src/serviceworker.js' + // serviceworker: './src/serviceworker.js' }, module: { rules: [ @@ -94,7 +97,11 @@ module.exports = { use: { loader: 'babel-loader', options: { - presets: ['@babel/preset-env', '@babel/typescript', '@babel/preset-react'], + presets: ['@babel/preset-env', '@babel/typescript', ['@babel/preset-react', { + runtime: 'automatic', + development: true, + importSource: '@welldone-software/why-did-you-render', + }]], plugins: [ '@babel/transform-runtime', '@babel/plugin-proposal-object-rest-spread', @@ -104,7 +111,7 @@ module.exports = { ] } }, - exclude: /node_modules\/(?!(@tlon\/indigo-dark|@tlon\/indigo-light)\/).*/ + exclude: /node_modules\/(?!(@tlon\/indigo-dark|@tlon\/indigo-light|@tlon\/indigo-react)\/).*/ }, { test: /\.css$/i, @@ -127,6 +134,7 @@ module.exports = { plugins: [ new UrbitShipPlugin(urbitrc), new webpack.DefinePlugin({ + 'process.env.LANDSCAPE_SHORTHASH': JSON.stringify(GIT_DESC), 'process.env.TUTORIAL_HOST': JSON.stringify('~difmex-passed'), 'process.env.TUTORIAL_GROUP': JSON.stringify('beginner-island'), 'process.env.TUTORIAL_CHAT': JSON.stringify('introduce-yourself-7010'), diff --git a/pkg/interface/config/webpack.prod.js b/pkg/interface/config/webpack.prod.js index 8d9f527090..e8f080c939 100644 --- a/pkg/interface/config/webpack.prod.js +++ b/pkg/interface/config/webpack.prod.js @@ -30,7 +30,7 @@ module.exports = { ] } }, - exclude: /node_modules\/(?!(@tlon\/indigo-dark|@tlon\/indigo-light)\/).*/ + exclude: /node_modules\/(?!(@tlon\/indigo-dark|@tlon\/indigo-light|@tlon\/indigo-react)\/).*/ }, { test: /\.css$/i, diff --git a/pkg/interface/package-lock.json b/pkg/interface/package-lock.json index b63ac3536b..1fe4199adc 100644 --- a/pkg/interface/package-lock.json +++ b/pkg/interface/package-lock.json @@ -1326,6 +1326,50 @@ "warning": "^4.0.3" } }, + "@react-spring/animated": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/@react-spring/animated/-/animated-9.1.1.tgz", + "integrity": "sha512-u8Assg5uySwqyoeb1f7eBAUSl8sleJTewdfhVi1EtcM9ngU2Snhcp6snF8NGxvf4gZp5z7v+Dfx3KdB2V8NnXQ==", + "requires": { + "@react-spring/shared": "~9.1.1", + "@react-spring/types": "~9.1.1" + } + }, + "@react-spring/core": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/@react-spring/core/-/core-9.1.1.tgz", + "integrity": "sha512-flHeLN56idxQ1YIpUYY1m3r2ZAM2xg7Zb/pHBFSCbnOKP7TtlhAAOfmrabERqaThmrqkFKiq9FjyF76d3OjE5g==", + "requires": { + "@react-spring/animated": "~9.1.1", + "@react-spring/shared": "~9.1.1", + "@react-spring/types": "~9.1.1" + } + }, + "@react-spring/shared": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/@react-spring/shared/-/shared-9.1.1.tgz", + "integrity": "sha512-GA9A9l5JxC50eDTDPu5IDUMhQ4MiBrXd3ZdlI6/wUCgAsZ1wPx77sxaccomxlUomRet0IUcXCEKcL1Flax7ZMQ==", + "requires": { + "@react-spring/types": "~9.1.1", + "rafz": "^0.1.14" + } + }, + "@react-spring/types": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/@react-spring/types/-/types-9.1.1.tgz", + "integrity": "sha512-GaesYowey+nmDw/yhZ5jErEH2UaDl4jxax8aQtW5h3OpNu/QS8swoEn/jxQgffLb0n6gjsER7QyIx/dmZIWlyw==" + }, + "@react-spring/web": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/@react-spring/web/-/web-9.1.1.tgz", + "integrity": "sha512-py4c/Agqz9Unf+Apame29XYTLqHnKXSwJA6Q44jcNtHRFMuRjIpCyhS13C1ZI5PcJT0g9b8CvtMQFiShne8wNQ==", + "requires": { + "@react-spring/animated": "~9.1.1", + "@react-spring/core": "~9.1.1", + "@react-spring/shared": "~9.1.1", + "@react-spring/types": "~9.1.1" + } + }, "@styled-system/background": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/@styled-system/background/-/background-5.1.2.tgz", @@ -1434,14 +1478,14 @@ "integrity": "sha512-/c+3/aC+gSnLHiLwTdje7pYS84ZAR3zyMJhp2mT9BIPtk7ek/EGsrrugZjVJxeKXqy+mQpFD5TXktgAEh0Ko1A==" }, "@tlon/indigo-light": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@tlon/indigo-light/-/indigo-light-1.0.6.tgz", - "integrity": "sha512-kBzJueOoGDVF2knGt+Kf5ylvil6+V1qn8/RqAj1S6wUTnfUfAMRzDp4LQI2MxLI8Is0OG3XCErVSOUImU6R3lg==" + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@tlon/indigo-light/-/indigo-light-1.0.7.tgz", + "integrity": "sha512-xO8hj2Ak6cEYe2QCM3w7UuaSB8ubg6G0G6/OkPVMVrz6b5ztccZmkbmYCYJ/Ot6976lGzKFsWFKRUhwRgCHfHQ==" }, "@tlon/indigo-react": { - "version": "1.2.19", - "resolved": "https://registry.npmjs.org/@tlon/indigo-react/-/indigo-react-1.2.19.tgz", - "integrity": "sha512-lcHtPIbKeXVDvqd9dkCswB++CLRB2TsYFoegRU5VX3A886R+larJP81CzmoAwmZiJL3OnwypRklyfAv41F6W2w==", + "version": "1.2.22", + "resolved": "https://registry.npmjs.org/@tlon/indigo-react/-/indigo-react-1.2.22.tgz", + "integrity": "sha512-8w2TkYicch+R0kkZT+MZ4oG0pIJFNjhmVlbXgqyXhOCPRJB2WrAh6OM5Cbb389r7lA+CXXfu3Nx7Rdiuxjf/vg==", "requires": { "@reach/menu-button": "^0.10.5", "react": "^16.13.1", @@ -1606,6 +1650,11 @@ "source-map": "^0.6.1" } }, + "@types/unist": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.3.tgz", + "integrity": "sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==" + }, "@types/warning": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/warning/-/warning-3.0.0.tgz", @@ -1778,47 +1827,1076 @@ "@types/lodash": "^4.14.168", "@urbit/eslint-config": "^1.0.0", "big-integer": "^1.6.48", + "immer": "^9.0.1", "lodash": "^4.17.20" }, "dependencies": { "@babel/runtime": { - "version": "7.12.5", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.12.5.tgz", - "integrity": "sha512-plcc+hbExy3McchJCEQG3knOsuh3HH+Prx1P6cLIkET/0dLuQDEnrT+s27Axgc9bqfsmNUNHfscgMUdBpC9xfg==", + "version": "7.14.0", + "bundled": true, "requires": { "regenerator-runtime": "^0.13.4" } }, + "@blakeembrey/deque": { + "version": "1.0.5", + "bundled": true + }, + "@blakeembrey/template": { + "version": "1.0.0", + "bundled": true + }, "@types/lodash": { "version": "4.14.168", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.168.tgz", - "integrity": "sha512-oVfRvqHV/V6D1yifJbVRU3TMp8OT6o6BG+U9MkwuJ3U8/CsDHvalRpsxBqivn71ztOFZBTfJMvETbqHiaNSj7Q==" + "bundled": true }, "@urbit/eslint-config": { "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@urbit/eslint-config/-/eslint-config-1.0.0.tgz", - "integrity": "sha512-Xmzb6MvM7KorlPJEq/hURZZ4BHSVy/7CoQXWogsBSTv5MOZnMqwNKw6yt24k2AO/2UpHwjGptimaNLqFfesJbw==" + "bundled": true + }, + "anymatch": { + "version": "3.1.2", + "bundled": true, + "requires": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "arg": { + "version": "4.1.3", + "bundled": true }, "big-integer": { "version": "1.6.48", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", - "integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==" + "bundled": true + }, + "binary-extensions": { + "version": "2.2.0", + "bundled": true + }, + "braces": { + "version": "3.0.2", + "bundled": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "chokidar": { + "version": "3.5.1", + "bundled": true, + "requires": { + "anymatch": "~3.1.1", + "braces": "~3.0.2", + "fsevents": "~2.3.1", + "glob-parent": "~5.1.0", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.5.0" + } + }, + "cross-spawn": { + "version": "7.0.3", + "bundled": true, + "requires": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "fill-range": { + "version": "7.0.1", + "bundled": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "fsevents": { + "version": "2.3.2", + "bundled": true, + "optional": true + }, + "glob-parent": { + "version": "5.1.2", + "bundled": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "ignore": { + "version": "5.1.8", + "bundled": true + }, + "immer": { + "version": "9.0.2", + "bundled": true + }, + "is-binary-path": { + "version": "2.1.0", + "bundled": true, + "requires": { + "binary-extensions": "^2.0.0" + } + }, + "is-extglob": { + "version": "2.1.1", + "bundled": true + }, + "is-glob": { + "version": "4.0.1", + "bundled": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-number": { + "version": "7.0.0", + "bundled": true + }, + "isexe": { + "version": "2.0.0", + "bundled": true }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + "version": "4.17.21", + "bundled": true + }, + "normalize-path": { + "version": "3.0.0", + "bundled": true + }, + "onchange": { + "version": "7.1.0", + "bundled": true, + "requires": { + "@blakeembrey/deque": "^1.0.5", + "@blakeembrey/template": "^1.0.0", + "arg": "^4.1.3", + "chokidar": "^3.3.1", + "cross-spawn": "^7.0.1", + "ignore": "^5.1.4", + "tree-kill": "^1.2.2" + } + }, + "path-key": { + "version": "3.1.1", + "bundled": true + }, + "picomatch": { + "version": "2.2.3", + "bundled": true + }, + "readdirp": { + "version": "3.5.0", + "bundled": true, + "requires": { + "picomatch": "^2.2.1" + } }, "regenerator-runtime": { "version": "0.13.7", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.7.tgz", - "integrity": "sha512-a54FxoJDIr27pgf7IgeQGxmqUNYrcV338lf/6gH456HZ/PhX+5BcwHXG9ajESmwe6WRO0tAzRUrRmNONWgkrew==" + "bundled": true + }, + "shebang-command": { + "version": "2.0.0", + "bundled": true, + "requires": { + "shebang-regex": "^3.0.0" + } + }, + "shebang-regex": { + "version": "3.0.0", + "bundled": true + }, + "to-regex-range": { + "version": "5.0.1", + "bundled": true, + "requires": { + "is-number": "^7.0.0" + } + }, + "tree-kill": { + "version": "1.2.2", + "bundled": true + }, + "which": { + "version": "2.0.2", + "bundled": true, + "requires": { + "isexe": "^2.0.0" + } } } }, "@urbit/eslint-config": { "version": "file:../npm/eslint-config", - "dev": true + "dev": true, + "dependencies": { + "@babel/code-frame": { + "version": "7.12.13", + "bundled": true, + "requires": { + "@babel/highlight": "^7.12.13" + } + }, + "@babel/generator": { + "version": "7.14.1", + "bundled": true, + "requires": { + "@babel/types": "^7.14.1", + "jsesc": "^2.5.1", + "source-map": "^0.5.0" + } + }, + "@babel/helper-function-name": { + "version": "7.12.13", + "bundled": true, + "requires": { + "@babel/helper-get-function-arity": "^7.12.13", + "@babel/template": "^7.12.13", + "@babel/types": "^7.12.13" + } + }, + "@babel/helper-get-function-arity": { + "version": "7.12.13", + "bundled": true, + "requires": { + "@babel/types": "^7.12.13" + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.12.13", + "bundled": true, + "requires": { + "@babel/types": "^7.12.13" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.14.0", + "bundled": true + }, + "@babel/highlight": { + "version": "7.14.0", + "bundled": true, + "requires": { + "@babel/helper-validator-identifier": "^7.14.0", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + } + }, + "@babel/parser": { + "version": "7.14.1", + "bundled": true + }, + "@babel/template": { + "version": "7.12.13", + "bundled": true, + "requires": { + "@babel/code-frame": "^7.12.13", + "@babel/parser": "^7.12.13", + "@babel/types": "^7.12.13" + } + }, + "@babel/traverse": { + "version": "7.14.0", + "bundled": true, + "requires": { + "@babel/code-frame": "^7.12.13", + "@babel/generator": "^7.14.0", + "@babel/helper-function-name": "^7.12.13", + "@babel/helper-split-export-declaration": "^7.12.13", + "@babel/parser": "^7.14.0", + "@babel/types": "^7.14.0", + "debug": "^4.1.0", + "globals": "^11.1.0" + } + }, + "@babel/types": { + "version": "7.14.1", + "bundled": true, + "requires": { + "@babel/helper-validator-identifier": "^7.14.0", + "to-fast-properties": "^2.0.0" + } + }, + "@nodelib/fs.scandir": { + "version": "2.1.4", + "bundled": true, + "requires": { + "@nodelib/fs.stat": "2.0.4", + "run-parallel": "^1.1.9" + } + }, + "@nodelib/fs.stat": { + "version": "2.0.4", + "bundled": true + }, + "@nodelib/fs.walk": { + "version": "1.2.6", + "bundled": true, + "requires": { + "@nodelib/fs.scandir": "2.1.4", + "fastq": "^1.6.0" + } + }, + "@types/json-schema": { + "version": "7.0.7", + "bundled": true + }, + "@typescript-eslint/eslint-plugin": { + "version": "4.22.1", + "bundled": true, + "requires": { + "@typescript-eslint/experimental-utils": "4.22.1", + "@typescript-eslint/scope-manager": "4.22.1", + "debug": "^4.1.1", + "functional-red-black-tree": "^1.0.1", + "lodash": "^4.17.15", + "regexpp": "^3.0.0", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + } + }, + "@typescript-eslint/experimental-utils": { + "version": "4.22.1", + "bundled": true, + "requires": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/scope-manager": "4.22.1", + "@typescript-eslint/types": "4.22.1", + "@typescript-eslint/typescript-estree": "4.22.1", + "eslint-scope": "^5.0.0", + "eslint-utils": "^2.0.0" + } + }, + "@typescript-eslint/parser": { + "version": "4.22.1", + "bundled": true, + "requires": { + "@typescript-eslint/scope-manager": "4.22.1", + "@typescript-eslint/types": "4.22.1", + "@typescript-eslint/typescript-estree": "4.22.1", + "debug": "^4.1.1" + } + }, + "@typescript-eslint/scope-manager": { + "version": "4.22.1", + "bundled": true, + "requires": { + "@typescript-eslint/types": "4.22.1", + "@typescript-eslint/visitor-keys": "4.22.1" + } + }, + "@typescript-eslint/types": { + "version": "4.22.1", + "bundled": true + }, + "@typescript-eslint/typescript-estree": { + "version": "4.22.1", + "bundled": true, + "requires": { + "@typescript-eslint/types": "4.22.1", + "@typescript-eslint/visitor-keys": "4.22.1", + "debug": "^4.1.1", + "globby": "^11.0.1", + "is-glob": "^4.0.1", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + } + }, + "@typescript-eslint/visitor-keys": { + "version": "4.22.1", + "bundled": true, + "requires": { + "@typescript-eslint/types": "4.22.1", + "eslint-visitor-keys": "^2.0.0" + } + }, + "ansi-styles": { + "version": "3.2.1", + "bundled": true, + "requires": { + "color-convert": "^1.9.0" + } + }, + "array-includes": { + "version": "3.1.3", + "bundled": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.2", + "get-intrinsic": "^1.1.1", + "is-string": "^1.0.5" + } + }, + "array-union": { + "version": "2.1.0", + "bundled": true + }, + "array.prototype.flatmap": { + "version": "1.2.4", + "bundled": true, + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.1", + "function-bind": "^1.1.1" + } + }, + "babel-eslint": { + "version": "10.1.0", + "bundled": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@babel/parser": "^7.7.0", + "@babel/traverse": "^7.7.0", + "@babel/types": "^7.7.0", + "eslint-visitor-keys": "^1.0.0", + "resolve": "^1.12.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "bundled": true + } + } + }, + "balanced-match": { + "version": "1.0.2", + "bundled": true + }, + "brace-expansion": { + "version": "1.1.11", + "bundled": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "braces": { + "version": "3.0.2", + "bundled": true, + "requires": { + "fill-range": "^7.0.1" + } + }, + "call-bind": { + "version": "1.0.2", + "bundled": true, + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, + "chalk": { + "version": "2.4.2", + "bundled": true, + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "bundled": true, + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "bundled": true + }, + "concat-map": { + "version": "0.0.1", + "bundled": true + }, + "debug": { + "version": "4.3.1", + "bundled": true, + "requires": { + "ms": "2.1.2" + } + }, + "define-properties": { + "version": "1.1.3", + "bundled": true, + "requires": { + "object-keys": "^1.0.12" + } + }, + "dir-glob": { + "version": "3.0.1", + "bundled": true, + "requires": { + "path-type": "^4.0.0" + } + }, + "doctrine": { + "version": "2.1.0", + "bundled": true, + "requires": { + "esutils": "^2.0.2" + } + }, + "es-abstract": { + "version": "1.18.0", + "bundled": true, + "requires": { + "call-bind": "^1.0.2", + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "get-intrinsic": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.2", + "is-callable": "^1.2.3", + "is-negative-zero": "^2.0.1", + "is-regex": "^1.1.2", + "is-string": "^1.0.5", + "object-inspect": "^1.9.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.2", + "string.prototype.trimend": "^1.0.4", + "string.prototype.trimstart": "^1.0.4", + "unbox-primitive": "^1.0.0" + } + }, + "es-to-primitive": { + "version": "1.2.1", + "bundled": true, + "requires": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "bundled": true + }, + "eslint-plugin-react": { + "version": "7.23.2", + "bundled": true, + "requires": { + "array-includes": "^3.1.3", + "array.prototype.flatmap": "^1.2.4", + "doctrine": "^2.1.0", + "has": "^1.0.3", + "jsx-ast-utils": "^2.4.1 || ^3.0.0", + "minimatch": "^3.0.4", + "object.entries": "^1.1.3", + "object.fromentries": "^2.0.4", + "object.values": "^1.1.3", + "prop-types": "^15.7.2", + "resolve": "^2.0.0-next.3", + "string.prototype.matchall": "^4.0.4" + }, + "dependencies": { + "resolve": { + "version": "2.0.0-next.3", + "bundled": true, + "requires": { + "is-core-module": "^2.2.0", + "path-parse": "^1.0.6" + } + } + } + }, + "eslint-scope": { + "version": "5.1.1", + "bundled": true, + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + } + }, + "eslint-utils": { + "version": "2.1.0", + "bundled": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "bundled": true + } + } + }, + "eslint-visitor-keys": { + "version": "2.1.0", + "bundled": true + }, + "esrecurse": { + "version": "4.3.0", + "bundled": true, + "requires": { + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "bundled": true + } + } + }, + "estraverse": { + "version": "4.3.0", + "bundled": true + }, + "esutils": { + "version": "2.0.3", + "bundled": true + }, + "fast-glob": { + "version": "3.2.5", + "bundled": true, + "requires": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.0", + "merge2": "^1.3.0", + "micromatch": "^4.0.2", + "picomatch": "^2.2.1" + } + }, + "fastq": { + "version": "1.11.0", + "bundled": true, + "requires": { + "reusify": "^1.0.4" + } + }, + "fill-range": { + "version": "7.0.1", + "bundled": true, + "requires": { + "to-regex-range": "^5.0.1" + } + }, + "function-bind": { + "version": "1.1.1", + "bundled": true + }, + "functional-red-black-tree": { + "version": "1.0.1", + "bundled": true + }, + "get-intrinsic": { + "version": "1.1.1", + "bundled": true, + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + } + }, + "glob-parent": { + "version": "5.1.2", + "bundled": true, + "requires": { + "is-glob": "^4.0.1" + } + }, + "globals": { + "version": "11.12.0", + "bundled": true + }, + "globby": { + "version": "11.0.3", + "bundled": true, + "requires": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.1.1", + "ignore": "^5.1.4", + "merge2": "^1.3.0", + "slash": "^3.0.0" + } + }, + "has": { + "version": "1.0.3", + "bundled": true, + "requires": { + "function-bind": "^1.1.1" + } + }, + "has-bigints": { + "version": "1.0.1", + "bundled": true + }, + "has-flag": { + "version": "3.0.0", + "bundled": true + }, + "has-symbols": { + "version": "1.0.2", + "bundled": true + }, + "ignore": { + "version": "5.1.8", + "bundled": true + }, + "internal-slot": { + "version": "1.0.3", + "bundled": true, + "requires": { + "get-intrinsic": "^1.1.0", + "has": "^1.0.3", + "side-channel": "^1.0.4" + } + }, + "is-bigint": { + "version": "1.0.2", + "bundled": true + }, + "is-boolean-object": { + "version": "1.1.0", + "bundled": true, + "requires": { + "call-bind": "^1.0.0" + } + }, + "is-callable": { + "version": "1.2.3", + "bundled": true + }, + "is-core-module": { + "version": "2.3.0", + "bundled": true, + "requires": { + "has": "^1.0.3" + } + }, + "is-date-object": { + "version": "1.0.3", + "bundled": true + }, + "is-extglob": { + "version": "2.1.1", + "bundled": true + }, + "is-glob": { + "version": "4.0.1", + "bundled": true, + "requires": { + "is-extglob": "^2.1.1" + } + }, + "is-negative-zero": { + "version": "2.0.1", + "bundled": true + }, + "is-number": { + "version": "7.0.0", + "bundled": true + }, + "is-number-object": { + "version": "1.0.4", + "bundled": true + }, + "is-regex": { + "version": "1.1.2", + "bundled": true, + "requires": { + "call-bind": "^1.0.2", + "has-symbols": "^1.0.1" + } + }, + "is-string": { + "version": "1.0.5", + "bundled": true + }, + "is-symbol": { + "version": "1.0.3", + "bundled": true, + "requires": { + "has-symbols": "^1.0.1" + } + }, + "js-tokens": { + "version": "4.0.0", + "bundled": true + }, + "jsesc": { + "version": "2.5.2", + "bundled": true + }, + "jsx-ast-utils": { + "version": "3.2.0", + "bundled": true, + "requires": { + "array-includes": "^3.1.2", + "object.assign": "^4.1.2" + } + }, + "lodash": { + "version": "4.17.21", + "bundled": true + }, + "loose-envify": { + "version": "1.4.0", + "bundled": true, + "requires": { + "js-tokens": "^3.0.0 || ^4.0.0" + } + }, + "lru-cache": { + "version": "6.0.0", + "bundled": true, + "requires": { + "yallist": "^4.0.0" + } + }, + "merge2": { + "version": "1.4.1", + "bundled": true + }, + "micromatch": { + "version": "4.0.4", + "bundled": true, + "requires": { + "braces": "^3.0.1", + "picomatch": "^2.2.3" + } + }, + "minimatch": { + "version": "3.0.4", + "bundled": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "ms": { + "version": "2.1.2", + "bundled": true + }, + "object-assign": { + "version": "4.1.1", + "bundled": true + }, + "object-inspect": { + "version": "1.10.2", + "bundled": true + }, + "object-keys": { + "version": "1.1.1", + "bundled": true + }, + "object.assign": { + "version": "4.1.2", + "bundled": true, + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "has-symbols": "^1.0.1", + "object-keys": "^1.1.1" + } + }, + "object.entries": { + "version": "1.1.3", + "bundled": true, + "requires": { + "call-bind": "^1.0.0", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.1", + "has": "^1.0.3" + } + }, + "object.fromentries": { + "version": "2.0.4", + "bundled": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.2", + "has": "^1.0.3" + } + }, + "object.values": { + "version": "1.1.3", + "bundled": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.2", + "has": "^1.0.3" + } + }, + "path-parse": { + "version": "1.0.6", + "bundled": true + }, + "path-type": { + "version": "4.0.0", + "bundled": true + }, + "picomatch": { + "version": "2.2.3", + "bundled": true + }, + "prop-types": { + "version": "15.7.2", + "bundled": true, + "requires": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.8.1" + } + }, + "queue-microtask": { + "version": "1.2.3", + "bundled": true + }, + "react-is": { + "version": "16.13.1", + "bundled": true + }, + "regexp.prototype.flags": { + "version": "1.3.1", + "bundled": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "regexpp": { + "version": "3.1.0", + "bundled": true + }, + "resolve": { + "version": "1.20.0", + "bundled": true, + "requires": { + "is-core-module": "^2.2.0", + "path-parse": "^1.0.6" + } + }, + "reusify": { + "version": "1.0.4", + "bundled": true + }, + "run-parallel": { + "version": "1.2.0", + "bundled": true, + "requires": { + "queue-microtask": "^1.2.2" + } + }, + "semver": { + "version": "7.3.5", + "bundled": true, + "requires": { + "lru-cache": "^6.0.0" + } + }, + "side-channel": { + "version": "1.0.4", + "bundled": true, + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, + "slash": { + "version": "3.0.0", + "bundled": true + }, + "source-map": { + "version": "0.5.7", + "bundled": true + }, + "string.prototype.matchall": { + "version": "4.0.4", + "bundled": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3", + "es-abstract": "^1.18.0-next.2", + "has-symbols": "^1.0.1", + "internal-slot": "^1.0.3", + "regexp.prototype.flags": "^1.3.1", + "side-channel": "^1.0.4" + } + }, + "string.prototype.trimend": { + "version": "1.0.4", + "bundled": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "string.prototype.trimstart": { + "version": "1.0.4", + "bundled": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.1.3" + } + }, + "supports-color": { + "version": "5.5.0", + "bundled": true, + "requires": { + "has-flag": "^3.0.0" + } + }, + "to-fast-properties": { + "version": "2.0.0", + "bundled": true + }, + "to-regex-range": { + "version": "5.0.1", + "bundled": true, + "requires": { + "is-number": "^7.0.0" + } + }, + "tslib": { + "version": "1.14.1", + "bundled": true + }, + "tsutils": { + "version": "3.21.0", + "bundled": true, + "requires": { + "tslib": "^1.8.1" + } + }, + "typescript": { + "version": "4.2.4", + "bundled": true + }, + "unbox-primitive": { + "version": "1.0.1", + "bundled": true, + "requires": { + "function-bind": "^1.1.1", + "has-bigints": "^1.0.1", + "has-symbols": "^1.0.2", + "which-boxed-primitive": "^1.0.2" + } + }, + "which-boxed-primitive": { + "version": "1.0.2", + "bundled": true, + "requires": { + "is-bigint": "^1.0.1", + "is-boolean-object": "^1.1.0", + "is-number-object": "^1.0.4", + "is-string": "^1.0.5", + "is-symbol": "^1.0.3" + } + }, + "yallist": { + "version": "4.0.0", + "bundled": true + } + } }, "@webassemblyjs/ast": { "version": "1.9.0", @@ -1995,6 +3073,15 @@ "@xtuc/long": "4.2.2" } }, + "@welldone-software/why-did-you-render": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/@welldone-software/why-did-you-render/-/why-did-you-render-6.1.0.tgz", + "integrity": "sha512-0s+PuKQ4v9VV1SZSM6iS7d2T7X288T3DF+K8yfkFAhI31HhJGGH1SY1ssVm+LqjSMyrVWT60ZF5r0qUsO0Z9Lw==", + "dev": true, + "requires": { + "lodash": "^4" + } + }, "@xtuc/ieee754": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", @@ -2077,6 +3164,11 @@ "color-convert": "^1.9.0" } }, + "any-ascii": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/any-ascii/-/any-ascii-0.1.7.tgz", + "integrity": "sha512-9zc8XIPeG9lDGtjiQGQtRF2+ow97/eTtZJR7K4UvciSC5GSOySYoytXeA2fSaY8pLhpRMcAsiZDEEkuU20HD8g==" + }, "anymatch": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", @@ -2891,6 +3983,11 @@ "integrity": "sha512-blMmO0QQujuUWZKyVrD1msR4WNDAqb/UPO1Sw2WWsQ7deoM5bJiicKnWJ1Y0NS/aGINSnKPIWBMw5luX+NDUCA==", "dev": true }, + "ccount": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-1.1.0.tgz", + "integrity": "sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg==" + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", @@ -2916,6 +4013,11 @@ "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-1.2.4.tgz", "integrity": "sha512-iBMyeEHxfVnIakwOuDXpVkc54HijNgCyQB2w0VfGQThle6NXn50zU6V/u+LDhxHcDUPojn6Kpga3PTAD8W1bQw==" }, + "character-entities-html4": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-1.1.4.tgz", + "integrity": "sha512-HRcDxZuZqMx3/a+qrzxdBKBPUpxWEq9xw2OPZ3a/174ihfrQKVsFhqtthBInFy1zZ9GgZyFXOatNujm8M+El3g==" + }, "character-entities-legacy": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-1.1.4.tgz", @@ -3862,18 +4964,18 @@ "dev": true }, "elliptic": { - "version": "6.5.3", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz", - "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==", + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", + "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", "dev": true, "requires": { - "bn.js": "^4.4.0", - "brorand": "^1.0.1", + "bn.js": "^4.11.9", + "brorand": "^1.1.0", "hash.js": "^1.0.0", - "hmac-drbg": "^1.0.0", - "inherits": "^2.0.1", - "minimalistic-assert": "^1.0.0", - "minimalistic-crypto-utils": "^1.0.0" + "hmac-drbg": "^1.0.1", + "inherits": "^2.0.4", + "minimalistic-assert": "^1.0.1", + "minimalistic-crypto-utils": "^1.0.1" } }, "emoji-regex": { @@ -3980,6 +5082,14 @@ "prr": "~1.0.1" } }, + "error-stack-parser": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/error-stack-parser/-/error-stack-parser-2.0.6.tgz", + "integrity": "sha512-d51brTeqC+BHlwF0BhPtcYgF5nlzf9ZZ0ZIUQNZpc9ZB9qw5IJ2diTrBY9jlCJkTLITYPjmiX6OWCwH+fuyNgQ==", + "requires": { + "stackframe": "^1.1.1" + } + }, "es-abstract": { "version": "1.18.0-next.2", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.0-next.2.tgz", @@ -5726,6 +6836,11 @@ "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-1.0.4.tgz", "integrity": "sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==" }, + "is-alphanumeric": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-alphanumeric/-/is-alphanumeric-1.0.0.tgz", + "integrity": "sha1-Spzvcdr0wAHB2B1j0UDPU/1oifQ=" + }, "is-alphanumerical": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-1.0.4.tgz", @@ -6077,9 +7192,9 @@ } }, "lodash": { - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash-es": { "version": "4.17.20", @@ -6113,6 +7228,11 @@ "integrity": "sha512-Hesni4s5UkWkwCGJMQGAh71PaLUmKFM60dHvq0zi/vDhhrzuk+4GgNbTXJ12YYQJn6ZKBDNIjYcuQGKudvqrIw==", "dev": true }, + "longest-streak": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-2.0.4.tgz", + "integrity": "sha512-vM6rUVCVUJJt33bnmHiZEvr7wPT78ztX7rojL+LW51bHtLh6HTjx84LA5W4+oa6aKEJA7jJu5LR6vQRBpA5DVg==" + }, "loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -6168,13 +7288,12 @@ "resolved": "https://registry.npmjs.org/markdown-escapes/-/markdown-escapes-1.0.4.tgz", "integrity": "sha512-8z4efJYk43E0upd0NbVXwgSTQs6cT3T06etieCMEg7dRbzCbxUCK/GHlX8mhHRDcp+OLlHkPKsvqQTCvsRl2cg==" }, - "markdown-to-jsx": { - "version": "6.11.4", - "resolved": "https://registry.npmjs.org/markdown-to-jsx/-/markdown-to-jsx-6.11.4.tgz", - "integrity": "sha512-3lRCD5Sh+tfA52iGgfs/XZiw33f7fFX9Bn55aNnVNUd2GzLDkOWyKYYD8Yju2B1Vn+feiEdgJs8T6Tg0xNokPw==", + "markdown-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz", + "integrity": "sha512-Ezda85ToJUBhM6WGaG6veasyym+Tbs3cMAw/ZhOPqXiYsr0jgocBV3j3nx+4lk47plLlIqjwuTm/ywVI+zjJ/A==", "requires": { - "prop-types": "^15.6.2", - "unquote": "^1.1.0" + "repeat-string": "^1.0.0" } }, "md5.js": { @@ -6196,6 +7315,40 @@ "unist-util-visit-parents": "1.1.2" } }, + "mdast-util-compact": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-compact/-/mdast-util-compact-2.0.1.tgz", + "integrity": "sha512-7GlnT24gEwDrdAwEHrU4Vv5lLWrEer4KOkAiKT9nYstsTad7Oc1TwqT2zIMKRdZF7cTuaf+GA1E4Kv7jJh8mPA==", + "requires": { + "unist-util-visit": "^2.0.0" + }, + "dependencies": { + "unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==" + }, + "unist-util-visit": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz", + "integrity": "sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0", + "unist-util-visit-parents": "^3.0.0" + } + }, + "unist-util-visit-parents": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz", + "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0" + } + } + } + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -7480,6 +8633,11 @@ "integrity": "sha512-dB15eXv3p2jDlbOiNLyMabYg1/sXvppd8DP2J3EOCQ0AkuSXCW2tP7mnVouVLJKgUMY6yP0kcQDVpLCN13h4Xg==", "dev": true }, + "rafz": { + "version": "0.1.14", + "resolved": "https://registry.npmjs.org/rafz/-/rafz-0.1.14.tgz", + "integrity": "sha512-YiQkedSt1urYtYbvHhTQR3l67M8SZbUvga5eJFM/v4vx/GmDdtXlE2hjJIyRjhhO/PjcdGC+CXCYOUA4onit8w==" + }, "ramda": { "version": "0.27.1", "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.27.1.tgz", @@ -7627,6 +8785,24 @@ "unified": "^6.1.5", "unist-util-visit": "^1.3.0", "xtend": "^4.0.1" + }, + "dependencies": { + "unist-util-visit": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz", + "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==", + "requires": { + "unist-util-visit-parents": "^2.0.0" + } + }, + "unist-util-visit-parents": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-2.1.2.tgz", + "integrity": "sha512-DyN5vD4NE3aSeB+PXYNKxzGsfocxp6asDc2XXE3b0ekO2BaRUpBicbbUygfSvYfUz1IkmjFR1YF7dPklraMZ2g==", + "requires": { + "unist-util-is": "^3.0.0" + } + } } }, "react-oembed-container": { @@ -7673,6 +8849,11 @@ "resolved": "https://registry.npmjs.org/react-side-effect/-/react-side-effect-2.1.1.tgz", "integrity": "sha512-2FoTQzRNTncBVtnzxFOk2mCpcfxQpenBMbk5kSVBg5UcPqV9fRbgY2zhb7GTWWOlpFmAxhClBDlIq8Rsubz1yQ==" }, + "react-use-gesture": { + "version": "9.1.3", + "resolved": "https://registry.npmjs.org/react-use-gesture/-/react-use-gesture-9.1.3.tgz", + "integrity": "sha512-CdqA2SmS/fj3kkS2W8ZU8wjTbVBAIwDWaRprX7OKaj7HlGwBasGEFggmk5qNklknqk9zK/h8D355bEJFTpqEMg==" + }, "react-virtuoso": { "version": "0.20.3", "resolved": "https://registry.npmjs.org/react-virtuoso/-/react-virtuoso-0.20.3.tgz", @@ -7815,15 +8996,180 @@ "integrity": "sha1-VNvzd+UUQKypCkzSdGANP/LYiKk=", "dev": true }, + "remark": { + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/remark/-/remark-12.0.1.tgz", + "integrity": "sha512-gS7HDonkdIaHmmP/+shCPejCEEW+liMp/t/QwmF0Xt47Rpuhl32lLtDV1uKWvGoq+kxr5jSgg5oAIpGuyULjUw==", + "requires": { + "remark-parse": "^8.0.0", + "remark-stringify": "^8.0.0", + "unified": "^9.0.0" + }, + "dependencies": { + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, + "is-plain-obj": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", + "integrity": "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA==" + }, + "parse-entities": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz", + "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==", + "requires": { + "character-entities": "^1.0.0", + "character-entities-legacy": "^1.0.0", + "character-reference-invalid": "^1.0.0", + "is-alphanumerical": "^1.0.0", + "is-decimal": "^1.0.0", + "is-hexadecimal": "^1.0.0" + } + }, + "remark-parse": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-8.0.3.tgz", + "integrity": "sha512-E1K9+QLGgggHxCQtLt++uXltxEprmWzNfg+MxpfHsZlrddKzZ/hZyWHDbK3/Ap8HJQqYJRXP+jHczdL6q6i85Q==", + "requires": { + "ccount": "^1.0.0", + "collapse-white-space": "^1.0.2", + "is-alphabetical": "^1.0.0", + "is-decimal": "^1.0.0", + "is-whitespace-character": "^1.0.0", + "is-word-character": "^1.0.0", + "markdown-escapes": "^1.0.0", + "parse-entities": "^2.0.0", + "repeat-string": "^1.5.4", + "state-toggle": "^1.0.0", + "trim": "0.0.1", + "trim-trailing-lines": "^1.0.0", + "unherit": "^1.0.4", + "unist-util-remove-position": "^2.0.0", + "vfile-location": "^3.0.0", + "xtend": "^4.0.1" + } + }, + "unified": { + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/unified/-/unified-9.2.1.tgz", + "integrity": "sha512-juWjuI8Z4xFg8pJbnEZ41b5xjGUWGHqXALmBZ3FC3WX0PIx1CZBIIJ6mXbYMcf6Yw4Fi0rFUTA1cdz/BglbOhA==", + "requires": { + "bail": "^1.0.0", + "extend": "^3.0.0", + "is-buffer": "^2.0.0", + "is-plain-obj": "^2.0.0", + "trough": "^1.0.0", + "vfile": "^4.0.0" + } + }, + "unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==" + }, + "unist-util-remove-position": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-2.0.1.tgz", + "integrity": "sha512-fDZsLYIe2uT+oGFnuZmy73K6ZxOPG/Qcm+w7jbEjaFcJgbQ6cqjs/eSPzXhsmGpAsWPkqZM9pYjww5QTn3LHMA==", + "requires": { + "unist-util-visit": "^2.0.0" + } + }, + "unist-util-stringify-position": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-2.0.3.tgz", + "integrity": "sha512-3faScn5I+hy9VleOq/qNbAd6pAx7iH5jYBMS9I1HgQVijz/4mv5Bvw5iw1sC/90CODiKo81G/ps8AJrISn687g==", + "requires": { + "@types/unist": "^2.0.2" + } + }, + "unist-util-visit": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz", + "integrity": "sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0", + "unist-util-visit-parents": "^3.0.0" + } + }, + "unist-util-visit-parents": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz", + "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0" + } + }, + "vfile": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-4.2.1.tgz", + "integrity": "sha512-O6AE4OskCG5S1emQ/4gl8zK586RqA3srz3nfK/Viy0UPToBc5Trp9BVFb1u0CjsKrAWwnpr4ifM/KBXPWwJbCA==", + "requires": { + "@types/unist": "^2.0.0", + "is-buffer": "^2.0.0", + "unist-util-stringify-position": "^2.0.0", + "vfile-message": "^2.0.0" + } + }, + "vfile-location": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-3.2.0.tgz", + "integrity": "sha512-aLEIZKv/oxuCDZ8lkJGhuhztf/BW4M+iHdCwglA/eWc+vtuRFJj8EtgceYFX4LRjOhCAAiNHsKGssC6onJ+jbA==" + }, + "vfile-message": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-2.0.4.tgz", + "integrity": "sha512-DjssxRGkMvifUOJre00juHoP9DPWuzjxKuMDrhNbk2TdaYYBNMStsNhEOt3idrtI12VQYM/1+iM0KOzXi4pxwQ==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-stringify-position": "^2.0.0" + } + } + } + }, "remark-breaks": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/remark-breaks/-/remark-breaks-2.0.1.tgz", - "integrity": "sha512-CZKI8xdPUnvMqPxYEIBBUg8C0B0kyn14lkW0abzhfh/P71YRIxCC3wvBh6AejQL602OxF6kNRl1x4HAZA07JyQ==" + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/remark-breaks/-/remark-breaks-2.0.2.tgz", + "integrity": "sha512-LsQnPPQ7Fzp9RTjj4IwdEmjPOr9bxe9zYKWhs9ZQOg9hMg8rOfeeqQ410cvVdIK87Famqza1CKRxNkepp2EvUA==", + "requires": { + "unist-util-visit": "^2.0.0" + }, + "dependencies": { + "unist-util-is": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-4.1.0.tgz", + "integrity": "sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==" + }, + "unist-util-visit": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-2.0.3.tgz", + "integrity": "sha512-iJ4/RczbJMkD0712mGktuGpm/U4By4FfDonL7N/9tATGIF4imikjOuagyMY53tnZq3NP6BcmlrHhEKAfGWjh7Q==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0", + "unist-util-visit-parents": "^3.0.0" + } + }, + "unist-util-visit-parents": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-3.1.1.tgz", + "integrity": "sha512-1KROIZWo6bcMrZEwiH2UrXDyalAa0uqzWCxCJj6lPOvTve2WkfgCytoDTPaMnodXh1WrXOq0haVYHj99ynJlsg==", + "requires": { + "@types/unist": "^2.0.0", + "unist-util-is": "^4.0.0" + } + } + } }, "remark-disable-tokenizers": { - "version": "1.0.24", - "resolved": "https://registry.npmjs.org/remark-disable-tokenizers/-/remark-disable-tokenizers-1.0.24.tgz", - "integrity": "sha512-HsAmBY5cNliHYAzba4zuskZzkDdp6sG+tRelDb4AoPo2YHNGHnxYsatShzTIsnRNLgCbsxycW5Ge6KigHn701A==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/remark-disable-tokenizers/-/remark-disable-tokenizers-1.1.0.tgz", + "integrity": "sha512-49cCg4uSVKVmDHWKT5w+2mpDQ3G+xvt/GjypOqjlS0qTSs6/aBxE3iNWgX6ls2upXY17EKVlU0UpGcZjmGWI6A==", "requires": { "clone": "^2.1.2" } @@ -7850,6 +9196,42 @@ "xtend": "^4.0.1" } }, + "remark-stringify": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/remark-stringify/-/remark-stringify-8.1.1.tgz", + "integrity": "sha512-q4EyPZT3PcA3Eq7vPpT6bIdokXzFGp9i85igjmhRyXWmPs0Y6/d2FYwUNotKAWyLch7g0ASZJn/KHHcHZQ163A==", + "requires": { + "ccount": "^1.0.0", + "is-alphanumeric": "^1.0.0", + "is-decimal": "^1.0.0", + "is-whitespace-character": "^1.0.0", + "longest-streak": "^2.0.1", + "markdown-escapes": "^1.0.0", + "markdown-table": "^2.0.0", + "mdast-util-compact": "^2.0.0", + "parse-entities": "^2.0.0", + "repeat-string": "^1.5.4", + "state-toggle": "^1.0.0", + "stringify-entities": "^3.0.0", + "unherit": "^1.0.4", + "xtend": "^4.0.1" + }, + "dependencies": { + "parse-entities": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-2.0.0.tgz", + "integrity": "sha512-kkywGpCcRYhqQIchaWqZ875wzpS/bMKhz5HnN3p7wveJTkTtyAB/AlnS0f8DFSqYW1T82t6yEAkEcB+A1I3MbQ==", + "requires": { + "character-entities": "^1.0.0", + "character-entities-legacy": "^1.0.0", + "character-reference-invalid": "^1.0.0", + "is-alphanumerical": "^1.0.0", + "is-decimal": "^1.0.0", + "is-hexadecimal": "^1.0.0" + } + } + } + }, "remove-trailing-separator": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", @@ -8741,14 +10123,53 @@ "dev": true }, "ssri": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.1.tgz", - "integrity": "sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==", + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.2.tgz", + "integrity": "sha512-cepbSq/neFK7xB6A50KHN0xHDotYzq58wWCa5LeWqnPrHG8GzfEjO/4O8kpmcGW+oaxkvhEJCWgbgNk4/ZV93Q==", "dev": true, "requires": { "figgy-pudding": "^3.5.1" } }, + "stack-generator": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/stack-generator/-/stack-generator-2.0.5.tgz", + "integrity": "sha512-/t1ebrbHkrLrDuNMdeAcsvynWgoH/i4o8EGGfX7dEYDoTXOYVAkEpFdtshlvabzc6JlJ8Kf9YdFEoz7JkzGN9Q==", + "requires": { + "stackframe": "^1.1.1" + } + }, + "stackframe": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.2.0.tgz", + "integrity": "sha512-GrdeshiRmS1YLMYgzF16olf2jJ/IzxXY9lhKOskuVziubpTYcYqyOwYeJKzQkwy7uN0fYSsbsC4RQaXf9LCrYA==" + }, + "stacktrace-gps": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/stacktrace-gps/-/stacktrace-gps-3.0.4.tgz", + "integrity": "sha512-qIr8x41yZVSldqdqe6jciXEaSCKw1U8XTXpjDuy0ki/apyTn/r3w9hDAAQOhZdxvsC93H+WwwEu5cq5VemzYeg==", + "requires": { + "source-map": "0.5.6", + "stackframe": "^1.1.1" + }, + "dependencies": { + "source-map": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.6.tgz", + "integrity": "sha1-dc449SvwczxafwwRjYEzSiu19BI=" + } + } + }, + "stacktrace-js": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/stacktrace-js/-/stacktrace-js-2.0.2.tgz", + "integrity": "sha512-Je5vBeY4S1r/RnLydLl0TBTi3F2qdfWmYsGvtfZgEI+SCprPppaIhQf5nGcal4gI4cGpCV/duLcAzT1np6sQqg==", + "requires": { + "error-stack-parser": "^2.0.6", + "stack-generator": "^2.0.5", + "stacktrace-gps": "^3.0.4" + } + }, "state-toggle": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/state-toggle/-/state-toggle-1.0.3.tgz", @@ -8946,6 +10367,16 @@ } } }, + "stringify-entities": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-3.1.0.tgz", + "integrity": "sha512-3FP+jGMmMV/ffZs86MoghGqAoqXAdxLrJP4GUdrDN1aIScYih5tuIO3eF4To5AJZ79KDZ8Fpdy7QJnK8SsL1Vg==", + "requires": { + "character-entities-html4": "^1.0.0", + "character-entities-legacy": "^1.0.0", + "xtend": "^4.0.0" + } + }, "strip-ansi": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", @@ -9440,9 +10871,9 @@ "dev": true }, "typescript": { - "version": "3.9.7", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", - "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.4.tgz", + "integrity": "sha512-V+evlYHZnQkaz8TRBuxTA92yZBPotr5H+WhQ7bD3hZUndx5tGOa1fuCgeSjxAzM1RiN5IzvadIXTVefuuwZCRg==", "dev": true }, "unherit": { @@ -9541,6 +10972,24 @@ "integrity": "sha512-tLqd653ArxJIPnKII6LMZwH+mb5q+n/GtXQZo6S6csPRs5zB0u79Yw8ouR3wTw8wxvdJFhpP6Y7jorWdCgLO0A==", "requires": { "unist-util-visit": "^1.1.0" + }, + "dependencies": { + "unist-util-visit": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz", + "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==", + "requires": { + "unist-util-visit-parents": "^2.0.0" + } + }, + "unist-util-visit-parents": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-2.1.2.tgz", + "integrity": "sha512-DyN5vD4NE3aSeB+PXYNKxzGsfocxp6asDc2XXE3b0ekO2BaRUpBicbbUygfSvYfUz1IkmjFR1YF7dPklraMZ2g==", + "requires": { + "unist-util-is": "^3.0.0" + } + } } }, "unist-util-stringify-position": { @@ -9549,19 +10998,27 @@ "integrity": "sha512-pNCVrk64LZv1kElr0N1wPiHEUoXNVFERp+mlTg/s9R5Lwg87f9bM/3sQB99w+N9D/qnM9ar3+AKDBwo/gm/iQQ==" }, "unist-util-visit": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-1.4.1.tgz", - "integrity": "sha512-AvGNk7Bb//EmJZyhtRUnNMEpId/AZ5Ph/KUpTI09WHQuDZHKovQ1oEv3mfmKpWKtoMzyMC4GLBm1Zy5k12fjIw==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-3.0.1.tgz", + "integrity": "sha512-DHyYg2LPkrzcdlhs2CJTWB1TWRRvah+CLiGjw5Ul9k13xPSEi+bK5EMFHVgSiyFNH2AS2/EinkWGZ05HKcXM1w==", "requires": { - "unist-util-visit-parents": "^2.0.0" + "@types/unist": "^2.0.0", + "unist-util-is": "^5.0.0", + "unist-util-visit-parents": "^4.0.0" }, "dependencies": { + "unist-util-is": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-5.0.0.tgz", + "integrity": "sha512-G4p13DhfdUNmlnJxd0uy5Skx1FG58LSDhX8h1xgpeSq0omOQ4ZN5BO54ToFlNX55NDTbRHMdwTOJXqAieInSEA==" + }, "unist-util-visit-parents": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-2.1.2.tgz", - "integrity": "sha512-DyN5vD4NE3aSeB+PXYNKxzGsfocxp6asDc2XXE3b0ekO2BaRUpBicbbUygfSvYfUz1IkmjFR1YF7dPklraMZ2g==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-4.0.0.tgz", + "integrity": "sha512-QyATSx30wHguIzI82+GVeuXGnFlh3AUVcyeZPOo5Paz2Z52zfRe3/0WLlBv6XlMWcr5xEdFqox6PteUL6hzEFA==", "requires": { - "unist-util-is": "^3.0.0" + "@types/unist": "^2.0.0", + "unist-util-is": "^5.0.0" } } } @@ -9577,11 +11034,6 @@ "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", "dev": true }, - "unquote": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/unquote/-/unquote-1.1.1.tgz", - "integrity": "sha1-j97XMk7G6IoP+LkF58CYzcCG1UQ=" - }, "unset-value": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-0.1.2.tgz", @@ -9645,9 +11097,9 @@ } }, "url-parse": { - "version": "1.4.7", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.7.tgz", - "integrity": "sha512-d3uaVyzDB9tQoSXFvuSUNFibTd9zxd2bkVrDRvF5TmvWWQwqE4lgYJ5m+x1DbecWkw+LK4RNl2CU1hHuOKPVlg==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.1.tgz", + "integrity": "sha512-HOfCOUJt7iSYzEx/UqgtwKRMC6EU91NFhsCHMv9oM03VJcVo2Qrp8T8kI9D7amFf1cu+/3CEhgb3rF9zL7k85Q==", "dev": true, "requires": { "querystringify": "^2.1.1", @@ -9861,7 +11313,6 @@ "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", "dev": true, - "optional": true, "requires": { "is-extendable": "^0.1.0" } @@ -9928,7 +11379,6 @@ "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", "dev": true, - "optional": true, "requires": { "kind-of": "^3.0.2" } diff --git a/pkg/interface/package.json b/pkg/interface/package.json index c7db782f95..858e8f1b9f 100644 --- a/pkg/interface/package.json +++ b/pkg/interface/package.json @@ -8,11 +8,13 @@ "@reach/disclosure": "^0.10.5", "@reach/menu-button": "^0.10.5", "@reach/tabs": "^0.10.5", + "@react-spring/web": "^9.1.1", "@tlon/indigo-dark": "^1.0.6", - "@tlon/indigo-light": "^1.0.6", - "@tlon/indigo-react": "^1.2.19", + "@tlon/indigo-light": "^1.0.7", + "@tlon/indigo-react": "^1.2.22", "@tlon/sigil-js": "^1.4.3", "@urbit/api": "file:../npm/api", + "any-ascii": "^0.1.7", "aws-sdk": "^2.830.0", "big-integer": "^1.6.48", "classnames": "^2.2.6", @@ -21,8 +23,7 @@ "file-saver": "^2.0.5", "formik": "^2.1.5", "immer": "^8.0.1", - "lodash": "^4.17.20", - "markdown-to-jsx": "^6.11.4", + "lodash": "^4.17.21", "moment": "^2.29.1", "mousetrap": "^1.6.5", "mousetrap-global-bind": "^1.1.0", @@ -37,14 +38,18 @@ "react-markdown": "^4.3.1", "react-oembed-container": "^1.0.0", "react-router-dom": "^5.2.0", + "react-use-gesture": "^9.1.3", "react-virtuoso": "^0.20.3", "react-visibility-sensor": "^5.1.1", - "remark-breaks": "^2.0.1", - "remark-disable-tokenizers": "^1.0.24", + "remark": "^12.0.0", + "remark-breaks": "^2.0.2", + "remark-disable-tokenizers": "1.1.0", + "stacktrace-js": "^2.0.2", "style-loader": "^1.3.0", "styled-components": "^5.1.1", "styled-system": "^5.1.5", "suncalc": "^1.8.0", + "unist-util-visit": "^3.0.0", "urbit-ob": "^5.0.1", "xterm": "^4.10.0", "xterm-addon-fit": "^0.5.0", @@ -73,6 +78,7 @@ "@types/yup": "^0.29.11", "@typescript-eslint/eslint-plugin": "^4.15.0", "@urbit/eslint-config": "file:../npm/eslint-config", + "@welldone-software/why-did-you-render": "^6.1.0", "babel-eslint": "^10.1.0", "babel-loader": "^8.2.2", "babel-plugin-lodash": "^3.3.4", @@ -87,7 +93,7 @@ "react-hot-loader": "^4.13.0", "sass": "^1.32.5", "sass-loader": "^8.0.2", - "typescript": "^3.9.7", + "typescript": "^4.2.4", "webpack": "^4.46.0", "webpack-cli": "^3.3.12", "webpack-dev-server": "^3.11.2" @@ -97,8 +103,9 @@ "lint-file": "eslint", "tsc": "tsc", "tsc:watch": "tsc --watch", + "preinstall": "./preinstall.sh", "build:dev": "cross-env NODE_ENV=development webpack --config config/webpack.dev.js", - "build:prod": "cd ../npm/api && npm i && cd ../../interface && cross-env NODE_ENV=production webpack --config config/webpack.prod.js", + "build:prod": "cd ../npm/api && npm ci && cd ../../interface && cross-env NODE_ENV=production webpack --config config/webpack.prod.js", "start": "webpack-dev-server --config config/webpack.dev.js", "test": "echo \"Error: no test specified\" && exit 1" }, diff --git a/pkg/interface/preinstall.sh b/pkg/interface/preinstall.sh new file mode 100755 index 0000000000..4f35cfc0f7 --- /dev/null +++ b/pkg/interface/preinstall.sh @@ -0,0 +1,12 @@ +#!/bin/sh +cd ../npm + +for i in $(find . -type d -maxdepth 1) ; do + packageJson="${i}/package.json" + if [ -f "${packageJson}" ]; then + echo "installing ${i}..." + cd ./${i} + npm ci + cd .. + fi +done \ No newline at end of file diff --git a/pkg/interface/src/index.js b/pkg/interface/src/index.js index ded85a8f58..7d908cd506 100644 --- a/pkg/interface/src/index.js +++ b/pkg/interface/src/index.js @@ -1,8 +1,7 @@ import * as React from 'react'; import * as ReactDOM from 'react-dom'; - import './register-sw'; - import App from './views/App'; +import './wdyr'; ReactDOM.render(, document.getElementById('root')); diff --git a/pkg/interface/src/logic/api/base.ts b/pkg/interface/src/logic/api/base.ts index d1122b4345..a5d021131c 100644 --- a/pkg/interface/src/logic/api/base.ts +++ b/pkg/interface/src/logic/api/base.ts @@ -1,5 +1,5 @@ +import { Path, Patp } from '@urbit/api'; import _ from 'lodash'; -import { Patp, Path } from '@urbit/api'; import BaseStore from '../store/base'; export default class BaseApi { diff --git a/pkg/interface/src/logic/api/contacts.ts b/pkg/interface/src/logic/api/contacts.ts index ae6a6972a0..4828bf8bea 100644 --- a/pkg/interface/src/logic/api/contacts.ts +++ b/pkg/interface/src/logic/api/contacts.ts @@ -1,7 +1,8 @@ -import BaseApi from './base'; -import { StoreState } from '../store/type'; import { Patp } from '@urbit/api'; -import { ContactEdit } from '@urbit/api/contacts'; +import { ContactEditField } from '@urbit/api/contacts'; +import _ from 'lodash'; +import { StoreState } from '../store/type'; +import BaseApi from './base'; export default class ContactsApi extends BaseApi { add(ship: Patp, contact: any) { @@ -13,7 +14,7 @@ export default class ContactsApi extends BaseApi { return this.storeAction({ remove: { ship } }); } - edit(ship: Patp, editField: ContactEdit) { + edit(ship: Patp, editField: ContactEditField) { /* editField can be... {nickname: ''} {email: ''} @@ -73,6 +74,28 @@ export default class ContactsApi extends BaseApi { ); } + async disallowedShipsForOurContact(ships: string[]): Promise { + return _.compact( + await Promise.all( + ships.map( + async (s) => { + const ship = `~${s}`; + if(s === window.ship) { + return null; + } + const allowed = await this.fetchIsAllowed( + `~${window.ship}`, + 'personal', + ship, + true + ); + return allowed ? null : ship; + } + ) + ) + ); + } + retrieve(ship: string) { const resource = { ship, name: '' }; return this.action('contact-pull-hook', 'pull-hook-action', { @@ -84,7 +107,7 @@ export default class ContactsApi extends BaseApi { } private storeAction(action: any): Promise { - return this.action('contact-store', 'contact-update', action); + return this.action('contact-store', 'contact-update-0', action); } private viewAction(threadName: string, action: any) { @@ -92,6 +115,6 @@ export default class ContactsApi extends BaseApi { } private hookAction(ship: Patp, action: any): Promise { - return this.action('contact-push-hook', 'contact-update', action); + return this.action('contact-push-hook', 'contact-update-0', action); } } diff --git a/pkg/interface/src/logic/api/gcp.ts b/pkg/interface/src/logic/api/gcp.ts index c6b955b22d..f8a72d6662 100644 --- a/pkg/interface/src/logic/api/gcp.ts +++ b/pkg/interface/src/logic/api/gcp.ts @@ -1,19 +1,14 @@ +import type { StoreState } from '../store/type'; import BaseApi from './base'; -import {StoreState} from '../store/type'; -import {GcpToken} from '../types/gcp-state'; - export default class GcpApi extends BaseApi { - isConfigured() { - return this.spider('noun', 'json', 'gcp-is-configured', {}) - .then((data) => { - this.store.handleEvent({ - data - }); - }); + // Does not touch the store; use the value manually. + async isConfigured(): Promise { + return this.spider('noun', 'json', 'gcp-is-configured', {}); } - getToken() { + // Does not return the token; read it out of the store. + async getToken(): Promise { return this.spider('noun', 'gcp-token', 'gcp-get-token', {}) .then((token) => { this.store.handleEvent({ @@ -21,4 +16,4 @@ export default class GcpApi extends BaseApi { }); }); } -}; +} diff --git a/pkg/interface/src/logic/api/global.ts b/pkg/interface/src/logic/api/global.ts index 5d26f0b7c9..1595094886 100644 --- a/pkg/interface/src/logic/api/global.ts +++ b/pkg/interface/src/logic/api/global.ts @@ -1,17 +1,17 @@ import { Patp } from '@urbit/api'; -import BaseApi from './base'; -import { StoreState } from '../store/type'; import GlobalStore from '../store/store'; -import LocalApi from './local'; -import InviteApi from './invite'; -import MetadataApi from './metadata'; +import { StoreState } from '../store/type'; +import BaseApi from './base'; import ContactsApi from './contacts'; -import GroupsApi from './groups'; -import LaunchApi from './launch'; -import GraphApi from './graph'; -import S3Api from './s3'; import GcpApi from './gcp'; +import GraphApi from './graph'; +import GroupsApi from './groups'; import { HarkApi } from './hark'; +import InviteApi from './invite'; +import LaunchApi from './launch'; +import LocalApi from './local'; +import MetadataApi from './metadata'; +import S3Api from './s3'; import SettingsApi from './settings'; import TermApi from './term'; diff --git a/pkg/interface/src/logic/api/graph.ts b/pkg/interface/src/logic/api/graph.ts index 3c9b71b43a..12fffce3a7 100644 --- a/pkg/interface/src/logic/api/graph.ts +++ b/pkg/interface/src/logic/api/graph.ts @@ -1,16 +1,16 @@ -import BaseApi from './base'; -import { StoreState } from '../store/type'; -import { Patp, Path } from '@urbit/api'; +import { Content, Enc, GraphNode, GroupPolicy, Path, Patp, Post, Resource } from '@urbit/api'; +import BigIntOrderedMap from '@urbit/api/lib/BigIntOrderedMap'; import _ from 'lodash'; +import { decToUd, deSig, resourceAsPath, unixToDa } from '~/logic/lib/util'; import { makeResource, resourceFromPath } from '../lib/group'; -import { GroupPolicy, Enc, Post, Content } from '@urbit/api'; -import { numToUd, unixToDa, decToUd, deSig, resourceAsPath } from '~/logic/lib/util'; +import { StoreState } from '../store/type'; +import BaseApi from './base'; export const createBlankNodeWithChildPost = ( parentIndex = '', childIndex = '', contents: Content[] -) => { +): GraphNode => { const date = unixToDa(Date.now()).toString(); const nodeIndex = parentIndex + '/' + date; @@ -36,7 +36,7 @@ export const createBlankNodeWithChildPost = ( hash: null, signatures: [] }, - children: childGraph + children: childGraph as BigIntOrderedMap }; }; @@ -83,7 +83,7 @@ export default class GraphApi extends BaseApi { joiningGraphs = new Set(); private storeAction(action: any): Promise { - return this.action('graph-store', 'graph-update', action); + return this.action('graph-store', 'graph-update-2', action); } private viewAction(threadName: string, action: any) { @@ -91,7 +91,7 @@ export default class GraphApi extends BaseApi { } private hookAction(ship: Patp, action: any): Promise { - return this.action('graph-push-hook', 'graph-update', action); + return this.action('graph-push-hook', 'graph-update-2', action); } createManagedGraph( @@ -185,13 +185,12 @@ export default class GraphApi extends BaseApi { }); } - eval(cord: string) { + eval(cord: string): Promise { return this.spider('graph-view-action', 'tang', 'graph-eval', { eval: cord }); } - addGraph(ship: Patp, name: string, graph: any, mark: any) { return this.storeAction({ 'add-graph': { @@ -211,7 +210,7 @@ export default class GraphApi extends BaseApi { return this.addNodes(ship, name, nodes); } - addNode(ship: Patp, name: string, node: Object) { + addNode(ship: Patp, name: string, node: GraphNode) { const nodes = {}; nodes[node.post.index] = node; @@ -227,7 +226,7 @@ export default class GraphApi extends BaseApi { }; const pendingPromise = this.spider( - 'graph-update', + 'graph-update-2', 'graph-view-action', 'graph-add-nodes', action @@ -259,9 +258,32 @@ export default class GraphApi extends BaseApi { */ } - removeNodes(ship: Patp, name: string, indices: string[]) { + async enableGroupFeed(group: Resource, vip: any = ''): Promise { + const { resource } = await this.spider( + 'graph-view-action', + 'resource', + 'graph-create-group-feed', + { + 'create-group-feed': { resource: group, vip } + } + ); + return resource; + } + + async disableGroupFeed(group: Resource): Promise { + await this.spider( + 'graph-view-action', + 'json', + 'graph-disable-group-feed', + { + 'disable-group-feed': { resource: group } + } + ); + } + + removePosts(ship: Patp, name: string, indices: string[]) { return this.hookAction(ship, { - 'remove-nodes': { + 'remove-posts': { resource: { ship, name }, indices } @@ -336,15 +358,17 @@ export default class GraphApi extends BaseApi { }); } - getNode(ship: string, resource: string, index: string) { - const idx = index.split('/').map(numToUd).join('/'); - return this.scry( + async getNode(ship: string, resource: string, index: string) { + const idx = index.split('/').map(decToUd).join('/'); + const data = await this.scry( 'graph-store', `/node/${ship}/${resource}${idx}` - ).then((node) => { - this.store.handleEvent({ - data: node - }); + ); + const node = data['graph-update']; + this.store.handleEvent({ + data: { + 'graph-update-loose': node + } }); } } diff --git a/pkg/interface/src/logic/api/groups.ts b/pkg/interface/src/logic/api/groups.ts index f0bab91f0b..ee1807c51b 100644 --- a/pkg/interface/src/logic/api/groups.ts +++ b/pkg/interface/src/logic/api/groups.ts @@ -1,14 +1,14 @@ -import BaseApi from './base'; -import { StoreState } from '../store/type'; -import { Path, Patp, Enc } from '@urbit/api'; +import { Enc, Patp } from '@urbit/api'; import { GroupAction, GroupPolicy, - Resource, - Tag, - GroupPolicyDiff + + GroupPolicyDiff, Resource, + Tag } from '@urbit/api/groups'; import { makeResource } from '../lib/group'; +import { StoreState } from '../store/type'; +import BaseApi from './base'; export default class GroupsApi extends BaseApi { remove(resource: Resource, ships: Patp[]) { @@ -78,12 +78,16 @@ export default class GroupsApi extends BaseApi { }); } + hide(resource: string) { + return this.viewAction({ hide: resource }); + } + private proxyAction(action: GroupAction) { - return this.action('group-push-hook', 'group-update', action); + return this.action('group-push-hook', 'group-update-0', action); } private storeAction(action: GroupAction) { - return this.action('group-store', 'group-update', action); + return this.action('group-store', 'group-update-0', action); } private viewThread(thread: string, action: any) { diff --git a/pkg/interface/src/logic/api/hark.ts b/pkg/interface/src/logic/api/hark.ts index b807201a1a..368eccb385 100644 --- a/pkg/interface/src/logic/api/hark.ts +++ b/pkg/interface/src/logic/api/hark.ts @@ -1,9 +1,14 @@ -import BaseApi from './base'; -import { StoreState } from '../store/type'; -import { dateToDa, decToUd } from '../lib/util'; -import { NotifIndex, IndexedNotification, Association, GraphNotifDescription } from '@urbit/api'; +import { Association, GraphNotifDescription, IndexedNotification, NotifIndex } from '@urbit/api'; import { BigInteger } from 'big-integer'; import { getParentIndex } from '../lib/notification'; +import { dateToDa, decToUd } from '../lib/util'; +import useHarkState from '../state/hark'; +import { StoreState } from '../store/type'; +import BaseApi from './base'; + +function getHarkSize() { + return useHarkState.getState().notifications.size ?? 0; +} export class HarkApi extends BaseApi { private harkAction(action: any): Promise { @@ -70,7 +75,6 @@ export class HarkApi extends BaseApi { graph: { graph: association.resource, group: association.group, - module: association.metadata.module, description, index: parent } } @@ -172,10 +176,10 @@ export class HarkApi extends BaseApi { } async getMore(): Promise { - const offset = this.store.state['notifications']?.size || 0; + const offset = getHarkSize(); const count = 3; await this.getSubset(offset, count, false); - return offset === (this.store.state.notifications?.size || 0); + return offset === getHarkSize(); } async getSubset(offset:number, count:number, isArchive: boolean) { diff --git a/pkg/interface/src/logic/api/invite.ts b/pkg/interface/src/logic/api/invite.ts index dc390d12d1..1588879fdf 100644 --- a/pkg/interface/src/logic/api/invite.ts +++ b/pkg/interface/src/logic/api/invite.ts @@ -1,6 +1,6 @@ -import BaseApi from './base'; +import { Serial } from '@urbit/api'; import { StoreState } from '../store/type'; -import { Serial, Path } from '@urbit/api'; +import BaseApi from './base'; export default class InviteApi extends BaseApi { accept(app: string, uid: Serial) { diff --git a/pkg/interface/src/logic/api/launch.ts b/pkg/interface/src/logic/api/launch.ts index 3bba3b3cc0..013928188e 100644 --- a/pkg/interface/src/logic/api/launch.ts +++ b/pkg/interface/src/logic/api/launch.ts @@ -1,5 +1,5 @@ -import BaseApi from './base'; import { StoreState } from '../store/type'; +import BaseApi from './base'; export default class LaunchApi extends BaseApi { add(name: string, tile = { basic : { title: '', linkedUrl: '', iconUrl: '' } }) { diff --git a/pkg/interface/src/logic/api/local.ts b/pkg/interface/src/logic/api/local.ts index 305ac71f89..240e2abd60 100644 --- a/pkg/interface/src/logic/api/local.ts +++ b/pkg/interface/src/logic/api/local.ts @@ -1,5 +1,5 @@ -import BaseApi from './base'; import { StoreState } from '../store/type'; +import BaseApi from './base'; export default class LocalApi extends BaseApi { getBaseHash() { @@ -8,7 +8,9 @@ export default class LocalApi extends BaseApi { }); } - dehydrate() { - this.store.dehydrate(); + getRuntimeLag() { + return this.scry('launch', '/runtime-lag').then((runtimeLag) => { + this.store.handleEvent({ data: { runtimeLag } }); + }); } } diff --git a/pkg/interface/src/logic/api/metadata.ts b/pkg/interface/src/logic/api/metadata.ts index c2d388dfc3..97cb1471e1 100644 --- a/pkg/interface/src/logic/api/metadata.ts +++ b/pkg/interface/src/logic/api/metadata.ts @@ -1,8 +1,8 @@ -import BaseApi from './base'; -import { StoreState } from '../store/type'; -import { Path, Patp, Association, Metadata, MetadataUpdatePreview } from '@urbit/api'; +import { Association, Metadata, MetadataUpdatePreview, Path } from '@urbit/api'; import { uxToHex } from '../lib/util'; +import { StoreState } from '../store/type'; +import BaseApi from './base'; export default class MetadataApi extends BaseApi { metadataAdd(appName: string, resource: Path, group: Path, title: string, description: string, dateCreated: string, color: string, moduleName: string) { @@ -20,8 +20,9 @@ export default class MetadataApi extends BaseApi { color, 'date-created': dateCreated, creator, - 'module': moduleName, + config: { graph: moduleName }, picture: '', + hidden: false, preview: false, vip: '' } @@ -77,7 +78,6 @@ export default class MetadataApi extends BaseApi { tempChannel.delete(); }, (ev: any) => { - console.log(ev); if ('metadata-hook-update' in ev) { done = true; tempChannel.delete(); @@ -103,6 +103,6 @@ export default class MetadataApi extends BaseApi { } private metadataAction(data) { - return this.action('metadata-push-hook', 'metadata-update', data); + return this.action('metadata-push-hook', 'metadata-update-1', data); } } diff --git a/pkg/interface/src/logic/api/s3.ts b/pkg/interface/src/logic/api/s3.ts index 1d775ef775..14b998a144 100644 --- a/pkg/interface/src/logic/api/s3.ts +++ b/pkg/interface/src/logic/api/s3.ts @@ -1,6 +1,5 @@ -import BaseApi from './base'; import { StoreState } from '../store/type'; -import { S3Update } from '../../types/s3-update'; +import BaseApi from './base'; export default class S3Api extends BaseApi { setCurrentBucket(bucket: string) { diff --git a/pkg/interface/src/logic/api/settings.ts b/pkg/interface/src/logic/api/settings.ts index 9a5812ed98..47be601ad8 100644 --- a/pkg/interface/src/logic/api/settings.ts +++ b/pkg/interface/src/logic/api/settings.ts @@ -1,12 +1,13 @@ -import BaseApi from './base'; -import { StoreState } from '../store/type'; -import { Key, - Value, - Bucket +import { + Bucket, Key, + + SettingsUpdate, Value } from '@urbit/api/settings'; +import { StoreState } from '../store/type'; +import BaseApi from './base'; export default class SettingsApi extends BaseApi { - private storeAction(action: SettingsEvent): Promise { + private storeAction(action: SettingsUpdate): Promise { return this.action('settings-store', 'settings-event', action); } @@ -47,14 +48,14 @@ export default class SettingsApi extends BaseApi { } async getAll() { - const { all } = await this.scry("settings-store", "/all"); - this.store.handleEvent({data: - {"settings-data": { all } } + const { all } = await this.scry('settings-store', '/all'); + this.store.handleEvent({ data: + { 'settings-data': { all } } }); } async getBucket(bucket: Key) { - const data = await this.scry('settings-store', `/bucket/${bucket}`); + const data: Record = await this.scry('settings-store', `/bucket/${bucket}`); this.store.handleEvent({ data: { 'settings-data': { 'bucket-key': bucket, 'bucket': data.bucket @@ -62,7 +63,7 @@ export default class SettingsApi extends BaseApi { } async getEntry(bucket: Key, entry: Key) { - const data = await this.scry('settings-store', `/entry/${bucket}/${entry}`); + const data: Record = await this.scry('settings-store', `/entry/${bucket}/${entry}`); this.store.handleEvent({ data: { 'settings-data': { 'bucket-key': bucket, 'entry-key': entry, diff --git a/pkg/interface/src/logic/lib/BigIntOrderedMap.ts b/pkg/interface/src/logic/lib/BigIntOrderedMap.ts deleted file mode 100644 index 21fc1c8cac..0000000000 --- a/pkg/interface/src/logic/lib/BigIntOrderedMap.ts +++ /dev/null @@ -1,232 +0,0 @@ -import bigInt, { BigInteger } from 'big-integer'; - -interface NonemptyNode { - n: [BigInteger, V]; - l: MapNode; - r: MapNode; -} - -type MapNode = NonemptyNode | null; - -/** - * An implementation of ordered maps for JS - * Plagiarised wholesale from sys/zuse - */ -export class BigIntOrderedMap implements Iterable<[BigInteger, V]> { - private root: MapNode = null; - size = 0; - - constructor(initial: [BigInteger, V][] = []) { - initial.forEach(([key, val]) => { - this.set(key, val); - }); - } - - /** - * Retrieve an value for a key - */ - get(key: BigInteger): V | null { - const inner = (node: MapNode) => { - if (!node) { - return node; - } - const [k, v] = node.n; - if (key.eq(k)) { - return v; - } - if (key.gt(k)) { - return inner(node.l); - } else { - return inner(node.r); - } - }; - - return inner(this.root); - } - - /** - * Put an item by a key - */ - set(key: BigInteger, value: V): void { - const inner = (node: MapNode) => { - if (!node) { - return { - n: [key, value], - l: null, - r: null - }; - } - const [k] = node.n; - if (key.eq(k)) { - this.size--; - return { - ...node, - n: [k, value] - }; - } - if (key.gt(k)) { - const l = inner(node.l); - if (!l) { - throw new Error('invariant violation'); - } - return { - ...node, - l - }; - } - const r = inner(node.r); - if (!r) { - throw new Error('invariant violation'); - } - - return { ...node, r }; - }; - this.size++; - this.root = inner(this.root); - } - - /** - * Remove all entries - */ - clear() { - this.root = null; - } - - /** - * Predicate testing if map contains key - */ - has(key: BigInteger): boolean { - const inner = (node: MapNode) => { - if (!node) { - return false; - } - const [k] = node.n; - - if (k.eq(key)) { - return true; - } - if (key.gt(k)) { - return inner(node.l); - } - return inner(node.r); - }; - return inner(this.root); - } - - /** - * Remove value associated with key, returning whether that key - * existed in the first place - */ - delete(key: BigInteger) { - const inner = (node: MapNode): [boolean, MapNode] => { - if (!node) { - return [false, null]; - } - const [k] = node.n; - if (k.eq(key)) { - return [true, this.nip(node)]; - } - if (key.gt(k)) { - const [bool, l] = inner(node.l); - return [ - bool, - { - ...node, - l - } - ]; - } - - const [bool, r] = inner(node.r); - return [ - bool, - { - ...node, - r - } - ]; - }; - const [ret, newRoot] = inner(this.root); - if(ret) { - this.size--; - } - this.root = newRoot; - return ret; - } - - private nip(nod: NonemptyNode): MapNode { - const inner = (node: NonemptyNode) => { - if (!node.l) { - return node.r; - } - if (!node.r) { - return node.l; - } - return { - ...node.l, - r: inner(node.r) - }; - }; - return inner(nod); - } - - peekLargest(): [BigInteger, V] | undefined { - const inner = (node: MapNode) => { - if(!node) { - return undefined; - } - if(node.l) { - return inner(node.l); - } - return node.n; - }; - return inner(this.root); - } - - peekSmallest(): [BigInteger, V] | undefined { - const inner = (node: MapNode) => { - if(!node) { - return undefined; - } - if(node.r) { - return inner(node.r); - } - return node.n; - }; - return inner(this.root); - } - - keys(): BigInteger[] { - const list = Array.from(this); - return list.map(([key]) => key); - } - - forEach(f: (value: V, key: BigInteger) => void) { - const list = Array.from(this); - return list.forEach(([k,v]) => f(v,k)); - } - - [Symbol.iterator](): IterableIterator<[BigInteger, V]> { - const result: [BigInteger, V][] = []; - const inner = (node: MapNode) => { - if (!node) { - return; - } - inner(node.l); - result.push(node.n); - inner(node.r); - }; - inner(this.root); - - let idx = 0; - return { - [Symbol.iterator]: this[Symbol.iterator], - next: (): IteratorResult<[BigInteger, V]> => { - if (idx < result.length) { - return { value: result[idx++], done: false }; - } - return { done: true, value: null }; - } - }; - } -} diff --git a/pkg/interface/src/logic/lib/GcpClient.ts b/pkg/interface/src/logic/lib/GcpClient.ts index b03c7fd0a6..793ea78f83 100644 --- a/pkg/interface/src/logic/lib/GcpClient.ts +++ b/pkg/interface/src/logic/lib/GcpClient.ts @@ -7,14 +7,12 @@ // import querystring from 'querystring'; import { - StorageAcl, StorageClient, StorageUpload, UploadParams, UploadResult } from './StorageClient'; - const ENDPOINT = 'storage.googleapis.com'; class GcpUpload implements StorageUpload { @@ -26,8 +24,8 @@ class GcpUpload implements StorageUpload { this.#accessKey = accessKey; } - async promise(): UploadResult { - const {Bucket, Key, ContentType, Body} = this.#params; + async promise(): Promise { + const { Bucket, Key, ContentType, Body } = this.#params; const urlParams = { uploadType: 'media', name: Key, @@ -50,7 +48,7 @@ class GcpUpload implements StorageUpload { console.error('GcpClient server error', await response.json()); throw new Error(`GcpClient: response ${response.status}`); } - return {Location: `https://${ENDPOINT}/${Bucket}/${Key}`}; + return { Location: `https://${ENDPOINT}/${Bucket}/${Key}` }; } } diff --git a/pkg/interface/src/logic/lib/StorageClient.ts b/pkg/interface/src/logic/lib/StorageClient.ts index 31e12f8233..182ef0b8f2 100644 --- a/pkg/interface/src/logic/lib/StorageClient.ts +++ b/pkg/interface/src/logic/lib/StorageClient.ts @@ -1,14 +1,13 @@ // Defines a StorageClient interface interoperable between S3 and GCP Storage. // - // XX kind of gross. S3 needs 'public-read', GCP needs 'publicRead'. // Rather than write a wrapper around S3, we offer this field here, which // should always be passed, and will be replaced by 'publicRead' in the // GCP client. export enum StorageAcl { PublicRead = 'public-read' -}; +} export interface UploadParams { Bucket: string; // the bucket to upload the object to @@ -16,17 +15,17 @@ export interface UploadParams { ContentType: string; // the object's mime-type ACL: StorageAcl; // ACL, always 'public-read' Body: File; // the object itself -}; +} export interface UploadResult { Location: string; -}; +} // Extra layer of indirection used by S3 client. export interface StorageUpload { promise(): Promise; -}; +} export interface StorageClient { upload(params: UploadParams): StorageUpload; -}; +} diff --git a/pkg/interface/src/logic/lib/bigInt.ts b/pkg/interface/src/logic/lib/bigInt.ts index 01da28b455..72b5e34c69 100644 --- a/pkg/interface/src/logic/lib/bigInt.ts +++ b/pkg/interface/src/logic/lib/bigInt.ts @@ -1,4 +1,4 @@ -import bigInt, { BigInteger } from 'big-integer'; +import { BigInteger } from 'big-integer'; export function max(a: BigInteger, b: BigInteger) { return a.gt(b) ? a : b; diff --git a/pkg/interface/src/logic/lib/formGroup.ts b/pkg/interface/src/logic/lib/formGroup.ts new file mode 100644 index 0000000000..db679f2d3d --- /dev/null +++ b/pkg/interface/src/logic/lib/formGroup.ts @@ -0,0 +1,18 @@ +import React from 'react'; + +export type SubmitHandler = () => Promise; +interface IFormGroupContext { + addSubmit: (id: string, submit: SubmitHandler) => void; + onDirty: (id: string, touched: boolean) => void; + onErrors: (id: string, errors: boolean) => void; + submitAll: () => Promise; +} + +const fallback: IFormGroupContext = { + addSubmit: () => {}, + onDirty: () => {}, + onErrors: () => {}, + submitAll: () => Promise.resolve() +}; + +export const FormGroupContext = React.createContext(fallback); diff --git a/pkg/interface/src/logic/lib/gcpManager.ts b/pkg/interface/src/logic/lib/gcpManager.ts index 94bfa9461f..9710e82d91 100644 --- a/pkg/interface/src/logic/lib/gcpManager.ts +++ b/pkg/interface/src/logic/lib/gcpManager.ts @@ -5,10 +5,9 @@ // 1. call configure with a GlobalApi and GlobalStore. // 2. call start() to start the token refresh loop. // -// If the ship does not have GCP storage configured, we don't try to get -// a token, but we keep checking at regular intervals to see if it gets -// configured. If GCP storage is configured, we try to invoke the GCP -// get-token thread on the ship until it gives us an access token. Once +// If the ship does not have GCP storage configured, we don't try to +// get a token. If GCP storage is configured, we try to invoke the GCP +// get-token thread on the ship until it gives us an access token. Once // we have a token, we refresh it every hour or so according to its // intrinsic expiry. // @@ -16,7 +15,6 @@ import GlobalApi from '../api/global'; import useStorageState from '../state/storage'; - class GcpManager { #api: GlobalApi | null = null; @@ -25,7 +23,7 @@ class GcpManager { } #running = false; - #timeoutId: number | null = null; + #timeoutId: ReturnType | null = null; start() { if (this.#running) { @@ -60,23 +58,22 @@ class GcpManager { this.start(); } - #consecutiveFailures: number = 0; - - private isConfigured() { - return useStorageState.getState().gcp.configured; - } + #consecutiveFailures = 0; + #configured = false; private refreshLoop() { - if (!this.isConfigured()) { - this.#api.gcp.isConfigured() - .then(() => { - if (this.isConfigured() === undefined) { - throw new Error("can't check whether GCP is configured?"); + if (!this.#configured) { + this.#api!.gcp.isConfigured() + .then((configured) => { + if (configured === undefined) { + throw new Error('can\'t check whether GCP is configured?'); } - if (this.isConfigured()) { + this.#configured = configured; + if (this.#configured) { this.refreshLoop(); } else { - this.refreshAfter(10_000); + console.log('GcpManager: GCP storage not configured; stopping.'); + this.stop(); } }) .catch((reason) => { @@ -85,7 +82,7 @@ class GcpManager { }); return; } - this.#api.gcp.getToken() + this.#api!.gcp.getToken() .then(() => { const token = useStorageState.getState().gcp.token; if (token) { diff --git a/pkg/interface/src/logic/lib/group.ts b/pkg/interface/src/logic/lib/group.ts index 2eb81e015b..7268bcc0db 100644 --- a/pkg/interface/src/logic/lib/group.ts +++ b/pkg/interface/src/logic/lib/group.ts @@ -1,7 +1,6 @@ +import { Path, PatpNoSig } from '@urbit/api'; +import { Group, Resource, roleTags, RoleTags } from '@urbit/api/groups'; import _ from 'lodash'; -import { roleTags, RoleTags, Group, Resource } from '@urbit/api/groups'; -import { PatpNoSig, Path } from '@urbit/api'; -import { deSig } from './util'; export function roleForShip( group: Group, diff --git a/pkg/interface/src/logic/lib/hark.ts b/pkg/interface/src/logic/lib/hark.ts index b65a09603d..a5fe5883ea 100644 --- a/pkg/interface/src/logic/lib/hark.ts +++ b/pkg/interface/src/logic/lib/hark.ts @@ -1,6 +1,7 @@ +import { IndexedNotification, NotificationGraphConfig, Unreads } from '@urbit/api'; import bigInt, { BigInteger } from 'big-integer'; +import _ from 'lodash'; import f from 'lodash/fp'; -import { Unreads } from '@urbit/api'; export function getLastSeen( unreads: Unreads, @@ -31,6 +32,29 @@ export function getNotificationCount( ): number { const unread = unreads.graph?.[path] || {}; return Object.keys(unread) - .map(index => unread[index]?.notifications || 0) + .map(index => _.get(unread[index], 'notifications.length', 0)) .reduce(f.add, 0); } + +export function isWatching( + config: NotificationGraphConfig, + graph: string, + index = '/' +) { + return Boolean(config.watching.find( + watch => watch.graph === graph && watch.index === index + )); +} + +export function getNotificationKey(time: BigInteger, notification: IndexedNotification): string { + const base = time.toString(); + if('graph' in notification.index) { + const { graph, index } = notification.index.graph; + return `${base}-${graph}-${index}`; + } else if('group' in notification.index) { + const { group } = notification.index.group; + return `${base}-${group}`; + } + return `${base}-unknown`; +} + diff --git a/pkg/interface/src/logic/lib/idling.ts b/pkg/interface/src/logic/lib/idling.ts index 9744e68dee..e4032413a5 100644 --- a/pkg/interface/src/logic/lib/idling.ts +++ b/pkg/interface/src/logic/lib/idling.ts @@ -1,9 +1,9 @@ -import { useState, useEffect } from 'react'; +import { useEffect, useState } from 'react'; export function useIdlingState() { const [idling, setIdling] = useState(false); - useEffect(() => { + useEffect(() => { function blur() { setIdling(true); } @@ -16,7 +16,7 @@ export function useIdlingState() { return () => { window.removeEventListener('blur', blur); window.removeEventListener('focus', focus); - } + }; }, []); return idling; diff --git a/pkg/interface/src/logic/lib/migrateSettings.ts b/pkg/interface/src/logic/lib/migrateSettings.ts index ec841af18f..d35f6212b5 100644 --- a/pkg/interface/src/logic/lib/migrateSettings.ts +++ b/pkg/interface/src/logic/lib/migrateSettings.ts @@ -1,15 +1,15 @@ -import useLocalState, { LocalState } from "~/logic/state/local"; -import useSettingsState from "~/logic/state/settings"; -import GlobalApi from "../api/global"; -import { BackgroundConfig, RemoteContentPolicy } from "~/types"; +import useLocalState from '~/logic/state/local'; +import useSettingsState from '~/logic/state/settings'; +import { BackgroundConfig, RemoteContentPolicy } from '~/types'; +import GlobalApi from '../api/global'; const getBackgroundString = (bg: BackgroundConfig) => { - if (bg?.type === "url") { + if (bg?.type === 'url') { return bg.url; - } else if (bg?.type === "color") { + } else if (bg?.type === 'color') { return bg.color; } else { - return ""; + return ''; } }; @@ -18,17 +18,17 @@ export function useMigrateSettings(api: GlobalApi) { const { display, remoteContentPolicy, calm } = useSettingsState(); return async () => { - let promises: Promise[] = []; + const promises: Promise[] = []; if (local.hideAvatars !== calm.hideAvatars) { promises.push( - api.settings.putEntry("calm", "hideAvatars", local.hideAvatars) + api.settings.putEntry('calm', 'hideAvatars', local.hideAvatars) ); } if (local.hideNicknames !== calm.hideNicknames) { promises.push( - api.settings.putEntry("calm", "hideNicknames", local.hideNicknames) + api.settings.putEntry('calm', 'hideNicknames', local.hideNicknames) ); } @@ -38,15 +38,15 @@ export function useMigrateSettings(api: GlobalApi) { ) { promises.push( api.settings.putEntry( - "display", - "background", + 'display', + 'background', getBackgroundString(local.background) ) ); promises.push( api.settings.putEntry( - "display", - "backgroundType", + 'display', + 'backgroundType', local.background?.type ) ); @@ -57,12 +57,12 @@ export function useMigrateSettings(api: GlobalApi) { const localVal = local.remoteContentPolicy[key]; if (localVal !== remoteContentPolicy[key]) { promises.push( - api.settings.putEntry("remoteContentPolicy", key, localVal) + api.settings.putEntry('remoteContentPolicy', key, localVal) ); } }); await Promise.all(promises); - localStorage.removeItem("localReducer"); + localStorage.removeItem('localReducer'); }; } diff --git a/pkg/interface/src/logic/lib/notification.ts b/pkg/interface/src/logic/lib/notification.ts index bd2979f409..7514d1e0c7 100644 --- a/pkg/interface/src/logic/lib/notification.ts +++ b/pkg/interface/src/logic/lib/notification.ts @@ -1,4 +1,4 @@ -import { GraphNotifIndex, GraphNotificationContents } from '@urbit/api'; +import { GraphNotificationContents, GraphNotifIndex } from '@urbit/api'; export function getParentIndex( idx: GraphNotifIndex, diff --git a/pkg/interface/src/logic/lib/omnibox.js b/pkg/interface/src/logic/lib/omnibox.js index 439b5a5e34..0da2641447 100644 --- a/pkg/interface/src/logic/lib/omnibox.js +++ b/pkg/interface/src/logic/lib/omnibox.js @@ -1,5 +1,5 @@ -import { cite } from '~/logic/lib/util'; import { isChannelAdmin } from '~/logic/lib/group'; +import { cite } from '~/logic/lib/util'; const makeIndexes = () => new Map([ ['ships', []], @@ -23,7 +23,7 @@ const result = function(title, link, app, host) { const shipIndex = function(contacts) { const ships = []; Object.keys(contacts).map((e) => { - return ships.push(result(e, `/~profile/${e}`, 'profile', contacts[e]?.status || "")); + return ships.push(result(e, `/~profile/${e}`, 'profile', contacts[e]?.status || '')); }); return ships; }; @@ -38,11 +38,11 @@ const commandIndex = function (currentGroup, groups, associations) { ? (association.metadata.vip === 'member-metadata' || isChannelAdmin(group, currentGroup)) : !currentGroup; // home workspace or hasn't loaded const workspace = currentGroup || '/home'; - commands.push(result(`Groups: Create`, `/~landscape/new`, 'Groups', null)); + commands.push(result('Groups: Create', '/~landscape/new', 'Groups', null)); if (canAdd) { - commands.push(result(`Channel: Create`, `/~landscape${workspace}/new`, 'Groups', null)); + commands.push(result('Channel: Create', `/~landscape${workspace}/new`, 'Groups', null)); } - commands.push(result(`Groups: Join`, `/~landscape/join`, 'Groups', null)); + commands.push(result('Groups: Join', '/~landscape/join', 'Groups', null)); return commands; }; @@ -80,7 +80,7 @@ const otherIndex = function(config) { logout: result('Log Out', '/~/logout', 'logout', null) }; other.push(result('Tutorial', '/?tutorial=true', 'tutorial', null)); - for(let cat of config.categories) { + for(const cat of config.categories) { if(idx[cat]) { other.push(idx[cat]); } @@ -99,9 +99,11 @@ export default function index(contacts, associations, apps, currentGroup, groups Object.keys(associations).filter((e) => { // skip apps with no metadata return Object.keys(associations[e]).length > 0; - }).map((e) => { - // iterate through each app's metadata object - Object.keys(associations[e]).map((association) => { + }).map((e) => { + // iterate through each app's metadata object + Object.keys(associations[e]) + .filter(association => !associations?.[e]?.[association]?.metadata?.hidden) + .map((association) => { const each = associations[e][association]; let title = each.resource; if (each.metadata.title !== '') { @@ -114,7 +116,7 @@ export default function index(contacts, associations, apps, currentGroup, groups }; if (each['app-name'] === 'graph') { - app = each.metadata.module; + app = each.metadata.config.graph; } const shipStart = each.resource.substr(each.resource.indexOf('~')); @@ -128,7 +130,7 @@ export default function index(contacts, associations, apps, currentGroup, groups ); landscape.push(obj); } else { - const app = each.metadata.module || each['app-name']; + const app = each.metadata.config.graph || each['app-name']; let group = each.group; if (groups[each.group]?.hidden && app === 'chat') { group = '/messages'; diff --git a/pkg/interface/src/logic/lib/permalinks.ts b/pkg/interface/src/logic/lib/permalinks.ts new file mode 100644 index 0000000000..d225dc317b --- /dev/null +++ b/pkg/interface/src/logic/lib/permalinks.ts @@ -0,0 +1,108 @@ + +import { + ReferenceContent, resourceFromPath +} from '@urbit/api'; + +export function getPermalinkForGraph( + group: string, + graph: string, + index = '' +) { + const groupLink = getPermalinkForAssociatedGroup(group); + const { ship, name } = resourceFromPath(graph); + return `${groupLink}/graph/${ship}/${name}${index}`; +} + +function getPermalinkForAssociatedGroup(group: string) { + const { ship, name } = resourceFromPath(group); + return `web+urbitgraph://group/${ship}/${name}`; +} + +type Permalink = GraphPermalink | GroupPermalink; + +export interface GroupPermalink { + type: 'group'; + group: string; + link: string; +} + +export interface GraphPermalink { + type: 'graph'; + link: string; + graph: string; + group: string; + index: string; +} + +function parseGraphPermalink( + link: string, + group: string, + segments: string[] +): GraphPermalink | null { + const [kind, ship, name, ...index] = segments; + if (kind !== 'graph') { + return null; + } + const graph = `/ship/${ship}/${name}`; + return { + type: 'graph', + link: link.slice(16), + graph, + group, + index: `/${index.join('/')}` + }; +} + +export function permalinkToReference(link: Permalink): ReferenceContent { + if(link.type === 'graph') { + const reference = { + graph: { + graph: link.graph, + group: link.group, + index: link.index + } + }; + return { reference }; + } else { + const reference = { + group: link.group + }; + return { reference }; + } +} + +export function referenceToPermalink({ reference }: ReferenceContent): Permalink { + if('graph' in reference) { + const { graph, group, index } = reference.graph; + const link = `web+urbitgraph://group${group.slice(5)}/graph${graph.slice(5)}${index}`; + return { + type: 'graph', + link, + ...reference.graph + }; + } else { + const link = `web+urbitgraph://group${reference.group.slice(5)}`; + return { + type: 'group', + link, + ...reference + }; + } +} + +export function parsePermalink(url: string): Permalink | null { + const [kind, ...rest] = url.slice(17).split('/'); + if (kind === 'group') { + const [ship, name, ...graph] = rest; + const group = `/ship/${ship}/${name}`; + if (graph.length > 0) { + return parseGraphPermalink(url, group, graph); + } + return { + type: 'group', + group, + link: url.slice(11) + }; + } + return null; +} diff --git a/pkg/interface/src/logic/lib/platform.ts b/pkg/interface/src/logic/lib/platform.ts index 3243cbf110..f76ecc6433 100644 --- a/pkg/interface/src/logic/lib/platform.ts +++ b/pkg/interface/src/logic/lib/platform.ts @@ -1,6 +1,10 @@ const ua = window.navigator.userAgent; -export const IS_IOS = ua.includes('iPhone'); +export const IS_IOS = ua.includes('iPhone') || ua.includes('iPad'); -console.log(IS_IOS); +export const IS_SAFARI = ua.includes('Safari') && !ua.includes('Chrome'); + +export const IS_ANDROID = ua.includes('Android'); + +export const IS_MOBILE = IS_IOS || IS_ANDROID; diff --git a/pkg/interface/src/logic/lib/post.ts b/pkg/interface/src/logic/lib/post.ts index 397563283a..8ba48e7210 100644 --- a/pkg/interface/src/logic/lib/post.ts +++ b/pkg/interface/src/logic/lib/post.ts @@ -1,4 +1,4 @@ -import { Post, GraphNode } from '@urbit/api'; +import { GraphNode, Post } from '@urbit/api'; export const buntPost = (): Post => ({ author: '', diff --git a/pkg/interface/src/logic/lib/publish.ts b/pkg/interface/src/logic/lib/publish.ts index ce616b3cf5..22fd166cca 100644 --- a/pkg/interface/src/logic/lib/publish.ts +++ b/pkg/interface/src/logic/lib/publish.ts @@ -1,13 +1,14 @@ -import { Post, GraphNode, TextContent, Graph, NodeMap } from '@urbit/api'; +import { Content, GraphNode, Post, TextContent } from '@urbit/api'; +import BigIntOrderedMap from '@urbit/api/lib/BigIntOrderedMap'; +import bigInt, { BigInteger } from 'big-integer'; import { buntPost } from '~/logic/lib/post'; import { unixToDa } from '~/logic/lib/util'; -import { BigIntOrderedMap } from './BigIntOrderedMap'; -import bigInt, { BigInteger } from 'big-integer'; +import tokenizeMessage from './tokenizeMessage'; export function newPost( title: string, body: string -): [BigInteger, NodeMap] { +): [BigInteger, any] { const now = Date.now(); const nowDa = unixToDa(now); const root: Post = { @@ -19,13 +20,15 @@ export function newPost( signatures: [] }; + const tokenisedBody = tokenizeMessage(body); + const revContainer: Post = { ...root, index: root.index + '/1' }; const commentsContainer = { ...root, index: root.index + '/2' }; const firstRevision: Post = { ...revContainer, index: revContainer.index + '/1', - contents: [{ text: title }, { text: body }] + contents: [{ text: title }, ...tokenisedBody] }; const nodes = { @@ -54,11 +57,12 @@ export function newPost( export function editPost(rev: number, noteId: BigInteger, title: string, body: string) { const now = Date.now(); + const tokenisedBody = tokenizeMessage(body); const newRev: Post = { author: `~${window.ship}`, index: `/${noteId.toString()}/1/${rev}`, 'time-sent': now, - contents: [{ text: title }, { text: body }], + contents: [{ text: title }, ...tokenisedBody], hash: null, signatures: [] }; @@ -73,41 +77,53 @@ export function editPost(rev: number, noteId: BigInteger, title: string, body: s } export function getLatestRevision(node: GraphNode): [number, string, string, Post] { - const revs = node.children.get(bigInt(1)); const empty = [1, '', '', buntPost()] as [number, string, string, Post]; + const revs = node.children?.get(bigInt(1)); if(!revs) { return empty; } - const [revNum, rev] = [...revs.children][0]; - if(!rev) { + let revNum, rev; + if (revs?.children !== null) { + [revNum, rev] = [...revs.children][0]; + } + if (!rev) { return empty; } - const [title, body] = rev.post.contents as TextContent[]; - return [revNum.toJSNumber(), title.text, body.text, rev.post]; + const title = rev.post.contents[0]; + const body = rev.post.contents.slice(1); + return [revNum.toJSNumber(), title.text, body, rev.post]; } export function getLatestCommentRevision(node: GraphNode): [number, Post] { const empty = [1, buntPost()] as [number, Post]; - if (node.children.size <= 0) { + const childSize = node?.children?.size ?? 0; + if (childSize <= 0) { return empty; } - const [revNum, rev] = [...node.children][0]; - if(!rev) { + let revNum, rev; + if (node?.children !== null) { + [revNum, rev] = [...node.children][0]; + } + if (!rev) { return empty; } return [revNum.toJSNumber(), rev.post]; } export function getComments(node: GraphNode): GraphNode { - const comments = node.children.get(bigInt(2)); + const comments = node.children?.get(bigInt(2)); if(!comments) { return { post: buntPost(), children: new BigIntOrderedMap() }; } return comments; } -export function getSnippet(body: string) { - const start = body.slice(0, body.indexOf('\n', 2)); - return (start === body || start.startsWith('![')) ? start : `${start}...`; -} +export function getSnippet(body: Content[]) { + const firstContent = body + .filter((c: Content): c is TextContent => 'text' in c).map(c => c.text)[0] ?? ''; + const newlineIdx = firstContent.indexOf('\n', 2); + const end = newlineIdx > -1 ? newlineIdx : firstContent.length; + const start = firstContent.substr(0, end); + return (start === firstContent || firstContent.startsWith('![')) ? start : `${start}...`; +} diff --git a/pkg/interface/src/logic/lib/sigil.js b/pkg/interface/src/logic/lib/sigil.js index f2444068ed..642c7cb0df 100644 --- a/pkg/interface/src/logic/lib/sigil.js +++ b/pkg/interface/src/logic/lib/sigil.js @@ -1,6 +1,6 @@ -import React, { memo } from 'react'; -import { sigil, reactRenderer } from '@tlon/sigil-js'; import { Box } from '@tlon/indigo-react'; +import { reactRenderer, sigil } from '@tlon/sigil-js'; +import React, { memo } from 'react'; export const foregroundFromBackground = (background) => { const rgb = { @@ -23,9 +23,10 @@ export const Sigil = memo( size, svgClass = '', icon = false, - padding = 0 + padding = 0, + display = 'inline-block' }) => { - const innerSize = Number(size) - 2*padding; + const innerSize = Number(size) - 2 * padding; const paddingPx = `${padding}px`; const foregroundColor = foreground ? foreground @@ -34,14 +35,14 @@ export const Sigil = memo( ) : ( { try { @@ -8,19 +11,33 @@ const isUrl = (string) => { } catch (e) { return false; } +}; + +const isRef = (str) => { + return isUrl(str) && str.startsWith('web+urbitgraph://'); +}; + +const isGroup = str => { + try { + return GROUP_REGEX.test(str); + } catch (e) { + return false; + } } +const convertToGroupRef = (group) => `web+urbitgraph://group/${group}`; + const tokenizeMessage = (text) => { let messages = []; - let message = []; + // by line + let currTextBlock = []; let isInCodeBlock = false; let endOfCodeBlock = false; text.split(/\r?\n/).forEach((line, index) => { - if (index !== 0) { - message.push('\n'); - } + // by space + let currTextLine = []; // A line of backticks enters and exits a codeblock - if (line.startsWith('```')) { + if (line.trim().startsWith('```')) { // But we need to check if we've ended a codeblock endOfCodeBlock = isInCodeBlock; isInCodeBlock = (!isInCodeBlock); @@ -29,48 +46,74 @@ const tokenizeMessage = (text) => { } if (isInCodeBlock || endOfCodeBlock) { - message.push(line); + currTextLine = [line]; } else { - line.split(/\s/).forEach((str) => { + const words = line.split(/\s/); + words.forEach((word, idx) => { + const str = isGroup(word) ? convertToGroupRef(word) : word; + + const last = words.length - 1 === idx; if ( (str.startsWith('`') && str !== '`') || (str === '`' && !isInCodeBlock) ) { isInCodeBlock = true; - } else if ( + } + if(isRef(str) && !isInCodeBlock) { + if (currTextLine.length > 0 || currTextBlock.length > 0) { + // If we're in the middle of a message, add it to the stack and reset + currTextLine.push(''); + messages.push({ text: currTextBlock.join('\n') + currTextLine.join(' ') }); + currTextBlock = last ? [''] : []; + currTextLine = []; + } + const link = parsePermalink(str); + if(!link) { + messages.push({ url: str }); + } else { + const reference = permalinkToReference(link); + messages.push(reference); + } + currTextLine = []; + } else if (isUrl(str) && !isInCodeBlock) { + if (currTextLine.length > 0 || currTextBlock.length > 0) { + // If we're in the middle of a message, add it to the stack and reset + currTextLine.push(''); + messages.push({ text: currTextBlock.join('\n') + currTextLine.join(' ') }); + currTextBlock = last ? [''] : []; + currTextLine = []; + } + messages.push({ url: str }); + currTextLine = []; + } else if(urbitOb.isValidPatp(str) && !isInCodeBlock) { + if (currTextLine.length > 0 || currTextBlock.length > 0) { + // If we're in the middle of a message, add it to the stack and reset + currTextLine.push(''); + messages.push({ text: currTextBlock.join('\n') + currTextLine.join(' ') }); + currTextBlock = last ? [''] : []; + currTextLine = []; + } + messages.push({ mention: str }); + currTextLine = []; + + } else { + currTextLine.push(str); + } + if ( (str.endsWith('`') && str !== '`') || (str === '`' && isInCodeBlock) ) { isInCodeBlock = false; } - if (isUrl(str) && !isInCodeBlock) { - if (message.length > 0) { - // If we're in the middle of a message, add it to the stack and reset - messages.push({ text: message.join(' ') }); - message = []; - } - messages.push({ url: str }); - message = []; - } else if(urbitOb.isValidPatp(str) && !isInCodeBlock) { - if (message.length > 0) { - // If we're in the middle of a message, add it to the stack and reset - messages.push({ text: message.join(' ') }); - message = []; - } - messages.push({ mention: str }); - message = []; - - } else { - message.push(str); - } }); } + currTextBlock.push(currTextLine.join(' ')) }); - if (message.length) { + if (currTextBlock.length) { // Add any remaining message - messages.push({ text: message.join(' ') }); + messages.push({ text: currTextBlock.join('\n') }); } return messages; }; diff --git a/pkg/interface/src/logic/lib/tutorialModal.ts b/pkg/interface/src/logic/lib/tutorialModal.ts index 507e9b5e15..81908f4ee5 100644 --- a/pkg/interface/src/logic/lib/tutorialModal.ts +++ b/pkg/interface/src/logic/lib/tutorialModal.ts @@ -1,5 +1,6 @@ -import { TutorialProgress, Associations } from '@urbit/api'; +import { Associations } from '@urbit/api'; import { AlignX, AlignY } from '~/logic/lib/relativePosition'; +import { TutorialProgress } from '~/types'; import { Direction } from '~/views/components/Triangle'; export const MODAL_WIDTH = 256; @@ -22,7 +23,7 @@ interface StepDetail { alignY: AlignY | AlignY[]; offsetX: number; offsetY: number; - arrow: Direction; + arrow?: Direction; } export function hasTutorialGroup(props: { associations: Associations }) { @@ -91,7 +92,7 @@ export const progressDetails: Record = { alignY: 'top', arrow: 'East', offsetX: MODAL_WIDTH + 24, - offsetY: 80, + offsetY: 80 }, channels: { title: 'Channels', @@ -156,17 +157,17 @@ export const progressDetails: Record = { alignX: 'right', arrow: 'South', offsetX: -300 + MODAL_WIDTH / 2, - offsetY: -4, + offsetY: -4 }, leap: { title: 'Leap', description: 'Leap allows you to go to a specific channel, message, collection, profile or group simply by typing in a command or selecting a shortcut from the dropdown menu.', url: `/~profile/~${window.ship}`, - alignY: "top", - alignX: "left", - arrow: "North", + alignY: 'top', + alignX: 'left', + arrow: 'North', offsetX: 76, - offsetY: -48, - }, + offsetY: -48 + } }; diff --git a/pkg/interface/src/logic/lib/useCopy.ts b/pkg/interface/src/logic/lib/useCopy.ts new file mode 100644 index 0000000000..6bccb0f536 --- /dev/null +++ b/pkg/interface/src/logic/lib/useCopy.ts @@ -0,0 +1,20 @@ +import { useCallback, useMemo, useState } from 'react'; +import { writeText } from './util'; + +export function useCopy(copied: string, display?: string) { + const [didCopy, setDidCopy] = useState(false); + const doCopy = useCallback(() => { + writeText(copied); + setDidCopy(true); + setTimeout(() => { + setDidCopy(false); + }, 2000); + }, [copied]); + + const copyDisplay = useMemo(() => (didCopy ? 'Copied' : display), [ + didCopy, + display + ]); + + return { copyDisplay, doCopy, didCopy }; +} diff --git a/pkg/interface/src/logic/lib/useDrag.ts b/pkg/interface/src/logic/lib/useDrag.ts index fb17928e96..706b9ddc95 100644 --- a/pkg/interface/src/logic/lib/useDrag.ts +++ b/pkg/interface/src/logic/lib/useDrag.ts @@ -1,4 +1,4 @@ -import { useState, useCallback, useMemo, useEffect } from 'react'; +import { useCallback, useEffect, useState } from 'react'; function validateDragEvent(e: DragEvent): FileList | File[] | true | null { const files: File[] = []; diff --git a/pkg/interface/src/logic/lib/useDropdown.ts b/pkg/interface/src/logic/lib/useDropdown.ts index 98a7359f32..eee41e5c70 100644 --- a/pkg/interface/src/logic/lib/useDropdown.ts +++ b/pkg/interface/src/logic/lib/useDropdown.ts @@ -1,4 +1,4 @@ -import { useState, useEffect, useMemo, useCallback } from 'react'; +import { useCallback, useState } from 'react'; export function useDropdown( candidates: C[], diff --git a/pkg/interface/src/logic/lib/useLazyScroll.ts b/pkg/interface/src/logic/lib/useLazyScroll.ts index 03a3ce432d..fe2d9ec101 100644 --- a/pkg/interface/src/logic/lib/useLazyScroll.ts +++ b/pkg/interface/src/logic/lib/useLazyScroll.ts @@ -1,5 +1,5 @@ -import { useEffect, RefObject, useRef, useState } from 'react'; import _ from 'lodash'; +import { RefObject, useEffect, useState } from 'react'; import usePreviousValue from './usePreviousValue'; export function distanceToBottom(el: HTMLElement) { @@ -11,6 +11,7 @@ export function distanceToBottom(el: HTMLElement) { export function useLazyScroll( ref: RefObject, + ready: boolean, margin: number, count: number, loadMore: () => Promise @@ -41,10 +42,15 @@ export function useLazyScroll( }, [count]); useEffect(() => { - if (!ref.current) { + if(!ready) { + setIsDone(false); + } + }, [ready]); + + useEffect(() => { + if (!ref.current || isDone || !ready) { return; } - setIsDone(false); const scroll = ref.current; loadUntil(scroll); @@ -58,7 +64,7 @@ export function useLazyScroll( return () => { ref.current?.removeEventListener('scroll', onScroll); }; - }, [ref?.current, count]); + }, [ref?.current, ready, isDone]); return { isDone, isLoading }; } diff --git a/pkg/interface/src/logic/lib/useLocalStorageState.ts b/pkg/interface/src/logic/lib/useLocalStorageState.ts index 13de71afe3..5e35a6a761 100644 --- a/pkg/interface/src/logic/lib/useLocalStorageState.ts +++ b/pkg/interface/src/logic/lib/useLocalStorageState.ts @@ -1,4 +1,4 @@ -import { useState, useCallback, useEffect } from 'react'; +import { useCallback, useEffect, useState } from 'react'; function retrieve(key: string, initial: T): T { const s = localStorage.getItem(key); @@ -15,7 +15,8 @@ function retrieve(key: string, initial: T): T { interface SetStateFunc { (t: T): T; } -type SetState = T | SetStateFunc; +// See microsoft/typescript#37663 for filed bug +type SetState = T extends any ? SetStateFunc : never; export function useLocalStorageState(key: string, initial: T) { const [state, _setState] = useState(() => retrieve(key, initial)); diff --git a/pkg/interface/src/logic/lib/useModal.tsx b/pkg/interface/src/logic/lib/useModal.tsx index 3d19343cd7..9edbf011ab 100644 --- a/pkg/interface/src/logic/lib/useModal.tsx +++ b/pkg/interface/src/logic/lib/useModal.tsx @@ -1,19 +1,13 @@ -import React, { - useState, - ReactNode, - useCallback, - SyntheticEvent, - useMemo, - useEffect, - useRef -} from 'react'; - import { Box } from '@tlon/indigo-react'; -import { useOutsideClick } from './useOutsideClick'; +import React, { + ReactNode, + useCallback, + useMemo, + useRef, useState +} from 'react'; +import { PropFunc } from '~/types'; import { ModalOverlay } from '~/views/components/ModalOverlay'; import { Portal } from '~/views/components/Portal'; -import { ModalPortal } from '~/views/components/ModalPortal'; -import { PropFunc } from '@urbit/api'; type ModalFunc = (dismiss: () => void) => JSX.Element; interface UseModalProps { @@ -63,7 +57,7 @@ export function useModal(props: UseModalProps & PropFunc): UseModalR display="flex" alignItems="stretch" flexDirection="column" - spacing="2" + spacing={2} dismiss={dismiss} {...rest} > diff --git a/pkg/interface/src/logic/lib/useOutsideClick.ts b/pkg/interface/src/logic/lib/useOutsideClick.ts index 481fa2a904..0ffabd2a04 100644 --- a/pkg/interface/src/logic/lib/useOutsideClick.ts +++ b/pkg/interface/src/logic/lib/useOutsideClick.ts @@ -1,4 +1,4 @@ -import { useEffect, RefObject } from 'react'; +import { RefObject, useEffect } from 'react'; export function useOutsideClick( ref: RefObject, diff --git a/pkg/interface/src/logic/lib/usePreviousValue.ts b/pkg/interface/src/logic/lib/usePreviousValue.ts index 109f9ffa86..e4ee5a8f51 100644 --- a/pkg/interface/src/logic/lib/usePreviousValue.ts +++ b/pkg/interface/src/logic/lib/usePreviousValue.ts @@ -1,5 +1,5 @@ import { useRef } from 'react'; -import { Primitive } from '@urbit/api'; +import { Primitive } from '~/types'; export default function usePreviousValue(value: T): T { const prev = useRef(null); diff --git a/pkg/interface/src/logic/lib/useQuery.ts b/pkg/interface/src/logic/lib/useQuery.ts index 735060a2da..142a15affd 100644 --- a/pkg/interface/src/logic/lib/useQuery.ts +++ b/pkg/interface/src/logic/lib/useQuery.ts @@ -1,30 +1,46 @@ -import { useMemo, useCallback } from 'react'; -import { useLocation } from 'react-router-dom'; import _ from 'lodash'; +import { useCallback, useMemo } from 'react'; +import { useLocation } from 'react-router-dom'; + +function mergeQuery(search: URLSearchParams, added: Record) { + _.forIn(added, (v, k) => { + if (v) { + search.append(k, v); + } else { + search.delete(k); + } + }); +} export function useQuery() { - const { search } = useLocation(); + const { search, pathname } = useLocation(); const query = useMemo(() => new URLSearchParams(search), [search]); const appendQuery = useCallback( - (q: Record) => { - const newQuery = new URLSearchParams(search); - _.forIn(q, (value, key) => { - if (!value) { - newQuery.delete(key); - } else { - newQuery.append(key, value); - } - }); - - return newQuery.toString(); + (added: Record) => { + const q = new URLSearchParams(search); + mergeQuery(q, added); + return q.toString(); }, [search] ); + const toQuery = useCallback( + (params: Record, path = pathname) => { + const q = new URLSearchParams(search); + mergeQuery(q, params); + return { + pathname: path, + search: q.toString() + }; + }, + [search, pathname] + ); + return { query, - appendQuery + appendQuery, + toQuery }; } diff --git a/pkg/interface/src/logic/lib/useRunIO.ts b/pkg/interface/src/logic/lib/useRunIO.ts new file mode 100644 index 0000000000..fadb377dba --- /dev/null +++ b/pkg/interface/src/logic/lib/useRunIO.ts @@ -0,0 +1,51 @@ +import { useEffect, useState } from 'react'; +import { unstable_batchedUpdates } from 'react-dom'; + +export type IOInstance = ( + input: I +) => (props: P) => Promise<[(p: P) => boolean, O]>; + +export function useRunIO( + io: (i: I) => Promise, + after: (o: O) => void, + key: string +): (i: I) => Promise { + const [resolve, setResolve] = useState<() => void>(() => () => {}); + const [reject, setReject] = useState<(e: any) => void>(() => () => {}); + const [output, setOutput] = useState(null); + const [done, setDone] = useState(false); + const run = (i: I) => + new Promise((res, rej) => { + setResolve(() => res); + setReject(() => rej); + io(i) + .then((o) => { + unstable_batchedUpdates(() => { + setOutput(o); + setDone(true); + }); + }) + .catch(rej); + }); + + useEffect(() => { + reject(new Error('useRunIO: key changed')); + setDone(false); + setOutput(null); + }, [key]); + + useEffect(() => { + if (!done) { + return; + } + try { + after(output!); + resolve(); + } catch (e) { + reject(e); + } + }, [done]); + + return run; +} + diff --git a/pkg/interface/src/logic/lib/useStatelessAsyncClickable.ts b/pkg/interface/src/logic/lib/useStatelessAsyncClickable.ts index 154f903baa..9a640b6cb4 100644 --- a/pkg/interface/src/logic/lib/useStatelessAsyncClickable.ts +++ b/pkg/interface/src/logic/lib/useStatelessAsyncClickable.ts @@ -1,11 +1,11 @@ -import { MouseEvent, useCallback, useState, useEffect } from 'react'; +import { MouseEvent, useCallback, useEffect, useState } from 'react'; export type AsyncClickableState = 'waiting' | 'error' | 'loading' | 'success'; export function useStatelessAsyncClickable( onClick: (e: MouseEvent) => Promise, name: string ) { - const [state, setState] = useState('waiting'); + const [state, setState] = useState('waiting'); const handleClick = useCallback( async (e: MouseEvent) => { try { diff --git a/pkg/interface/src/logic/lib/useStorage.ts b/pkg/interface/src/logic/lib/useStorage.ts index 64f9bb4505..33026add9e 100644 --- a/pkg/interface/src/logic/lib/useStorage.ts +++ b/pkg/interface/src/logic/lib/useStorage.ts @@ -1,22 +1,16 @@ -import { useCallback, useMemo, useEffect, useRef, useState } from 'react'; -import { - GcpState, - S3State, - StorageState -} from '../../types'; import S3 from 'aws-sdk/clients/s3'; -import GcpClient from './GcpClient'; -import { StorageClient, StorageAcl } from './StorageClient'; -import { dateToDa, deSig } from './util'; +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import useStorageState from '../state/storage'; - +import GcpClient from './GcpClient'; +import { StorageAcl, StorageClient } from './StorageClient'; +import { dateToDa, deSig } from './util'; export interface IuseStorage { canUpload: boolean; upload: (file: File, bucket: string) => Promise; uploadDefault: (file: File) => Promise; uploading: boolean; - promptUpload: () => Promise; + promptUpload: () => Promise; } const useStorage = ({ accept = '*' } = { accept: '*' }): IuseStorage => { @@ -54,7 +48,7 @@ const useStorage = ({ accept = '*' } = { accept: '*' }): IuseStorage => { ); const upload = useCallback( - async (file: File, bucket: string) => { + async (file: File, bucket: string): Promise => { if (client.current === null) { throw new Error('Storage not ready'); } @@ -83,7 +77,7 @@ const useStorage = ({ accept = '*' } = { accept: '*' }): IuseStorage => { [client, setUploading] ); - const uploadDefault = useCallback(async (file: File) => { + const uploadDefault = useCallback(async (file: File): Promise => { if (s3.configuration.currentBucket === '') { throw new Error('current bucket not set'); } @@ -91,7 +85,7 @@ const useStorage = ({ accept = '*' } = { accept: '*' }): IuseStorage => { }, [s3, upload]); const promptUpload = useCallback( - () => { + (): Promise => { return new Promise((resolve, reject) => { const fileSelector = document.createElement('input'); fileSelector.setAttribute('type', 'file'); @@ -101,10 +95,10 @@ const useStorage = ({ accept = '*' } = { accept: '*' }): IuseStorage => { const files = fileSelector.files; if (!files || files.length <= 0) { reject(); - return; + } else { + uploadDefault(files[0]).then(resolve); + document.body.removeChild(fileSelector); } - uploadDefault(files[0]).then(resolve); - document.body.removeChild(fileSelector); }); document.body.appendChild(fileSelector); fileSelector.click(); @@ -113,7 +107,7 @@ const useStorage = ({ accept = '*' } = { accept: '*' }): IuseStorage => { [uploadDefault] ); - return {canUpload, upload, uploadDefault, uploading, promptUpload}; + return { canUpload, upload, uploadDefault, uploading, promptUpload }; }; export default useStorage; diff --git a/pkg/interface/src/logic/lib/useToggleState.ts b/pkg/interface/src/logic/lib/useToggleState.ts new file mode 100644 index 0000000000..b38a18043d --- /dev/null +++ b/pkg/interface/src/logic/lib/useToggleState.ts @@ -0,0 +1,11 @@ +import { useCallback, useState } from 'react'; + +export function useToggleState(initial: boolean) { + const [state, setState] = useState(initial); + + const toggle = useCallback(() => { + setState(s => !s); + }, [setState]); + + return [state, toggle] as const; +} diff --git a/pkg/interface/src/logic/lib/useWaitForProps.ts b/pkg/interface/src/logic/lib/useWaitForProps.ts index 8a81b0b0ae..6d32901b58 100644 --- a/pkg/interface/src/logic/lib/useWaitForProps.ts +++ b/pkg/interface/src/logic/lib/useWaitForProps.ts @@ -1,4 +1,4 @@ -import { useState, useEffect, useCallback } from 'react'; +import { useCallback, useEffect, useState } from 'react'; export function useWaitForProps

(props: P, timeout = 0) { const [resolve, setResolve] = useState<() => void>(() => () => {}); diff --git a/pkg/interface/src/logic/lib/util.ts b/pkg/interface/src/logic/lib/util.tsx similarity index 79% rename from pkg/interface/src/logic/lib/util.ts rename to pkg/interface/src/logic/lib/util.tsx index e0c92ea9cc..1dc01f9cf2 100644 --- a/pkg/interface/src/logic/lib/util.ts +++ b/pkg/interface/src/logic/lib/util.tsx @@ -1,14 +1,13 @@ -import { useEffect, useState } from 'react'; -import _ from 'lodash'; -import f, { compose, memoize } from 'lodash/fp'; -import bigInt, { BigInteger } from 'big-integer'; +/* eslint-disable max-lines */ import { Association, Contact } from '@urbit/api'; -import useLocalState from '../state/local'; -import produce, { enableMapSet } from 'immer'; +import anyAscii from 'any-ascii'; +import bigInt, { BigInteger } from 'big-integer'; +import { enableMapSet } from 'immer'; +import _ from 'lodash'; +import f from 'lodash/fp'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import { IconRef } from '~/types'; import useSettingsState from '../state/settings'; -import { State, UseStore } from 'zustand'; -import { Cage } from '~/types/cage'; -import { BaseState } from '../state/base'; enableMapSet(); @@ -23,11 +22,18 @@ export const MOMENT_CALENDAR_DATE = { sameElse: '~YYYY.M.D' }; -export const getModuleIcon = (mod: string) => { - if (mod === 'link') { +export type GraphModule = 'link' | 'post' | 'chat' | 'publish'; + +export const getModuleIcon = (mod: GraphModule): IconRef => { + if (mod === 'link') { return 'Collection'; } - return _.capitalize(mod); + + if (mod === 'post') { + return 'Dashboard'; + } + + return _.capitalize(mod) as IconRef; }; export function wait(ms: number) { @@ -36,10 +42,6 @@ export function wait(ms: number) { }); } -export function appIsGraph(app: string) { - return app === 'publish' || app == 'link'; -} - export function parentPath(path: string) { return _.dropRight(path.split('/'), 1).join('/'); } @@ -57,10 +59,20 @@ export function daToUnix(da: BigInteger) { } export function unixToDa(unix: number) { - const timeSinceEpoch = bigInt(unix).multiply(DA_SECOND).divide(bigInt(1000)); + const timeSinceEpoch = bigInt(unix).multiply(DA_SECOND).divide(bigInt(1000)); return DA_UNIX_EPOCH.add(timeSinceEpoch); } +export function dmCounterparty(resource: string) { + const [,,ship,name] = resource.split('/'); + return ship === `~${window.ship}` ? `~${name.slice(4)}` : ship; +} + +export function isDm(resource: string) { + const [,,,name] = resource.split('/'); + return name.startsWith('dm--'); +} + export function makePatDa(patda: string) { return bigInt(udToDec(patda)); } @@ -160,9 +172,9 @@ export function dateToDa(d: Date, mil = false) { ); } -export function deSig(ship: string) { +export function deSig(ship: string): string { if (!ship) { - return null; + return ''; } return ship.replace('~', ''); } @@ -180,13 +192,16 @@ export function uxToHex(ux: string) { export const hexToUx = (hex) => { const ux = f.flow( f.chunk(4), - f.map(x => _.dropWhile(x, y => y === 0).join('')), + // eslint-disable-next-line prefer-arrow-callback + f.map(x => _.dropWhile(x, function(y: unknown) { + return y === 0; + }).join('')), f.join('.') )(hex.split('')); return `0x${ux}`; }; -export function writeText(str: string) { +export function writeText(str: string | null): Promise { return new Promise((resolve, reject) => { const range = document.createRange(); range.selectNodeContents(document.body); @@ -211,11 +226,11 @@ export function writeText(str: string) { } // trim patps to match dojo, chat-cli -export function cite(ship: string) { +export function cite(ship: string): string { let patp = ship, shortened = ''; if (patp === null || patp === '') { - return null; + return ''; } if (patp.startsWith('~')) { patp = patp.substr(1); @@ -307,7 +322,7 @@ export function stringToTa(str: string) { export function amOwnerOfGroup(groupPath: string) { if (!groupPath) -return false; + return false; const groupOwner = /(\/~)?\/~([a-z-]{3,})\/.*/.exec(groupPath)?.[2]; return window.ship === groupOwner; } @@ -326,11 +341,12 @@ export function getContactDetails(contact: any) { } export function stringToSymbol(str: string) { + const ascii = anyAscii(str); let result = ''; - for (let i = 0; i < str.length; i++) { - const n = str.charCodeAt(i); + for (let i = 0; i < ascii.length; i++) { + const n = ascii.charCodeAt(i); if ((n >= 97 && n <= 122) || (n >= 48 && n <= 57)) { - result += str[i]; + result += ascii[i]; } else if (n >= 65 && n <= 90) { result += String.fromCharCode(n + 32); } else { @@ -344,7 +360,6 @@ export function stringToSymbol(str: string) { } return result; } - /** * Formats a numbers as a `@ud` inserting dot where needed */ @@ -362,7 +377,7 @@ export function numToUd(num: number) { export function usePreventWindowUnload(shouldPreventDefault: boolean, message = 'You have unsaved changes. Are you sure you want to exit?') { useEffect(() => { if (!shouldPreventDefault) -return; + return; const handleBeforeUnload = (event) => { event.preventDefault(); return message; @@ -378,13 +393,14 @@ return; } export function pluralize(text: string, isPlural = false, vowel = false) { - return isPlural ? `${text}s`: `${vowel ? 'an' : 'a'} ${text}`; + return isPlural ? `${text}s` : `${vowel ? 'an' : 'a'} ${text}`; } // Hide is an optional second parameter for when this function is used in class components export function useShowNickname(contact: Contact | null, hide?: boolean): boolean { - const hideNicknames = typeof hide !== 'undefined' ? hide : useSettingsState(state => state.calm.hideNicknames); - return !!(contact && contact.nickname && !hideNicknames); + const hideState = useSettingsState(state => state.calm.hideNicknames); + const hideNicknames = typeof hide !== 'undefined' ? hide : hideState; + return Boolean(contact && contact.nickname && !hideNicknames); } interface useHoveringInterface { @@ -397,22 +413,32 @@ interface useHoveringInterface { export const useHovering = (): useHoveringInterface => { const [hovering, setHovering] = useState(false); - const bind = { - onMouseOver: () => setHovering(true), - onMouseLeave: () => setHovering(false) - }; - return { hovering, bind }; + const onMouseOver = useCallback(() => setHovering(true), []); + const onMouseLeave = useCallback(() => setHovering(false), []); + const bind = useMemo(() => ({ + onMouseOver, + onMouseLeave + }), [onMouseLeave, onMouseOver]); + + return useMemo(() => ({ hovering, bind }), [hovering, bind]); }; +export function withHovering(Component: React.ComponentType) { + return React.forwardRef((props, ref) => { + const { hovering, bind } = useHovering(); + return + }) +} + const DM_REGEX = /ship\/~([a-z]|-)*\/dm--/; -export function getItemTitle(association: Association) { - if(DM_REGEX.test(association.resource)) { - const [,,ship,name] = association.resource.split('/'); - if(ship.slice(1) === window.ship) { +export function getItemTitle(association: Association): string { + if (DM_REGEX.test(association.resource)) { + const [, , ship, name] = association.resource.split('/'); + if (ship.slice(1) === window.ship) { return cite(`~${name.slice(4)}`); } return cite(ship); } - return association.metadata.title || association.resource; + return association.metadata.title ?? association.resource ?? ''; } diff --git a/pkg/interface/src/logic/lib/virtualContext.tsx b/pkg/interface/src/logic/lib/virtualContext.tsx index fb22350c86..61117bfc21 100644 --- a/pkg/interface/src/logic/lib/virtualContext.tsx +++ b/pkg/interface/src/logic/lib/virtualContext.tsx @@ -1,9 +1,10 @@ import React, { - useContext, - useState, - useCallback, - useLayoutEffect, -} from "react"; + useCallback, useContext, + + useEffect, useState +} from 'react'; +import { Primitive } from '~/types'; +import usePreviousValue from './usePreviousValue'; export interface VirtualContextProps { save: () => void; @@ -11,7 +12,7 @@ export interface VirtualContextProps { } const fallback: VirtualContextProps = { save: () => {}, - restore: () => {}, + restore: () => {} }; export const VirtualContext = React.createContext(fallback); @@ -23,7 +24,7 @@ export function useVirtual() { export const withVirtual =

(Component: React.ComponentType

) => React.forwardRef((props: P, ref) => ( - {(context) => } + {context => } )); @@ -39,9 +40,22 @@ export function useVirtualResizeState(s: boolean) { [_setState, save] ); - useLayoutEffect(() => { - restore(); + useEffect(() => { + requestAnimationFrame(restore); }, [state]); return [state, setState] as const; } + +export function useVirtualResizeProp(prop: Primitive) { + const { save, restore } = useVirtual(); + const oldProp = usePreviousValue(prop); + + if(prop !== oldProp) { + save(); + } + + useEffect(() => { + requestAnimationFrame(restore); + }, [prop]); +} diff --git a/pkg/interface/src/logic/lib/withState.tsx b/pkg/interface/src/logic/lib/withState.tsx index 6ada75c616..80fdce4c15 100644 --- a/pkg/interface/src/logic/lib/withState.tsx +++ b/pkg/interface/src/logic/lib/withState.tsx @@ -1,7 +1,6 @@ -import React from "react"; -import { ReactElement } from "react"; -import { UseStore } from "zustand"; -import { BaseState } from "../state/base"; +import React from 'react'; +import { UseStore } from 'zustand'; +import { BaseState } from '../state/base'; const withStateo = < StateType extends BaseState @@ -16,19 +15,21 @@ const withStateo = < (object, key) => ({ ...object, [key]: state[key] }), {} ) ) : useState(); - return - }) + return ; + }); }; -const withState = < - StateType extends BaseState, - stateKey extends keyof StateType - >( +interface StatePicker extends Array { + 0: UseStore; + 1?: string[]; +} + +const withState = ( Component: any, - stores: ([UseStore, stateKey[]])[], + stores: StatePicker[] ) => { return React.forwardRef((props, ref) => { - let stateProps: unknown = {}; + const stateProps: unknown = {}; stores.forEach(([store, keys]) => { const storeProps = Array.isArray(keys) ? store(state => keys.reduce( @@ -37,8 +38,8 @@ const withState = < : store(); Object.assign(stateProps, storeProps); }); - return + return ; }); -} +}; -export default withState; \ No newline at end of file +export default withState; diff --git a/pkg/interface/src/logic/lib/workspace.ts b/pkg/interface/src/logic/lib/workspace.ts index f13df61956..2d8a93b12c 100644 --- a/pkg/interface/src/logic/lib/workspace.ts +++ b/pkg/interface/src/logic/lib/workspace.ts @@ -1,4 +1,5 @@ -import { Associations, Workspace } from '@urbit/api'; +import { Associations } from '@urbit/api'; +import { Workspace } from '~/types'; export function getTitleFromWorkspace( associations: Associations, @@ -17,10 +18,10 @@ export function getTitleFromWorkspace( export function getGroupFromWorkspace( workspace: Workspace -): string | undefined { +): string { if (workspace.type === 'group') { return workspace.group; } - return undefined; + return ''; } diff --git a/pkg/interface/src/logic/reducers/connection.ts b/pkg/interface/src/logic/reducers/connection.ts index 958465433d..529376c04b 100644 --- a/pkg/interface/src/logic/reducers/connection.ts +++ b/pkg/interface/src/logic/reducers/connection.ts @@ -1,6 +1,5 @@ -import _ from 'lodash'; -import { StoreState } from '../../store/type'; import { Cage } from '~/types/cage'; +import { StoreState } from '../store/type'; type LocalState = Pick; diff --git a/pkg/interface/src/logic/reducers/contact-update.ts b/pkg/interface/src/logic/reducers/contact-update.ts index 160aff708e..c998d584f2 100644 --- a/pkg/interface/src/logic/reducers/contact-update.ts +++ b/pkg/interface/src/logic/reducers/contact-update.ts @@ -1,11 +1,7 @@ -import _ from 'lodash'; -import { compose } from 'lodash/fp'; - import { ContactUpdate } from '@urbit/api'; - -import useContactState, { ContactState } from '../state/contact'; +import _ from 'lodash'; import { reduceState } from '../state/base'; - +import useContactState, { ContactState } from '../state/contact'; export const ContactReducer = (json) => { const data: ContactUpdate = _.get(json, 'contact-update', false); @@ -69,7 +65,7 @@ const edit = (json: ContactUpdate, state: ContactState): ContactState => { } const value = data['edit-field'][field]; - + if(field === 'add-group') { state.contacts[ship].groups.push(value); } else if (field === 'remove-group') { diff --git a/pkg/interface/src/logic/reducers/gcp-reducer.ts b/pkg/interface/src/logic/reducers/gcp-reducer.ts index c7280d6cf8..f809c1a9f6 100644 --- a/pkg/interface/src/logic/reducers/gcp-reducer.ts +++ b/pkg/interface/src/logic/reducers/gcp-reducer.ts @@ -1,43 +1,32 @@ -import _ from 'lodash'; -import {StoreState} from '../store/type'; -import {GcpToken} from '../../types/gcp-state'; -import { Cage } from '~/types/cage'; -import useStorageState, { StorageState } from '../state/storage'; +import type { Cage } from '~/types/cage'; +import type { GcpToken } from '../../types/gcp-state'; import { reduceState } from '../state/base'; +import useStorageState, { StorageState } from '../state/storage'; export default class GcpReducer { reduce(json: Cage) { reduceState(useStorageState, json, [ - reduceConfigured, reduceToken ]); } } -const reduceConfigured = (json, state: StorageState): StorageState => { - let data = json['gcp-configured']; - if (data !== undefined) { - state.gcp.configured = data; - } - return state; -} - const reduceToken = (json: Cage, state: StorageState): StorageState => { - let data = json['gcp-token']; + const data = json['gcp-token']; if (data) { - state = setToken(data, state); + setToken(data, state); } return state; -} +}; const setToken = (data: any, state: StorageState): StorageState => { if (isToken(data)) { state.gcp.token = data; } return state; -} +}; -const isToken = (token: any): boolean => { +const isToken = (token: any): token is GcpToken => { return (typeof(token.accessKey) === 'string' && typeof(token.expiresIn) === 'number'); -} +}; diff --git a/pkg/interface/src/logic/reducers/graph-update.ts b/pkg/interface/src/logic/reducers/graph-update.ts index 21022f9820..b049119131 100644 --- a/pkg/interface/src/logic/reducers/graph-update.ts +++ b/pkg/interface/src/logic/reducers/graph-update.ts @@ -1,80 +1,87 @@ +import { GraphNode } from '@urbit/api'; +import BigIntOrderedMap from '@urbit/api/lib/BigIntOrderedMap'; +import bigInt, { BigInteger } from 'big-integer'; +import produce from 'immer'; import _ from 'lodash'; -import { BigIntOrderedMap } from "~/logic/lib/BigIntOrderedMap"; -import bigInt, { BigInteger } from "big-integer"; -import useGraphState, { GraphState } from '../state/graph'; import { reduceState } from '../state/base'; +import useGraphState, { GraphState } from '../state/graph'; export const GraphReducer = (json) => { const data = _.get(json, 'graph-update', false); - + if (data) { reduceState(useGraphState, data, [ keys, addGraph, removeGraph, addNodes, - removeNodes + removePosts ]); } + const loose = _.get(json, 'graph-update-loose', false); + if(loose) { + reduceState(useGraphState, loose, [addNodesLoose]); + } +}; + +const addNodesLoose = (json: any, state: GraphState): GraphState => { + const data = _.get(json, 'add-nodes', false); + if(data) { + const { resource: { ship, name }, nodes } = data; + const resource = `${ship}/${name}`; + + const indices = _.get(state.looseNodes, [resource], {}); + _.forIn(nodes, (node, index) => { + indices[index] = processNode(node); + }); + _.set(state.looseNodes, [resource], indices); + } + return state; }; const keys = (json, state: GraphState): GraphState => { const data = _.get(json, 'keys', false); if (data) { state.graphKeys = new Set(data.map((res) => { - let resource = res.ship + '/' + res.name; + const resource = res.ship + '/' + res.name; return resource; })); } return state; }; +const processNode = (node) => { + // is empty + if (!node.children) { + return produce(node, (draft: GraphNode) => { + draft.children = new BigIntOrderedMap(); + }); + } + + // is graph + return produce(node, (draft: GraphNode) => { + draft.children = new BigIntOrderedMap() + .gas(_.map(draft.children, (item, idx) => + [bigInt(idx), processNode(item)] as [BigInteger, any] + )); + }); +}; + const addGraph = (json, state: GraphState): GraphState => { - - const _processNode = (node) => { - // is empty - if (!node.children) { - node.children = new BigIntOrderedMap(); - return node; - } - - // is graph - let converted = new BigIntOrderedMap(); - for (let idx in node.children) { - let item = node.children[idx]; - let index = bigInt(idx); - - converted.set( - index, - _processNode(item) - ); - } - node.children = converted; - return node; - }; - const data = _.get(json, 'add-graph', false); if (data) { if (!('graphs' in state)) { state.graphs = {}; } - let resource = data.resource.ship + '/' + data.resource.name; + const resource = data.resource.ship + '/' + data.resource.name; state.graphs[resource] = new BigIntOrderedMap(); state.graphTimesentMap[resource] = {}; + state.graphs[resource] = state.graphs[resource].gas(Object.keys(data.graph).map((idx) => { + return [bigInt(idx), processNode(data.graph[idx])]; + })); - for (let idx in data.graph) { - let item = data.graph[idx]; - let index = bigInt(idx); - - let node = _processNode(item); - - state.graphs[resource].set( - index, - node - ); - } state.graphKeys.add(resource); } return state; @@ -83,11 +90,10 @@ const addGraph = (json, state: GraphState): GraphState => { const removeGraph = (json, state: GraphState): GraphState => { const data = _.get(json, 'remove-graph', false); if (data) { - if (!('graphs' in state)) { state.graphs = {}; } - let resource = data.ship + '/' + data.name; + const resource = data.ship + '/' + data.name; state.graphKeys.delete(resource); delete state.graphs[resource]; } @@ -95,10 +101,10 @@ const removeGraph = (json, state: GraphState): GraphState => { }; const mapifyChildren = (children) => { - return new BigIntOrderedMap( + return new BigIntOrderedMap().gas( _.map(children, (node, idx) => { idx = idx && idx.startsWith('/') ? idx.slice(1) : idx; - const nd = {...node, children: mapifyChildren(node.children || {}) }; + const nd = { ...node, children: mapifyChildren(node.children || {}) }; return [bigInt(idx), nd]; })); }; @@ -107,29 +113,29 @@ const addNodes = (json, state) => { const _addNode = (graph, index, node) => { // set child of graph if (index.length === 1) { - graph.set(index[0], node); - return graph; + return graph.set(index[0], node); } // set parent of graph - let parNode = graph.get(index[0]); + const parNode = graph.get(index[0]); if (!parNode) { console.error('parent node does not exist, cannot add child'); - return; + return graph; } - parNode.children = _addNode(parNode.children, index.slice(1), node); - graph.set(index[0], parNode); - return graph; + return graph.set(index[0], produce(parNode, (draft) => { + draft.children = _addNode(draft.children, index.slice(1), node); + })); }; const _remove = (graph, index) => { if (index.length === 1) { - graph.delete(index[0]); + return graph.delete(index[0]); } else { const child = graph.get(index[0]); if (child) { - child.children = _remove(child.children, index.slice(1)); - graph.set(index[0], child); + return graph.set(index[0], produce(child, (draft) => { + draft.children = _remove(draft.children, index.slice(1)); + })); } } @@ -138,17 +144,18 @@ const addNodes = (json, state) => { const _killByFuzzyTimestamp = (graph, resource, timestamp) => { if (state.graphTimesentMap[resource][timestamp]) { - let index = state.graphTimesentMap[resource][timestamp]; + const index = state.graphTimesentMap[resource][timestamp]; - if (index.split('/').length === 0) { return; } - let indexArr = index.split('/').slice(1).map((ind) => { + if (index.split('/').length === 0) { + return graph; +} + const indexArr = index.split('/').slice(1).map((ind) => { return bigInt(ind); }); - graph = _remove(graph, indexArr); delete state.graphTimesentMap[resource][timestamp]; + return _remove(graph, indexArr); } - return graph; }; @@ -160,16 +167,17 @@ const addNodes = (json, state) => { graph = _killByFuzzyTimestamp(graph, resource, post['time-sent']); graph = _killByFuzzyTimestamp(graph, resource, post['time-sent'] - 1); graph = _killByFuzzyTimestamp(graph, resource, post['time-sent'] + 1); - return graph; }; const data = _.get(json, 'add-nodes', false); if (data) { - if (!('graphs' in state)) { return state; } + if (!('graphs' in state)) { + return state; +} - let resource = data.resource.ship + '/' + data.resource.name; - if (!(resource in state.graphs)) { + const resource = data.resource.ship + '/' + data.resource.name; + if (!(resource in state.graphs)) { state.graphs[resource] = new BigIntOrderedMap(); } @@ -178,72 +186,93 @@ const addNodes = (json, state) => { } state.graphKeys.add(resource); - - let indices = Array.from(Object.keys(data.nodes)); + + const indices = Array.from(Object.keys(data.nodes)); indices.sort((a, b) => { - let aArr = a.split('/'); - let bArr = b.split('/'); - return bArr.length < aArr.length; + const aArr = a.split('/'); + const bArr = b.split('/'); + return aArr.length - bArr.length; }); - let graph = state.graphs[resource]; - indices.forEach((index) => { - let node = data.nodes[index]; - graph = _removePending(graph, node.post, resource); - - if (index.split('/').length === 0) { return; } - let indexArr = index.split('/').slice(1).map((ind) => { + const node = data.nodes[index]; + const old = state.graphs[resource].size; + state.graphs[resource] = _removePending(state.graphs[resource], node.post, resource); + const newSize = state.graphs[resource].size; + + if (index.split('/').length === 0) { + return; +} + const indexArr = index.split('/').slice(1).map((ind) => { return bigInt(ind); }); - if (indexArr.length === 0) { return state; } + if (indexArr.length === 0) { + return state; +} if (node.post.pending) { state.graphTimesentMap[resource][node.post['time-sent']] = index; } - node.children = mapifyChildren(node?.children || {}); - - graph = _addNode( - graph, + state.graphs[resource] = _addNode( + state.graphs[resource], indexArr, - node + produce(node, (draft) => { + draft.children = mapifyChildren(draft?.children || {}); + }) ); - + if(newSize !== old) { + console.log(`${resource}, (${old}, ${newSize}, ${state.graphs[resource].size})`); + } }); - - state.graphs[resource] = graph; } return state; }; -const removeNodes = (json, state: GraphState): GraphState => { +const removePosts = (json, state: GraphState): GraphState => { const _remove = (graph, index) => { + const child = graph.get(index[0]); if (index.length === 1) { - graph.delete(index[0]); + if (child) { + return graph.set(index[0], { + post: child.post.hash || '', + children: child.children + }); + } + } else { + if (child) { + _remove(child.children, index.slice(1)); + return graph.set(index[0], child); } else { const child = graph.get(index[0]); if (child) { - _remove(child.children, index.slice(1)); - graph.set(index[0], child); + return graph.set(index[0], produce((draft) => { + draft.children = _remove(draft.children, index.slice(1)); + })); } + return graph; } + } }; - const data = _.get(json, 'remove-nodes', false); + const data = _.get(json, 'remove-posts', false); if (data) { const { ship, name } = data.resource; const res = `${ship}/${name}`; - if (!(res in state.graphs)) { return state; } + if (!(res in state.graphs)) { + return state; +} data.indices.forEach((index) => { - if (index.split('/').length === 0) { return; } - let indexArr = index.split('/').slice(1).map((ind) => { + if (index.split('/').length === 0) { + return; +} + const indexArr = index.split('/').slice(1).map((ind) => { return bigInt(ind); }); - _remove(state.graphs[res], indexArr); + state.graphs[res] = _remove(state.graphs[res], indexArr); }); } return state; diff --git a/pkg/interface/src/logic/reducers/group-update.ts b/pkg/interface/src/logic/reducers/group-update.ts index 6b8c25c92d..eb4d242ae5 100644 --- a/pkg/interface/src/logic/reducers/group-update.ts +++ b/pkg/interface/src/logic/reducers/group-update.ts @@ -1,21 +1,16 @@ +import { Enc } from '@urbit/api'; +import { + Group, + + GroupPolicy, GroupUpdate, + + InvitePolicy, InvitePolicyDiff, OpenPolicy, OpenPolicyDiff, Tags +} from '@urbit/api/groups'; import _ from 'lodash'; import { Cage } from '~/types/cage'; -import { - GroupUpdate, - Group, - Tags, - GroupPolicy, - GroupPolicyDiff, - OpenPolicyDiff, - OpenPolicy, - InvitePolicyDiff, - InvitePolicy -} from '@urbit/api/groups'; -import { Enc, PatpNoSig } from '@urbit/api'; import { resourceAsPath } from '../lib/util'; -import useGroupState, { GroupState } from '../state/group'; -import { compose } from 'lodash/fp'; import { reduceState } from '../state/base'; +import useGroupState, { GroupState } from '../state/group'; function decodeGroup(group: Enc): Group { const members = new Set(group.members); @@ -43,7 +38,7 @@ function decodePolicy(policy: Enc): GroupPolicy { function decodeTags(tags: Enc): Tags { return _.reduce( tags, - (acc, ships, key): Tags => { + (acc, ships: any, key): Tags => { if (key.search(/\\/) === -1) { acc.role[key] = new Set(ships); return acc; @@ -71,11 +66,10 @@ export default class GroupReducer { addGroup, removeGroup, changePolicy, - expose, + expose ]); } } - } const initial = (json: GroupUpdate, state: GroupState): GroupState => { const data = json['initial']; @@ -83,7 +77,7 @@ const initial = (json: GroupUpdate, state: GroupState): GroupState => { state.groups = _.mapValues(data, decodeGroup); } return state; -} +}; const initialGroup = (json: GroupUpdate, state: GroupState): GroupState => { if ('initialGroup' in json) { @@ -92,7 +86,7 @@ const initialGroup = (json: GroupUpdate, state: GroupState): GroupState => { state.groups[path] = decodeGroup(group); } return state; -} +}; const addGroup = (json: GroupUpdate, state: GroupState): GroupState => { if ('addGroup' in json) { @@ -106,7 +100,7 @@ const addGroup = (json: GroupUpdate, state: GroupState): GroupState => { }; } return state; -} +}; const removeGroup = (json: GroupUpdate, state: GroupState): GroupState => { if('removeGroup' in json) { @@ -115,7 +109,7 @@ const removeGroup = (json: GroupUpdate, state: GroupState): GroupState => { delete state.groups[resourcePath]; } return state; -} +}; const addMembers = (json: GroupUpdate, state: GroupState): GroupState => { if ('addMembers' in json) { @@ -125,14 +119,14 @@ const addMembers = (json: GroupUpdate, state: GroupState): GroupState => { state.groups[resourcePath].members.add(member); if ( 'invite' in state.groups[resourcePath].policy && - state.groups[resourcePath].policy.invite.pending.has(member) + state.groups[resourcePath].policy['invite'].pending.has(member) ) { - state.groups[resourcePath].policy.invite.pending.delete(member) + state.groups[resourcePath].policy['invite'].pending.delete(member); } } } return state; -} +}; const removeMembers = (json: GroupUpdate, state: GroupState): GroupState => { if ('removeMembers' in json) { @@ -143,7 +137,7 @@ const removeMembers = (json: GroupUpdate, state: GroupState): GroupState => { } } return state; -} +}; const addTag = (json: GroupUpdate, state: GroupState): GroupState => { if ('addTag' in json) { @@ -159,7 +153,7 @@ const addTag = (json: GroupUpdate, state: GroupState): GroupState => { _.set(tags, tagAccessors, tagged); } return state; -} +}; const removeTag = (json: GroupUpdate, state: GroupState): GroupState => { if ('removeTag' in json) { @@ -179,7 +173,7 @@ const removeTag = (json: GroupUpdate, state: GroupState): GroupState => { _.set(tags, tagAccessors, tagged); } return state; -} +}; const changePolicy = (json: GroupUpdate, state: GroupState): GroupState => { if ('changePolicy' in json && state) { @@ -197,7 +191,7 @@ const changePolicy = (json: GroupUpdate, state: GroupState): GroupState => { } } return state; -} +}; const expose = (json: GroupUpdate, state: GroupState): GroupState => { if( 'expose' in json && state) { @@ -206,7 +200,7 @@ const expose = (json: GroupUpdate, state: GroupState): GroupState => { state.groups[resourcePath].hidden = false; } return state; -} +}; const inviteChangePolicy = (diff: InvitePolicyDiff, policy: InvitePolicy) => { if ('addInvites' in diff) { @@ -222,7 +216,7 @@ const inviteChangePolicy = (diff: InvitePolicyDiff, policy: InvitePolicy) => { } else { console.log('bad policy change'); } -} +}; const openChangePolicy = (diff: OpenPolicyDiff, policy: OpenPolicy) => { if ('allowRanks' in diff) { @@ -248,4 +242,4 @@ const openChangePolicy = (diff: OpenPolicyDiff, policy: OpenPolicy) => { } else { console.log('bad policy change'); } -} +}; diff --git a/pkg/interface/src/logic/reducers/group-view.ts b/pkg/interface/src/logic/reducers/group-view.ts index b6f5ec5725..fae75a0a6b 100644 --- a/pkg/interface/src/logic/reducers/group-view.ts +++ b/pkg/interface/src/logic/reducers/group-view.ts @@ -1,5 +1,4 @@ import { GroupUpdate } from '@urbit/api/groups'; -import { resourceAsPath } from '~/logic/lib/util'; import { reduceState } from '../state/base'; import useGroupState, { GroupState } from '../state/group'; @@ -11,11 +10,20 @@ const initial = (json: any, state: GroupState): GroupState => { return state; }; +const started = (json: any, state: GroupState): GroupState => { + const data = json.started; + if(data) { + const { resource, request } = data; + state.pendingJoin[resource] = request; + } + return state; +}; + const progress = (json: any, state: GroupState): GroupState => { const data = json.progress; if(data) { const { progress, resource } = data; - state.pendingJoin = { ...state.pendingJoin, [resource]: progress }; + state.pendingJoin[resource].progress = progress; if(progress === 'done') { setTimeout(() => { delete state.pendingJoin[resource]; @@ -25,12 +33,22 @@ const progress = (json: any, state: GroupState): GroupState => { return state; }; +const hide = (json: any, state: GroupState) => { + const data = json.hide; + if(data) { + state.pendingJoin[data].hidden = true; + } + return state; +}; + export const GroupViewReducer = (json: any) => { const data = json['group-view-update']; if (data) { reduceState(useGroupState, data, [ progress, + hide, + started, initial ]); } -}; \ No newline at end of file +}; diff --git a/pkg/interface/src/logic/reducers/hark-update.ts b/pkg/interface/src/logic/reducers/hark-update.ts index aa80250738..50978b4e3a 100644 --- a/pkg/interface/src/logic/reducers/hark-update.ts +++ b/pkg/interface/src/logic/reducers/hark-update.ts @@ -1,23 +1,19 @@ import { - Notifications, NotifIndex, - NotificationGraphConfig, - GroupNotificationsConfig, - UnreadStats, Timebox } from '@urbit/api'; -import { makePatDa } from '~/logic/lib/util'; +import BigIntOrderedMap from '@urbit/api/lib/BigIntOrderedMap'; +import { BigInteger } from 'big-integer'; import _ from 'lodash'; -import { StoreState } from '../store/type'; -import { BigIntOrderedMap } from '../lib/BigIntOrderedMap'; -import useHarkState, { HarkState } from '../state/hark'; import { compose } from 'lodash/fp'; +import { makePatDa } from '~/logic/lib/util'; import { reduceState } from '../state/base'; +import useHarkState, { HarkState } from '../state/hark'; export const HarkReducer = (json: any) => { const data = _.get(json, 'harkUpdate', false); if (data) { - reduce(data); + reduceState(useHarkState, data, [reduce]); } const graphHookData = _.get(json, 'hark-graph-hook-update', false); if (graphHookData) { @@ -26,7 +22,7 @@ export const HarkReducer = (json: any) => { graphIgnore, graphListen, graphWatchSelf, - graphMentions, + graphMentions ]); } const groupHookData = _.get(json, 'hark-group-hook-update', false); @@ -34,13 +30,14 @@ export const HarkReducer = (json: any) => { reduceState(useHarkState, groupHookData, [ groupInitial, groupListen, - groupIgnore, + groupIgnore ]); } }; -function reduce(data) { - reduceState(useHarkState, data, [ +function reduce(data, state) { + const reducers = [ + calculateCount, unread, read, archive, @@ -55,8 +52,34 @@ function reduce(data) { unreadEach, seenIndex, removeGraph, - readAll, - ]); + readAll + ]; + const reducer = compose(reducers.map(r => (s) => { + return r(data, s); + })); + return reducer(state); +} + +function calculateCount(json: any, state: HarkState) { + let count = 0; + _.forEach(state.unreads.graph, (graphs) => { + _.forEach(graphs, (graph) => { + if (typeof graph?.notifications === 'object') { + count += graph?.notifications.length; + } else { + count += 0; + } + }); + }); + _.forEach(state.unreads.group, (group) => { + if (typeof group?.notifications === 'object') { + count += group?.notifications.length; + } else { + count += 0; + } + }); + state.notificationsCount = count; + return state; } function groupInitial(json: any, state: HarkState): HarkState { @@ -132,7 +155,7 @@ function graphWatchSelf(json: any, state: HarkState): HarkState { function readAll(json: any, state: HarkState): HarkState { const data = _.get(json, 'read-all'); - if(data) { + if(data) { clearState(state); } return state; @@ -156,7 +179,7 @@ function seenIndex(json: any, state: HarkState): HarkState { function readEach(json: any, state: HarkState): HarkState { const data = _.get(json, 'read-each'); - if(data) { + if (data) { updateUnreads(state, data.index, u => u.delete(data.target)); } return state; @@ -189,13 +212,17 @@ function unreadEach(json: any, state: HarkState): HarkState { function unreads(json: any, state: HarkState): HarkState { const data = _.get(json, 'unreads'); if(data) { + clearState(state); data.forEach(({ index, stats }) => { const { unreads, notifications, last } = stats; - updateNotificationStats(state, index, 'notifications', x => x + notifications); updateNotificationStats(state, index, 'last', () => last); + _.each(notifications, ({ time, index }) => { + addNotificationToUnread(state, index, makePatDa(time)); + }); if('count' in unreads) { updateUnreadCount(state, index, (u = 0) => u + unreads.count); } else { + updateUnreads(state, index, s => new Set()); unreads.each.forEach((u: string) => { updateUnreads(state, index, s => s.add(u)); }); @@ -205,7 +232,7 @@ function unreads(json: any, state: HarkState): HarkState { return state; } -function clearState(state: HarkState) { +function clearState(state: HarkState): HarkState { const initialState = { notifications: new BigIntOrderedMap(), archivedNotifications: new BigIntOrderedMap(), @@ -221,10 +248,9 @@ function clearState(state: HarkState) { }, notificationsCount: 0 }; + Object.assign(state, initialState); - Object.keys(initialState).forEach((key) => { - state[key] = initialState[key]; - }); + return state; } function updateUnreadCount(state: HarkState, index: NotifIndex, count: (c: number) => number): HarkState { @@ -242,26 +268,59 @@ function updateUnreads(state: HarkState, index: NotifIndex, f: (us: Set) if(!('graph' in index)) { return state; } - const unreads = _.get(state.unreads.graph, [index.graph.graph, index.graph.index, 'unreads'], new Set()); - const oldSize = unreads.size; + const unreads: any = _.get(state.unreads.graph, [index.graph.graph, index.graph.index, 'unreads'], new Set()); f(unreads); - const newSize = unreads.size; + _.set(state.unreads.graph, [index.graph.graph, index.graph.index, 'unreads'], unreads); return state; } -function updateNotificationStats(state: HarkState, index: NotifIndex, statField: 'notifications' | 'unreads' | 'last', f: (x: number) => number): HarkState { - if(statField === 'notifications') { - state.notificationsCount = f(state.notificationsCount); - } +function addNotificationToUnread(state: HarkState, index: NotifIndex, time: BigInteger) { if('graph' in index) { - const curr = _.get(state.unreads.graph, [index.graph.graph, index.graph.index, statField], 0); - _.set(state.unreads.graph, [index.graph.graph, index.graph.index, statField], f(curr)); - } else if('group' in index) { - const curr = _.get(state.unreads.group, [index.group, statField], 0); - _.set(state.unreads.group, [index.group, statField], f(curr)); + const path = [index.graph.graph, index.graph.index, 'notifications']; + const curr = _.get(state.unreads.graph, path, []); + _.set(state.unreads.graph, path, + [ + ...curr.filter(c => !(c.time.eq(time) && notifIdxEqual(c.index, index))), + { time, index } + ] + ); + } else if ('group' in index) { + const path = [index.group.group, 'notifications']; + const curr = _.get(state.unreads.group, path, []); + _.set(state.unreads.group, path, + [ + ...curr.filter(c => !(c.time.eq(time) && notifIdxEqual(c.index, index))), + { time, index } + ] + ); } - return state; +} + +function removeNotificationFromUnread(state: HarkState, index: NotifIndex, time: BigInteger) { + if('graph' in index) { + const path = [index.graph.graph, index.graph.index, 'notifications']; + const curr = _.get(state.unreads.graph, path, []); + _.set(state.unreads.graph, path, + curr.filter(c => !(c.time.eq(time) && notifIdxEqual(c.index, index))) + ); + } else if ('group' in index) { + const path = [index.group.group, 'notifications']; + const curr = _.get(state.unreads.group, path, []); + _.set(state.unreads.group, path, + curr.filter(c => !(c.time.eq(time) && notifIdxEqual(c.index, index))) + ); + } +} + +function updateNotificationStats(state: HarkState, index: NotifIndex, statField: 'unreads' | 'last', f: (x: number) => number) { + if('graph' in index) { + const curr: any = _.get(state.unreads.graph, [index.graph.graph, index.graph.index, statField], 0); + _.set(state.unreads.graph, [index.graph.graph, index.graph.index, statField], f(curr)); + } else if('group' in index) { + const curr: any = _.get(state.unreads.group, [index.group.group, statField], 0); + _.set(state.unreads.group, [index.group.group, statField], f(curr)); + } } function added(json: any, state: HarkState): HarkState { @@ -270,20 +329,16 @@ function added(json: any, state: HarkState): HarkState { const { index, notification } = data; const time = makePatDa(data.time); const timebox = state.notifications.get(time) || []; + addNotificationToUnread(state, index, time); const arrIdx = timebox.findIndex(idxNotif => notifIdxEqual(index, idxNotif.index) ); if (arrIdx !== -1) { - if (timebox[arrIdx]?.notification?.read) { - // TODO this is additive, and with a persistent state it keeps incrementing - state = updateNotificationStats(state, index, 'notifications', x => x+1); - } timebox[arrIdx] = { index, notification }; - state.notifications.set(time, timebox); + state.notifications = state.notifications.set(time, timebox); } else { - state = updateNotificationStats(state, index, 'notifications', x => x+1); - state.notifications.set(time, [...timebox, { index, notification }]); + state.notifications = state.notifications.set(time, [...timebox, { index, notification }]); } } return state; @@ -302,7 +357,7 @@ const timebox = (json: any, state: HarkState): HarkState => { if (data) { const time = makePatDa(data.time); if (!data.archive) { - state.notifications.set(time, data.notifications); + state.notifications = state.notifications.set(time, data.notifications); } } return state; @@ -311,7 +366,9 @@ const timebox = (json: any, state: HarkState): HarkState => { function more(json: any, state: HarkState): HarkState { const data = _.get(json, 'more', false); if (data) { - _.forEach(data, d => reduce(d)); + _.forEach(data, (d) => { + reduce(d, state); + }); } return state; } @@ -353,7 +410,7 @@ function setRead( return state; } timebox[arrIdx].notification.read = read; - state.notifications.set(patDa, timebox); + state.notifications = state.notifications.set(patDa, timebox); return state; } @@ -361,7 +418,7 @@ function read(json: any, state: HarkState): HarkState { const data = _.get(json, 'read-note', false); if (data) { const { time, index } = data; - updateNotificationStats(state, index, 'notifications', x => x-1); + removeNotificationFromUnread(state, index, makePatDa(time)); setRead(time, index, true, state); } return state; @@ -371,7 +428,7 @@ function unread(json: any, state: HarkState): HarkState { const data = _.get(json, 'unread-note', false); if (data) { const { time, index } = data; - updateNotificationStats(state, index, 'notifications', x => x+1); + addNotificationToUnread(state, index, makePatDa(time)); setRead(time, index, false, state); } return state; @@ -381,6 +438,7 @@ function archive(json: any, state: HarkState): HarkState { const data = _.get(json, 'archive', false); if (data) { const { index } = data; + removeNotificationFromUnread(state, index, makePatDa(data.time)); const time = makePatDa(data.time); const timebox = state.notifications.get(time); if (!timebox) { @@ -392,12 +450,10 @@ function archive(json: any, state: HarkState): HarkState { ); if(unarchived.length === 0) { console.log('deleting entire timebox'); - state.notifications.delete(time); + state.notifications = state.notifications.delete(time); } else { - state.notifications.set(time, unarchived); + state.notifications = state.notifications.set(time, unarchived); } - const newlyRead = archived.filter(x => !x.notification.read).length; - updateNotificationStats(state, index, 'notifications', x => x - newlyRead); } return state; } diff --git a/pkg/interface/src/logic/reducers/invite-update.ts b/pkg/interface/src/logic/reducers/invite-update.ts index 2dff0e740f..8ab979b860 100644 --- a/pkg/interface/src/logic/reducers/invite-update.ts +++ b/pkg/interface/src/logic/reducers/invite-update.ts @@ -1,11 +1,8 @@ -import _ from 'lodash'; -import { compose } from 'lodash/fp'; - import { InviteUpdate } from '@urbit/api/invite'; - +import _ from 'lodash'; import { Cage } from '~/types/cage'; -import useInviteState, { InviteState } from '../state/invite'; import { reduceState } from '../state/base'; +import useInviteState, { InviteState } from '../state/invite'; export default class InviteReducer { reduce(json: Cage) { @@ -17,7 +14,7 @@ export default class InviteReducer { deleteInvite, invite, accepted, - decline, + decline ]); } } @@ -29,7 +26,7 @@ const initial = (json: InviteUpdate, state: InviteState): InviteState => { state.invites = data; } return state; -} +}; const create = (json: InviteUpdate, state: InviteState): InviteState => { const data = _.get(json, 'create', false); @@ -37,7 +34,7 @@ const create = (json: InviteUpdate, state: InviteState): InviteState => { state.invites[data] = {}; } return state; -} +}; const deleteInvite = (json: InviteUpdate, state: InviteState): InviteState => { const data = _.get(json, 'delete', false); @@ -45,7 +42,7 @@ const deleteInvite = (json: InviteUpdate, state: InviteState): InviteState => { delete state.invites[data]; } return state; -} +}; const invite = (json: InviteUpdate, state: InviteState): InviteState => { const data = _.get(json, 'invite', false); @@ -53,7 +50,7 @@ const invite = (json: InviteUpdate, state: InviteState): InviteState => { state.invites[data.term][data.uid] = data.invite; } return state; -} +}; const accepted = (json: InviteUpdate, state: InviteState): InviteState => { const data = _.get(json, 'accepted', false); @@ -61,7 +58,7 @@ const accepted = (json: InviteUpdate, state: InviteState): InviteState => { delete state.invites[data.term][data.uid]; } return state; -} +}; const decline = (json: InviteUpdate, state: InviteState): InviteState => { const data = _.get(json, 'decline', false); @@ -69,4 +66,4 @@ const decline = (json: InviteUpdate, state: InviteState): InviteState => { delete state.invites[data.term][data.uid]; } return state; -} \ No newline at end of file +}; diff --git a/pkg/interface/src/logic/reducers/launch-update.ts b/pkg/interface/src/logic/reducers/launch-update.ts index 598731bf82..441a3b0a05 100644 --- a/pkg/interface/src/logic/reducers/launch-update.ts +++ b/pkg/interface/src/logic/reducers/launch-update.ts @@ -1,9 +1,8 @@ import _ from 'lodash'; -import { LaunchUpdate, WeatherState } from '~/types/launch-update'; import { Cage } from '~/types/cage'; -import useLaunchState, { LaunchState } from '../state/launch'; -import { compose } from 'lodash/fp'; +import { LaunchUpdate, WeatherState } from '~/types/launch-update'; import { reduceState } from '../state/base'; +import useLaunchState, { LaunchState } from '../state/launch'; export default class LaunchReducer { reduce(json: Cage) { @@ -14,29 +13,36 @@ export default class LaunchReducer { changeFirstTime, changeOrder, changeFirstTime, - changeIsShown, + changeIsShown ]); } - const weatherData: WeatherState = _.get(json, 'weather', false); + const weatherData: WeatherState | boolean | Record = _.get(json, 'weather', false); if (weatherData) { - useLaunchState.getState().set(state => { + useLaunchState.getState().set((state) => { state.weather = weatherData; }); } const locationData = _.get(json, 'location', false); if (locationData) { - useLaunchState.getState().set(state => { + useLaunchState.getState().set((state) => { state.userLocation = locationData; }); } const baseHash = _.get(json, 'baseHash', false); if (baseHash) { - useLaunchState.getState().set(state => { + useLaunchState.getState().set((state) => { state.baseHash = baseHash; - }) + }); + } + + const runtimeLag = _.get(json, 'runtimeLag', null); + if (runtimeLag !== null) { + useLaunchState.getState().set(state => { + state.runtimeLag = runtimeLag; + }); } } } @@ -44,12 +50,12 @@ export default class LaunchReducer { export const initial = (json: LaunchUpdate, state: LaunchState): LaunchState => { const data = _.get(json, 'initial', false); if (data) { - Object.keys(data).forEach(key => { + Object.keys(data).forEach((key) => { state[key] = data[key]; }); } return state; -} +}; export const changeFirstTime = (json: LaunchUpdate, state: LaunchState): LaunchState => { const data = _.get(json, 'changeFirstTime', false); @@ -57,7 +63,7 @@ export const changeFirstTime = (json: LaunchUpdate, state: LaunchState): LaunchS state.firstTime = data; } return state; -} +}; export const changeOrder = (json: LaunchUpdate, state: LaunchState): LaunchState => { const data = _.get(json, 'changeOrder', false); @@ -65,7 +71,7 @@ export const changeOrder = (json: LaunchUpdate, state: LaunchState): LaunchState state.tileOrdering = data; } return state; -} +}; export const changeIsShown = (json: LaunchUpdate, state: LaunchState): LaunchState => { const data = _.get(json, 'changeIsShown', false); @@ -76,4 +82,4 @@ export const changeIsShown = (json: LaunchUpdate, state: LaunchState): LaunchSta } } return state; -} \ No newline at end of file +}; diff --git a/pkg/interface/src/logic/reducers/metadata-update.ts b/pkg/interface/src/logic/reducers/metadata-update.ts index 6001bc6bc4..9049a19f38 100644 --- a/pkg/interface/src/logic/reducers/metadata-update.ts +++ b/pkg/interface/src/logic/reducers/metadata-update.ts @@ -1,11 +1,8 @@ -import _ from 'lodash'; -import { compose } from 'lodash/fp'; - import { MetadataUpdate } from '@urbit/api/metadata'; - +import _ from 'lodash'; import { Cage } from '~/types/cage'; -import useMetadataState, { MetadataState } from '../state/metadata'; import { reduceState } from '../state/base'; +import useMetadataState, { MetadataState } from '../state/metadata'; export default class MetadataReducer { reduce(json: Cage) { @@ -16,7 +13,7 @@ export default class MetadataReducer { add, update, remove, - groupInitial, + groupInitial ]); } } @@ -25,10 +22,10 @@ export default class MetadataReducer { const groupInitial = (json: MetadataUpdate, state: MetadataState): MetadataState => { const data = _.get(json, 'initial-group', false); if(data) { - state = associations(data, state); + associations(data, state); } return state; -} +}; const associations = (json: MetadataUpdate, state: MetadataState): MetadataState => { const data = _.get(json, 'associations', false); @@ -50,7 +47,7 @@ const associations = (json: MetadataUpdate, state: MetadataState): MetadataState state.associations = metadata; } return state; -} +}; const add = (json: MetadataUpdate, state: MetadataState): MetadataState => { const data = _.get(json, 'add', false); @@ -70,7 +67,7 @@ const add = (json: MetadataUpdate, state: MetadataState): MetadataState => { state.associations = metadata; } return state; -} +}; const update = (json: MetadataUpdate, state: MetadataState): MetadataState => { const data = _.get(json, 'update-metadata', false); @@ -90,7 +87,7 @@ const update = (json: MetadataUpdate, state: MetadataState): MetadataState => { state.associations = metadata; } return state; -} +}; const remove = (json: MetadataUpdate, state: MetadataState): MetadataState => { const data = _.get(json, 'remove', false); @@ -105,4 +102,4 @@ const remove = (json: MetadataUpdate, state: MetadataState): MetadataState => { state.associations = metadata; } return state; -} +}; diff --git a/pkg/interface/src/logic/reducers/s3-update.ts b/pkg/interface/src/logic/reducers/s3-update.ts index 538f12d5cb..b67ff3d607 100644 --- a/pkg/interface/src/logic/reducers/s3-update.ts +++ b/pkg/interface/src/logic/reducers/s3-update.ts @@ -1,11 +1,9 @@ import _ from 'lodash'; -import { compose } from 'lodash/fp'; import { Cage } from '~/types/cage'; import { S3Update } from '~/types/s3-update'; import { reduceState } from '../state/base'; import useStorageState, { StorageState } from '../state/storage'; - export default class S3Reducer { reduce(json: Cage) { const data = _.get(json, 's3-update', false); @@ -18,7 +16,7 @@ export default class S3Reducer { removeBucket, endpoint, accessKeyId, - secretAccessKey, + secretAccessKey ]); } } @@ -30,7 +28,7 @@ const credentials = (json: S3Update, state: StorageState): StorageState => { state.s3.credentials = data; } return state; -} +}; const configuration = (json: S3Update, state: StorageState): StorageState => { const data = _.get(json, 'configuration', false); @@ -41,7 +39,7 @@ const configuration = (json: S3Update, state: StorageState): StorageState => { }; } return state; -} +}; const currentBucket = (json: S3Update, state: StorageState): StorageState => { const data = _.get(json, 'setCurrentBucket', false); @@ -49,7 +47,7 @@ const currentBucket = (json: S3Update, state: StorageState): StorageState => { state.s3.configuration.currentBucket = data; } return state; -} +}; const addBucket = (json: S3Update, state: StorageState): StorageState => { const data = _.get(json, 'addBucket', false); @@ -58,7 +56,7 @@ const addBucket = (json: S3Update, state: StorageState): StorageState => { state.s3.configuration.buckets.add(data); } return state; -} +}; const removeBucket = (json: S3Update, state: StorageState): StorageState => { const data = _.get(json, 'removeBucket', false); @@ -66,7 +64,7 @@ const removeBucket = (json: S3Update, state: StorageState): StorageState => { state.s3.configuration.buckets.delete(data); } return state; -} +}; const endpoint = (json: S3Update, state: StorageState): StorageState => { const data = _.get(json, 'setEndpoint', false); @@ -74,7 +72,7 @@ const endpoint = (json: S3Update, state: StorageState): StorageState => { state.s3.credentials.endpoint = data; } return state; -} +}; const accessKeyId = (json: S3Update , state: StorageState): StorageState => { const data = _.get(json, 'setAccessKeyId', false); @@ -82,7 +80,7 @@ const accessKeyId = (json: S3Update , state: StorageState): StorageState => { state.s3.credentials.accessKeyId = data; } return state; -} +}; const secretAccessKey = (json: S3Update, state: StorageState): StorageState => { const data = _.get(json, 'setSecretAccessKey', false); @@ -90,4 +88,4 @@ const secretAccessKey = (json: S3Update, state: StorageState): StorageState => { state.s3.credentials.secretAccessKey = data; } return state; -} \ No newline at end of file +}; diff --git a/pkg/interface/src/logic/reducers/settings-update.ts b/pkg/interface/src/logic/reducers/settings-update.ts index 1d43f517b0..1a5e165bb0 100644 --- a/pkg/interface/src/logic/reducers/settings-update.ts +++ b/pkg/interface/src/logic/reducers/settings-update.ts @@ -1,25 +1,25 @@ +import { SettingsUpdate } from '@urbit/api/settings'; import _ from 'lodash'; -import useSettingsState, { SettingsState } from "~/logic/state/settings"; -import { SettingsUpdate } from '@urbit/api/dist/settings'; +import useSettingsState, { SettingsState } from '~/logic/state/settings'; import { reduceState } from '../state/base'; export default class SettingsReducer { reduce(json: any) { - let data = json["settings-event"]; + let data = json['settings-event']; if (data) { reduceState(useSettingsState, data, [ this.putBucket, this.delBucket, this.putEntry, - this.delEntry, + this.delEntry ]); } - data = json["settings-data"]; + data = json['settings-data']; if (data) { reduceState(useSettingsState, data, [ this.getAll, this.getBucket, - this.getEntry, + this.getEntry ]); } } @@ -27,7 +27,7 @@ export default class SettingsReducer { putBucket(json: SettingsUpdate, state: SettingsState): SettingsState { const data = _.get(json, 'put-bucket', false); if (data) { - state[data["bucket-key"]] = data.bucket; + state[data['bucket-key']] = data.bucket; } return state; } @@ -40,21 +40,21 @@ export default class SettingsReducer { return state; } - putEntry(json: SettingsUpdate, state: SettingsState): SettingsState { - const data = _.get(json, 'put-entry', false); + putEntry(json: SettingsUpdate, state: any): SettingsState { + const data: Record = _.get(json, 'put-entry', false); if (data) { - if (!state[data["bucket-key"]]) { - state[data["bucket-key"]] = {}; + if (!state[data['bucket-key']]) { + state[data['bucket-key']] = {}; } - state[data["bucket-key"]][data["entry-key"]] = data.value; + state[data['bucket-key']][data['entry-key']] = data.value; } return state; } - delEntry(json: SettingsUpdate, state: SettingsState): SettingsState { + delEntry(json: SettingsUpdate, state: any): SettingsState { const data = _.get(json, 'del-entry', false); if (data) { - delete state[data["bucket-key"]][data["entry-key"]]; + delete state[data['bucket-key']][data['entry-key']]; } return state; } @@ -62,7 +62,7 @@ export default class SettingsReducer { getAll(json: any, state: SettingsState): SettingsState { const data = _.get(json, 'all'); if(data) { - _.mergeWith(state, data, (obj, src) => _.isArray(src) ? src : undefined) + _.mergeWith(state, data, (obj, src) => _.isArray(src) ? src : undefined); } return state; } @@ -76,7 +76,7 @@ export default class SettingsReducer { return state; } - getEntry(json: any, state: SettingsState) { + getEntry(json: any, state: any) { const bucketKey = _.get(json, 'bucket-key', false); const entryKey = _.get(json, 'entry-key', false); const entry = _.get(json, 'entry', false); diff --git a/pkg/interface/src/logic/state/base.ts b/pkg/interface/src/logic/state/base.ts index e1add80e3c..c987df77a7 100644 --- a/pkg/interface/src/logic/state/base.ts +++ b/pkg/interface/src/logic/state/base.ts @@ -1,20 +1,15 @@ -import produce from "immer"; -import { compose } from "lodash/fp"; -import create, { State, UseStore } from "zustand"; -import { persist } from "zustand/middleware"; +import produce, { setAutoFreeze } from 'immer'; +import { compose } from 'lodash/fp'; +import create, { State, UseStore } from 'zustand'; +import { persist } from 'zustand/middleware'; +setAutoFreeze(false); export const stateSetter = ( fn: (state: StateType) => void, set ): void => { - // fn = (state: StateType) => { - // // TODO this is a stub for the store debugging - // fn(state); - // } - return set(fn); - // TODO we want to use the below, but it makes everything read-only - return set(produce(fn)); + set(produce(fn)); }; export const reduceState = < @@ -25,40 +20,39 @@ export const reduceState = < data: UpdateType, reducers: ((data: UpdateType, state: StateType) => StateType)[] ): void => { - const oldState = state.getState(); - const reducer = compose(reducers.map(reducer => reducer.bind(reducer, data))); - const newState = reducer(oldState); - state.getState().set(state => state = newState); + const reducer = compose(reducers.map(r => sta => r(data, sta))); + state.getState().set((state) => { + reducer(state); + }); }; export let stateStorageKeys: string[] = []; export const stateStorageKey = (stateName: string) => { - stateName = `Landcape${stateName}State`; + stateName = `Landscape${stateName}State`; stateStorageKeys = [...new Set([...stateStorageKeys, stateName])]; return stateName; }; (window as any).clearStates = () => { - stateStorageKeys.forEach(key => { + stateStorageKeys.forEach((key) => { localStorage.removeItem(key); }); -} +}; export interface BaseState extends State { set: (fn: (state: StateType) => void) => void; } -export const createState = >( +export const createState = >( name: string, - properties: Omit, + properties: { [K in keyof Omit]: T[K] }, blacklist: string[] = [] -): UseStore => create(persist((set, get) => ({ - // TODO why does this typing break? +): UseStore => create(persist((set, get) => ({ set: fn => stateSetter(fn, set), - ...properties + ...properties as any }), { blacklist, name: stateStorageKey(name), - version: 1, // TODO version these according to base hash -})); \ No newline at end of file + version: process.env.LANDSCAPE_SHORTHASH as any +})); diff --git a/pkg/interface/src/logic/state/contact.ts b/pkg/interface/src/logic/state/contact.ts index eaceac3e6c..c28ccb996d 100644 --- a/pkg/interface/src/logic/state/contact.ts +++ b/pkg/interface/src/logic/state/contact.ts @@ -1,18 +1,18 @@ -import { Patp, Rolodex, Scry } from "@urbit/api"; - -import { BaseState, createState } from "./base"; +import { Contact, Patp, Rolodex } from '@urbit/api'; +import { useCallback } from 'react'; +import { BaseState, createState } from './base'; export interface ContactState extends BaseState { contacts: Rolodex; isContactPublic: boolean; nackedContacts: Set; // fetchIsAllowed: (entity, name, ship, personal) => Promise; -}; +} const useContactState = createState('Contact', { contacts: {}, nackedContacts: new Set(), - isContactPublic: false, + isContactPublic: false // fetchIsAllowed: async ( // entity, // name, @@ -28,4 +28,14 @@ const useContactState = createState('Contact', { // }, }, ['nackedContacts']); -export default useContactState; \ No newline at end of file +export function useContact(ship: string) { + return useContactState( + useCallback(s => s.contacts[ship] as Contact | null, [ship]) + ); +} + +export function useOurContact() { + return useContact(`~${window.ship}`); +} + +export default useContactState; diff --git a/pkg/interface/src/logic/state/graph.ts b/pkg/interface/src/logic/state/graph.ts index 2f8aab6920..569efc2164 100644 --- a/pkg/interface/src/logic/state/graph.ts +++ b/pkg/interface/src/logic/state/graph.ts @@ -1,10 +1,15 @@ -import { Graphs, decToUd, numToUd } from "@urbit/api"; - -import { BaseState, createState } from "./base"; +import { Association, deSig, GraphNode, Graphs, resourceFromPath } from '@urbit/api'; +import { useCallback } from 'react'; +import { BaseState, createState } from './base'; export interface GraphState extends BaseState { graphs: Graphs; graphKeys: Set; + looseNodes: { + [graph: string]: { + [index: string]: GraphNode; + } + }; pendingIndices: Record; graphTimesentMap: Record; // getKeys: () => Promise; @@ -16,13 +21,14 @@ export interface GraphState extends BaseState { // getYoungerSiblings: (ship: string, resource: string, count: number, index?: string) => Promise; // getGraphSubset: (ship: string, resource: string, start: string, end: string) => Promise; // getNode: (ship: string, resource: string, index: string) => Promise; -}; +} const useGraphState = createState('Graph', { graphs: {}, graphKeys: new Set(), + looseNodes: {}, pendingIndices: {}, - graphTimesentMap: {}, + graphTimesentMap: {} // getKeys: async () => { // const api = useApi(); // const keys = await api.scry({ @@ -122,6 +128,20 @@ const useGraphState = createState('Graph', { // }); // graphReducer(node); // }, -}, ['graphs', 'graphKeys']); +}, ['graphs', 'graphKeys', 'looseNodes', 'graphTimesentMap']); -export default useGraphState; \ No newline at end of file +export function useGraph(ship: string, name: string) { + return useGraphState( + useCallback(s => s.graphs[`${deSig(ship)}/${name}`], [ship, name]) + ); +} + +export function useGraphForAssoc(association: Association) { + const { resource } = association; + const { ship, name } = resourceFromPath(resource); + return useGraph(ship, name); +} + +window.useGraphState = useGraphState; + +export default useGraphState; diff --git a/pkg/interface/src/logic/state/group.ts b/pkg/interface/src/logic/state/group.ts index 218567ce20..bd6c478eb2 100644 --- a/pkg/interface/src/logic/state/group.ts +++ b/pkg/interface/src/logic/state/group.ts @@ -1,15 +1,27 @@ -import { Path, JoinRequests } from "@urbit/api"; - -import { BaseState, createState } from "./base"; +import { Association, Group, JoinRequests } from '@urbit/api'; +import { useCallback } from 'react'; +import { BaseState, createState } from './base'; export interface GroupState extends BaseState { - groups: Set; + groups: { + [group: string]: Group; + } pendingJoin: JoinRequests; -}; +} const useGroupState = createState('Group', { - groups: new Set(), - pendingJoin: {}, + groups: {}, + pendingJoin: {} }, ['groups']); -export default useGroupState; \ No newline at end of file +export function useGroup(group: string) { + return useGroupState(useCallback(s => s.groups[group] as Group | undefined, [group])); +} + +export function useGroupForAssoc(association: Association) { + return useGroupState( + useCallback(s => s.groups[association.group] as Group | undefined, [association]) + ); +} + +export default useGroupState; diff --git a/pkg/interface/src/logic/state/hark.ts b/pkg/interface/src/logic/state/hark.ts index d02402c098..792844d719 100644 --- a/pkg/interface/src/logic/state/hark.ts +++ b/pkg/interface/src/logic/state/hark.ts @@ -1,8 +1,7 @@ -import { NotificationGraphConfig, Timebox, Unreads, dateToDa } from "@urbit/api"; -import BigIntOrderedMap from "@urbit/api/lib/BigIntOrderedMap"; - +import { NotificationGraphConfig, Timebox, Unreads } from '@urbit/api'; +import BigIntOrderedMap from '@urbit/api/lib/BigIntOrderedMap'; // import { harkGraphHookReducer, harkGroupHookReducer, harkReducer } from "~/logic/subscription/hark"; -import { BaseState, createState } from "./base"; +import { BaseState, createState } from './base'; export const HARK_FETCH_MORE_COUNT = 3; @@ -15,9 +14,9 @@ export interface HarkState extends BaseState { notifications: BigIntOrderedMap; notificationsCount: number; notificationsGraphConfig: NotificationGraphConfig; // TODO unthread this everywhere - notificationsGroupConfig: []; // TODO type this + notificationsGroupConfig: string[]; unreads: Unreads; -}; +} const useHarkState = createState('Hark', { archivedNotifications: new BigIntOrderedMap(), @@ -62,8 +61,7 @@ const useHarkState = createState('Hark', { unreads: { graph: {}, group: {} - }, + } }, ['notifications', 'archivedNotifications', 'unreads', 'notificationsCount']); - export default useHarkState; diff --git a/pkg/interface/src/logic/state/invite.ts b/pkg/interface/src/logic/state/invite.ts index 4da81eb53a..0aab291b32 100644 --- a/pkg/interface/src/logic/state/invite.ts +++ b/pkg/interface/src/logic/state/invite.ts @@ -3,10 +3,10 @@ import { BaseState, createState } from './base'; export interface InviteState extends BaseState { invites: Invites; -}; +} const useInviteState = createState('Invite', { - invites: {}, + invites: {} }); -export default useInviteState; \ No newline at end of file +export default useInviteState; diff --git a/pkg/interface/src/logic/state/launch.ts b/pkg/interface/src/logic/state/launch.ts index 14f2113e58..adeb8a721a 100644 --- a/pkg/interface/src/logic/state/launch.ts +++ b/pkg/interface/src/logic/state/launch.ts @@ -1,7 +1,5 @@ -import { Tile, WeatherState } from "~/types/launch-update"; - -import { BaseState, createState } from "./base"; - +import { Tile, WeatherState } from '~/types/launch-update'; +import { BaseState, createState } from './base'; export interface LaunchState extends BaseState { firstTime: boolean; @@ -9,9 +7,10 @@ export interface LaunchState extends BaseState { tiles: { [app: string]: Tile; }, - weather: WeatherState | null, + weather: WeatherState | null | Record | boolean, userLocation: string | null; baseHash: string | null; + runtimeLag: boolean; }; const useLaunchState = createState('Launch', { @@ -20,8 +19,8 @@ const useLaunchState = createState('Launch', { tiles: {}, weather: null, userLocation: null, - baseHash: null + baseHash: null, + runtimeLag: false, }); - -export default useLaunchState; \ No newline at end of file +export default useLaunchState; diff --git a/pkg/interface/src/logic/state/local.tsx b/pkg/interface/src/logic/state/local.tsx index c5a187d11b..5d6b883719 100644 --- a/pkg/interface/src/logic/state/local.tsx +++ b/pkg/interface/src/logic/state/local.tsx @@ -1,13 +1,12 @@ -import React, { ReactNode } from 'react'; -import f from 'lodash/fp'; -import create, { State } from 'zustand'; -import { persist } from 'zustand/middleware'; import produce from 'immer'; -import { BackgroundConfig, RemoteContentPolicy, TutorialProgress, tutorialProgress, LeapCategories } from "~/types/local-update"; - +import f from 'lodash/fp'; +import React from 'react'; +import create, { State } from 'zustand'; +import { persist } from 'zustand/middleware'; +import { BackgroundConfig, LeapCategories, RemoteContentPolicy, TutorialProgress, tutorialProgress } from '~/types/local-update'; export interface LocalState { - theme: "light" | "dark" | "auto"; + theme: 'light' | 'dark' | 'auto'; hideAvatars: boolean; hideNicknames: boolean; remoteContentPolicy: RemoteContentPolicy; @@ -21,12 +20,13 @@ export interface LocalState { hideLeapCats: LeapCategories[]; setTutorialRef: (el: HTMLElement | null) => void; dark: boolean; + mobile: boolean; background: BackgroundConfig; omniboxShown: boolean; suspendedFocus?: HTMLElement; toggleOmnibox: () => void; set: (fn: (state: LocalState) => void) => void -}; +} type LocalStateZus = LocalState & State; @@ -35,8 +35,9 @@ export const selectLocalState = const useLocalState = create(persist((set, get) => ({ dark: false, + mobile: false, background: undefined, - theme: "auto", + theme: 'auto', hideAvatars: false, hideNicknames: false, hideLeapCats: [], @@ -90,8 +91,8 @@ const useLocalState = create(persist((set, get) => ({ name: 'localReducer' })); -function withLocalState(Component: any, stateMemberKeys?: S[]) { - return React.forwardRef((props: Omit, ref) => { +function withLocalState>(Component: C, stateMemberKeys?: S[]) { + return React.forwardRef>((props, ref) => { const localState = stateMemberKeys ? useLocalState( state => stateMemberKeys.reduce( (object, key) => ({ ...object, [key]: state[key] }), {} diff --git a/pkg/interface/src/logic/state/metadata.ts b/pkg/interface/src/logic/state/metadata.ts index fabffca86e..38a4a627ee 100644 --- a/pkg/interface/src/logic/state/metadata.ts +++ b/pkg/interface/src/logic/state/metadata.ts @@ -1,21 +1,35 @@ -import { MetadataUpdatePreview, Associations } from "@urbit/api"; - -import { BaseState, createState } from "./base"; +import { Association, Associations } from '@urbit/api'; +import _ from 'lodash'; +import { useCallback } from 'react'; +import { BaseState, createState } from './base'; export const METADATA_MAX_PREVIEW_WAIT = 150000; export interface MetadataState extends BaseState { associations: Associations; // preview: (group: string) => Promise; -}; +} + +export function useAssocForGraph(graph: string) { + return useMetadataState(useCallback(s => s.associations.graph[graph] as Association | undefined, [graph])); +} + +export function useAssocForGroup(group: string) { + return useMetadataState(useCallback(s => s.associations.groups[group] as Association | undefined, [group])); +} + +export function useGraphsForGroup(group: string) { + const graphs = useMetadataState(s => s.associations.graph); + return _.pickBy(graphs, (a: Association) => a.group === group); +} const useMetadataState = createState('Metadata', { - associations: { groups: {}, graph: {}, contacts: {}, chat: {}, link: {}, publish: {} }, + associations: { groups: {}, graph: {}, contacts: {}, chat: {}, link: {}, publish: {} } // preview: async (group): Promise => { // return new Promise((resolve, reject) => { // const api = useApi(); // let done = false; - + // setTimeout(() => { // if (done) { // return; @@ -23,7 +37,7 @@ const useMetadataState = createState('Metadata', { // done = true; // reject(new Error('offline')); // }, METADATA_MAX_PREVIEW_WAIT); - + // api.subscribe({ // app: 'metadata-pull-hook', // path: `/preview${group}`, @@ -53,5 +67,4 @@ const useMetadataState = createState('Metadata', { // }, }); - -export default useMetadataState; \ No newline at end of file +export default useMetadataState; diff --git a/pkg/interface/src/logic/state/settings.ts b/pkg/interface/src/logic/state/settings.ts index 02e96fffea..ae802ed15d 100644 --- a/pkg/interface/src/logic/state/settings.ts +++ b/pkg/interface/src/logic/state/settings.ts @@ -1,14 +1,13 @@ import f from 'lodash/fp'; -import { RemoteContentPolicy, LeapCategories, leapCategories } from "~/types/local-update"; import { BaseState, createState } from '~/logic/state/base'; - +import { LeapCategories, leapCategories, RemoteContentPolicy } from '~/types/local-update'; export interface SettingsState extends BaseState { display: { backgroundType: 'none' | 'url' | 'color'; background?: string; dark: boolean; - theme: "light" | "dark" | "auto"; + theme: 'light' | 'dark' | 'auto'; }; calm: { hideNicknames: boolean; @@ -25,7 +24,7 @@ export interface SettingsState extends BaseState { seen: boolean; joined?: number; }; -}; +} export const selectSettingsState = (keys: K[]) => f.pick(keys); @@ -39,7 +38,7 @@ const useSettingsState = createState('Settings', { backgroundType: 'none', background: undefined, dark: false, - theme: "auto" + theme: 'auto' }, calm: { hideNicknames: false, @@ -55,10 +54,10 @@ const useSettingsState = createState('Settings', { videoShown: true }, leap: { - categories: leapCategories, + categories: leapCategories }, tutorial: { - seen: false, + seen: true, joined: undefined } }); diff --git a/pkg/interface/src/logic/state/storage.ts b/pkg/interface/src/logic/state/storage.ts index bceb96f46e..d803ade400 100644 --- a/pkg/interface/src/logic/state/storage.ts +++ b/pkg/interface/src/logic/state/storage.ts @@ -1,4 +1,4 @@ -import { BaseState, createState } from "./base"; +import { BaseState, createState } from './base'; export interface GcpToken { accessKey: string; @@ -17,7 +17,7 @@ export interface StorageState extends BaseState { }; credentials: any | null; // TODO better type } -}; +} const useStorageState = createState('Storage', { gcp: {}, @@ -26,8 +26,8 @@ const useStorageState = createState('Storage', { buckets: new Set(), currentBucket: '' }, - credentials: null, + credentials: null } }, ['s3']); -export default useStorageState; \ No newline at end of file +export default useStorageState; diff --git a/pkg/interface/src/logic/store/base.ts b/pkg/interface/src/logic/store/base.ts index b2c894f34e..f798f228f6 100644 --- a/pkg/interface/src/logic/store/base.ts +++ b/pkg/interface/src/logic/store/base.ts @@ -32,7 +32,9 @@ export default class BaseStore { } this.reduce(json, this.state); - this.setState(this.state); + if('connection' in json) { + this.setState(this.state); + } } reduce(data, state) { diff --git a/pkg/interface/src/logic/store/store.ts b/pkg/interface/src/logic/store/store.ts index a938869ea6..cfc943d7fa 100644 --- a/pkg/interface/src/logic/store/store.ts +++ b/pkg/interface/src/logic/store/store.ts @@ -1,25 +1,20 @@ import _ from 'lodash'; - -import BaseStore from './base'; -import InviteReducer from '../reducers/invite-update'; -import MetadataReducer from '../reducers/metadata-update'; -import LocalReducer from '../reducers/local'; - -import { StoreState } from './type'; -import { Timebox } from '@urbit/api'; +import { unstable_batchedUpdates } from 'react-dom'; import { Cage } from '~/types/cage'; -import S3Reducer from '../reducers/s3-update'; -import { GraphReducer } from '../reducers/graph-update'; -import { HarkReducer } from '../reducers/hark-update'; -import { ContactReducer } from '../reducers/contact-update'; -import GroupReducer from '../reducers/group-update'; -import LaunchReducer from '../reducers/launch-update'; import ConnectionReducer from '../reducers/connection'; -import SettingsReducer from '../reducers/settings-update'; +import { ContactReducer } from '../reducers/contact-update'; import GcpReducer from '../reducers/gcp-reducer'; -import { OrderedMap } from '../lib/OrderedMap'; -import { BigIntOrderedMap } from '../lib/BigIntOrderedMap'; +import { GraphReducer } from '../reducers/graph-update'; +import GroupReducer from '../reducers/group-update'; import { GroupViewReducer } from '../reducers/group-view'; +import { HarkReducer } from '../reducers/hark-update'; +import InviteReducer from '../reducers/invite-update'; +import LaunchReducer from '../reducers/launch-update'; +import MetadataReducer from '../reducers/metadata-update'; +import S3Reducer from '../reducers/s3-update'; +import SettingsReducer from '../reducers/settings-update'; +import BaseStore from './base'; +import { StoreState } from './type'; export default class GlobalStore extends BaseStore { inviteReducer = new InviteReducer(); @@ -45,25 +40,28 @@ export default class GlobalStore extends BaseStore { initialState(): StoreState { return { - connection: 'connected', + connection: 'connected' }; } reduce(data: Cage, state: StoreState) { - // debug shim - const tag = Object.keys(data)[0]; - const oldActions = this.pastActions[tag] || []; - this.pastActions[tag] = [data[tag], ...oldActions.slice(0,14)]; - this.inviteReducer.reduce(data); - this.metadataReducer.reduce(data); - this.s3Reducer.reduce(data); - this.groupReducer.reduce(data); - this.launchReducer.reduce(data); - this.connReducer.reduce(data, this.state); - GraphReducer(data); - HarkReducer(data); - ContactReducer(data); - this.settingsReducer.reduce(data); - this.gcpReducer.reduce(data); + unstable_batchedUpdates(() => { + // debug shim + const tag = Object.keys(data)[0]; + const oldActions = this.pastActions[tag] || []; + this.pastActions[tag] = [data[tag], ...oldActions.slice(0, 14)]; + this.inviteReducer.reduce(data); + this.metadataReducer.reduce(data); + this.s3Reducer.reduce(data); + this.groupReducer.reduce(data); + GroupViewReducer(data); + this.launchReducer.reduce(data); + this.connReducer.reduce(data, this.state); + GraphReducer(data); + HarkReducer(data); + ContactReducer(data); + this.settingsReducer.reduce(data); + this.gcpReducer.reduce(data); + }); } } diff --git a/pkg/interface/src/logic/subscription/base.ts b/pkg/interface/src/logic/subscription/base.ts index debf665415..8100f0a612 100644 --- a/pkg/interface/src/logic/subscription/base.ts +++ b/pkg/interface/src/logic/subscription/base.ts @@ -1,6 +1,6 @@ -import BaseStore from '../store/base'; -import BaseApi from '../api/base'; import { Path } from '@urbit/api'; +import BaseApi from '../api/base'; +import BaseStore from '../store/base'; export default class BaseSubscription { private errorCount = 0; diff --git a/pkg/interface/src/logic/subscription/global.ts b/pkg/interface/src/logic/subscription/global.ts index 5b444f06df..584c2c889b 100644 --- a/pkg/interface/src/logic/subscription/global.ts +++ b/pkg/interface/src/logic/subscription/global.ts @@ -1,31 +1,9 @@ -import BaseSubscription from './base'; -import { StoreState } from '../store/type'; import { Path } from '@urbit/api'; -import _ from 'lodash'; - -/** - * Path to subscribe on and app to subscribe to - */ -type AppSubscription = [Path, string]; - -const groupSubscriptions: AppSubscription[] = [ -]; - -const graphSubscriptions: AppSubscription[] = [ - ['/updates', 'graph-store'] -]; - -type AppName = 'groups' | 'graph'; -const appSubscriptions: Record = { - groups: groupSubscriptions, - graph: graphSubscriptions -}; +import { StoreState } from '../store/type'; +import BaseSubscription from './base'; export default class GlobalSubscription extends BaseSubscription { - openSubscriptions: Record = { - groups: [], - graph: [] - }; + openSubscriptions: any = {}; start() { this.subscribe('/all', 'metadata-store'); @@ -35,7 +13,6 @@ export default class GlobalSubscription extends BaseSubscription { this.subscribe('/groups', 'group-store'); this.clearQueue(); - // TODO: update to get /updates this.subscribe('/all', 'contact-store'); this.subscribe('/all', 's3-store'); this.subscribe('/keys', 'graph-store'); @@ -45,29 +22,32 @@ export default class GlobalSubscription extends BaseSubscription { this.subscribe('/all', 'settings-store'); this.subscribe('/all', 'group-view'); this.subscribe('/nacks', 'contact-pull-hook'); + this.clearQueue(); + + this.subscribe('/updates', 'graph-store'); + } + + subscribe(path: Path, app: string) { + if (`${app}${path}` in this.openSubscriptions) { + return; + } + + const id = super.subscribe(path, app); + this.openSubscriptions[`${app}${path}`] = { app, path, id }; + } + + unsubscribe(id) { + for (const key in Object.keys(this.openSubscriptions)) { + const val = this.openSubscriptions[key]; + if (id === val.id) { + delete this.openSubscriptions[`${val.app}${val.path}`]; + super.unsubscribe(id); + } + } } restart() { + this.openSubscriptions = {}; super.restart(); - _.mapValues(this.openSubscriptions, (subs, app: AppName) => { - if(subs.length > 0) { - this.stopApp(app); - this.startApp(app); - } - }); - } - - startApp(app: AppName) { - if(this.openSubscriptions[app].length > 0) { - console.log(`${app} already started`); - return; - } - this.openSubscriptions[app] = - appSubscriptions[app].map(([path, agent]) => this.subscribe(path, agent)); - } - - stopApp(app: AppName) { - this.openSubscriptions[app].map(id => this.unsubscribe(id)); - this.openSubscriptions[app] = []; } } diff --git a/pkg/interface/src/register-sw.js b/pkg/interface/src/register-sw.js index d4a2ed09c2..1d6bb05180 100644 --- a/pkg/interface/src/register-sw.js +++ b/pkg/interface/src/register-sw.js @@ -1,10 +1,8 @@ - - -if ("serviceWorker" in navigator) { - window.addEventListener("load", () => { - navigator.serviceWorker.register("/~landscape/js/bundle/serviceworker.js", { - scope: "/", - }).then(reg => { +if ('serviceWorker' in navigator && process.env.NODE_ENV !== 'development') { + window.addEventListener('load', () => { + navigator.serviceWorker.register('/~landscape/js/bundle/serviceworker.js', { + scope: '/' + }).then((reg) => { }); }); } diff --git a/pkg/interface/src/serviceworker.js b/pkg/interface/src/serviceworker.js index 76858dba63..cd83dee192 100644 --- a/pkg/interface/src/serviceworker.js +++ b/pkg/interface/src/serviceworker.js @@ -1,24 +1,21 @@ -import { registerRoute } from 'workbox-routing'; -import { - NetworkFirst, - StaleWhileRevalidate, - CacheFirst, -} from 'workbox-strategies'; - // Used for filtering matches based on status code, header, or both import { CacheableResponsePlugin } from 'workbox-cacheable-response'; // Used to limit entries in cache, remove entries after a certain period of time import { ExpirationPlugin } from 'workbox-expiration'; - +import { registerRoute } from 'workbox-routing'; +import { + CacheFirst, NetworkFirst, + StaleWhileRevalidate +} from 'workbox-strategies'; // generate a different sw for every build, to bust cache properly const hash = process.env.LANDSCAPE_SHORTHASH; -self.addEventListener("install", ev => { +self.addEventListener('install', (ev) => { self.skipWaiting(); }); -self.addEventListener('activate', ev => { +self.addEventListener('activate', (ev) => { ev.waitUntil(clients.claim()); }); @@ -33,10 +30,10 @@ registerRoute( plugins: [ // Ensure that only requests that result in a 200 status are cached new CacheableResponsePlugin({ - statuses: [200], - }), - ], - }), + statuses: [200] + }) + ] + }) ); // Cache CSS, JS, and Web Worker requests with a Stale While Revalidate strategy @@ -44,7 +41,7 @@ registerRoute( // Check to see if the request's destination is style for stylesheets, script for JavaScript, or worker for web worker ({ request }) => request.destination === 'style' || - request.destination === 'script' || +// request.destination === 'script' || request.destination === 'worker', // Use a Stale While Revalidate caching strategy new StaleWhileRevalidate({ @@ -53,12 +50,12 @@ registerRoute( plugins: [ // Ensure that only requests that result in a 200 status are cached new CacheableResponsePlugin({ - statuses: [200], - }), - ], - }), + statuses: [200] + }) + ] + }) ); - + // Cache images with a Cache First strategy registerRoute( // Check to see if the request's destination is style for an image @@ -70,13 +67,13 @@ registerRoute( plugins: [ // Ensure that only requests that result in a 200 status are cached new CacheableResponsePlugin({ - statuses: [200], + statuses: [200] }), // Don't cache more than 50 items, and expire them after 30 days new ExpirationPlugin({ maxEntries: 50, - maxAgeSeconds: 60 * 60 * 24 * 30, // 30 Days - }), - ], - }), + maxAgeSeconds: 60 * 60 * 24 * 30 // 30 Days + }) + ] + }) ); diff --git a/pkg/interface/src/types/cage.ts b/pkg/interface/src/types/cage.ts index 044fb46212..09a141aaab 100644 --- a/pkg/interface/src/types/cage.ts +++ b/pkg/interface/src/types/cage.ts @@ -1,8 +1,8 @@ -import { LocalUpdate } from './local-update'; -import { LaunchUpdate, WeatherState } from './launch-update'; -import { ConnectionStatus } from './connection'; import { ContactUpdate, GroupUpdate, InviteUpdate, MetadataUpdate } from '@urbit/api'; import { SettingsUpdate } from '@urbit/api/settings'; +import { ConnectionStatus } from './connection'; +import { LaunchUpdate, WeatherState } from './launch-update'; +import { LocalUpdate } from './local-update'; interface MarksToTypes { readonly json: any; diff --git a/pkg/interface/src/types/gcp-state.ts b/pkg/interface/src/types/gcp-state.ts index cdbcbc1c97..ba6ff5479d 100644 --- a/pkg/interface/src/types/gcp-state.ts +++ b/pkg/interface/src/types/gcp-state.ts @@ -1,9 +1,8 @@ export interface GcpToken { accessKey: string; expiresIn: number; -}; +} export interface GcpState { - configured?: boolean; token?: GcpToken -}; +} diff --git a/pkg/interface/src/types/index.ts b/pkg/interface/src/types/index.ts index 40d45c7ff9..3f0c3c741e 100644 --- a/pkg/interface/src/types/index.ts +++ b/pkg/interface/src/types/index.ts @@ -1,10 +1,11 @@ export * from './cage'; export * from './connection'; +export * from './gcp-state'; export * from './global'; export * from './launch-update'; export * from './local-update'; -export * from './storage-state'; -export * from './gcp-state'; export * from './s3-update'; -export * from './workspace'; +export * from './storage-state'; export * from './util'; +export * from './workspace'; + diff --git a/pkg/interface/src/types/local-update.ts b/pkg/interface/src/types/local-update.ts index 9813729f37..31e99560d6 100644 --- a/pkg/interface/src/types/local-update.ts +++ b/pkg/interface/src/types/local-update.ts @@ -1,6 +1,6 @@ export const tutorialProgress = ['hidden', 'start', 'group-desc', 'channels', 'chat', 'link', 'publish', 'profile', 'leap', 'notifications', 'done', 'exit'] as const; -export const leapCategories = ["mychannel", "messages", "updates", "profile", "logout"] as const; +export const leapCategories = ['mychannel', 'messages', 'updates', 'profile', 'logout']; export type LeapCategories = typeof leapCategories[number]; @@ -13,6 +13,10 @@ interface LocalUpdateBaseHash { baseHash: string; } +interface LocalUpdateRuntimeLag { + runtimeLag: boolean; +} + interface LocalUpdateBackgroundConfig { backgroundConfig: BackgroundConfig; } @@ -51,6 +55,7 @@ export type BackgroundConfig = BackgroundConfigUrl | BackgroundConfigColor | und export type LocalUpdate = | LocalUpdateSetDark | LocalUpdateBaseHash +| LocalUpdateRuntimeLag | LocalUpdateBackgroundConfig | LocalUpdateHideAvatars | LocalUpdateHideNicknames diff --git a/pkg/interface/src/types/storage-state.ts b/pkg/interface/src/types/storage-state.ts index 61bf612ada..e36cdfff63 100644 --- a/pkg/interface/src/types/storage-state.ts +++ b/pkg/interface/src/types/storage-state.ts @@ -1,8 +1,7 @@ -import {GcpState} from './gcp-state'; -import {S3State} from './s3-update'; - +import { GcpState } from './gcp-state'; +import { S3State } from './s3-update'; export interface StorageState { gcp: GcpState; s3: S3State; -}; +} diff --git a/pkg/interface/src/views/App.js b/pkg/interface/src/views/App.js index 86c82e306a..0bc6d7adb9 100644 --- a/pkg/interface/src/views/App.js +++ b/pkg/interface/src/views/App.js @@ -1,39 +1,34 @@ -import { hot } from 'react-hot-loader/root'; -import 'react-hot-loader'; -import * as React from 'react'; -import { BrowserRouter as Router, withRouter } from 'react-router-dom'; -import styled, { ThemeProvider, createGlobalStyle } from 'styled-components'; +import dark from '@tlon/indigo-dark'; +import light from '@tlon/indigo-light'; import { sigil as sigiljs, stringRenderer } from '@tlon/sigil-js'; -import Helmet from 'react-helmet'; - import Mousetrap from 'mousetrap'; import 'mousetrap-global-bind'; - -import './css/indigo-static.css'; -import './css/fonts.css'; -import light from '@tlon/indigo-light'; -import dark from '@tlon/indigo-dark'; - -import { Text, Anchor, Row } from '@tlon/indigo-react'; - -import { Content } from './landscape/components/Content'; -import StatusBar from './components/StatusBar'; -import Omnibox from './components/leap/Omnibox'; -import ErrorBoundary from '~/views/components/ErrorBoundary'; -import { TutorialModal } from '~/views/landscape/components/TutorialModal'; - -import GlobalStore from '~/logic/store/store'; -import GlobalSubscription from '~/logic/subscription/global'; +import * as React from 'react'; +import Helmet from 'react-helmet'; +import 'react-hot-loader'; +import { hot } from 'react-hot-loader/root'; +import { BrowserRouter as Router, withRouter } from 'react-router-dom'; +import styled, { ThemeProvider } from 'styled-components'; import GlobalApi from '~/logic/api/global'; -import { uxToHex } from '~/logic/lib/util'; +import gcpManager from '~/logic/lib/gcpManager'; import { foregroundFromBackground } from '~/logic/lib/sigil'; +import { uxToHex } from '~/logic/lib/util'; import withState from '~/logic/lib/withState'; -import useLocalState from '~/logic/state/local'; import useContactState from '~/logic/state/contact'; import useGroupState from '~/logic/state/group'; +import useLocalState from '~/logic/state/local'; import useSettingsState from '~/logic/state/settings'; -import gcpManager from '~/logic/lib/gcpManager'; - +import GlobalStore from '~/logic/store/store'; +import GlobalSubscription from '~/logic/subscription/global'; +import ErrorBoundary from '~/views/components/ErrorBoundary'; +import { TutorialModal } from '~/views/landscape/components/TutorialModal'; +import './apps/chat/css/custom.css'; +import Omnibox from './components/leap/Omnibox'; +import StatusBar from './components/StatusBar'; +import './css/fonts.css'; +import './css/indigo-static.css'; +import { Content } from './landscape/components/Content'; +import './landscape/css/custom.css'; const Root = withState(styled.div` font-family: ${p => p.theme.fonts.sans}; @@ -50,6 +45,7 @@ const Root = withState(styled.div` } display: flex; flex-flow: column nowrap; + touch-action: none; * { scrollbar-width: thin; @@ -89,19 +85,25 @@ class App extends React.Component { new GlobalSubscription(this.store, this.api, this.appChannel); this.updateTheme = this.updateTheme.bind(this); + this.updateMobile = this.updateMobile.bind(this); this.faviconString = this.faviconString.bind(this); } componentDidMount() { this.subscription.start(); + const theme = this.getTheme(); this.themeWatcher = window.matchMedia('(prefers-color-scheme: dark)'); + this.mobileWatcher = window.matchMedia(`(max-width: ${theme.breakpoints[0]})`); this.themeWatcher.onchange = this.updateTheme; + this.mobileWatcher.onchange = this.updateMobile; setTimeout(() => { // Something about how the store works doesn't like changing it // before the app has actually rendered, hence the timeout. + this.updateMobile(this.mobileWatcher); this.updateTheme(this.themeWatcher); }, 500); this.api.local.getBaseHash(); + this.api.local.getRuntimeLag(); //TODO consider polling periodically this.api.settings.getAll(); gcpManager.start(); Mousetrap.bindGlobal(['command+/', 'ctrl+/'], (e) => { @@ -113,14 +115,21 @@ class App extends React.Component { componentWillUnmount() { this.themeWatcher.onchange = undefined; + this.mobileWatcher.onchange = undefined; } updateTheme(e) { - this.props.set(state => { + this.props.set((state) => { state.dark = e.matches; }); } + updateMobile(e) { + this.props.set((state) => { + state.mobile = e.matches; + }); + } + faviconString() { let background = '#ffffff'; if (this.props.contacts.hasOwnProperty(`~${window.ship}`)) { @@ -137,12 +146,16 @@ class App extends React.Component { return dataurl; } - render() { - const { state, props } = this; - const theme = - ((props.dark && props?.display?.theme == "auto") || - props?.display?.theme == "dark" + getTheme() { + const { props } = this; + return ((props.dark && props?.display?.theme == 'auto') || + props?.display?.theme == 'dark' ) ? dark : light; + } + + render() { + const { state } = this; + const theme = this.getTheme(); const ourContact = this.props.contacts[`~${this.ship}`] || null; return ( @@ -178,6 +191,7 @@ class App extends React.Component { ship={this.ship} api={this.api} subscription={this.subscription} + connection={this.state.connection} /> @@ -193,4 +207,4 @@ export default withState(process.env.NODE_ENV === 'production' ? App : hot(App), [useContactState], [useSettingsState, ['display']], [useLocalState] -]); \ No newline at end of file +]); diff --git a/pkg/interface/src/views/apps/chat/ChatResource.tsx b/pkg/interface/src/views/apps/chat/ChatResource.tsx index 8e3aa063e6..3bac9adb1f 100644 --- a/pkg/interface/src/views/apps/chat/ChatResource.tsx +++ b/pkg/interface/src/views/apps/chat/ChatResource.tsx @@ -1,194 +1,161 @@ -import React, { useRef, useCallback, useEffect, useState } from 'react'; -import { RouteComponentProps } from 'react-router-dom'; -import { Col } from '@tlon/indigo-react'; -import _ from 'lodash'; - +import { Content, createPost, Post } from '@urbit/api'; import { Association } from '@urbit/api/metadata'; -import { StoreState } from '~/logic/store/type'; -import { useFileDrag } from '~/logic/lib/useDrag'; -import ChatWindow from './components/ChatWindow'; -import ChatInput from './components/ChatInput'; -import GlobalApi from '~/logic/api/global'; -import { ShareProfile } from '~/views/apps/chat/components/ShareProfile'; -import SubmitDragger from '~/views/components/SubmitDragger'; -import { useLocalStorageState } from '~/logic/lib/useLocalStorageState'; -import { Loading } from '~/views/components/Loading'; -import { isWriter, resourceFromPath } from '~/logic/lib/group'; +import { BigInteger } from 'big-integer'; +import React, { + ReactElement, useCallback, + useEffect, -import './css/custom.css'; -import useContactState from '~/logic/state/contact'; -import useGraphState from '~/logic/state/graph'; -import useGroupState from '~/logic/state/group'; + useMemo, useState +} from 'react'; +import GlobalApi from '~/logic/api/global'; +import { isWriter, resourceFromPath } from '~/logic/lib/group'; +import { getPermalinkForGraph } from '~/logic/lib/permalinks'; +import useGraphState, { useGraphForAssoc } from '~/logic/state/graph'; +import { useGroupForAssoc } from '~/logic/state/group'; import useHarkState from '~/logic/state/hark'; +import { StoreState } from '~/logic/store/type'; +import { Loading } from '~/views/components/Loading'; +import { ChatPane } from './components/ChatPane'; + +const getCurrGraphSize = (ship: string, name: string) => { + const { graphs } = useGraphState.getState(); + const graph = graphs[`${ship}/${name}`]; + return graph?.size ?? 0; +}; type ChatResourceProps = StoreState & { association: Association; api: GlobalApi; baseUrl: string; -} & RouteComponentProps; +}; -export function ChatResource(props: ChatResourceProps) { - const station = props.association.resource; - const groupPath = props.association.group; - const groups = useGroupState(state => state.groups); - const group = groups[groupPath]; - const contacts = useContactState(state => state.contacts); - const graphs = useGraphState(state => state.graphs); - const graphPath = station.slice(7); - const graph = graphs[graphPath]; +const ChatResource = (props: ChatResourceProps): ReactElement => { + const { association, api } = props; + const { resource } = association; + const [toShare, setToShare] = useState(); + const group = useGroupForAssoc(association)!; + const graph = useGraphForAssoc(association); const unreads = useHarkState(state => state.unreads); - const unreadCount = unreads.graph?.[station]?.['/']?.unreads || 0; - const graphTimesentMap = useGraphState(state => state.graphTimesentMap); - const [,, owner, name] = station.split('/'); - const ourContact = contacts?.[`~${window.ship}`]; - const chatInput = useRef(); - const canWrite = isWriter(group, station); + const unreadCount = + (unreads.graph?.[resource]?.['/']?.unreads as number) || 0; + const canWrite = group ? isWriter(group, resource) : false; useEffect(() => { - const count = 100 + unreadCount; - props.api.graph.getNewest(owner, name, count); - }, [station]); - - const onFileDrag = useCallback( - (files: FileList | File[]) => { - if (!chatInput.current) { - return; - } - chatInput.current?.uploadFiles(files); - }, - [chatInput.current] - ); - - const { bind, dragging } = useFileDrag(onFileDrag); - - const [unsent, setUnsent] = useLocalStorageState>( - 'chat-unsent', - {} - ); - - const appendUnsent = useCallback( - (u: string) => setUnsent(s => ({ ...s, [station]: u })), - [station] - ); - - const clearUnsent = useCallback( - () => setUnsent(s => _.omit(s, station)), - [station] - ); - - const scrollTo = new URLSearchParams(location.search).get('msg'); - - useEffect(() => { - const clear = () => { - props.history.replace(location.pathname); - }; - setTimeout(clear, 10000); - return clear; - }, [station]); - - const [showBanner, setShowBanner] = useState(false); - const [hasLoadedAllowed, setHasLoadedAllowed] = useState(false); - const [recipients, setRecipients] = useState([]); - - const res = resourceFromPath(groupPath); - - useEffect(() => { - (async () => { - if (!res) { return; } - if (!group) { return; } + const count = Math.min(400, 100 + unreadCount); + const { ship, name } = resourceFromPath(resource); + props.api.graph.getNewest(ship, name, count); + setToShare(undefined); + (async function () { if (group.hidden) { - const members = _.compact(await Promise.all( + const members = await props.api.contacts.disallowedShipsForOurContact( Array.from(group.members) - .map(s => { - const ship = `~${s}`; - if(s === window.ship) { - return Promise.resolve(null); - } - return props.api.contacts.fetchIsAllowed( - `~${window.ship}`, - 'personal', - ship, - true - ).then(isAllowed => { - return isAllowed ? null : ship; - }); - }) - )); - - if(members.length > 0) { - setShowBanner(true); - setRecipients(members); - } else { - setShowBanner(false); + ); + if (members.length > 0) { + setToShare(members); } } else { - const groupShared = await props.api.contacts.fetchIsAllowed( + const { ship: groupHost } = resourceFromPath(association.group); + const shared = await props.api.contacts.fetchIsAllowed( `~${window.ship}`, 'personal', - res.ship, + groupHost, true ); - setShowBanner(!groupShared); + if (!shared) { + setToShare(association.group); + } } - - setHasLoadedAllowed(true); })(); - }, [groupPath, group]); + }, [resource]); - if(!graph) { + const onReply = useCallback( + (msg: Post) => { + const url = getPermalinkForGraph( + props.association.group, + props.association.resource, + msg.index + ); + return `${url}\n~${msg.author} : `; + }, + [association] + ); + + const isAdmin = useMemo( + () => (group ? group.tags.role.admin.has(`~${window.ship}`) : false), + [group] + ); + + const fetchMessages = useCallback(async (newer: boolean) => { + const { api } = props; + const pageSize = 100; + + const [, , ship, name] = resource.split('/'); + const graphSize = graph?.size ?? 0; + const expectedSize = graphSize + pageSize; + if (newer) { + const index = graph.peekLargest()?.[0]; + if (!index) { + return true; + } + await api.graph.getYoungerSiblings( + ship, + name, + pageSize, + `/${index.toString()}` + ); + return expectedSize !== getCurrGraphSize(ship.slice(1), name); + } else { + const index = graph.peekSmallest()?.[0]; + if (!index) { + return true; + } + await api.graph.getOlderSiblings(ship, name, pageSize, `/${index.toString()}`); + const done = expectedSize !== getCurrGraphSize(ship.slice(1), name); + return done; + } + }, [graph, resource]); + + const onSubmit = useCallback((contents: Content[]) => { + const { ship, name } = resourceFromPath(resource); + api.graph.addPost(ship, name, createPost(window.ship, contents)); + }, [resource]); + + const onDelete = useCallback((msg: Post) => { + const { ship, name } = resourceFromPath(resource); + api.graph.removePosts(ship, name, [msg.index]); + }, [resource]); + + const dismissUnread = useCallback(() => { + api.hark.markCountAsRead(association, '/', 'message'); + }, [association]); + + const getPermalink = useCallback( + (index: BigInteger) => + getPermalinkForGraph(association.group, resource, `/${index.toString()}`), + [association] + ); + + if (!graph) { return ; } - const modifiedContacts = { ...contacts }; - delete modifiedContacts[`~${window.ship}`]; - return ( - - - {dragging && } - - { canWrite && ( - )} - + ); -} +}; + +export { ChatResource }; diff --git a/pkg/interface/src/views/apps/chat/components/chat-editor.js b/pkg/interface/src/views/apps/chat/components/ChatEditor.tsx similarity index 67% rename from pkg/interface/src/views/apps/chat/components/chat-editor.js rename to pkg/interface/src/views/apps/chat/components/ChatEditor.tsx index 03910afc2b..2f7a0b99d6 100644 --- a/pkg/interface/src/views/apps/chat/components/chat-editor.js +++ b/pkg/interface/src/views/apps/chat/components/ChatEditor.tsx @@ -1,22 +1,15 @@ +/* eslint-disable max-lines-per-function */ +import { BaseTextArea, Box, Row } from '@tlon/indigo-react'; +import 'codemirror/addon/display/placeholder'; +import 'codemirror/addon/hint/show-hint'; +import 'codemirror/lib/codemirror.css'; +import 'codemirror/mode/markdown/markdown'; import React, { Component } from 'react'; import { UnControlled as CodeEditor } from 'react-codemirror2'; -import { MOBILE_BROWSER_REGEX } from "~/logic/lib/util"; -import CodeMirror from 'codemirror'; -import styled from "styled-components"; - -import { Row, BaseTextArea, Box } from '@tlon/indigo-react'; - -import 'codemirror/mode/markdown/markdown'; -import 'codemirror/addon/display/placeholder'; - -import 'codemirror/lib/codemirror.css'; - +import styled from 'styled-components'; +import { MOBILE_BROWSER_REGEX } from '~/logic/lib/util'; import '../css/custom.css'; -const BROWSER_REGEX = - new RegExp(String(/Android|webOS|iPhone|iPad|iPod|BlackBerry/i)); - - const MARKDOWN_CONFIG = { name: 'markdown', tokenTypeOverrides: { @@ -39,11 +32,23 @@ const MARKDOWN_CONFIG = { // Until CodeMirror supports options.inputStyle = 'textarea' on mobile, // we need to hack this into a regular input that has some funny behaviors -const inputProxy = (input) => new Proxy(input, { +const inputProxy = input => new Proxy(input, { get(target, property) { + if(property === 'focus') { + return () => { + target.focus(); + }; + } if (property in target) { return target[property]; } + if (property === 'execCommand') { + return () => { + target.setSelectionRange(target.value.length, target.value.length); + input.blur(); + input.focus(); + }; + } if (property === 'setOption') { return () => {}; } @@ -51,7 +56,9 @@ const inputProxy = (input) => new Proxy(input, { return () => target.value; } if (property === 'setValue') { - return (val) => target.value = val; + return (val) => { + target.value = val; + }; } if (property === 'element') { return input; @@ -87,26 +94,55 @@ const MobileBox = styled(Box)` } `; -export default class ChatEditor extends Component { - constructor(props) { +interface ChatEditorProps { + message: string; + inCodeMode: boolean; + submit: (message: string) => void; + onUnmount: (message: string) => void; + onPaste: () => void; + changeEvent: (message: string) => void; + placeholder: string; +} + +interface ChatEditorState { + message: string; +} + +interface CodeMirrorShim { + setValue: (string) => void; + setOption: (option: string, property: any) => void; + focus: () => void; + execCommand: (string) => void; + getValue: () => string; + getInputField: () => HTMLInputElement; + element: HTMLElement; +} + +export default class ChatEditor extends Component { + editor: CodeMirrorShim; + constructor(props: ChatEditorProps) { super(props); this.state = { message: props.message }; + this.editor = null; } - componentWillUnmount() { + componentWillUnmount(): void { this.props.onUnmount(this.state.message); } - componentDidUpdate(prevProps) { + componentDidUpdate(prevProps: ChatEditorProps): void { const { props } = this; - if (prevProps.message !== props.message) { + if (prevProps.message !== props.message && this.editor) { this.editor.setValue(props.message); this.editor.setOption('mode', MARKDOWN_CONFIG); + this.editor?.focus(); + this.editor.execCommand('goDocEnd'); + this.editor?.focus(); return; } @@ -131,7 +167,7 @@ export default class ChatEditor extends Component { return; } - let editorMessage = this.editor.getValue(); + const editorMessage = this.editor.getValue(); if (editorMessage === '') { return; } @@ -142,7 +178,11 @@ export default class ChatEditor extends Component { } messageChange(editor, data, value) { + if(value.endsWith('/')) { + editor.showHint(['test', 'foo']); + } if (this.state.message !== '' && value == '') { + this.props.changeEvent(value); this.setState({ message: value }); @@ -150,6 +190,7 @@ export default class ChatEditor extends Component { if (value == this.props.message || value == '' || value == ' ') { return; } + this.props.changeEvent(value); this.setState({ message: value }); @@ -182,16 +223,16 @@ export default class ChatEditor extends Component { } }, // The below will ony work once codemirror's bug is fixed - spellcheck: !!MOBILE_BROWSER_REGEX.test(navigator.userAgent), - autocorrect: !!MOBILE_BROWSER_REGEX.test(navigator.userAgent), - autocapitalize: !!MOBILE_BROWSER_REGEX.test(navigator.userAgent) + spellcheck: Boolean(MOBILE_BROWSER_REGEX.test(navigator.userAgent)), + autocorrect: Boolean(MOBILE_BROWSER_REGEX.test(navigator.userAgent)), + autocapitalize: Boolean(MOBILE_BROWSER_REGEX.test(navigator.userAgent)) }; return ( {MOBILE_BROWSER_REGEX.test(navigator.userAgent) ? { + onClick={(event) => { if (this.editor) { this.editor.element.focus(); } @@ -213,24 +255,20 @@ export default class ChatEditor extends Component { > { - this.messageChange(null, null, event.target.value); - }} - onKeyDown={event => { - if (event.key === 'Enter') { - event.preventDefault(); - this.submit(); - } else { - this.messageChange(null, null, event.target.value); - } - }} - ref={input => { - if (!input) return; + placeholder={inCodeMode ? 'Code...' : 'Message...'} + onChange={event => + this.messageChange(null, null, event.target.value) + } + onKeyDown={event => + this.messageChange(null, null, (event.target as any).value) + } + ref={(input) => { + if (!input) +return; this.editor = inputProxy(input); }} {...props} @@ -246,7 +284,7 @@ export default class ChatEditor extends Component { editor.focus(); }} {...props} - /> + /> } diff --git a/pkg/interface/src/views/apps/chat/components/ChatInput.tsx b/pkg/interface/src/views/apps/chat/components/ChatInput.tsx index 65803ebc31..5d0524ed43 100644 --- a/pkg/interface/src/views/apps/chat/components/ChatInput.tsx +++ b/pkg/interface/src/views/apps/chat/components/ChatInput.tsx @@ -1,38 +1,35 @@ -import React, { Component } from 'react'; -import ChatEditor from './chat-editor'; -import { IuseStorage } from '~/logic/lib/useStorage'; -import { uxToHex } from '~/logic/lib/util'; -import { Sigil } from '~/logic/lib/sigil'; -import { createPost } from '~/logic/api/graph'; -import tokenizeMessage, { isUrl } from '~/logic/lib/tokenizeMessage'; +import { BaseImage, Box, Icon, LoadingSpinner, Row } from '@tlon/indigo-react'; +import { Contact, Content } from '@urbit/api'; +import React, { Component, ReactNode } from 'react'; import GlobalApi from '~/logic/api/global'; -import { Envelope } from '~/types/chat-update'; -import { StorageState } from '~/types'; -import { Contacts, Content } from '@urbit/api'; -import { Row, BaseImage, Box, Icon, LoadingSpinner } from '@tlon/indigo-react'; -import withStorage from '~/views/components/withStorage'; +import { Sigil } from '~/logic/lib/sigil'; +import tokenizeMessage from '~/logic/lib/tokenizeMessage'; +import { IuseStorage } from '~/logic/lib/useStorage'; +import { MOBILE_BROWSER_REGEX, uxToHex } from '~/logic/lib/util'; import { withLocalState } from '~/logic/state/local'; +import withStorage from '~/views/components/withStorage'; +import ChatEditor from './ChatEditor'; type ChatInputProps = IuseStorage & { api: GlobalApi; - numMsgs: number; - station: unknown; - ourContact: unknown; - envelopes: Envelope[]; + ourContact?: Contact; onUnmount(msg: string): void; placeholder: string; message: string; deleteMessage(): void; hideAvatars: boolean; -} + onSubmit: (contents: Content[]) => void; + children?: ReactNode; +}; interface ChatInputState { inCodeMode: boolean; submitFocus: boolean; uploadingPaste: boolean; + currentInput: string; } -class ChatInput extends Component { +export class ChatInput extends Component { private chatEditor: React.RefObject; constructor(props) { @@ -41,7 +38,8 @@ class ChatInput extends Component { this.state = { inCodeMode: false, submitFocus: false, - uploadingPaste: false + uploadingPaste: false, + currentInput: props.message }; this.chatEditor = React.createRef(); @@ -50,6 +48,7 @@ class ChatInput extends Component { this.toggleCode = this.toggleCode.bind(this); this.uploadSuccess = this.uploadSuccess.bind(this); this.uploadError = this.uploadError.bind(this); + this.eventHandler = this.eventHandler.bind(this); } toggleCode() { @@ -58,36 +57,30 @@ class ChatInput extends Component { }); } - submit(text) { + async submit(text) { const { props, state } = this; - const [,,ship,name] = props.station.split('/'); - if (state.inCodeMode) { - this.setState({ - inCodeMode: false - }, async () => { - const output = await props.api.graph.eval(text); - const contents: Content[] = [{ code: { output, expression: text } }]; - const post = createPost(contents); - props.api.graph.addPost(ship, name, post); - }); - return; - } - - const post = createPost(tokenizeMessage((text))); - + const { onSubmit, api } = this.props; + this.setState({ + inCodeMode: false + }); props.deleteMessage(); - - props.api.graph.addPost(ship, name, post); + if(state.inCodeMode) { + const output = await api.graph.eval(text) as string[]; + onSubmit([{ code: { output, expression: text } }]); + } else { + onSubmit(tokenizeMessage(text)); + } + this.chatEditor.current.editor.focus(); + this.setState({ currentInput: '' }); } - uploadSuccess(url) { + uploadSuccess(url: string) { const { props } = this; if (this.state.uploadingPaste) { this.chatEditor.current.editor.setValue(url); this.setState({ uploadingPaste: false }); } else { - const [,,ship,name] = props.station.split('/'); - props.api.graph.addPost(ship,name, createPost([{ url }])); + props.onSubmit([{ url }]); } } @@ -110,41 +103,55 @@ class ChatInput extends Component { return; } Array.from(files).forEach((file) => { - this.props.uploadDefault(file) + this.props + .uploadDefault(file) .then(this.uploadSuccess) .catch(this.uploadError); }); } + eventHandler(value) { + this.setState({ currentInput: value }); + } + render() { const { props, state } = this; - const color = props.ourContact - ? uxToHex(props.ourContact.color) : '000000'; + const color = props.ourContact ? uxToHex(props.ourContact.color) : '000000'; - const sigilClass = props.ourContact - ? '' : 'mix-blend-diff'; + const sigilClass = props.ourContact ? '' : 'mix-blend-diff'; - const avatar = ( - props.ourContact && - ((props.ourContact?.avatar) && !props.hideAvatars) - ) - ? - : ; + ) : ( + + + + ); return ( { flexGrow={1} flexShrink={0} borderTop={1} - borderTopColor='washedGray' + borderTopColor='lightGray' backgroundColor='white' className='cf' zIndex={0} > - + {avatar} { onUnmount={props.onUnmount} message={props.message} onPaste={this.onPaste.bind(this)} + changeEvent={this.eventHandler} placeholder='Message...' /> - - {this.props.canUpload - ? this.props.uploading - ? - : this.props.promptUpload().then(this.uploadSuccess)} - /> - : null - } - - + + + {this.props.canUpload ? ( + this.props.uploading ? ( + + ) : ( + + this.props.promptUpload().then(this.uploadSuccess) + } + /> + ) + ) : null} + + {MOBILE_BROWSER_REGEX.test(navigator.userAgent) ? + this.chatEditor.current.submit()} + > + + + : null} ); } } -export default withLocalState(withStorage(ChatInput, { accept: 'image/*' }), ['hideAvatars']); +export default withLocalState, 'hideAvatars', ChatInput>( + withStorage(ChatInput, { accept: 'image/*' }), + ['hideAvatars'] +); diff --git a/pkg/interface/src/views/apps/chat/components/ChatMessage.tsx b/pkg/interface/src/views/apps/chat/components/ChatMessage.tsx index c3359bdf5b..6bee90d048 100644 --- a/pkg/interface/src/views/apps/chat/components/ChatMessage.tsx +++ b/pkg/interface/src/views/apps/chat/components/ChatMessage.tsx @@ -1,49 +1,33 @@ /* eslint-disable max-lines-per-function */ -import React, { - useState, - useEffect, - useRef, - Component, - PureComponent, - useCallback -} from 'react'; +import { BaseImage, Box, Col, Icon, Row, Rule, Text } from '@tlon/indigo-react'; +import { Contact, MentionContent, Post } from '@urbit/api'; +import bigInt from 'big-integer'; import moment from 'moment'; -import _ from 'lodash'; +import React, { + Ref, + useEffect, + useMemo, useState +} from 'react'; import VisibilitySensor from 'react-visibility-sensor'; -import { Box, Row, Text, Rule, BaseImage } from '@tlon/indigo-react'; +import GlobalApi from '~/logic/api/global'; +import { useIdlingState } from '~/logic/lib/idling'; import { Sigil } from '~/logic/lib/sigil'; -import OverlaySigil from '~/views/components/OverlaySigil'; +import { useCopy } from '~/logic/lib/useCopy'; import { - uxToHex, - cite, - writeText, - useShowNickname, - useHideAvatar, - useHovering + cite, daToUnix, useHovering, useShowNickname, uxToHex } from '~/logic/lib/util'; -import { - Group, - Association, - Contacts, - Post, - Groups, - Associations -} from '~/types'; -import TextContent from './content/text'; -import CodeContent from './content/code'; -import RemoteContent from '~/views/components/RemoteContent'; -import { Mention } from '~/views/components/MentionText'; -import styled from 'styled-components'; +import { useContact } from '~/logic/state/contact'; import useLocalState from '~/logic/state/local'; -import useSettingsState, {selectCalmState} from "~/logic/state/settings"; -import Timestamp from '~/views/components/Timestamp'; -import useContactState from '~/logic/state/contact'; -import {useIdlingState} from '~/logic/lib/idling'; +import useSettingsState, { selectCalmState } from '~/logic/state/settings'; +import { Dropdown } from '~/views/components/Dropdown'; +import ProfileOverlay from '~/views/components/ProfileOverlay'; +import { GraphContent } from '~/views/landscape/components/Graph/GraphContent'; + export const DATESTAMP_FORMAT = '[~]YYYY.M.D'; interface DayBreakProps { - when: string; + when: string | number; shimTop?: boolean; } @@ -57,201 +41,335 @@ export const DayBreak = ({ when, shimTop = false }: DayBreakProps) => ( mt={shimTop ? '-8px' : '0'} > - + {moment(when).calendar(null, { sameElse: DATESTAMP_FORMAT })} ); -export const UnreadMarker = React.forwardRef(({ dayBreak, when, api, association }, ref) => { - const [visible, setVisible] = useState(false); - const idling = useIdlingState(); - const dismiss = useCallback(() => { - api.hark.markCountAsRead(association, '/', 'message'); - }, [api, association]); +export const UnreadMarker = React.forwardRef( + ({ dismissUnread }: any, ref: Ref) => { + const [visible, setVisible] = useState(false); + const idling = useIdlingState(); - useEffect(() => { - if(visible && !idling) { - dismiss(); - } - }, [visible, idling]); + useEffect(() => { + if (visible && !idling) { + dismissUnread(); + } + }, [visible, idling]); + + return ( + + + + + New messages below + + + + + ); + } +); + +const MessageActionItem = (props) => { + return ( + + + {props.children} + + + ); +}; + +const MessageActions = ({ onReply, onDelete, msg, isAdmin, permalink }) => { + const isOwn = () => msg.author === window.ship; + const { doCopy, copyDisplay } = useCopy(permalink, 'Copy Message Link'); return ( - - - - - New messages below - - - - -)}); + + + onReply(msg)} + > + + + + onReply(msg)}> + Reply + + + {copyDisplay} + + {(isAdmin || isOwn()) ? ( + onDelete(msg)} color='red'> + Delete Message + + ) : null} + {false && ( + console.log(e)}> + View Signature + + )} + + } + > + + + + + + + ); +}; + +const MessageWrapper = (props) => { + const { hovering, bind } = useHovering(); + const showHover = (props.transcluded === 0) && hovering && !props.hideHover; + return ( + + {props.children} + {showHover ? : null} + + ); +}; interface ChatMessageProps { msg: Post; previousMsg?: Post; nextMsg?: Post; - isLastRead: boolean; - group: Group; - association: Association; + isLastRead?: boolean; + permalink?: string; + transcluded?: number; className?: string; - isPending: boolean; + isPending?: boolean; style?: unknown; - scrollWindow: HTMLDivElement; isLastMessage?: boolean; - unreadMarkerRef: React.RefObject; - history: unknown; + dismissUnread?: () => void; api: GlobalApi; highlighted?: boolean; renderSigil?: boolean; + hideHover?: boolean; innerRef: (el: HTMLDivElement | null) => void; + onReply?: (msg: Post) => void; + showOurContact: boolean; + onDelete?: () => void; } -class ChatMessage extends Component { - private divRef: React.RefObject; - - constructor(props) { - super(props); - this.divRef = React.createRef(); - } - - componentDidMount() { - } - - render() { - const { - msg, - previousMsg, - nextMsg, - isLastRead, - group, - association, - className = '', - isPending, - style, - scrollWindow, - isLastMessage, - unreadMarkerRef, - history, - api, - highlighted, - fontSize, - } = this.props; - - let { renderSigil } = this.props; - - if (renderSigil === undefined) { - renderSigil = Boolean( - (nextMsg && msg.author !== nextMsg.author) || - !nextMsg || - msg.number === 1 - ); - } - - const dayBreak = - nextMsg && - new Date(msg['time-sent']).getDate() !== - new Date(nextMsg['time-sent']).getDate(); - - const containerClass = `${isPending ? 'o-40' : ''} ${className}`; - - const timestamp = moment - .unix(msg['time-sent'] / 1000) - .format(renderSigil ? 'h:mm A' : 'h:mm'); - - - const messageProps = { - msg, - timestamp, - association, - group, - style, - containerClass, - isPending, - history, - api, - scrollWindow, - highlighted, - fontSize, - }; - - const unreadContainerStyle = { - height: isLastRead ? '2rem' : '0' - }; +function ChatMessage(props: ChatMessageProps) { + let { highlighted } = props; + const { + msg, + nextMsg, + isLastRead = false, + className = '', + isPending = false, + style, + isLastMessage, + api, + showOurContact, + hideHover, + dismissUnread = () => null, + permalink = '' + } = props; + if (typeof msg === 'string' || !msg) { return ( - - {dayBreak && !isLastRead ? ( - - ) : null} - {renderSigil ? ( - <> - - - - ) : ( - - )} - - {isLastRead ? ( - - ) : null} - - + This message has been deleted. ); } + + let onReply = props?.onReply ?? (() => {}); + let onDelete = props?.onDelete ?? (() => {}); + const transcluded = props?.transcluded ?? 0; + const renderSigil = props.renderSigil ?? (Boolean(nextMsg && msg.author !== nextMsg.author) || + !nextMsg || + msg.number === 1 + ); + + const ourMention = msg?.contents?.some((e: MentionContent) => { + return e?.mention && e?.mention === window.ship; + }); + + if (!highlighted) { + if (ourMention) { + highlighted = true; + } + } + + const date = useMemo(() => daToUnix(bigInt(msg.index.split('/')[1])), [msg.index]); + const nextDate = useMemo(() => nextMsg && typeof nextMsg !== 'string' ? ( + daToUnix(bigInt(nextMsg.index.split('/')[1])) + ) : null, + [nextMsg] + ); + + const dayBreak = useMemo(() => + nextDate && + new Date(date).getDate() !== + new Date(nextDate).getDate() + , [nextDate, date]); + + const containerClass = `${isPending ? 'o-40' : ''} ${className}`; + + const timestamp = useMemo(() => moment + .unix(date / 1000) + .format(renderSigil ? 'h:mm A' : 'h:mm'), + [date, renderSigil] + ); + + const messageProps = { + msg, + timestamp, + isPending, + showOurContact, + api, + highlighted, + hideHover, + transcluded, + onReply, + onDelete + }; + + const message = useMemo(() => ( + + ), [renderSigil, msg, timestamp, api, transcluded, showOurContact]); + + const unreadContainerStyle = { + height: isLastRead ? '2rem' : '0' + }; + + return ( + + {dayBreak && !isLastRead ? ( + + ) : null} + + { renderSigil && } + {message} + + + {isLastRead ? ( + + ) : null} + + + ); } -export default React.forwardRef((props, ref) => ); +export default React.forwardRef((props: Omit, ref: any) => ( + +)); export const MessageAuthor = ({ timestamp, msg, - group, api, - history, - scrollWindow, - ...rest + showOurContact, + ...props }) => { - const osDark = useLocalState((state) => state.dark); + const osDark = useLocalState(state => state.dark); const theme = useSettingsState(s => s.display.theme); const dark = theme === 'dark' || (theme === 'auto' && osDark); - const contacts = useContactState(state => state.contacts); + let contact: Contact | null = useContact(`~${msg.author}`); + + const date = daToUnix(bigInt(msg.index.split('/')[1])); const datestamp = moment - .unix(msg['time-sent'] / 1000) + .unix(date / 1000) .format(DATESTAMP_FORMAT); - const contact = - `~${msg.author}` in contacts ? contacts[`~${msg.author}`] : false; + contact = + ((msg.author === window.ship && showOurContact) || + msg.author !== window.ship) + ? contact + : null; + const showNickname = useShowNickname(contact); const { hideAvatars } = useSettingsState(selectCalmState); - const shipName = showNickname ? contact.nickname : cite(msg.author); - const copyNotice = 'Copied'; + const shipName = showNickname && contact?.nickname || cite(msg.author) || `~${msg.author}`; const color = contact ? `#${uxToHex(contact.color)}` : dark @@ -262,76 +380,56 @@ export const MessageAuthor = ({ : dark ? 'mix-blend-diff' : 'mix-blend-darken'; - const [displayName, setDisplayName] = useState(shipName); - const [nameMono, setNameMono] = useState(showNickname ? false : true); + + const { copyDisplay, doCopy, didCopy } = useCopy(`~${msg.author}`, shipName); const { hovering, bind } = useHovering(); - const [showOverlay, setShowOverlay] = useState(false); - - const toggleOverlay = () => { - setShowOverlay((value) => !value); - }; - - const showCopyNotice = () => { - setDisplayName(copyNotice); - setNameMono(false); - }; - - useEffect(() => { - const resetDisplay = () => { - setDisplayName(shipName); - setNameMono(showNickname ? false : true); - }; - const timer = setTimeout(() => resetDisplay(), 800); - return () => clearTimeout(timer); - }, [shipName, displayName]); + const nameMono = !(showNickname || didCopy); const img = contact?.avatar && !hideAvatars ? ( ) : ( - + + + ); return ( - + { - setShowOverlay(true); - }} - height={16} + height={24} pr={2} - pl={2} + mt={'1px'} + pl={props.transcluded ? '11px' : '12px'} cursor='pointer' position='relative' > - {showOverlay && ( - toggleOverlay()} - history={history} - className='relative' - scrollWindow={scrollWindow} - api={api} - /> - )} - {img} + + {img} + { - writeText(`~${msg.author}`); - showCopyNotice(); - }} + onClick={doCopy} title={`~${msg.author}`} > - {displayName} + {copyDisplay} {timestamp} @@ -375,25 +470,29 @@ export const MessageAuthor = ({ ); }; -export const Message = ({ +type MessageProps = { timestamp: string; timestampHover: boolean; } + & Pick + +export const Message = React.memo(({ timestamp, msg, - group, api, - scrollWindow, timestampHover, - ...rest -}) => { + transcluded, + showOurContact +}: MessageProps) => { const { hovering, bind } = useHovering(); - const contacts = useContactState(state => state.contacts); return ( - + {timestampHover ? ( @@ -402,74 +501,19 @@ export const Message = ({ ) : ( <> )} - - {msg.contents.map((content, i) => { - switch (Object.keys(content)[0]) { - case 'text': - return ( - - ); - case 'code': - return ; - case 'url': - return ( - - - - ); - case 'mention': - const first = (i) => (i === 0); - return ( - - ); - default: - return null; - } - })} - + ); -}; +}); + +Message.displayName = 'Message'; export const MessagePlaceholder = ({ height, @@ -480,10 +524,10 @@ export const MessagePlaceholder = ({ }) => ( diff --git a/pkg/interface/src/views/apps/chat/components/ChatPane.tsx b/pkg/interface/src/views/apps/chat/components/ChatPane.tsx new file mode 100644 index 0000000000..dd6ebf46fb --- /dev/null +++ b/pkg/interface/src/views/apps/chat/components/ChatPane.tsx @@ -0,0 +1,177 @@ +import { Col } from '@tlon/indigo-react'; +import { Content, Graph, Post } from '@urbit/api'; +import bigInt, { BigInteger } from 'big-integer'; +import _ from 'lodash'; +import React, { ReactElement, useCallback, useEffect, useRef, useState } from 'react'; +import GlobalApi from '~/logic/api/global'; +import { useFileDrag } from '~/logic/lib/useDrag'; +import { useLocalStorageState } from '~/logic/lib/useLocalStorageState'; +import { useOurContact } from '~/logic/state/contact'; +import useGraphState from '~/logic/state/graph'; +import ShareProfile from '~/views/apps/chat/components/ShareProfile'; +import { Loading } from '~/views/components/Loading'; +import SubmitDragger from '~/views/components/SubmitDragger'; +import ChatInput, { ChatInput as NakedChatInput } from './ChatInput'; +import ChatWindow from './ChatWindow'; + +interface ChatPaneProps { + /** + * A key to uniquely identify a ChatPane instance. Should be either the + * resource for group chats or the @p for DMs + */ + id: string; + /** + * The graph of the chat to render + */ + graph: Graph; + unreadCount: number; + /** + * User able to write to chat + */ + canWrite: boolean; + api: GlobalApi; + /** + * Get contents of reply message + */ + onReply: (msg: Post) => string; + onDelete: (msg: Post) => void; + /** + * Fetch more messages + * + * @param newer Get newer or older backlog + * @returns Whether backlog is finished loading in that direction + */ + fetchMessages: (newer: boolean) => Promise; + /** + * Dismiss unreads for chat + */ + dismissUnread: () => void; + /** + * Get permalink for a node + */ + getPermalink: (idx: BigInteger) => string; + isAdmin: boolean; + /** + * Post message with contents to channel + */ + onSubmit: (contents: Content[]) => void; + /** + * + * Users or group we haven't shared our contact with yet + * + * string[] - array of ships + * string - path of group + */ + promptShare?: string[] | string; +} + +export function ChatPane(props: ChatPaneProps): ReactElement { + const { + api, + graph, + unreadCount, + canWrite, + id, + getPermalink, + isAdmin, + dismissUnread, + onSubmit, + onDelete, + promptShare = [], + fetchMessages + } = props; + const graphTimesentMap = useGraphState(state => state.graphTimesentMap); + const ourContact = useOurContact(); + const chatInput = useRef(); + + const onFileDrag = useCallback( + (files: FileList | File[]) => { + if (!chatInput.current) { + return; + } + (chatInput.current as NakedChatInput)?.uploadFiles(files); + }, + [chatInput.current] + ); + + const { bind, dragging } = useFileDrag(onFileDrag); + + const [unsent, setUnsent] = useLocalStorageState>( + 'chat-unsent', + {} + ); + + const appendUnsent = useCallback( + (u: string) => setUnsent(s => ({ ...s, [id]: u })), + [id] + ); + + const clearUnsent = useCallback(() => { + setUnsent((s) => { + if (id in s) { + return _.omit(s, id); + } + return s; + }); + }, [id]); + + const scrollTo = new URLSearchParams(location.search).get('msg'); + + const [showBanner, setShowBanner] = useState(false); + + useEffect(() => { + setShowBanner(promptShare.length > 0); + }, [promptShare]); + + const onReply = useCallback( + (msg: Post) => { + const message = props.onReply(msg); + setUnsent(s => ({ ...s, [id]: message })); + }, + [id, props.onReply] + ); + + if (!graph) { + return ; + } + + return ( + + setShowBanner(false)} + /> + {dragging && } + + {canWrite && ( + + )} + + ); +} diff --git a/pkg/interface/src/views/apps/chat/components/ChatWindow.tsx b/pkg/interface/src/views/apps/chat/components/ChatWindow.tsx index 00fa30140f..b721601665 100644 --- a/pkg/interface/src/views/apps/chat/components/ChatWindow.tsx +++ b/pkg/interface/src/views/apps/chat/components/ChatWindow.tsx @@ -1,47 +1,32 @@ -import React, { Component } from 'react'; -import { RouteComponentProps } from 'react-router-dom'; -import _ from 'lodash'; -import bigInt, { BigInteger } from 'big-integer'; - -import { Col } from '@tlon/indigo-react'; +import { Col, Text } from '@tlon/indigo-react'; import { - Patp, - Contacts, - Association, - Associations, - Group, - Groups, - Graph + Graph, + GraphNode, Post } from '@urbit/api'; - +import bigInt, { BigInteger } from 'big-integer'; +import React, { Component } from 'react'; import GlobalApi from '~/logic/api/global'; - import VirtualScroller from '~/views/components/VirtualScroller'; - import ChatMessage, { MessagePlaceholder } from './ChatMessage'; -import { UnreadNotice } from './unread-notice'; -import withState from '~/logic/lib/withState'; -import useGroupState from '~/logic/state/group'; -import useMetadataState from '~/logic/state/metadata'; -import useGraphState from '~/logic/state/graph'; +import UnreadNotice from './UnreadNotice'; -const INITIAL_LOAD = 20; -const DEFAULT_BACKLOG_SIZE = 100; const IDLE_THRESHOLD = 64; -const MAX_BACKLOG_SIZE = 1000; -type ChatWindowProps = RouteComponentProps<{ - ship: Patp; - station: string; -}> & { +type ChatWindowProps = { unreadCount: number; graph: Graph; - association: Association; - group: Group; - ship: Patp; - station: any; + graphSize: number; + station?: unknown; + fetchMessages: (newer: boolean) => Promise; api: GlobalApi; - scrollTo?: number; + scrollTo?: BigInteger; + onReply: (msg: Post) => void; + onDelete: (msg: Post) => void; + dismissUnread: () => void; + pendingSize?: number; + showOurContact: boolean; + getPermalink: (index: BigInteger) => string; + isAdmin: boolean; }; interface ChatWindowState { @@ -51,18 +36,20 @@ interface ChatWindowState { unreadIndex: BigInteger; } +interface RendererProps { + index: bigInt.BigInteger; + scrollWindow: any; +} + const virtScrollerStyle = { height: '100%' }; class ChatWindow extends Component< ChatWindowProps, ChatWindowState > { - private virtualList: VirtualScroller | null; - private unreadMarkerRef: React.RefObject; + private virtualList: VirtualScroller | null; private prevSize = 0; - private loadedNewest = false; - private loadedOldest = false; - private fetchPending = false; + private unreadSet = false; INITIALIZATION_MAX_TIME = 100; @@ -76,35 +63,40 @@ class ChatWindow extends Component< unreadIndex: bigInt.zero }; - this.dismissUnread = this.dismissUnread.bind(this); this.scrollToUnread = this.scrollToUnread.bind(this); this.handleWindowBlur = this.handleWindowBlur.bind(this); this.handleWindowFocus = this.handleWindowFocus.bind(this); this.stayLockedIfActive = this.stayLockedIfActive.bind(this); this.virtualList = null; - this.unreadMarkerRef = React.createRef(); this.prevSize = props.graph.size; } componentDidMount() { this.calculateUnreadIndex(); setTimeout(() => { - if (this.props.scrollTo) { - this.scrollToUnread(); - } - this.setState({ initialized: true }); - + this.setState({ initialized: true }, () => { + if(this.props.scrollTo) { + this.virtualList!.scrollLocked = false; + this.virtualList!.scrollToIndex(this.props.scrollTo); + } + }); }, this.INITIALIZATION_MAX_TIME); } calculateUnreadIndex() { const { graph, unreadCount } = this.props; + const { state } = this; + if(state.unreadIndex.neq(bigInt.zero)) { + return; + } const unreadIndex = graph.keys()[unreadCount]; if (!unreadIndex || unreadCount === 0) { - this.setState({ - unreadIndex: bigInt.zero - }); + if(state.unreadIndex.neq(bigInt.zero)) { + this.setState({ + unreadIndex: bigInt.zero + }); + } return; } this.setState({ @@ -112,22 +104,41 @@ class ChatWindow extends Component< }); } - handleWindowBlur() { + dismissedInitialUnread(): boolean { + const { unreadCount, graph } = this.props; + + return this.state.unreadIndex.eq(bigInt.zero) + ? unreadCount > graph.size + : this.state.unreadIndex.neq(graph.keys()?.[unreadCount]?.[0] ?? bigInt.zero); + } + + handleWindowBlur(): void { this.setState({ idle: true }); } - handleWindowFocus() { + handleWindowFocus(): void { this.setState({ idle: false }); if (this.virtualList?.window?.scrollTop === 0) { - this.dismissUnread(); + this.props.dismissUnread(); } } - componentDidUpdate(prevProps: ChatWindowProps, prevState) { - const { history, graph, unreadCount, station } = this.props; + componentDidUpdate(prevProps: ChatWindowProps): void { + const { unreadCount, graphSize, station } = this.props; + if(unreadCount === 0 && prevProps.unreadCount !== unreadCount) { + this.unreadSet = true; + } - if (graph.size !== prevProps.graph.size && this.fetchPending) { - this.fetchPending = false; + if(this.prevSize !== graphSize) { + this.prevSize = graphSize; + if(this.state.unreadIndex.eq(bigInt.zero)) { + this.calculateUnreadIndex(); + } + if(this.unreadSet && + this.dismissedInitialUnread() && + this.virtualList!.startOffset() < 5) { + this.props.dismissUnread(); + } } if (unreadCount > prevProps.unreadCount) { @@ -140,14 +151,20 @@ class ChatWindow extends Component< } } - stayLockedIfActive() { + stayLockedIfActive(): void { if (this.virtualList && !this.state.idle) { this.virtualList.resetScroll(); - this.dismissUnread(); + this.props.dismissUnread(); } } - scrollToUnread() { + onBottomLoaded = () => { + if(this.state.unreadIndex.eq(bigInt.zero)) { + this.calculateUnreadIndex(); + } + } + + scrollToUnread(): void { const { unreadIndex } = this.state; if (unreadIndex.eq(bigInt.zero)) { return; @@ -156,78 +173,42 @@ class ChatWindow extends Component< this.virtualList?.scrollToIndex(this.state.unreadIndex); } - dismissUnread() { - const { association } = this.props; - if (this.state.fetchPending) return; - if (this.props.unreadCount === 0) return; - this.props.api.hark.markCountAsRead(association, '/', 'message'); - } - - setActive = () => { - if(this.state.idle) { - this.setState({ idle: false }); - } - } - - fetchMessages = async (newer: boolean): Promise => { - const { api, station, graph } = this.props; - if(this.fetchPending) { - return false; - } - - - this.fetchPending = true; - - const [, , ship, name] = station.split('/'); - const currSize = graph.size; - if (newer) { - const [index] = graph.peekLargest()!; - await api.graph.getYoungerSiblings( - ship, - name, - 100, - `/${index.toString()}` - ); - } else { - const [index] = graph.peekSmallest()!; - await api.graph.getOlderSiblings(ship, name, 100, `/${index.toString()}`); - this.calculateUnreadIndex(); - } - this.fetchPending = false; - return currSize === graph.size; - } - - onScroll = ({ scrollTop, scrollHeight, windowHeight }) => { + onScroll = ({ scrollTop }) => { if (!this.state.idle && scrollTop > IDLE_THRESHOLD) { this.setState({ idle: true }); } } - - renderer = React.forwardRef(({ index, scrollWindow }, ref) => { + renderer = React.forwardRef(({ index, scrollWindow }: RendererProps, ref) => { const { api, - association, - group, - contacts, + showOurContact, graph, - history, - groups, - associations + onReply, + onDelete, + getPermalink, + dismissUnread, + isAdmin } = this.props; - const { unreadMarkerRef } = this; + const permalink = getPermalink(index); const messageProps = { - association, - group, - contacts, - unreadMarkerRef, - history, + showOurContact, api, - groups, - associations + onReply, + onDelete, + permalink, + dismissUnread, + isAdmin }; + const msg = graph.get(index)?.post; - if (!msg) return null; + if (!msg || typeof msg === 'string') { + return ( + + This message has been deleted. + + ); + }; if (!this.state.initialized) { return ( idx.eq(index)); - const prevIdx = keys[graphIdx + 1]; - const nextIdx = keys[graphIdx - 1]; + const highlighted = index.eq(this.props.scrollTo ?? bigInt.zero); + const keys = graph.keys(); + const graphIdx = keys.findIndex(idx => idx.eq(index)); + const prevIdx = keys[graphIdx - 1]; + const nextIdx = keys[graphIdx + 1]; const isLastRead: boolean = this.state.unreadIndex.eq(index); const props = { highlighted, @@ -256,6 +237,7 @@ class ChatWindow extends Component< msg, ...messageProps }; + return ( - - )} + ref={(list) => { this.virtualList = list; }} offset={unreadCount} origin='bottom' style={virtScrollerStyle} - onStartReached={this.setActive} + onBottomLoaded={this.onBottomLoaded} onScroll={this.onScroll} data={graph} size={graph.size} pendingSize={pendingSize} - id={association.resource} averageHeight={22} renderer={this.renderer} - loadRows={this.fetchMessages} + loadRows={this.props.fetchMessages} /> ); } } -export default withState(ChatWindow, [ - [useGroupState, ['groups']], - [useMetadataState, ['associations']], - [useGraphState, ['pendingSize']] -]); \ No newline at end of file +export default ChatWindow; diff --git a/pkg/interface/src/views/apps/chat/components/ShareProfile.js b/pkg/interface/src/views/apps/chat/components/ShareProfile.tsx similarity index 55% rename from pkg/interface/src/views/apps/chat/components/ShareProfile.js rename to pkg/interface/src/views/apps/chat/components/ShareProfile.tsx index 13c9751a26..b24cb50b35 100644 --- a/pkg/interface/src/views/apps/chat/components/ShareProfile.js +++ b/pkg/interface/src/views/apps/chat/components/ShareProfile.tsx @@ -1,19 +1,20 @@ -import React, { - useState, - useEffect -} from 'react'; -import _ from 'lodash'; -import { Box, Row, Text, BaseImage } from '@tlon/indigo-react'; -import { uxToHex } from '~/logic/lib/util'; +import { BaseImage, Box, Row, Text } from '@tlon/indigo-react'; +import { Contact } from '@urbit/api'; +import React, { ReactElement } from 'react'; +import GlobalApi from '~/logic/api/global'; import { Sigil } from '~/logic/lib/sigil'; +import { uxToHex } from '~/logic/lib/util'; -export const ShareProfile = (props) => { +interface ShareProfileProps { + our?: Contact; + api: GlobalApi; + recipients: string | string[]; + onShare: () => void; +} + +const ShareProfile = (props: ShareProfileProps): ReactElement | null => { const { api, - showBanner, - setShowBanner, - group, - groupPath, recipients } = props; @@ -24,43 +25,46 @@ export const ShareProfile = (props) => { width='24px' height='24px' borderRadius={2} - style={{ objectFit: 'cover' }} /> + style={{ objectFit: 'cover' }} + /> ) : ( + backgroundColor={props.our ? `#${uxToHex(props.our.color)}` : '#000000'} + > + color={props.our ? `#${uxToHex(props.our.color)}` : '#000000'} + icon + /> ); const onClick = async () => { - if(group.hidden && recipients.length > 0) { - await api.contacts.allowShips(recipients); - await Promise.all(recipients.map(r => api.contacts.share(r))) - setShowBanner(false); - } else if (!group.hidden) { - const [,,ship,name] = groupPath.split('/'); + if(typeof recipients === 'string') { + const [,,ship,name] = recipients.split('/'); await api.contacts.allowGroup(ship,name); if(ship !== `~${window.ship}`) { await api.contacts.share(ship); } - setShowBanner(false); + } else if(recipients.length > 0) { + await api.contacts.allowShips(recipients); + await Promise.all(recipients.map(r => api.contacts.share(r))); } + props.onShare(); }; - return showBanner ? ( + return props.recipients?.length > 0 ? ( {image} @@ -72,3 +76,5 @@ export const ShareProfile = (props) => { ) : null; }; + +export default ShareProfile; diff --git a/pkg/interface/src/views/apps/chat/components/UnreadNotice.tsx b/pkg/interface/src/views/apps/chat/components/UnreadNotice.tsx new file mode 100644 index 0000000000..a81acf014b --- /dev/null +++ b/pkg/interface/src/views/apps/chat/components/UnreadNotice.tsx @@ -0,0 +1,68 @@ +import { Box, Center, Icon, Text } from '@tlon/indigo-react'; +import moment from 'moment'; +import React, { ReactElement } from 'react'; +import Timestamp from '~/views/components/Timestamp'; + +const UnreadNotice = (props): ReactElement | null => { + const { unreadCount, unreadMsg, dismissUnread, onClick } = props; + + if (unreadCount === 0) { + return null; + } + + const stamp = unreadMsg && moment.unix(unreadMsg.post['time-sent'] / 1000); + + return ( + +

+ + + + {unreadCount} new message{unreadCount > 1 ? 's' : ''} + {unreadMsg && ( + <> + {' '}since{' '} + + + )} + + + + +
+ + ); +}; + +export default UnreadNotice; diff --git a/pkg/interface/src/views/apps/chat/components/unread-notice.js b/pkg/interface/src/views/apps/chat/components/unread-notice.js deleted file mode 100644 index ff81a7b30b..0000000000 --- a/pkg/interface/src/views/apps/chat/components/unread-notice.js +++ /dev/null @@ -1,58 +0,0 @@ -import React, { useEffect, useState } from 'react'; -import moment from 'moment'; -import { Box, Text } from '@tlon/indigo-react'; -import VisibilitySensor from 'react-visibility-sensor'; - -import Timestamp from '~/views/components/Timestamp'; - -export const UnreadNotice = (props) => { - const { unreadCount, unreadMsg, dismissUnread, onClick } = props; - - if (!unreadMsg || (unreadCount === 0)) { - return null; - } - - const stamp = moment.unix(unreadMsg.post['time-sent'] / 1000); - - let datestamp = moment.unix(unreadMsg.post['time-sent'] / 1000).format('YYYY.M.D'); - const timestamp = moment.unix(unreadMsg.post['time-sent'] / 1000).format('HH:mm'); - - if (datestamp === moment().format('YYYY.M.D')) { - datestamp = null; - } - - return ( - - - - {unreadCount} new message{unreadCount > 1 ? 's' : ''} since{' '} - - - - Mark as Read - - - - ); -} diff --git a/pkg/interface/src/views/apps/chat/css/custom.css b/pkg/interface/src/views/apps/chat/css/custom.css index 6042e6dd07..794356a265 100644 --- a/pkg/interface/src/views/apps/chat/css/custom.css +++ b/pkg/interface/src/views/apps/chat/css/custom.css @@ -98,8 +98,15 @@ h2 { font-family: 'Inter', sans-serif; } +.embed-container:not(.embed-container .embed-container):not(.links) { + padding: 0px 8px 8px 8px; +} + .embed-container iframe { max-width: 100%; + width: 100%; + height: 100%; + margin-top: 8px; } .mh-16 { @@ -168,6 +175,7 @@ blockquote { } .chat .react-codemirror2 { width: 100%; + height: 100%; } .chat .CodeMirror { @@ -186,6 +194,18 @@ blockquote { font-size: 14px; } +.chat .CodeMirror-scroll { + display: flex; + flex-direction: column; + justify-content: center; + height: 100%; +} + +.chat .CodeMirror-sizer { + min-height: 0 !important; + max-height: 100%; +} + .chat.code .react-codemirror2 .CodeMirror * { font-family: 'Source Code Pro'; } diff --git a/pkg/interface/src/views/apps/graph/app.js b/pkg/interface/src/views/apps/graph/App.tsx similarity index 69% rename from pkg/interface/src/views/apps/graph/app.js rename to pkg/interface/src/views/apps/graph/App.tsx index 6daf7c449c..9cb22a7158 100644 --- a/pkg/interface/src/views/apps/graph/app.js +++ b/pkg/interface/src/views/apps/graph/App.tsx @@ -1,11 +1,17 @@ -import React, { PureComponent } from 'react'; -import { Switch, Route, useHistory } from 'react-router-dom'; -import { Center, Text } from "@tlon/indigo-react"; +import { Center, Text } from '@tlon/indigo-react'; +import { GraphConfig } from '@urbit/api'; +import React, { ReactElement } from 'react'; +import { Route, Switch, useHistory } from 'react-router-dom'; +import GlobalApi from '~/logic/api/global'; import { deSig } from '~/logic/lib/util'; import useGraphState from '~/logic/state/graph'; import useMetadataState from '~/logic/state/metadata'; -const GraphApp = (props) => { +interface GraphAppProps { + api: GlobalApi; +} + +const GraphApp = (props: GraphAppProps): ReactElement => { const associations= useMetadataState(state => state.associations); const graphKeys = useGraphState(state => state.graphKeys); const history = useHistory(); @@ -22,15 +28,12 @@ const GraphApp = (props) => { const path = `/ship/~${deSig(ship)}/${name}`; const association = associations.graph[path]; - const autoJoin = () => { try { api.graph.joinGraph( `~${deSig(props.match.params.ship)}`, props.match.params.name ); - - } catch(err) { setTimeout(autoJoin, 2000); } @@ -38,8 +41,8 @@ const GraphApp = (props) => { if(!graphKeys.has(resource)) { autoJoin(); - } else if(!!association) { - history.push(`/~landscape/home/resource/${association.metadata.module}${path}`); + } else if(Boolean(association) && 'graph' in association.metadata.config) { + history.push(`/~landscape/home/resource/${(association.metadata.config as GraphConfig).graph}${path}`); } return (
@@ -50,6 +53,6 @@ const GraphApp = (props) => { /> ); -} +}; -export default GraphApp; \ No newline at end of file +export default GraphApp; diff --git a/pkg/interface/src/views/apps/launch/app.js b/pkg/interface/src/views/apps/launch/App.tsx similarity index 60% rename from pkg/interface/src/views/apps/launch/app.js rename to pkg/interface/src/views/apps/launch/App.tsx index 3e6570adf9..f84d4592d6 100644 --- a/pkg/interface/src/views/apps/launch/app.js +++ b/pkg/interface/src/views/apps/launch/App.tsx @@ -1,38 +1,37 @@ -import React, { useState, useMemo, useEffect } from 'react'; -import styled from 'styled-components'; +/* eslint-disable max-lines-per-function */ +import { Box, Button, Col, Icon, Row, Text } from '@tlon/indigo-react'; import f from 'lodash/fp'; -import _ from 'lodash'; +import React, { ReactElement, useEffect, useMemo, useState } from 'react'; +import { Helmet } from 'react-helmet'; +import styled from 'styled-components'; +import GlobalApi from '~/logic/api/global'; +import { + hasTutorialGroup, -import { Col, Button, Box, Row, Icon, Text } from '@tlon/indigo-react'; + TUTORIAL_BOOK, + TUTORIAL_CHAT, TUTORIAL_GROUP, + TUTORIAL_HOST, -import './css/custom.css'; -import useContactState from '~/logic/state/contact'; -import Tiles from './components/tiles'; -import Tile from './components/tiles/tile'; + TUTORIAL_LINKS +} from '~/logic/lib/tutorialModal'; +import { useModal } from '~/logic/lib/useModal'; +import { useQuery } from '~/logic/lib/useQuery'; +import { useWaitForProps } from '~/logic/lib/useWaitForProps'; +import { writeText } from '~/logic/lib/util'; +import useHarkState from '~/logic/state/hark'; +import useLaunchState from '~/logic/state/launch'; +import useLocalState from '~/logic/state/local'; +import useMetadataState from '~/logic/state/metadata'; +import useSettingsState, { selectCalmState } from '~/logic/state/settings'; +import { StarIcon } from '~/views/components/StarIcon'; +import { StatelessAsyncButton } from '~/views/components/StatelessAsyncButton'; +import { JoinGroup } from '~/views/landscape/components/JoinGroup'; +import { NewGroup } from '~/views/landscape/components/NewGroup'; import Groups from './components/Groups'; import ModalButton from './components/ModalButton'; -import { StatelessAsyncButton } from '~/views/components/StatelessAsyncButton'; -import { StarIcon } from '~/views/components/StarIcon'; -import { writeText } from '~/logic/lib/util'; -import { useModal } from "~/logic/lib/useModal"; -import { NewGroup } from "~/views/landscape/components/NewGroup"; -import { JoinGroup } from "~/views/landscape/components/JoinGroup"; -import { Helmet } from 'react-helmet'; -import useLocalState from "~/logic/state/local"; -import useHarkState from '~/logic/state/hark'; -import { useWaitForProps } from '~/logic/lib/useWaitForProps'; -import { useQuery } from "~/logic/lib/useQuery"; -import { - hasTutorialGroup, - TUTORIAL_GROUP, - TUTORIAL_HOST, - TUTORIAL_BOOK, - TUTORIAL_CHAT, - TUTORIAL_LINKS -} from '~/logic/lib/tutorialModal'; -import useLaunchState from '~/logic/state/launch'; -import useSettingsState, { selectCalmState } from '~/logic/state/settings'; - +import Tiles from './components/tiles'; +import Tile from './components/tiles/tile'; +import './css/custom.css'; const ScrollbarLessBox = styled(Box)` scrollbar-width: none !important; @@ -44,15 +43,32 @@ const ScrollbarLessBox = styled(Box)` const tutSelector = f.pick(['tutorialProgress', 'nextTutStep', 'hideGroups']); -export default function LaunchApp(props) { - const baseHash = useLaunchState(state => state.baseHash); +interface LaunchAppProps { + connection: string; + api: GlobalApi; +} + +export const LaunchApp = (props: LaunchAppProps): ReactElement | null => { + const { connection } = props; + const { baseHash, runtimeLag } = useLaunchState(state => state); const [hashText, setHashText] = useState(baseHash); const [exitingTut, setExitingTut] = useState(false); + const seen = useSettingsState(s => s?.tutorial?.seen) ?? true; + const associations = useMetadataState(s => s.associations); + const hasLoaded = useMemo(() => Boolean(connection === 'connected'), [connection]); + const notificationsCount = useHarkState(state => state.notificationsCount); + const calmState = useSettingsState(selectCalmState); + const { hideUtilities } = calmState; + const { tutorialProgress, nextTutStep } = useLocalState(tutSelector); + let { hideGroups } = useLocalState(tutSelector); + !hideGroups ? { hideGroups } = calmState : null; + + const waiter = useWaitForProps({ ...props, associations }); const hashBox = ( - + {hashText || baseHash} @@ -76,27 +95,10 @@ export default function LaunchApp(props) { const { query } = useQuery(); - useEffect(() => { - if(query.get('tutorial')) { - if(hasTutorialGroup(props)) { - nextTutStep(); - } else { - showModal(); - } - } - }, [query]); - - const { hideUtilities } = useSettingsState(selectCalmState); - const { tutorialProgress, nextTutStep } = useLocalState(tutSelector); - let { hideGroups } = useLocalState(tutSelector); - !hideGroups ? { hideGroups } = useSettingsState(selectCalmState) : null; - - const waiter = useWaitForProps(props); - const { modal, showModal } = useModal({ position: 'relative', maxWidth: '350px', - modal: (dismiss) => { + modal: function modal(dismiss) { const onDismiss = (e) => { e.stopPropagation(); props.api.settings.putEntry('tutorial', 'seen', true); @@ -104,48 +106,47 @@ export default function LaunchApp(props) { }; const onContinue = async (e) => { e.stopPropagation(); - if(!hasTutorialGroup(props)) { + if (!hasTutorialGroup({ associations })) { await props.api.groups.join(TUTORIAL_HOST, TUTORIAL_GROUP); await props.api.settings.putEntry('tutorial', 'joined', Date.now()); await waiter(hasTutorialGroup); await Promise.all( - [TUTORIAL_BOOK, TUTORIAL_CHAT, TUTORIAL_LINKS].map(graph => - props.api.graph.joinGraph(TUTORIAL_HOST, graph))); + [TUTORIAL_BOOK, TUTORIAL_CHAT, TUTORIAL_LINKS].map(graph => props.api.graph.joinGraph(TUTORIAL_HOST, graph))); - await waiter(p => { + await waiter((p) => { return `/ship/${TUTORIAL_HOST}/${TUTORIAL_CHAT}` in p.associations.graph && - `/ship/${TUTORIAL_HOST}/${TUTORIAL_BOOK}` in p.associations.graph && - `/ship/${TUTORIAL_HOST}/${TUTORIAL_LINKS}` in p.associations.graph; + `/ship/${TUTORIAL_HOST}/${TUTORIAL_BOOK}` in p.associations.graph && + `/ship/${TUTORIAL_HOST}/${TUTORIAL_LINKS}` in p.associations.graph; }); } nextTutStep(); dismiss(); - } - return exitingTut ? ( - + }; + return exitingTut ? ( + - - You can always restart the tutorial by typing "tutorial" in Leap + + You can always restart the tutorial by typing “tutorial” in Leap - - + + ) : ( - + - Welcome - - You have been invited to use Landscape, an interface to chat + Welcome + + You have been invited to use Landscape, an interface to chat and interact with communities
Would you like a tour of Landscape? - + @@ -153,19 +154,27 @@ export default function LaunchApp(props) { - )} + ); + } }); - const contacts = useContactState(state => state.contacts); - const hasLoaded = useMemo(() => Object.keys(contacts).length > 0, [contacts]); - - const notificationsCount = useHarkState(state => state.notificationsCount); useEffect(() => { - const seenTutorial = _.get(props.settings, ['tutorial', 'seen'], true); - if(hasLoaded && !seenTutorial && tutorialProgress === 'hidden') { + if(query.get('tutorial')) { + if (hasTutorialGroup({ associations })) { + if (nextTutStep) { + nextTutStep(); + } + } else { + showModal(); + } + } + }, [query, showModal]); + + useEffect(() => { + if(hasLoaded && !seen && tutorialProgress === 'hidden') { showModal(); } - }, [props.settings, hasLoaded]); + }, [seen, hasLoaded]); return ( <> @@ -175,7 +184,7 @@ export default function LaunchApp(props) { {modal} - + - My Channels + My Channels @@ -212,7 +228,7 @@ export default function LaunchApp(props) { ) } - {hashBox} + {hashBox} - {hashBox} + {hashBox} ); -} +}; + +export default LaunchApp; diff --git a/pkg/interface/src/views/apps/launch/components/Groups.tsx b/pkg/interface/src/views/apps/launch/components/Groups.tsx index 7f52fce4e9..7656895893 100644 --- a/pkg/interface/src/views/apps/launch/components/Groups.tsx +++ b/pkg/interface/src/views/apps/launch/components/Groups.tsx @@ -1,19 +1,17 @@ -import React, { useRef } from 'react'; -import { Box, Text, Col } from '@tlon/indigo-react'; +import { Box, Col, Text } from '@tlon/indigo-react'; +import { Association, Associations, Unreads } from '@urbit/api'; import f from 'lodash/fp'; -import _ from 'lodash'; import moment from 'moment'; - -import { Associations, Association, Unreads, UnreadStats } from '@urbit/api'; +import React, { useRef } from 'react'; +import { getNotificationCount, getUnreadCount } from '~/logic/lib/hark'; +import { TUTORIAL_GROUP, TUTORIAL_GROUP_RESOURCE, TUTORIAL_HOST } from '~/logic/lib/tutorialModal'; import { alphabeticalOrder } from '~/logic/lib/util'; -import { getUnreadCount, getNotificationCount } from '~/logic/lib/hark'; -import Tile from '../components/tiles/tile'; -import { useTutorialModal } from '~/views/components/useTutorialModal'; import useGroupState from '~/logic/state/group'; import useHarkState from '~/logic/state/hark'; import useMetadataState from '~/logic/state/metadata'; -import { TUTORIAL_HOST, TUTORIAL_GROUP, TUTORIAL_GROUP_RESOURCE } from '~/logic/lib/tutorialModal'; import useSettingsState, { selectCalmState, SettingsState } from '~/logic/state/settings'; +import { useTutorialModal } from '~/views/components/useTutorialModal'; +import Tile from '../components/tiles/tile'; interface GroupsProps {} @@ -49,8 +47,8 @@ export default function Groups(props: GroupsProps & Parameters[0]) { const groups = Object.values(associations?.groups || {}) .filter(e => e?.group in groupState) .sort(sortGroupsAlph); - const graphUnreads = getGraphUnreads(associations || {}, unreads); - const graphNotifications = getGraphNotifications(associations || {}, unreads); + const graphUnreads = getGraphUnreads(associations || {} as Associations, unreads); + const graphNotifications = getGraphNotifications(associations || {} as Associations, unreads); return ( <> @@ -84,25 +82,29 @@ interface GroupProps { const selectJoined = (s: SettingsState) => s.tutorial.joined; function Group(props: GroupProps) { const { path, title, unreads, updates, first = false } = props; - const anchorRef = useRef(null); + const anchorRef = useRef(null); const isTutorialGroup = path === `/ship/${TUTORIAL_HOST}/${TUTORIAL_GROUP}`; useTutorialModal( 'start', isTutorialGroup, anchorRef ); - const { hideUnreads } = useSettingsState(selectCalmState) + const { hideUnreads } = useSettingsState(selectCalmState); const joined = useSettingsState(selectJoined); + const days = Math.max(0, Math.floor(moment.duration(moment(joined) + .add(14, 'days') + .diff(moment())) + .as('days'))) || 0; return ( {title} {!hideUnreads && ( - {isTutorialGroup && joined && - ({Math.floor(moment.duration(moment(joined).add(14, 'days').diff(moment())).as('days'))} days remaining) + {isTutorialGroup && joined && + ({days} day{days !== 1 && 's'} remaining) } {updates > 0 && - ({updates} update{updates !== 1 && 's'} ) + ({updates} update{updates !== 1 && 's'} ) } {unreads > 0 && ({unreads}) diff --git a/pkg/interface/src/views/apps/launch/components/ModalButton.tsx b/pkg/interface/src/views/apps/launch/components/ModalButton.tsx index 190b746a06..b7b73fb1e7 100644 --- a/pkg/interface/src/views/apps/launch/components/ModalButton.tsx +++ b/pkg/interface/src/views/apps/launch/components/ModalButton.tsx @@ -1,5 +1,5 @@ +import { Button, Icon, Row, Text } from '@tlon/indigo-react'; import React from 'react'; -import { Row, Button, Icon, Text } from '@tlon/indigo-react'; import { useModal } from '~/logic/lib/useModal'; const ModalButton = (props) => { diff --git a/pkg/interface/src/views/apps/launch/components/tiles.js b/pkg/interface/src/views/apps/launch/components/tiles.tsx similarity index 79% rename from pkg/interface/src/views/apps/launch/components/tiles.js rename to pkg/interface/src/views/apps/launch/components/tiles.tsx index da8b2326ac..377e0d2674 100644 --- a/pkg/interface/src/views/apps/launch/components/tiles.js +++ b/pkg/interface/src/views/apps/launch/components/tiles.tsx @@ -1,14 +1,18 @@ -import React from 'react'; - +import React, { ReactElement } from 'react'; +import GlobalApi from '~/logic/api/global'; +import useLaunchState from '~/logic/state/launch'; +import { WeatherState } from '~/types'; import BasicTile from './tiles/basic'; -import CustomTile from './tiles/custom'; import ClockTile from './tiles/clock'; +import CustomTile from './tiles/custom'; import WeatherTile from './tiles/weather'; -import useLaunchState from '~/logic/state/launch'; +export interface TileProps { + api: GlobalApi; +} -const Tiles = (props) => { - const weather = useLaunchState(state => state.weather); +const Tiles = (props: TileProps): ReactElement => { + const weather = useLaunchState(state => state.weather) as WeatherState; const tileOrdering = useLaunchState(state => state.tileOrdering); const tileState = useLaunchState(state => state.tiles); const tiles = tileOrdering.filter((key) => { @@ -23,7 +27,6 @@ const Tiles = (props) => { ); @@ -44,11 +47,12 @@ const Tiles = (props) => { } else { return ; } + return null; }); return ( <>{tiles} ); -} +}; export default Tiles; diff --git a/pkg/interface/src/views/apps/launch/components/tiles/basic.js b/pkg/interface/src/views/apps/launch/components/tiles/basic.js deleted file mode 100644 index d90dc29b01..0000000000 --- a/pkg/interface/src/views/apps/launch/components/tiles/basic.js +++ /dev/null @@ -1,32 +0,0 @@ -import React from 'react'; -import { Text, Icon } from '@tlon/indigo-react'; - -import Tile from './tile'; - -export default class BasicTile extends React.PureComponent { - render() { - const { props } = this; - - return ( - - - {props.title === 'Terminal' - ? - : null - }{props.title} - - - ); - } -} diff --git a/pkg/interface/src/views/apps/launch/components/tiles/basic.tsx b/pkg/interface/src/views/apps/launch/components/tiles/basic.tsx new file mode 100644 index 0000000000..7aa6aec34e --- /dev/null +++ b/pkg/interface/src/views/apps/launch/components/tiles/basic.tsx @@ -0,0 +1,32 @@ +import { Icon, Text } from '@tlon/indigo-react'; +import React, { ReactElement } from 'react'; +import Tile from './tile'; + +export interface BasicTileProps { + title: string; + linkedUrl: string; +} + +const BasicTile = (props: BasicTileProps): ReactElement => ( + + + {props.title === 'Terminal' + ? + : null + }{props.title} + + +); + +export default BasicTile; diff --git a/pkg/interface/src/views/apps/launch/components/tiles/clock.js b/pkg/interface/src/views/apps/launch/components/tiles/clock.tsx similarity index 83% rename from pkg/interface/src/views/apps/launch/components/tiles/clock.js rename to pkg/interface/src/views/apps/launch/components/tiles/clock.tsx index 90113d90d9..d9b48c2840 100644 --- a/pkg/interface/src/views/apps/launch/components/tiles/clock.js +++ b/pkg/interface/src/views/apps/launch/components/tiles/clock.tsx @@ -1,8 +1,8 @@ -import React from 'react'; +/* eslint-disable max-lines-per-function */ import moment from 'moment'; -import SunCalc from 'suncalc'; +import React from 'react'; import styled from 'styled-components'; - +import SunCalc from 'suncalc'; import Tile from './tile'; const VIEWBOX_SIZE = 100; @@ -32,8 +32,6 @@ const minsToDegs = (mins) => { return (mins / 1440) * 360; }; -const radToDeg = rad => rad * (180 / Math.PI); - const degToRad = deg => deg * (Math.PI / 180); const convert = (date, referenceTime) => { @@ -42,25 +40,25 @@ const convert = (date, referenceTime) => { // https://github.com/tingletech/moon-phase export const dFromPhase = (moonPhase) => { - let mag, sweep, d = "m50,0"; + let mag, sweep, d = 'm50,0'; if (moonPhase <= 0.25) { - sweep = [ 1, 0 ]; + sweep = [1, 0]; mag = 20 - 20 * moonPhase * 4; } else if (moonPhase <= 0.50) { - sweep = [ 0, 0 ]; + sweep = [0, 0]; mag = 20 * (moonPhase - 0.25) * 4; } else if (moonPhase <= 0.75) { - sweep = [ 1, 1 ]; + sweep = [1, 1]; mag = 20 - 20 * (moonPhase - 0.50) * 4; } else if (moonPhase <= 1) { - sweep = [ 0, 1 ]; + sweep = [0, 1]; mag = 20 * (moonPhase - 0.75) * 4; } - d = d + "a" + mag + ",20 0 1," + sweep[0] + " 0,100 "; - d = d + "a20,20 0 1," + sweep[1] + " 0,-100"; + d = d + 'a' + mag + ',20 0 1,' + sweep[0] + ' 0,100 '; + d = d + 'a20,20 0 1,' + sweep[1] + ' 0,-100'; return d; -} +}; const Moon = ({ angle, ...props }) => { const phase = SunCalc.getMoonIllumination(moment().toDate()).phase.toFixed(2); @@ -70,7 +68,7 @@ const Moon = ({ angle, ...props }) => { - + { { /> ); -} +}; -const Sun = ({ angle, ...props}) => ( +const Sun = ({ angle, ...props }) => ( { const x2 = CX + RADIUS * Math.cos(degToRad(end)); const y2 = CY + RADIUS * Math.sin(degToRad(end)); - const isLarge = Math.abs((start > 360 ? start - 360 : start) - end) > 180; + // const isLarge = Math.abs((start > 360 ? start - 360 : start) - end) > 180; const d = [ 'M', CX, CY, @@ -120,21 +118,29 @@ const SvgArc = ({ start, end, ...rest }) => { ].join(' '); return ; +}; + +interface ClockTextState { + time: number; } -class ClockText extends React.Component { +class ClockText extends React.Component<{}, ClockTextState> { + interval?: NodeJS.Timeout; constructor(props) { super(props); this.state = { time: Date.now() - } + }; } + componentDidMount() { this.interval = setInterval(() => this.setState({ time: Date.now() }), 1000); } componentWillUnmount() { - clearInterval(this.interval); + if (this.interval) { + clearInterval(this.interval); + } } render() { @@ -156,7 +162,8 @@ class ClockText extends React.Component { begin="0s" dur="1s" calcMode="discrete" - repeatCount="indefinite"/> + repeatCount="indefinite" + /> {now.format('mm A')} @@ -167,13 +174,40 @@ class ClockText extends React.Component { fontSize="10" fontFamily="Inter" className="date" - >{now.format('MMM D')}{now.format('Do').replace(now.format('D'), '')} + >{now.format('MMM D')}{now.format('Do').replace(now.format('D'), '')} ); } } -class Clock extends React.PureComponent { +interface ClockProps { + data: { + latitude?: number; + longitude?: number; + } +} + +interface ClockState { + time: number; + lat: number; + lon: number; + geolocationSuccess: boolean; + sunrise: number; + sunsetStart: number; + sunset: number; + sunriseEnd: number; + dusk: number; + dawn: number; + night: number; + nightEnd: number; + nauticalDawn: number; + nauticalDusk: number; +} + +class Clock extends React.PureComponent { + angle: number; + referenceTime: moment.Moment; + interval: NodeJS.Timeout; constructor(props) { super(props); this.angle = 0; @@ -185,7 +219,7 @@ class Clock extends React.PureComponent { geolocationSuccess: false, sunrise: 0, sunsetStart: 0, - sunset:0, + sunset: 0, sunriseEnd: 0, dusk: 0, dawn: 0, @@ -203,7 +237,6 @@ class Clock extends React.PureComponent { initGeolocation() { if (typeof this.props.data === 'object') { - const { latitude: lat, longitude: lon } = this.props.data; const suncalc = SunCalc.getTimes(new Date(), lat, lon); @@ -252,7 +285,7 @@ class Clock extends React.PureComponent { return ( - + - + - + @@ -313,12 +346,10 @@ class Clock extends React.PureComponent { } } - - const ClockTile = ({ location = {} }) => ( ); -export default ClockTile; \ No newline at end of file +export default ClockTile; diff --git a/pkg/interface/src/views/apps/launch/components/tiles/custom.js b/pkg/interface/src/views/apps/launch/components/tiles/custom.tsx similarity index 79% rename from pkg/interface/src/views/apps/launch/components/tiles/custom.js rename to pkg/interface/src/views/apps/launch/components/tiles/custom.tsx index 7ede9f3239..0699c9fb44 100644 --- a/pkg/interface/src/views/apps/launch/components/tiles/custom.js +++ b/pkg/interface/src/views/apps/launch/components/tiles/custom.tsx @@ -1,5 +1,5 @@ -import React from 'react'; -import { Box, BaseImage } from '@tlon/indigo-react'; +import { BaseImage, Box } from '@tlon/indigo-react'; +import React from 'react'; import Tile from './tile'; export default class CustomTile extends React.PureComponent { @@ -12,8 +12,8 @@ export default class CustomTile extends React.PureComponent { position='relative' backgroundColor='white' border='1px solid' - borderColor='washedGray' - borderRadius='2' + borderColor='lightGray' + borderRadius={2} > `/~${a}`); -const Tile = React.forwardRef((props, ref) => { +type TileProps = BoxProps & { + bg?: string; + to?: string; + href?: string; + p?: PaddingProps; + children: any; + gridColumnStart?: number; + color?: string; +} + +const Tile = React.forwardRef((props: TileProps, ref: RefObject) => { const { bg, to, href, p, boxShadow, gridColumnStart, ...rest } = props; let childElement = ( @@ -37,18 +46,16 @@ const Tile = React.forwardRef((props, ref) => { } else { childElement= ({childElement}); } - } - return ( diff --git a/pkg/interface/src/views/apps/launch/components/tiles/weather.js b/pkg/interface/src/views/apps/launch/components/tiles/weather.tsx similarity index 87% rename from pkg/interface/src/views/apps/launch/components/tiles/weather.js rename to pkg/interface/src/views/apps/launch/components/tiles/weather.tsx index 04c44726a7..9fa4f59329 100644 --- a/pkg/interface/src/views/apps/launch/components/tiles/weather.js +++ b/pkg/interface/src/views/apps/launch/components/tiles/weather.tsx @@ -1,12 +1,13 @@ -import React from 'react'; +import { BaseInput, Box, Icon, Text } from '@tlon/indigo-react'; import moment from 'moment'; -import { Box, Icon, Text, BaseAnchor, BaseInput } from '@tlon/indigo-react'; -import ErrorBoundary from '~/views/components/ErrorBoundary'; +import React from 'react'; +import GlobalApi from '~/logic/api/global'; import withState from '~/logic/lib/withState'; import useLaunchState from '~/logic/state/launch'; - +import ErrorBoundary from '~/views/components/ErrorBoundary'; import Tile from './tile'; + export const weatherStyleMap = { Clear: 'rgba(67, 169, 255, 0.4)', Sunny: 'rgba(67, 169, 255, 0.4)', @@ -36,8 +37,20 @@ const imperialCountries = [ 'Liberia', ]; -class WeatherTile extends React.Component { - constructor(props) { +interface WeatherTileProps { + weather: any; + api: GlobalApi; + location: string; +} + +interface WeatherTileState { + location: string; + manualEntry: boolean; + error: boolean; +} + +class WeatherTile extends React.Component { + constructor(props: WeatherTileProps) { super(props); this.state = { location: '', @@ -50,11 +63,7 @@ class WeatherTile extends React.Component { locationSubmit() { navigator.geolocation.getCurrentPosition((res) => { const location = `${res.coords.latitude},${res.coords.longitude}`; - this.setState({ - location - }, (err) => { - console.log(err); - }, { maximumAge: Infinity, timeout: 10000 }); + this.setState({ location }); this.props.api.launch.weather(location); this.setState({ manualEntry: !this.state.manualEntry }); }); @@ -62,10 +71,8 @@ class WeatherTile extends React.Component { manualLocationSubmit(event) { event.preventDefault(); - const location = document.getElementById('location').value; - this.setState({ location }, (err) => { - console.log(err); - }, { maximumAge: Infinity, timeout: 10000 }); + const location = (document.getElementById('location') as HTMLInputElement).value; + this.setState({ location }); this.props.api.launch.weather(location); this.setState({ manualEntry: !this.state.manualEntry }); } @@ -91,14 +98,14 @@ class WeatherTile extends React.Component { let secureCheck; let error; if (this.state.error === true) { - error = Please try again.; + error = Please try again.; } if (location.protocol === 'https:') { secureCheck = ( this.locationSubmit()} > - Detect -> + Detect {'->'} ); } @@ -128,15 +135,15 @@ class WeatherTile extends React.Component { {locationName ? ` Current location is near ${locationName}.` : ''} {error} - + this.setState({ manualEntry: !this.state.manualEntry })} > - + Weather - -> Set location + {'->'} Set location ); @@ -219,14 +226,14 @@ class WeatherTile extends React.Component { title={`${locationName} Weather`} > - + this.setState({ manualEntry: !this.state.manualEntry }) } > - Weather -> + Weather {'->'} @@ -269,7 +276,7 @@ class WeatherTile extends React.Component { flexDirection="column" justifyContent="flex-start" > - Weather + Weather Loading, please check again later... @@ -281,7 +288,7 @@ class WeatherTile extends React.Component { this.setState({ manualEntry: !this.state.manualEntry }) } > - -> + {'->'} diff --git a/pkg/interface/src/views/apps/links/LinkResource.tsx b/pkg/interface/src/views/apps/links/LinkResource.tsx index b45ffa31ca..5074d297db 100644 --- a/pkg/interface/src/views/apps/links/LinkResource.tsx +++ b/pkg/interface/src/views/apps/links/LinkResource.tsx @@ -1,21 +1,17 @@ -import React, { useEffect } from 'react'; -import { Box, Col, Center, LoadingSpinner, Text } from '@tlon/indigo-react'; -import { Switch, Route, Link } from 'react-router-dom'; -import bigInt from 'big-integer'; - -import GlobalApi from '~/logic/api/global'; -import { StoreState } from '~/logic/store/type'; -import { RouteComponentProps } from 'react-router-dom'; - -import { LinkItem } from './components/LinkItem'; -import LinkWindow from './LinkWindow'; -import { Comments } from '~/views/components/Comments'; - -import './css/custom.css'; +import { Box, Center, Col, LoadingSpinner, Text } from '@tlon/indigo-react'; import { Association } from '@urbit/api/metadata'; +import bigInt from 'big-integer'; +import React, { useEffect } from 'react'; +import { Link, Route, Switch } from 'react-router-dom'; +import GlobalApi from '~/logic/api/global'; import useGraphState from '~/logic/state/graph'; import useMetadataState from '~/logic/state/metadata'; +import { StoreState } from '~/logic/store/type'; +import { Comments } from '~/views/components/Comments'; import useGroupState from '../../../logic/state/group'; +import { LinkItem } from './components/LinkItem'; +import './css/custom.css'; +import LinkWindow from './LinkWindow'; const emptyMeasure = () => {}; @@ -23,13 +19,13 @@ type LinkResourceProps = StoreState & { association: Association; api: GlobalApi; baseUrl: string; -} & RouteComponentProps; +}; export function LinkResource(props: LinkResourceProps) { const { association, api, - baseUrl, + baseUrl } = props; const rid = association.resource; @@ -82,7 +78,7 @@ export function LinkResource(props: LinkResourceProps) { }} /> { const index = bigInt(props.match.params.index); const editCommentId = props.match.params.commentId || null; @@ -96,6 +92,14 @@ export function LinkResource(props: LinkResourceProps) { if (!node) { return Not found; } + + if (typeof node.post === 'string') { + return ( + + This link has been deleted. + + ); + } return ( @@ -105,6 +109,7 @@ export function LinkResource(props: LinkResourceProps) { resource={resourcePath} node={node} baseUrl={resourceUrl} + association={association} group={group} path={resource?.group} api={api} diff --git a/pkg/interface/src/views/apps/links/LinkWindow.tsx b/pkg/interface/src/views/apps/links/LinkWindow.tsx index 7b44aa08a3..05a13e5c75 100644 --- a/pkg/interface/src/views/apps/links/LinkWindow.tsx +++ b/pkg/interface/src/views/apps/links/LinkWindow.tsx @@ -1,23 +1,14 @@ +import { Box, Col, Text } from '@tlon/indigo-react'; +import { Association, Graph, Group } from '@urbit/api'; +import bigInt from 'big-integer'; import React, { - useRef, - useCallback, - useEffect, - useMemo, - Component, -} from "react"; - -import { Col, Text } from "@tlon/indigo-react"; -import bigInt from "big-integer"; -import { Association, Graph, Unreads, Group, Rolodex } from "@urbit/api"; - -import GlobalApi from "~/logic/api/global"; -import VirtualScroller from "~/views/components/VirtualScroller"; -import { LinkItem } from "./components/LinkItem"; -import LinkSubmit from "./components/LinkSubmit"; -import { isWriter } from "~/logic/lib/group"; -import { StorageState } from "~/types"; -import withState from "~/logic/lib/withState"; -import useGraphState from "~/logic/state/graph"; + Component, ReactNode +} from 'react'; +import GlobalApi from '~/logic/api/global'; +import { isWriter } from '~/logic/lib/group'; +import VirtualScroller from '~/views/components/VirtualScroller'; +import { LinkItem } from './components/LinkItem'; +import LinkSubmit from './components/LinkSubmit'; interface LinkWindowProps { association: Association; @@ -33,13 +24,18 @@ interface LinkWindowProps { } const style = { - height: "100%", - width: "100%", - display: "flex", - flexDirection: "column", - alignItems: "center", + height: '100%', + width: '100%', + display: 'flex', + flexDirection: 'column', + alignItems: 'center' }; +interface RendererProps { + index: BigInteger; + children?: ReactNode; +} + class LinkWindow extends Component { fetchLinks = async () => true; @@ -48,10 +44,10 @@ class LinkWindow extends Component { return isWriter(group, association.resource); } - renderItem = ({ index, scrollWindow }) => { + renderItem = React.forwardRef(({ index }: RendererProps, ref) => { const { props } = this; const { association, graph, api } = props; - const [, , ship, name] = association.resource.split("/"); + const [, , ship, name] = association.resource.split('/'); const node = graph.get(index); const first = graph.peekLargest()?.[0]; const post = node?.post; @@ -60,15 +56,16 @@ class LinkWindow extends Component { } const linkProps = { ...props, - node, + node }; if (this.canWrite() && index.eq(first ?? bigInt.zero)) { return ( { api={api} /> - + { typeof post !== 'string' && } ); } - return ; - }; + + if (typeof post === 'string') { + return null; + } + return ( + + + + ); + }); render() { const { graph, api, association } = this.props; const first = graph.peekLargest()?.[0]; - const [, , ship, name] = association.resource.split("/"); + const [, , ship, name] = association.resource.split('/'); if (!first) { return ( { } } -export default LinkWindow; \ No newline at end of file +export default LinkWindow; diff --git a/pkg/interface/src/views/apps/links/components/LinkItem.tsx b/pkg/interface/src/views/apps/links/components/LinkItem.tsx index d1922b0f36..034d65438e 100644 --- a/pkg/interface/src/views/apps/links/components/LinkItem.tsx +++ b/pkg/interface/src/views/apps/links/components/LinkItem.tsx @@ -1,27 +1,29 @@ -import React, { useState, useEffect, useRef, useCallback, ReactElement } from 'react'; -import { Link } from 'react-router-dom'; - -import { Row, Col, Anchor, Box, Text, Icon, Action } from '@tlon/indigo-react'; -import { GraphNode, Group, Rolodex, Unreads } from '@urbit/api'; - -import { writeText } from '~/logic/lib/util'; -import Author from '~/views/components/Author'; -import { roleForShip } from '~/logic/lib/group'; +import { Action, Anchor, Box, Col, Icon, Row, Rule, Text } from '@tlon/indigo-react'; +import { Association, GraphNode, Group, TextContent, UrlContent } from '@urbit/api'; +import React, { ReactElement, RefObject, useCallback, useEffect, useRef } from 'react'; +import { Link, Redirect } from 'react-router-dom'; import GlobalApi from '~/logic/api/global'; +import { roleForShip } from '~/logic/lib/group'; +import { getPermalinkForGraph, referenceToPermalink } from '~/logic/lib/permalinks'; +import { useCopy } from '~/logic/lib/useCopy'; +import useHarkState from '~/logic/state/hark'; +import Author from '~/views/components/Author'; import { Dropdown } from '~/views/components/Dropdown'; import RemoteContent from '~/views/components/RemoteContent'; -import useHarkState from '~/logic/state/hark'; +import { PermalinkEmbed } from '../../permalinks/embed'; interface LinkItemProps { node: GraphNode; + association: Association; resource: string; api: GlobalApi; group: Group; path: string; + baseUrl: string; } - -export const LinkItem = (props: LinkItemProps): ReactElement => { +export const LinkItem = React.forwardRef((props: LinkItemProps, ref: RefObject): ReactElement => { const { + association, node, resource, api, @@ -30,12 +32,16 @@ export const LinkItem = (props: LinkItemProps): ReactElement => { ...rest } = props; - const ref = useRef(null); + if (typeof node.post === 'string' || !node.post) { + return ; + } + const remoteRef = useRef(null); + const index = node.post.index.split('/')[1]; const markRead = useCallback(() => { api.hark.markEachAsRead(props.association, '/', `/${index}`, 'link', 'link'); - }, [props.association, index]); + }, [association, index]); useEffect(() => { function onBlur() { @@ -59,30 +65,35 @@ export const LinkItem = (props: LinkItemProps): ReactElement => { ); const author = node.post.author; - const index = node.post.index.split('/')[1]; const size = node.children ? node.children.size : 0; - const contents = node.post.contents; + const contents = node.post.contents as [TextContent, UrlContent]; const hostname = URLparser.exec(contents[1].url) ? URLparser.exec(contents[1].url)[4] : null; - const href = URLparser.exec(contents[1].url) ? contents[1].url : `http://${contents[1].url}` + const href = URLparser.exec(contents[1].url) ? contents[1].url : `http://${contents[1].url}`; const baseUrl = props.baseUrl || `/~404/${resource}`; const ourRole = group ? roleForShip(group, window.ship) : undefined; const [ship, name] = resource.split('/'); - const [locationText, setLocationText] = useState('Copy Link Location'); + const permalink = getPermalinkForGraph( + association.group, + association.resource, + `/${index}` + ); - const copyLocation = () => { - setLocationText('Copied'); - writeText(contents[1].url); - setTimeout(() => { - setLocationText('Copy Link Location'); - }, 2000); - }; + const { doCopy: doCopyLink, copyDisplay: locationText } = useCopy( + contents[1].url, + 'Copy block source' + ); + + const { doCopy: doCopyNode, copyDisplay: nodeText } = useCopy( + permalink, + 'Copy reference' + ); const deleteLink = () => { if (confirm('Are you sure you want to delete this link?')) { - api.graph.removeNodes(`~${ship}`, name, [node.post.index]); + api.graph.removePosts(`~${ship}`, name, [node.post.index]); } }; @@ -99,7 +110,8 @@ export const LinkItem = (props: LinkItemProps): ReactElement => { ref={ref} width="100%" opacity={node.post.pending ? '0.5' : '1'} - {...rest}> + {...rest} + > { width="100%" color='washedGray' border={1} - borderColor={isUnread ? 'blue' : 'washedGray'} + borderColor={isUnread ? 'blue' : 'lightGray'} borderRadius={2} alignItems="flex-start" overflow="hidden" onClick={markRead} > {contents[0].text} + { 'reference' in contents[1] ? ( + <> + + + + ) : ( + <> { remoteRef.current = r }} + ref={(r) => { + remoteRef.current = r; + }} renderUrl={false} url={href} text={contents[0].text} @@ -141,28 +162,29 @@ export const LinkItem = (props: LinkItemProps): ReactElement => { }} /> - - - {hostname} - - - + + + {hostname} + + + + + )} - - - + - + lineHeight={1} + /> + to={node.post.pending ? '#' : `${baseUrl}/index/${index}`} + style={{ cursor: node.post.pending ? 'default' : 'pointer' }} + > {size} @@ -177,8 +199,12 @@ export const LinkItem = (props: LinkItemProps): ReactElement => { options={ - {locationText} + {locationText} + + {nodeText} + + {(ourRole === 'admin' || node.post.author === window.ship) && Delete Link @@ -187,10 +213,10 @@ export const LinkItem = (props: LinkItemProps): ReactElement => { } > - + ); -}; +}); diff --git a/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx b/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx index 4064582724..b5c7b415bc 100644 --- a/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx +++ b/pkg/interface/src/views/apps/links/components/LinkSubmit.tsx @@ -1,12 +1,12 @@ import { BaseInput, Box, Button, LoadingSpinner, Text } from '@tlon/indigo-react'; +import { hasProvider } from 'oembed-parser'; import React, { useCallback, useState } from 'react'; import GlobalApi from '~/logic/api/global'; +import { createPost } from '~/logic/api/graph'; +import { parsePermalink, permalinkToReference } from '~/logic/lib/permalinks'; import { useFileDrag } from '~/logic/lib/useDrag'; import useStorage from '~/logic/lib/useStorage'; -import { StorageState } from '~/types'; import SubmitDragger from '~/views/components/SubmitDragger'; -import { createPost } from '~/logic/api/graph'; -import { hasProvider } from 'oembed-parser'; interface LinkSubmitProps { api: GlobalApi; @@ -28,12 +28,13 @@ const LinkSubmit = (props: LinkSubmitProps) => { const doPost = () => { const url = linkValue; const text = linkTitle ? linkTitle : linkValue; + const contents = url.startsWith('web+urbitgraph:/') + ? [{ text }, permalinkToReference(parsePermalink(url)!)] + : [{ text }, { url }]; + setDisabled(true); const parentIndex = props.parentIndex || ''; - const post = createPost([ - { text }, - { url } - ], parentIndex); + const post = createPost(contents, parentIndex); props.api.graph.addPost( `~${props.ship}`, @@ -61,6 +62,13 @@ const LinkSubmit = (props: LinkSubmitProps) => { setLinkValue(link); } } + if(link.startsWith('web+urbitgraph://')) { + const permalink = parsePermalink(link); + if(!permalink) { + setLinkValid(false); + return; + } + } if (linkValid) { if (hasProvider(linkValue)) { @@ -153,7 +161,7 @@ const LinkSubmit = (props: LinkSubmitProps) => { flexShrink={0} position='relative' border='1px solid' - borderColor={submitFocused ? 'black' : 'washedGray'} + borderColor={submitFocused ? 'black' : 'lightGray'} width='100%' borderRadius={2} {...bind} diff --git a/pkg/interface/src/views/apps/notifications/graph.tsx b/pkg/interface/src/views/apps/notifications/graph.tsx index b382f5cf8d..1498c01cb7 100644 --- a/pkg/interface/src/views/apps/notifications/graph.tsx +++ b/pkg/interface/src/views/apps/notifications/graph.tsx @@ -1,148 +1,155 @@ -import React, { ReactNode, useCallback } from 'react'; -import moment from 'moment'; -import { Row, Box, Col, Text, Anchor, Icon, Action } from '@tlon/indigo-react'; -import { Link, useHistory } from 'react-router-dom'; +import { Anchor, Box, Col, Icon, Row, Text } from '@tlon/indigo-react'; +import { Association, GraphNotificationContents, GraphNotifIndex, Post } from '@urbit/api'; +import { BigInteger } from 'big-integer'; import _ from 'lodash'; -import { - GraphNotifIndex, - GraphNotificationContents, - Associations, - Rolodex, - Groups -} from '~/types'; -import { Header } from './header'; -import { cite, deSig, pluralize, useShowNickname } from '~/logic/lib/util'; -import Author from '~/views/components/Author'; -import GlobalApi from '~/logic/api/global'; -import { getSnippet } from '~/logic/lib/publish'; +import React, { useCallback } from 'react'; +import { Link, useHistory } from 'react-router-dom'; import styled from 'styled-components'; -import { MentionText } from '~/views/components/MentionText'; -import ChatMessage from '../chat/components/ChatMessage'; -import useContactState from '~/logic/state/contact'; +import GlobalApi from '~/logic/api/global'; +import { referenceToPermalink } from '~/logic/lib/permalinks'; +import { + isDm, pluralize +} from '~/logic/lib/util'; import useGroupState from '~/logic/state/group'; +import { + useAssocForGraph, + useAssocForGroup +} from '~/logic/state/metadata'; +import Author from '~/views/components/Author'; +import { GraphContent } from '~/views/landscape/components/Graph/GraphContent'; +import { PermalinkEmbed } from '../permalinks/embed'; +import { Header } from './header'; + +const TruncBox = styled(Box)<{ truncate?: number }>` + -webkit-line-clamp: ${p => p.truncate ?? 'unset'}; + display: -webkit-box; + overflow: hidden; + -webkit-box-orient: vertical; + color: ${p => p.theme.colors.black}; +`; function getGraphModuleIcon(module: string) { if (module === 'link') { return 'Collection'; } + if (module === 'post') { + return 'Groups'; + } return _.capitalize(module); } -const FilterBox = styled(Box)` - background: linear-gradient( - to bottom, - transparent, - ${p => p.theme.colors.white} - ); -`; - -function describeNotification(description: string, plural: boolean): string { +function describeNotification( + description: string, + plural: boolean, + isDm: boolean, + singleAuthor: boolean +): string { switch (description) { + case 'post': + return singleAuthor ? 'replied to you' : 'Your post received replies'; case 'link': - return `added ${pluralize('new link', plural)} to`; + return `New link${plural ? 's' : ''} in`; case 'comment': - return `left ${pluralize('comment', plural)} on`; - case 'edit-comment': - return `updated ${pluralize('comment', plural)} on`; + return `New comment${plural ? 's' : ''} on`; case 'note': - return `posted ${pluralize('note', plural)} to`; + return `New Note${plural ? 's' : ''} in`; case 'edit-note': return `updated ${pluralize('note', plural)} in`; case 'mention': - return 'mentioned you on'; + return singleAuthor ? 'mentioned you in' : 'You were mentioned in'; case 'message': - return `sent ${pluralize('message', plural)} to`; + if (isDm) { + return 'messaged you'; + } + return `New message${plural ? 's' : ''} in`; default: return description; } } -const GraphUrl = ({ url, title }) => ( - - - - {title} - - -); +const GraphUrl = ({ contents, api }) => { + const [{ text }, link] = contents; -const GraphNodeContent = ({ - group, - post, - mod, - description, - index, - remoteContentPolicy -}) => { - const { contents } = post; - const idx = index.slice(1).split('/'); - if (mod === 'link') { - if (idx.length === 1) { - const [{ text }, { url }] = contents; - return ; - } else if (idx.length === 3) { - return ( - - ); - } - return null; - } - if (mod === 'publish') { - if (idx[1] === '2') { - return ( - - ); - } else if (idx[1] === '1') { - const [{ text: header }, { text: body }] = contents; - const snippet = getSnippet(body); - return ( - - - {header} - - - {snippet} - - - - ); - } - } - - if (mod === 'chat') { + if ('reference' in link) { return ( - - - + ); } - return null; + return ( + + + + {text} + + + ); +}; + +function ContentSummary({ icon, name, author, to }) { + return ( + + + + + + {name} + + + + + by + + + + + + ); +} + +export const GraphNodeContent = ({ post, mod, index, hidden, association }) => { + const { contents } = post; + const idx = index.slice(1).split('/'); + const url = getNodeUrl(mod, hidden, association?.group, association?.resource, index); + if (mod === 'link' && idx.length === 1) { + const [{ text: title }] = contents; + return ( + + ); + } + if (mod === 'publish' && idx[1] === '1') { + const [{ text: title }] = contents; + return ( + + ); + } + return ( + + + + ); }; function getNodeUrl( @@ -160,68 +167,89 @@ function getNodeUrl( const graphUrl = `/~landscape${groupPath}/resource/${mod}${graph}`; const idx = index.slice(1).split('/'); if (mod === 'publish') { - const [noteId] = idx; - return `${graphUrl}/note/${noteId}`; + const [noteId, kind, commId] = idx; + const selected = kind === '2' ? `?selected=${commId}` : ''; + return `${graphUrl}/note/${noteId}${selected}`; } else if (mod === 'link') { - const [linkId] = idx; - return `${graphUrl}/${linkId}`; + const [linkId, commId] = idx; + return `${graphUrl}/index/${linkId}${commId ? `?selected=${commId}` : ''}`; } else if (mod === 'chat') { + if (idx.length > 0) { + return `${graphUrl}?msg=${idx[0]}`; + } return graphUrl; + } else if (mod === 'post') { + return `/~landscape${groupPath}/feed${index}`; } return ''; } -const GraphNode = ({ - post, - author, - mod, - description, - time, - index, - graph, - groupPath, - group, - read, - onRead, - showContact = false, + +interface PostsByAuthor { + author: string; + posts: Post[]; +} +const GraphNodes = (props: { + posts: Post[]; + hideAuthors?: boolean; + index: string; + mod: string; + association: Association; + hidden: boolean; }) => { - author = deSig(author); - const history = useHistory(); - const contacts = useContactState(state => state.contacts); + const { + posts, + mod, + hidden, + index, + hideAuthors = false, + association + } = props; - const nodeUrl = getNodeUrl(mod, group?.hidden, groupPath, graph, index); + const postsByConsecAuthor = _.reduce( + posts, + (acc: PostsByAuthor[], val: Post, key: number) => { + const lent = acc.length; + if (lent > 0 && acc?.[lent - 1]?.author === val.author) { + const last = acc[lent - 1]; + const rest = acc.slice(0, -1); + return [...rest, { ...last, posts: [...last.posts, val] }]; + } + return [...acc, { author: val.author, posts: [val] }]; + }, + [] + ); - const onClick = useCallback(() => { - if (!read) { - onRead(); - } - history.push(nodeUrl); - }, [read, onRead]); - - const showNickname = useShowNickname(contacts?.[`~${author}`]); - const nickname = (contacts?.[`~${author}`]?.nickname && showNickname) ? contacts[`~${author}`].nickname : cite(author); return ( - - - {showContact && ( - - )} - - - - - + <> + {_.map(postsByConsecAuthor, ({ posts, author }, idx) => { + const time = posts[0]?.['time-sent']; + return ( + + {!hideAuthors && ( + + )} + + {_.map(posts, post => ( +