From fbbda575527e5f327bfc3c7ede8b3cc3e57b6f4a Mon Sep 17 00:00:00 2001 From: ixv Date: Fri, 11 Jun 2021 16:10:56 -0700 Subject: [PATCH 1/4] btc: use blockfilter when requesting address info --- pkg/arvo/app/btc-provider.hoon | 8 ++ pkg/arvo/app/btc-wallet.hoon | 230 ++++++++++++++++++------------- pkg/arvo/lib/bip/b158.hoon | 16 ++- pkg/arvo/lib/btc.hoon | 4 +- pkg/arvo/sur/btc-provider.hoon | 4 +- pkg/arvo/tests/lib/bip/b158.hoon | 11 +- 6 files changed, 168 insertions(+), 105 deletions(-) diff --git a/pkg/arvo/app/btc-provider.hoon b/pkg/arvo/app/btc-provider.hoon index e88fe55a4e..ff681b1b9b 100644 --- a/pkg/arvo/app/btc-provider.hoon +++ b/pkg/arvo/app/btc-provider.hoon @@ -195,6 +195,9 @@ :: %ping [%get-block-info ~] + :: + %block-info + [%get-block-info block.act] == [~[(req-card act ract)] state] :: @@ -284,6 +287,11 @@ ?: =(block.host-info block.r) ~[(send-status [%connected network.host-info block.r fee.r])] ~[(send-status [%new-block network.host-info block.r fee.r blockhash.r blockfilter.r])] + :: + %block-info + ?> ?=([%get-block-info *] r) + :_ state + ~[(send-update [%.y %block-info network.host-info +.r])] == :: ++ send-status diff --git a/pkg/arvo/app/btc-wallet.hoon b/pkg/arvo/app/btc-wallet.hoon index bd46aad8d7..ea6a4dd89a 100644 --- a/pkg/arvo/app/btc-wallet.hoon +++ b/pkg/arvo/app/btc-wallet.hoon @@ -4,7 +4,7 @@ :: x/scanned: (list xpub) of all scanned wallets :: x/balance/xpub: balance (in sats) of wallet /- *btc-wallet, bp=btc-provider, file-server, launch-store -/+ dbug, default-agent, bl=btc, bc=bitcoin, bip32 +/+ dbug, default-agent, bl=btc, bc=bitcoin, bip32, bip-b158 |% ++ defaults |% @@ -685,7 +685,6 @@ ++ handle-provider-status |= s=status:bp ^- (quip card _state) - |^ =^ cards state ?~ prov `state ?. =(host.u.prov src.bowl) `state @@ -704,102 +703,143 @@ (give-update %change-provider prov) cards == - :: - ++ on-connected - |= $: p=provider - =network - block=@ud - fee=(unit sats) - blockhash=(unit hexb) - blockfilter=(unit hexb) - == - ^- (quip card _state) - :_ %_ state - prov `p(connected %.y) - btc-state [block fee now.bowl] - == - ?: ?|(?!(connected.p) (lth block.btc-state block)) - ;: weld - (retry-pend-piym network) - (retry-poym network) - (retry-addrs network) - (retry-txs network) - (retry-scans network) - retry-ahistorical-txs +:: +++ on-connected + |= $: p=provider + =network + block=@ud + fee=(unit sats) + blockhash=(unit hexb) + blockfilter=(unit hexb) == + ^- (quip card _state) + :: request block-info for missing blocks + :: if blockhash or blockfilter are ~ request block-info for current block + :: + =| blocks=(list @ud) + =/ gap (sub block block.btc-state) + =? blocks (gth gap 1) + (gulf +(block.btc-state) (dec block)) + =? blocks ?|(?=(~ blockhash) ?=(~ blockfilter)) + (snoc blocks block) + :: + =/ cards=(list card) ;: weld -:: (retry-addrs network) retry-ahistorical-txs (retry-pend-piym network) + (retry-block-info blocks) == - :: - ++ retry-ahistorical-txs - ^- (list card) - %+ turn ~(tap in ahistorical-txs) - |= =txid - (poke-provider [%tx-info txid]) - - :: - ++ retry-scans - |= =network - ^- (list card) - %- zing - %+ murn ~(tap by scans) - |= [[=xpub:bc =chyg] =batch] - ?. =(network network:(~(got by walts) xpub)) ~ - `-:(req-scan batch xpub chyg) - :: +retry-addrs: get info on addresses with unconfirmed UTXOs - :: - ++ retry-addrs - |= =network - ^- (list card) - %- zing - %+ murn ~(val by walts) - |= w=walt - ?. =(network network.w) ~ - ^- (unit (list card)) - :- ~ - %+ turn ~(tap by wach.w) - |= [a=address *] - (poke-provider [%address-info a]) - :: +retry-txs: get info on txs without enough confirmations - :: - ++ retry-txs - |= =network - ^- (list card) - %+ murn ~(tap by history) - |= [=txid =hest] - =/ w (~(get by walts) xpub.hest) - ?~ w ~ - ?. =(network network.u.w) ~ - ?: (gte confs.hest confs.u.w) ~ - `(poke-provider [%tx-info txid]) - :: - ++ retry-poym - |= =network - ^- (list card) - ?~ txbu.poym ~ - =/ w (~(get by walts) xpub.u.txbu.poym) - ?~ w ~ - ?. =(network network.u.w) ~ - %+ weld - ?~ signed-tx.u.txbu.poym ~ - ~[(poke-provider [%broadcast-tx u.signed-tx.u.txbu.poym])] - %+ turn txis.u.txbu.poym - |= =txi - (poke-provider [%raw-tx ~(get-txid txb:bl u.txbu.poym)]) - :: +retry-pend-piym: check whether txids in pend-piym are in mempool - :: - ++ retry-pend-piym - |= =network - ^- (list card) - %+ murn ~(tap by pend.piym) - |= [=txid p=payment] - =/ w (~(get by walts) xpub.p) - ?~ w ~ - ?. =(network network.u.w) ~ - `(poke-provider [%tx-info txid]) - -- + =? cards ?|(!connected.p (gth gap 0)) + ;: weld cards + (retry-poym network) + (retry-txs network) + (retry-scans network) + == + =? cards ?&(?=(^ blockhash) ?=(^ blockfilter) (gth gap 0)) + ;: weld cards + (retry-filtered-addrs network u.blockhash u.blockfilter) + == + :- cards + %_ state + prov `p(connected %.y) + btc-state [block fee now.bowl] + == +:: +++ retry-block-info + |= blocks=(list @ud) + %+ turn blocks + |= block=@ud + (poke-provider %block-info `block) +:: +++ retry-ahistorical-txs + ^- (list card) + %+ turn ~(tap in ahistorical-txs) + |= =txid + (poke-provider [%tx-info txid]) +:: +++ retry-scans + |= =network + ^- (list card) + %- zing + %+ murn ~(tap by scans) + |= [[=xpub:bc =chyg] =batch] + ?. =(network network:(~(got by walts) xpub)) ~ + `-:(req-scan batch xpub chyg) +:: +retry-addrs: get info on addresses with unconfirmed UTXOs +:: +++ retry-addrs + |= =network + ^- (list card) + %- zing + %+ murn ~(val by walts) + |= w=walt + ?. =(network network.w) ~ + ^- (unit (list card)) + :- ~ + %+ turn ~(tap by wach.w) + |= [a=address *] + (poke-provider [%address-info a]) +:: +:: +++ retry-filtered-addrs + |= [=network blockhash=hexb blockfilter=hexb] + ^- (list card) + %- zing + %+ murn ~(val by walts) + |= w=walt + ^- (unit (list card)) + ?. =(network network.w) ~ + :- ~ + %+ murn + %~ tap in + %: all-match:bip-b158 + blockfilter + blockhash + :: + %+ turn ~(tap by wach.w) + |= [a=address *] + [a (to-script-pubkey:adr:bc a)] + == + |= [a=address spk=hexb] + ^- (unit card) + `(poke-provider [%address-info a]) +:: +retry-txs: get info on txs without enough confirmations +:: +++ retry-txs + |= =network + ^- (list card) + %+ murn ~(tap by history) + |= [=txid =hest] + =/ w (~(get by walts) xpub.hest) + ?~ w ~ + ?. =(network network.u.w) ~ + ?: (gte confs.hest confs.u.w) ~ + `(poke-provider [%tx-info txid]) +:: +++ retry-poym + |= =network + ^- (list card) + ?~ txbu.poym ~ + =/ w (~(get by walts) xpub.u.txbu.poym) + ?~ w ~ + ?. =(network network.u.w) ~ + %+ weld + ?~ signed-tx.u.txbu.poym ~ + ~[(poke-provider [%broadcast-tx u.signed-tx.u.txbu.poym])] + %+ turn txis.u.txbu.poym + |= =txi + (poke-provider [%raw-tx ~(get-txid txb:bl u.txbu.poym)]) +:: +retry-pend-piym: check whether txids in pend-piym are in mempool +:: +++ retry-pend-piym + |= =network + ^- (list card) + %+ murn ~(tap by pend.piym) + |= [=txid p=payment] + =/ w (~(get by walts) xpub.p) + ?~ w ~ + ?. =(network network.u.w) ~ + `(poke-provider [%tx-info txid]) :: ++ handle-provider-update |= upd=update:bp @@ -833,6 +873,10 @@ :~ (poke-internal [%fail-broadcast-tx txid.p.upd]) (give-update %cancel-tx txid.p.upd) == + :: + %block-info + :_ state + (retry-filtered-addrs network.p.upd blockhash.p.upd blockfilter.p.upd) == :: ++ handle-tx-info diff --git a/pkg/arvo/lib/bip/b158.hoon b/pkg/arvo/lib/bip/b158.hoon index bf7ac0badc..eb0eb7f0f8 100644 --- a/pkg/arvo/lib/bip/b158.hoon +++ b/pkg/arvo/lib/bip/b158.hoon @@ -211,19 +211,20 @@ $(last-val (add delta last-val)) :: +all-match: returns all target byts that match :: - filter: full block filter, with leading N -:: - k: key for siphash (end of blockhash, reversed) :: - targets: scriptpubkeys to match :: ++ all-match - |= [filter=hexb:bc k=byts targets=(list byts)] - ^- (set hexb:bc) - %- ~(gas in *(set hexb:bc)) + |= [filter=hexb:bc blockhash=hexb:bc targets=(list [address:bc byts])] + ^- (set [address:bc hexb:bc]) + =/ k (to-key (trip (to-cord:hxb:bcu blockhash))) + %- ~(gas in *(set [address:bc hexb:bc])) =/ [p=@ m=@] [p:params m:params] =/ [n=@ux gcs-set=bits:bc] (parse-filter filter) - =/ target-map=(map @ hexb:bc) - %- ~(gas by *(map @ hexb:bc)) + =/ target-map=(map @ [address:bc hexb:bc]) + %- ~(gas by *(map @ [address:bc hexb:bc])) %+ turn targets - |=(t=hexb:bc [(to-range:hsh t (mul n m) k) t]) + |= [a=address:bc t=hexb:bc] + [(to-range:hsh t (mul n m) k) a t] =+ target-hs=(sort ~(tap in ~(key by target-map)) lth) =+ last-val=0 =| matches=(list @) @@ -244,4 +245,5 @@ =^ delta gcs-set (de:gol gcs-set p) $(last-val (add delta last-val)) +:: -- diff --git a/pkg/arvo/lib/btc.hoon b/pkg/arvo/lib/btc.hoon index 179431c3d1..285148f652 100644 --- a/pkg/arvo/lib/btc.hoon +++ b/pkg/arvo/lib/btc.hoon @@ -505,8 +505,10 @@ (mk-url '/getblockcount' '') :: %get-block-info + =/ param=@t + ?~(block.ract '' (rsh [3 2] (scot %ui u.block.ract))) %- get-request - (mk-url '/getblockinfo' '') + (mk-url '/getblockinfo/' param) == ++ mk-url |= [base=@t params=@t] diff --git a/pkg/arvo/sur/btc-provider.hoon b/pkg/arvo/sur/btc-provider.hoon index 65c59be46f..25da6488d4 100644 --- a/pkg/arvo/sur/btc-provider.hoon +++ b/pkg/arvo/sur/btc-provider.hoon @@ -31,6 +31,7 @@ [%raw-tx txid=hexb] [%broadcast-tx rawtx=hexb] [%ping ~] + [%block-info block=(unit @ud)] == :: +$ result @@ -38,6 +39,7 @@ [%tx-info =info:tx] [%raw-tx txid=hexb rawtx=hexb] [%broadcast-tx txid=hexb broadcast=? included=?] + [%block-info =network block=@ud fee=(unit sats) blockhash=hexb blockfilter=hexb] == +$ error $% [%not-connected status=@ud] @@ -60,7 +62,7 @@ [%get-raw-tx txid=hexb] [%broadcast-tx rawtx=hexb] [%get-block-count ~] - [%get-block-info ~] + [%get-block-info block=(unit @ud)] == :: +$ result diff --git a/pkg/arvo/tests/lib/bip/b158.hoon b/pkg/arvo/tests/lib/bip/b158.hoon index 925ca1cfb5..cc330d8be5 100644 --- a/pkg/arvo/tests/lib/bip/b158.hoon +++ b/pkg/arvo/tests/lib/bip/b158.hoon @@ -151,9 +151,14 @@ :: ++ check-all-match |= v=match-vector - =+ k=(to-key blockhash.v) %+ expect-eq - !>(`(set hexb)`(sy inc-spks.v)) - !>(`(set hexb)`(all-match filter.v k (weld inc-spks.v exc-spks.v))) + !>(`(set hexb)`(sy [*address inc-spks.v])) + !> ^- (set hexb) + %: all-match + filter.v + blockhash.v + %+ turn (weld inc-spks.v exc-spks.v) + |=(h=hexb [*address a]) + == -- -- From 690971c9a96b6353b2812fca5830e92ce010b38f Mon Sep 17 00:00:00 2001 From: ixv Date: Fri, 11 Jun 2021 17:42:37 -0700 Subject: [PATCH 2/4] btc: fix b158 test --- pkg/arvo/tests/lib/bip/b158.hoon | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/pkg/arvo/tests/lib/bip/b158.hoon b/pkg/arvo/tests/lib/bip/b158.hoon index cc330d8be5..ed8cdfce33 100644 --- a/pkg/arvo/tests/lib/bip/b158.hoon +++ b/pkg/arvo/tests/lib/bip/b158.hoon @@ -151,14 +151,11 @@ :: ++ check-all-match |= v=match-vector + =/ b=hexb (from-cord:hxb (crip blockhash.v)) + =/ inc=(list [address hexb]) (turn inc-spks.v |=(h=hexb [*address h])) + =/ exc=(list [address hexb]) (turn exc-spks.v |=(h=hexb [*address h])) %+ expect-eq - !>(`(set hexb)`(sy [*address inc-spks.v])) - !> ^- (set hexb) - %: all-match - filter.v - blockhash.v - %+ turn (weld inc-spks.v exc-spks.v) - |=(h=hexb [*address a]) - == + !>(`(set [address hexb])`(sy inc)) + !>(`(set [address hexb])`(all-match filter.v b (weld inc exc))) -- -- From 0fe385df1fe1f5a21098387eaa8810c622a2202f Mon Sep 17 00:00:00 2001 From: ixv Date: Thu, 17 Jun 2021 03:33:22 -0700 Subject: [PATCH 3/4] btc: don't use blockfilter if gap > 50; resubscribe on kick --- pkg/arvo/app/btc-wallet.hoon | 43 ++++++++++++++++++++---------------- pkg/arvo/lib/bitcoin.hoon | 2 +- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pkg/arvo/app/btc-wallet.hoon b/pkg/arvo/app/btc-wallet.hoon index ea6a4dd89a..2049298c4d 100644 --- a/pkg/arvo/app/btc-wallet.hoon +++ b/pkg/arvo/app/btc-wallet.hoon @@ -4,7 +4,7 @@ :: x/scanned: (list xpub) of all scanned wallets :: x/balance/xpub: balance (in sats) of wallet /- *btc-wallet, bp=btc-provider, file-server, launch-store -/+ dbug, default-agent, bl=btc, bc=bitcoin, bip32, bip-b158 +/+ dbug, default-agent, bl=btc, bc=bitcoin, bip32 |% ++ defaults |% @@ -168,12 +168,14 @@ ^- (quip card _this) ?+ -.sign (on-agent:def wire sign) %kick - ~& >>> "kicked from prov {}" ?~ prov `this ?: ?& ?=(%set-provider -.wire) =(host.u.prov src.bowl) == - `this(prov ~) + :_ this(prov [~ src.bowl %.n]) + :~ (watch-provider src.bowl) + (give-update %change-provider `[src.bowl %.n]) + == `this :: %fact @@ -238,7 +240,6 @@ ?> (team:title our.bowl src.bowl) ?- -.comm %set-provider - |^ ?~ provider.comm ?~ prov `state :_ state(prov ~) @@ -252,20 +253,6 @@ (watch-provider u.provider.comm) (give-update %change-provider `[u.provider.comm %.n]) == - :: - ++ watch-provider - |= who=@p - ^- card - :* %pass /set-provider/[(scot %p who)] %agent [who %btc-provider] - %watch /clients - == - ++ leave-provider - |= who=@p - ^- card - :* %pass /set-provider/[(scot %p who)] %agent [who %btc-provider] - %leave ~ - == - -- :: %check-provider =/ pax /permitted/(scot %p provider.comm) @@ -722,6 +709,7 @@ (gulf +(block.btc-state) (dec block)) =? blocks ?|(?=(~ blockhash) ?=(~ blockfilter)) (snoc blocks block) + =? blocks (gth gap 50) ~ :: =/ cards=(list card) ;: weld @@ -739,6 +727,10 @@ ;: weld cards (retry-filtered-addrs network u.blockhash u.blockfilter) == + =? cards (gth gap 50) + ;: weld cards + (retry-addrs network) + == :- cards %_ state prov `p(connected %.y) @@ -792,7 +784,7 @@ :- ~ %+ murn %~ tap in - %: all-match:bip-b158 + %: all-match:bip-b158:bc blockfilter blockhash :: @@ -1126,6 +1118,19 @@ ^- card [%give %fact ~[/all] %btc-wallet-update !>(upd)] :: +++ watch-provider + |= who=@p + ^- card + :* %pass /set-provider/[(scot %p who)] %agent [who %btc-provider] + %watch /clients + == +++ leave-provider + |= who=@p + ^- card + :* %pass /set-provider/[(scot %p who)] %agent [who %btc-provider] + %leave ~ + == +:: ++ give-initial ^- card =^ a=(unit address) state diff --git a/pkg/arvo/lib/bitcoin.hoon b/pkg/arvo/lib/bitcoin.hoon index d790ff5a03..eb09a4f6be 100644 --- a/pkg/arvo/lib/bitcoin.hoon +++ b/pkg/arvo/lib/bitcoin.hoon @@ -3,7 +3,7 @@ :: expose BIP libraries :: /- sur=bitcoin -/+ bech32=bip-b173, pbt=bip-b174, bcu=bitcoin-utils +/+ bech32=bip-b173, pbt=bip-b174, bcu=bitcoin-utils, bip-b158 =, sur =, bcu |% From ec8ca6f0cf1b8abba2c7bceaf5d6cf2df15874cc Mon Sep 17 00:00:00 2001 From: ixv Date: Thu, 17 Jun 2021 15:53:12 -0700 Subject: [PATCH 4/4] btc: style nits, and fix watch-ack handling --- pkg/arvo/app/btc-provider.hoon | 3 +-- pkg/arvo/app/btc-wallet.hoon | 17 ++++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/pkg/arvo/app/btc-provider.hoon b/pkg/arvo/app/btc-provider.hoon index ff681b1b9b..d8cc3fd3e5 100644 --- a/pkg/arvo/app/btc-provider.hoon +++ b/pkg/arvo/app/btc-provider.hoon @@ -85,8 +85,7 @@ :: ?> ?=([%clients *] pax) ?. (is-whitelisted:hc src.bowl) - ~& >>> "btc-provider: blocked client {}" - [~[[%give %kick ~ ~]] this] + ~|("btc-provider: blocked client {}" !!) ~& > "btc-provider: accepted client {}" :- [do-ping:hc]~ this(clients.host-info (~(put in clients.host-info) src.bowl)) diff --git a/pkg/arvo/app/btc-wallet.hoon b/pkg/arvo/app/btc-wallet.hoon index 2049298c4d..ed59eef8ae 100644 --- a/pkg/arvo/app/btc-wallet.hoon +++ b/pkg/arvo/app/btc-wallet.hoon @@ -167,6 +167,11 @@ |= [=wire =sign:agent:gall] ^- (quip card _this) ?+ -.sign (on-agent:def wire sign) + %watch-ack + ?~ p.sign `this + %- (slog leaf+"connection rejected by provider ({})" u.p.sign) + `this + :: %kick ?~ prov `this ?: ?& ?=(%set-provider -.wire) @@ -724,13 +729,9 @@ (retry-scans network) == =? cards ?&(?=(^ blockhash) ?=(^ blockfilter) (gth gap 0)) - ;: weld cards - (retry-filtered-addrs network u.blockhash u.blockfilter) - == + (weld cards (retry-filtered-addrs network u.blockhash u.blockfilter)) =? cards (gth gap 50) - ;: weld cards - (retry-addrs network) - == + (weld cards (retry-addrs network)) :- cards %_ state prov `p(connected %.y) @@ -755,7 +756,9 @@ %- zing %+ murn ~(tap by scans) |= [[=xpub:bc =chyg] =batch] - ?. =(network network:(~(got by walts) xpub)) ~ + =/ w (~(get by walts) xpub) + ?~ w ~ + ?. =(network network.u.w) ~ `-:(req-scan batch xpub chyg) :: +retry-addrs: get info on addresses with unconfirmed UTXOs ::