roller: add batch-trimming functionality

If a batch gets bigger than a max size defined by the ethereum node
the raw transaction is sent to, the /ted/roller/send thread will crash and
the batch will be blocked, stopping any subsequent batches to be sent.

This detects when the current batch reaches a certain threshold and only
includes transactions up to that point, moving the ones that are not sent
back to the pending queue, adjusting their history and finding status.

pkg/arvo/app/roller.hoon

@@ -724,7 +724,7 @@
           [cards this]
         ::
             %thread-done
-          =+   !<(result=(each @ud [term @t]) q.cage.sign)
+          =+   !<(result=(each [@ud @ud] [term @t]) q.cage.sign)
           =^  cards  state
             (on-batch-result:do address nonce result)
           [cards this]
@@ -1341,8 +1341,9 @@
 ::  +on-batch-result: await resend after thread success or failure
 ::
 ++  on-batch-result
-  |=  [=address:ethereum nonce=@ud result=(each @ud [term @t])]
+  |=  [=address:ethereum nonce=@ud result=(each [@ud @ud] [term @t])]
   ^-  (quip card _state)
+  |^
   ::  print error if there was one
   ::
   ~?  ?=(%| -.result)  [dap.bowl %send-error +.p.result]
@@ -1360,6 +1361,24 @@
       (del:ors:dice sending [address nonce])
     `state
   =/  =send-tx  (got:ors:dice sending [address nonce])
+  ::  if the number of txs sent is less than the ones in sending, we remove
+  ::  them from the latest sending batch and add them on top of the pending list
+  ::
+  =/  n-txs=@ud  ?:(?=(%& -.result) -.p.result (lent txs.send-tx))
+  =/  not-sent=(list [=address:naive force=? =raw-tx:naive])
+    (slag n-txs txs.send-tx)
+  =/  partial-send=?  &(?=(%& -.result) (lth n-txs (lent txs.send-tx)))
+  =?  txs.send-tx   partial-send
+    (oust [n-txs (lent txs.send-tx)] txs.send-tx)
+  =?  pending       partial-send
+    (fix-not-sent-pending not-sent)
+  =/  [nif=_finding sih=_history]
+    (fix-not-sent-status not-sent)
+  =:  finding  nif
+      history  sih
+    ==
+  ~?  partial-send  [%extracting-txs-from-batch (lent not-sent)]
+  ::
   =?  sending  ?|  ?=(%& -.result)
                    ?=([%| %crash *] result)
                ==
@@ -1368,9 +1387,16 @@
     ::  update gas price for this tx in state
     ::
     ?:  ?=(%& -.result)
-      send-tx(next-gas-price p.result, sent &)
-    ::  if the thread crashed, we don't know the gas used,
-    ::  so we udpate it manually, same as the thread would do
+      send-tx(next-gas-price +.p.result, sent &)
+    ::  if the thread crashed, we don't know the gas used, so we udpate it
+    ::  manually, same as the thread would do. this has the problem of causing
+    ::  the batch to be blocked if the thread keeps crashing, and we don't have
+    ::  enough funds to pay.
+    ::
+    ::  on the other hand if the thread fails because +fetch-gas-price fails
+    ::  (e.g. API change), and our fallback gas price is too low, the batch will
+    ::  also be blocked, if we don't increase the next-gas-price, so either way
+    ::  the batch will be stuck because of another underlying issue.
     ::
     %_    send-tx
         next-gas-price
@@ -1400,6 +1426,44 @@
   %+  wait:b:sys
     /resend/(scot %ux address)/(scot %ud nonce)
   (add resend-time now.bowl)
+  ::
+  ++  fix-not-sent-pending
+    |=  not-sent=(list [=address:naive force=? =raw-tx:naive])
+    =;  txs=(list pend-tx)
+      (weld txs pending)
+    ::  TODO: this would not be needed if txs.send-tx was a (list pend-tx)
+    ::
+    %+  murn  not-sent
+    |=  [=address:naive force=? =raw-tx:naive]
+    =/  =keccak  (hash-raw-tx:lib raw-tx)
+    ?~  wer=(~(get by finding) keccak)
+      ~&  >>>  %missing-tx-in-finding
+      ~
+    ?@  u.wer
+      ~&  >>>  %missing-tx-in-finding
+      ~
+    `[force address time.u.wer raw-tx]
+  ::
+  ++  fix-not-sent-status
+    |=  not-sent=(list [=address:naive force=? =raw-tx:naive])
+    %+  roll  not-sent
+    |=  [[@ @ =raw-tx:naive] nif=_finding sih=_history]
+    =/  =keccak  (hash-raw-tx:lib raw-tx)
+    ?~  val=(~(get by nif) keccak)
+      [nif sih]
+    ?.  ?=(^ u.val)
+      [nif sih]
+    =*  time      time.u.val
+    =*  address   address.u.val
+    =*  ship      ship.from.tx.raw-tx
+    =/  l2-tx     (l2-tx +<.tx.raw-tx)
+    =/  =roll-tx  [ship %pending keccak l2-tx]
+    =+  txs=(~(got by sih) address)
+    =.  txs  +:(del:orh:dice txs time)
+    :-  (~(del by nif) keccak)
+    %+  ~(put by sih)  address
+    (put:orh:dice txs [time roll-tx])
+  --
 ::  +on-naive-diff: process l2 tx confirmations
 ::
 ++  on-naive-diff

pkg/arvo/sur/dice.hoon

@@ -88,6 +88,8 @@
 +$  send-tx
   $:  next-gas-price=@ud
       sent=?
+      ::  TODO: make txs as (list pend-tx)?
+      ::
       txs=(list [=address:naive force=? =raw-tx:naive])
   ==
 +$  part-tx

pkg/arvo/ted/roller/send.hoon

@@ -13,12 +13,26 @@
 =/  =address:ethereum  (address-from-prv:key:ethereum pk)
 ;<  expected-nonce=@ud  bind:m
   (get-next-nonce:ethio endpoint address)
-=/  batch-data=octs
-  %+  cad:naive  3
-  %-  flop
-  %+  roll  txs
-  |=  [=raw-tx:naive out=(list octs)]
-  [raw.raw-tx 65^sig.raw-tx out]
+::  Infura enforces a max calldata size (32, 64, 128 Kb?) so we calculate how
+::  many txs are included in a batch of that size, and only send those
+::
+=/  max-calldata=@ud  128.000
+=/  [n-txs=@ud batch-data=octs]
+  =|  n-txs=@ud
+  =|  size=@ud
+  =|  out=(list octs)
+  |-  ^-  [@ud octs]
+  ?~  txs
+    [n-txs (cad:naive 3 (flop out))]
+  =*  raw-tx  i.txs
+  =.  size    :(add 65 p.raw.raw-tx size)
+  ?:  (gth size max-calldata)
+    [n-txs (cad:naive 3 (flop out))]
+  %_  $
+    n-txs  +(n-txs)
+    txs    t.txs
+    out    [raw.raw-tx 65^sig.raw-tx out]
+  ==
 ::  if the batch is malformed, emit error to kick it out of sending
 ::
 ?~  (parse-roll:naive q.batch-data)
@@ -49,22 +63,23 @@
 ::  gasLimit = G_transaction + G_txdatanonzero × dataByteLength
 ::  where
 ::      G_transaction = 21000 gas (base fee)
-::    + G_txdatanonzero = 68 gas
+::    + G_txdatanonzero = 16 gas (previously 68; see EIP-2028)
 ::    * dataByteLength = (65 + raw) * (lent txs) bytes
 ::
-:: TODO: enforce max number of tx in batch?
+::  1.000 gas are added to the base fee as extra, for emitting the log
 ::
-=/  gas-limit=@ud  (add 21.000 (mul 68 p.batch-data))
-::  if we cannot pay for the transaction, don't bother sending it out
-::
-=/  max-cost=@ud  (mul gas-limit use-gas-price)
+=/  gas-limit=@ud  (add 22.000 (mul 16 p.batch-data))
+=/  max-cost=@ud   (mul gas-limit use-gas-price)
 ;<  balance=@ud  bind:m
   (get-balance:ethio endpoint address)
 ?:  (gth max-cost balance)
+  ::  if we cannot pay for the transaction, don't bother sending it out
+  ::
   (pure:m !>(%.n^[%not-sent %insufficient-roller-balance]))
 ::
 ::NOTE  this fails the thread if sending fails, which in the app gives us
 ::      the "retry with same gas price" behavior we want
+::
 ;<  =response:rpc  bind:m
   %+  send-batch  endpoint
   =;  tx=transaction:rpc:ethereum
@@ -80,19 +95,23 @@
 ::  log batch tx-hash to getTransactionReceipt(tx-hash)
 ::
 ~?  &(?=(%result -.response) ?=(%s -.res.response))
-  ^-  [nonce=@ud batch-hash=@t gas=@ud]
-  nonce^(so:dejs:format res.response)^use-gas-price
+  ^-  [nonce=@ud batch-hash=@t gas=@ud sent-txs=@ud bytes=@ud]
+  :*  nonce
+      (so:dejs:format res.response)
+      use-gas-price
+      n-txs
+      p.batch-data
+  ==
 %-  pure:m
-!>  ^-  (each @ud [term @t])
+!>  ^-  (each [@ud @ud] [term @t])
 ::  TODO: capture if the tx fails (e.g. Runtime Error: revert)
 ::  check that tx-hash in +.response is non-zero?
-::  enforce max here, or in app?
 ::
 ?+  -.response  %.n^[%error 'unexpected rpc response']
   %error   %.n^[%error message.response]
   ::  add five gwei to gas price of next attempt
   ::
-  %result  %.y^(add use-gas-price 5.000.000.000)
+  %result  %.y^[n-txs (add use-gas-price 5.000.000.000)]
 ==
 ::
 ::TODO  should be distilled further, partially added to strandio?