"scripthash.get_history": handle client_statushash and client_height

Author: SomberNight

electrumx/server/history.py

@@ -13,6 +13,7 @@
 from collections import defaultdict
 from typing import TYPE_CHECKING, Type, Optional, Dict, Sequence, Tuple, List
 import itertools
+from functools import partial
 
 from aiorpcx import run_in_thread
 
@@ -323,6 +324,7 @@ def get_txnums(
             it.seek(prefix + pack_txnum(txnum_min))
         txnum_min = txnum_min if txnum_min is not None else 0
         txnum_max = txnum_max if txnum_max is not None else float('inf')
+        assert txnum_min <= txnum_max, f"txnum_min={txnum_min}, txnum_max={txnum_max}"
         for db_key, db_val in it:
             tx_numb = db_key[-TXNUM_LEN:]
             if limit == 0:
@@ -359,29 +361,46 @@ def get_spender_txnum_for_txo(self, prev_txnum: int, txout_idx: int) -> Optional
                 spender_txnum = unpack_txnum(spender_txnumb)
         return spender_txnum
 
-    def fs_get_intermediate_statushash_for_hashx(self, hashX: bytes) -> Tuple[int, bytes]:
+    def fs_get_intermediate_statushash_for_hashx(
+            self,
+            *,
+            hashX: bytes,
+            txnum_max: int = None,
+    ) -> Tuple[int, bytes]:
         '''For a hashX, returns (tx_num, status), with the latest stored statushash
-        and corresponding tx_num.
+        and corresponding tx_num, where tx_num < txnum_max.
         This can be used to efficiently calculate the status of a hashX as
         only the txs mined after(>) tx_num will need to be hashed.
         '''
+        # first, search in-memory, among the unflushed statuses
         unflushed_statushashes = self._unflushed_hashx_to_statushash.get(hashX, [])
         if len(unflushed_statushashes) > 0:
-            tx_num, status = unflushed_statushashes[-1]
+            for tx_num, status in reversed(unflushed_statushashes):
+                if txnum_max is None or tx_num < txnum_max:
+                    return tx_num, status
+        # second, search in the on-disk DB
+        prefix = b'S' + hashX
+        it = self.db.iterator(prefix=prefix, reverse=True)
+        if txnum_max is not None:
+            it.seek(prefix + pack_txnum(txnum_max))
+        for db_key, db_val in it:
+            tx_numb = db_key[-TXNUM_LEN:]
+            tx_num = unpack_txnum(tx_numb)
+            status = db_val
+            break
         else:
-            prefix = b'S' + hashX
-            for db_key, db_val in self.db.iterator(prefix=prefix, reverse=True):
-                tx_numb = db_key[-TXNUM_LEN:]
-                tx_num = unpack_txnum(tx_numb)
-                status = db_val
-                break
-            else:
-                tx_num = -1
-                status = bytes(32)
+            tx_num = 0
+            status = bytes(32)
         return tx_num, status
 
-    async def get_intermediate_statushash_for_hashx(self, hashX: bytes) -> Tuple[int, bytes]:
-        return await run_in_thread(self.fs_get_intermediate_statushash_for_hashx, hashX)
+    async def get_intermediate_statushash_for_hashx(
+            self,
+            *,
+            hashX: bytes,
+            txnum_max: int = None,
+    ) -> Tuple[int, bytes]:
+        f = partial(self.fs_get_intermediate_statushash_for_hashx, hashX=hashX, txnum_max=txnum_max)
+        return await run_in_thread(f)
 
     def store_intermediate_statushash_for_hashx(
             self,
@@ -397,6 +416,7 @@ def store_intermediate_statushash_for_hashx(
         '''
         if hashX not in self._unflushed_hashx_to_statushash:
             self._unflushed_hashx_to_statushash[hashX] = []
+        # maintain invariant that unflushed statuses are in order (increasing tx_num):
         if len(self._unflushed_hashx_to_statushash[hashX]) > 0:
             tx_num_last, status_last = self._unflushed_hashx_to_statushash[hashX][-1]
             if tx_num <= tx_num_last:

electrumx/server/session.py

@@ -1218,12 +1218,19 @@ async def _address_status_proto_legacy(self, hashX: bytes) -> Optional[str]:
 
         return status
 
-    async def _address_status_proto_1_5(self, hashX: bytes) -> str:
-        '''Returns an address status, as per protocol newer than >=1.5'''
-        # first, consider confirmed history
-        tx_num_calced, status = await self.db.history.get_intermediate_statushash_for_hashx(hashX)
+    async def _calc_intermediate_status_for_hashX(
+            self,
+            *,
+            hashX: bytes,
+            txnum_max: int = None,
+    ) -> bytes:
+        '''Returns the status of a hashX, considering only confirmed history
+        up to (<) txnum_max.
+        '''
+        tx_num_calced, status = await self.db.history.get_intermediate_statushash_for_hashx(
+            hashX=hashX, txnum_max=txnum_max)
         db_history = await self.db.limited_history_triples(
-            hashX=hashX, limit=None, txnum_min=tx_num_calced+1)
+            hashX=hashX, limit=None, txnum_min=tx_num_calced+1, txnum_max=txnum_max)
         self.bump_cost(0.3 + len(db_history) * 0.001)  # cost of history-lookup
         self.bump_cost(36 * len(db_history) * 0.00002)  # cost of hashing mined txs
         reorgsafe_height = self.db.db_height - self.env.reorg_limit
@@ -1234,6 +1241,12 @@ async def _address_status_proto_1_5(self, hashX: bytes) -> str:
             if cnt % storestatus_period == 0 and height < reorgsafe_height:
                 self.db.history.store_intermediate_statushash_for_hashx(
                     hashX=hashX, tx_num=tx_num, status=status)
+        return status
+
+    async def _address_status_proto_1_5(self, hashX: bytes) -> str:
+        '''Returns an address status, as per protocol newer than >=1.5'''
+        # first, consider confirmed history
+        status = await self._calc_intermediate_status_for_hashX(hashX=hashX)
 
         # second, consider mempool txs
         mempool = await self.mempool.transaction_summaries(hashX)
@@ -1389,7 +1402,16 @@ async def scripthash_get_history_proto_1_5(
                 raise RPCError(BAD_REQUEST, f'from_height={from_height} '
                                             f'<= client_height={client_height} '
                                             f'< to_height={to_height} must hold.')
-            # TODO implement handling. they are ignored for now
+        # Done sanitising args; start handling
+        # Check if client status is consistent with server; if so we can fast-forward from_height
+        if client_statushash is not None:
+            client_txnum = self.db.get_next_tx_num_after_blockheight(client_height)
+            server_statushash = await self._calc_intermediate_status_for_hashX(
+                hashX=hashX,
+                txnum_max=client_txnum + 1,
+            )
+            if server_statushash == client_statushash:
+                from_height = client_height + 1
 
         # Limit size of returned history to ensure it fits within a response.
         limit_bytes = self.env.max_send - HISTORY_OVER_WIRE_OVERHEAD_BYTES