(bugfix) db: change tx_num endianness (LE->BE) to match db comparator

SomberNight · SomberNight · commit 532cacb1512a · 2020-11-16T19:23:15.000+01:00
History.get_txnums and History.backup depend on ordering of tx_nums,
so we want the lexicographical order (used by leveldb comparator) to
match the numerical order.
diff --git a/electrumx/lib/util.py b/electrumx/lib/util.py
@@ -317,6 +317,7 @@ def protocol_version(client_req, min_tuple, max_tuple):
 struct_le_Q = Struct('<Q')
 struct_be_H = Struct('>H')
 struct_be_I = Struct('>I')
+struct_be_Q = Struct('>Q')
 structB = Struct('B')
 
 unpack_le_int32_from = struct_le_i.unpack_from
@@ -330,6 +331,7 @@ def protocol_version(client_req, min_tuple, max_tuple):
 unpack_le_uint32 = struct_le_I.unpack
 unpack_le_uint64 = struct_le_Q.unpack
 unpack_be_uint32 = struct_be_I.unpack
+unpack_be_uint64 = struct_be_Q.unpack
 
 pack_le_int32 = struct_le_i.pack
 pack_le_int64 = struct_le_q.pack
@@ -338,6 +340,7 @@ def protocol_version(client_req, min_tuple, max_tuple):
 pack_le_uint64 = struct_le_Q.pack
 pack_be_uint16 = struct_be_H.pack
 pack_be_uint32 = struct_be_I.pack
+pack_be_uint64 = struct_be_Q.pack
 pack_byte = structB.pack
 
 hex_to_bytes = bytes.fromhex
diff --git a/electrumx/server/block_processor.py b/electrumx/server/block_processor.py
@@ -25,7 +25,7 @@
 )
 from electrumx.lib.tx import Tx
 from electrumx.server.db import FlushData, DB
-from electrumx.server.history import TXNUM_LEN, TXOUTIDX_LEN, TXOUTIDX_PADDING
+from electrumx.server.history import TXNUM_LEN, TXOUTIDX_LEN, TXOUTIDX_PADDING, pack_txnum
 
 if TYPE_CHECKING:
     from electrumx.lib.coins import Coin
@@ -457,11 +457,12 @@ def advance_txs(
         put_txo_to_spender_map = txo_to_spender_map.__setitem__
         to_le_uint32 = pack_le_uint32
         to_le_uint64 = pack_le_uint64
+        _pack_txnum = pack_txnum
 
         for tx, tx_hash in txs:
             hashXs = []
             append_hashX = hashXs.append
-            tx_numb = to_le_uint64(tx_num)[:TXNUM_LEN]
+            tx_numb = _pack_txnum(tx_num)
 
             # Spend the inputs
             for txin in tx.inputs:
@@ -652,7 +653,7 @@ def spend_utxo(self, tx_hash: bytes, txout_idx: int) -> bytes:
         if tx_num is None:
             raise ChainError(f'UTXO {hash_to_hex_str(tx_hash)} / {txout_idx:,d} has '
                              f'no corresponding tx_num in DB')
-        tx_numb = pack_le_uint64(tx_num)[:TXNUM_LEN]
+        tx_numb = pack_txnum(tx_num)
 
         # Key: b'h' + tx_num + txout_idx
         # Value: hashX
@@ -765,10 +766,10 @@ def advance_txs(self, txs, is_unspendable):
         update_touched_hashxs = self.touched_hashxs.update
         hashx_fundings = defaultdict(bytearray)
         to_le_uint32 = pack_le_uint32
-        to_le_uint64 = pack_le_uint64
+        _pack_txnum = pack_txnum
 
         for tx, _tx_hash in txs:
-            tx_numb = to_le_uint64(tx_num)[:TXNUM_LEN]
+            tx_numb = _pack_txnum(tx_num)
             hashXs = []
             append_hashX = hashXs.append
 
@@ -813,13 +814,14 @@ def advance_txs(self, txs, is_unspendable):
         put_txo_to_spender_map = txo_to_spender_map.__setitem__
         to_le_uint32 = pack_le_uint32
         to_le_uint64 = pack_le_uint64
+        _pack_txnum = pack_txnum
 
         hashXs_by_tx = [set() for _ in txs]
 
         # Add the new UTXOs
         for (tx, tx_hash), hashXs in zip(txs, hashXs_by_tx):
             add_hashXs = hashXs.add
-            tx_numb = to_le_uint64(tx_num)[:TXNUM_LEN]
+            tx_numb = _pack_txnum(tx_num)
 
             for idx, txout in enumerate(tx.outputs):
                 # Ignore unspendable outputs
diff --git a/electrumx/server/db.py b/electrumx/server/db.py
@@ -30,7 +30,7 @@
 )
 from electrumx.server.storage import db_class, Storage
 from electrumx.server.history import (
-    History, TXNUM_LEN, TXNUM_PADDING, TXOUTIDX_LEN, TXOUTIDX_PADDING,
+    History, TXNUM_LEN, TXNUM_PADDING, TXOUTIDX_LEN, TXOUTIDX_PADDING, pack_txnum, unpack_txnum,
 )
 
 if TYPE_CHECKING:
@@ -744,7 +744,7 @@ def read_utxos():
             for db_key, db_value in self.utxo_db.iterator(prefix=prefix):
                 txout_idx, = unpack_le_uint32(db_key[-TXOUTIDX_LEN:] + TXOUTIDX_PADDING)
                 tx_numb = db_key[-TXOUTIDX_LEN-TXNUM_LEN:-TXOUTIDX_LEN]
-                tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING)
+                tx_num = unpack_txnum(tx_numb)
                 value, = unpack_le_uint64(db_value)
                 tx_hash, height = self.fs_tx_hash(tx_num)
                 utxos_append(UTXO(tx_num, txout_idx, tx_hash, height, value))
@@ -765,7 +765,7 @@ def _get_hashX_for_utxo(
         tx_num = self.fs_txnum_for_txhash(tx_hash)
         if tx_num is None:
             return None, None
-        tx_numb = pack_le_uint64(tx_num)[:TXNUM_LEN]
+        tx_numb = pack_txnum(tx_num)
 
         # Key: b'h' + tx_num + txout_idx
         # Value: hashX
diff --git a/electrumx/server/history.py b/electrumx/server/history.py
@@ -9,18 +9,16 @@
 '''History by script hash (address).'''
 
 import ast
-import bisect
 import time
-from array import array
 from collections import defaultdict
 from typing import TYPE_CHECKING, Type, Optional, Dict, Sequence, Tuple
 import itertools
 
 import electrumx.lib.util as util
 from electrumx.lib.hash import HASHX_LEN, hash_to_hex_str
-from electrumx.lib.util import (pack_be_uint16, pack_le_uint64,
-                                unpack_be_uint16_from, unpack_le_uint64,
-                                pack_le_uint32, unpack_le_uint32)
+from electrumx.lib.util import (pack_le_uint64, unpack_le_uint64,
+                                pack_le_uint32, unpack_le_uint32,
+                                pack_be_uint64, unpack_be_uint64)
 
 if TYPE_CHECKING:
     from electrumx.server.storage import Storage
@@ -32,6 +30,14 @@
 TXOUTIDX_PADDING = bytes(4 - TXOUTIDX_LEN)
 
 
+def unpack_txnum(tx_numb: bytes) -> int:
+    return unpack_be_uint64(TXNUM_PADDING + tx_numb)[0]
+
+
+def pack_txnum(tx_num: int) -> bytes:
+    return pack_be_uint64(tx_num)[-TXNUM_LEN:]
+
+
 class History:
 
     DB_VERSIONS = (3, )
@@ -115,23 +121,23 @@ def clear_excess(self, utxo_db_tx_count: int) -> None:
         hkeys = []
         for db_key, db_val in self.db.iterator(prefix=b'H'):
             tx_numb = db_key[1+HASHX_LEN: 1+HASHX_LEN+TXNUM_LEN]
-            tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING)
+            tx_num = unpack_txnum(tx_numb)
             if tx_num >= utxo_db_tx_count:
                 hkeys.append(db_key)
 
         tkeys = []
         for db_key, db_val in self.db.iterator(prefix=b't'):
             tx_numb = db_val
-            tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING)
+            tx_num = unpack_txnum(tx_numb)
             if tx_num >= utxo_db_tx_count:
                 tkeys.append(db_key)
 
         skeys = []
         for db_key, db_val in self.db.iterator(prefix=b's'):
             tx_numb1 = db_key[1:1+TXNUM_LEN]
             tx_numb2 = db_val
-            tx_num1, = unpack_le_uint64(tx_numb1 + TXNUM_PADDING)
-            tx_num2, = unpack_le_uint64(tx_numb2 + TXNUM_PADDING)
+            tx_num1 = unpack_txnum(tx_numb1)
+            tx_num2 = unpack_txnum(tx_numb2)
             if max(tx_num1, tx_num2) >= utxo_db_tx_count:
                 skeys.append(db_key)
 
@@ -183,7 +189,7 @@ def add_unflushed(
             spender_txnum = get_txnum_for_txhash(spender_hash)
             assert spender_txnum is not None
             prev_idx_packed = pack_le_uint32(prev_idx)[:TXOUTIDX_LEN]
-            prev_txnumb = pack_le_uint64(prev_txnum)[:TXNUM_LEN]
+            prev_txnumb = pack_txnum(prev_txnum)
             unflushed_spenders[prev_txnumb+prev_idx_packed] = spender_txnum
 
     def unflushed_memsize(self):
@@ -210,11 +216,11 @@ def flush(self):
                     batch.put(db_key, b'')
             for tx_hash, tx_num in sorted(self._unflushed_txhash_to_txnum_map.items()):
                 db_key = b't' + tx_hash
-                tx_numb = pack_le_uint64(tx_num)[:TXNUM_LEN]
+                tx_numb = pack_txnum(tx_num)
                 batch.put(db_key, tx_numb)
             for prevout, spender_txnum in sorted(self._unflushed_txo_to_spender.items()):
                 db_key = b's' + prevout
-                db_val = pack_le_uint64(spender_txnum)[:TXNUM_LEN]
+                db_val = pack_txnum(spender_txnum)
                 batch.put(db_key, db_val)
             self.hist_db_tx_count = self.hist_db_tx_count_next
             self.write_state(batch)
@@ -242,11 +248,12 @@ def backup(self, *, hashXs, tx_count, tx_hashes: Sequence[bytes], spends: Sequen
                 prefix = b'H' + hashX
                 for db_key, db_val in self.db.iterator(prefix=prefix, reverse=True):
                     tx_numb = db_key[1+HASHX_LEN: 1+HASHX_LEN+TXNUM_LEN]
-                    tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING)
+                    tx_num = unpack_txnum(tx_numb)
                     if tx_num >= tx_count:
                         nremoves_addr += 1
                         deletes.append(db_key)
                     else:
+                        # note: we can break now, due to 'reverse=True' and txnums being big endian
                         break
                 for key in deletes:
                     batch.delete(key)
@@ -256,7 +263,7 @@ def backup(self, *, hashXs, tx_count, tx_hashes: Sequence[bytes], spends: Sequen
                 assert len(prev_idx) == TXOUTIDX_LEN
                 prev_txnum = get_txnum_for_txhash(prev_hash)
                 assert prev_txnum is not None
-                prev_txnumb = pack_le_uint64(prev_txnum)[:TXNUM_LEN]
+                prev_txnumb = pack_txnum(prev_txnum)
                 db_key = b's' + prev_txnumb + prev_idx
                 batch.delete(db_key)
             for tx_hash in sorted(tx_hashes):
@@ -282,7 +289,7 @@ def get_txnums(self, hashX: bytes, limit: Optional[int] = 1000) -> Sequence[int]
                 break
             prev_txnumb = db_key[1+HASHX_LEN: 1+HASHX_LEN+TXNUM_LEN]
             prev_idx_packed = db_key[-TXOUTIDX_LEN:]
-            prev_txnum, = unpack_le_uint64(prev_txnumb + TXNUM_PADDING)
+            prev_txnum = unpack_txnum(prev_txnumb)
             prev_idx, = unpack_le_uint32(prev_idx_packed + TXOUTIDX_PADDING)
             txnum_set.add(prev_txnum)
             spender_txnum = self.get_spender_txnum_for_txo(prev_txnum, prev_idx)
@@ -299,20 +306,20 @@ def get_txnum_for_txhash(self, tx_hash: bytes) -> Optional[int]:
             db_key = b't' + tx_hash
             tx_numb = self.db.get(db_key)
             if tx_numb:
-                tx_num, = unpack_le_uint64(tx_numb + TXNUM_PADDING)
+                tx_num = unpack_txnum(tx_numb)
         return tx_num
 
     def get_spender_txnum_for_txo(self, prev_txnum: int, txout_idx: int) -> Optional[int]:
         '''For an outpoint, returns the tx_num that spent it.
         If the outpoint is unspent, or even if it never existed (!), returns None.
         '''
         prev_idx_packed = pack_le_uint32(txout_idx)[:TXOUTIDX_LEN]
-        prev_txnumb = pack_le_uint64(prev_txnum)[:TXNUM_LEN]
+        prev_txnumb = pack_txnum(prev_txnum)
         prevout = prev_txnumb + prev_idx_packed
         spender_txnum = self._unflushed_txhash_to_txnum_map.get(prevout)
         if spender_txnum is None:
             db_key = b's' + prevout
             spender_txnumb = self.db.get(db_key)
             if spender_txnumb:
-                spender_txnum, = unpack_le_uint64(spender_txnumb + TXNUM_PADDING)
+                spender_txnum = unpack_txnum(spender_txnumb)
         return spender_txnum
