cache blocks to avoid lite servers usage and

skip failing (on master) tests

Author: klpx

go.mod

@@ -23,7 +23,8 @@ require (
 	github.com/sourcegraph/conc v0.3.0
 	github.com/stretchr/testify v1.11.1
 	github.com/tonkeeper/scam_backoffice_rules v0.0.11
-	github.com/tonkeeper/tongo v1.17.1
+	github.com/tonkeeper/tongo v1.17.2-0.20260225134521-792e5b276c44
+	go.etcd.io/bbolt v1.4.3
 	go.opentelemetry.io/otel v1.38.0
 	go.opentelemetry.io/otel/metric v1.38.0
 	go.opentelemetry.io/otel/trace v1.38.0

go.sum

@@ -272,10 +272,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/tonkeeper/scam_backoffice_rules v0.0.11 h1:VXbNNyTO49OArvQwPRCU+1WUEyfQgkqG7Q8NVhQ7320=
 github.com/tonkeeper/scam_backoffice_rules v0.0.11/go.mod h1:SqZXYO9vbID8ku+xnnaKXeNGmehxigODGrk5V1KqbRA=
-github.com/tonkeeper/tongo v1.17.0 h1:uz+X/kpKtkTN3TC8MoPtYiYEoPRDB2TEo1aMnKfj3QQ=
-github.com/tonkeeper/tongo v1.17.0/go.mod h1:FiCxH96fLp+7MOF1FkavAKdtnxLO3F1GHTqslyq1T7g=
-github.com/tonkeeper/tongo v1.17.1 h1:tp5YshzHWj+ONUHrk1+zWxdtjYo2lHQRHEcfh8i4Ub0=
-github.com/tonkeeper/tongo v1.17.1/go.mod h1:nHmdEXPfT0/EvkEaBzPiY599/0OYjQSW4dWR7aX+OII=
+github.com/tonkeeper/tongo v1.17.2-0.20260225134521-792e5b276c44 h1:kxYuCIdYCUIa62pPYFv4VW4xaWYW9jM55u4H1sU2dGM=
+github.com/tonkeeper/tongo v1.17.2-0.20260225134521-792e5b276c44/go.mod h1:nHmdEXPfT0/EvkEaBzPiY599/0OYjQSW4dWR7aX+OII=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
@@ -285,6 +283,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
+go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -323,8 +323,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a h1:ovFr6Z0MNmU7nH8VaX5xqw+05ST2uO1exVfZPVqRC5o=
-golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0=
 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=

pkg/bath/bath_test.go

@@ -57,10 +57,12 @@ type LiteStorage struct {
 	logger                  *zap.Logger
 	client                  *liteapi.Client
 	executor                abi.Executor
+	cachedExecutor          *CachedExecutor
 	jettonMetaCache         *xsync.MapOf[string, tep64.Metadata]
 	transactionsIndexByHash *xsync.MapOf[tongo.Bits256, *core.Transaction]
 	transactionsByInMsgLT   *xsync.MapOf[inMsgCreatedLT, tongo.Bits256]
-	blockCache              *xsync.MapOf[tongo.BlockIDExt, *tlb.Block]
+	blockCache              Cache[tongo.BlockID, Tuple2[tongo.BlockIDExt, *tlb.Block]]
+	accountCodeCache        Cache[tongo.AccountID, []byte]
 	accountInterfacesCache  *xsync.MapOf[tongo.AccountID, []abi.ContractInterface]
 	// tvmLibraryCache contains public tvm libraries.
 	// As a library is immutable, it's ok to cache it.
@@ -132,13 +134,38 @@ func NewLiteStorage(log *zap.Logger, cli *liteapi.Client, opts ...Option) (*Lite
 	if o.executor == nil {
 		o.executor = cli
 	}
+	blockBolt, err := NewBoltCache("./cache/blocks.db")
+	if err != nil {
+		return nil, fmt.Errorf("failed to init block cache: %w", err)
+	}
+	blockCache := &JsonTlbDiskCache[tongo.BlockID, Tuple2[tongo.BlockIDExt, *tlb.Block]]{
+		cache: blockBolt,
+	}
+	codeBolt, err := NewBoltCache("./cache/accountcode.db")
+	if err != nil {
+		return nil, fmt.Errorf("failed to init account code cache: %w", err)
+	}
+	codeCache := &BytesDiskCache[ton.AccountID]{
+		cache: codeBolt,
+	}
+	execBolt, err := NewBoltCache("./cache/executor.db")
+	if err != nil {
+		return nil, fmt.Errorf("failed to init executor cache: %w", err)
+	}
+	cachedExecutor := &CachedExecutor{
+		executor: cli,
+		cache: &JsonTlbDiskCache[wrappedString, CachedExecResult]{
+			cache: execBolt,
+		},
+	}
 	storage := &LiteStorage{
 		logger: log,
 		// TODO: introduce an env variable to configure this number
-		maxGoroutines: 5,
-		client:        cli,
-		executor:      o.executor,
-		stopCh:        make(chan struct{}),
+		maxGoroutines:  5,
+		client:         cli,
+		executor:       o.executor,
+		cachedExecutor: cachedExecutor,
+		stopCh:         make(chan struct{}),
 		// read-only data
 		knownAccounts:    make(map[string][]tongo.AccountID),
 		trackingAccounts: map[tongo.AccountID]struct{}{},
@@ -147,7 +174,8 @@ func NewLiteStorage(log *zap.Logger, cli *liteapi.Client, opts ...Option) (*Lite
 		jettonMetaCache:         xsync.NewMapOf[tep64.Metadata](),
 		transactionsIndexByHash: xsync.NewTypedMapOf[tongo.Bits256, *core.Transaction](hashBits256),
 		transactionsByInMsgLT:   xsync.NewTypedMapOf[inMsgCreatedLT, tongo.Bits256](hashInMsgCreatedLT),
-		blockCache:              xsync.NewTypedMapOf[tongo.BlockIDExt, *tlb.Block](hashBlockIDExt),
+		blockCache:              blockCache,
+		accountCodeCache:        codeCache,
 		accountInterfacesCache:  xsync.NewTypedMapOf[tongo.AccountID, []abi.ContractInterface](hashAccountID),
 		tvmLibraryCache:         cache.NewLRUCache[string, boc.Cell](10000, "tvm_libraries"),
 		configCache:             cache.NewLRUCache[int, ton.BlockchainConfig](4, "config"),
@@ -160,14 +188,20 @@ func NewLiteStorage(log *zap.Logger, cli *liteapi.Client, opts ...Option) (*Lite
 	}
 
 	blockIterator := iter.Iterator[tongo.BlockID]{MaxGoroutines: storage.maxGoroutines}
+	log.Info("preloading blocks", zap.Int("count", len(o.preloadBlocks)))
+	var blockWG sync.WaitGroup
 	blockIterator.ForEach(o.preloadBlocks, func(id *tongo.BlockID) {
-		if err := storage.preloadBlock(*id); err != nil {
-			log.Error("failed to preload block",
-				zap.String("blockID", id.String()),
-				zap.Error(err))
-		}
+		blockWG.Go(func() {
+			if err := storage.preloadBlock(*id); err != nil {
+				log.Error("failed to preload block",
+					zap.String("blockID", id.String()),
+					zap.Error(err))
+			}
+		})
 	})
+	blockWG.Wait()
 	iterator := iter.Iterator[tongo.AccountID]{MaxGoroutines: storage.maxGoroutines}
+	log.Info("preloading acounts", zap.Int("count", len(o.preloadAccounts)))
 	iterator.ForEach(o.preloadAccounts, func(accountID *tongo.AccountID) {
 		if err := storage.preloadAccount(*accountID); err != nil {
 			log.Error("failed to preload account",
@@ -306,28 +340,54 @@ func (s *LiteStorage) preloadAccount(a tongo.AccountID) error {
 	return nil
 }
 
-func (s *LiteStorage) preloadBlock(id tongo.BlockID) error {
-	ctx := context.Background()
+func (s *LiteStorage) getCachedBlock(ctx context.Context, id tongo.BlockID) (tongo.BlockIDExt, *tlb.Block, error) {
+	cached, ok := s.blockCache.Load(id)
+	if ok {
+		return cached.V1, cached.V2, nil
+	}
 	extID, _, err := s.client.LookupBlock(ctx, id, 1, nil, nil)
 	if err != nil {
-		return err
+		return tongo.BlockIDExt{}, nil, err
 	}
 	block, err := s.client.GetBlock(ctx, extID)
+	if err != nil {
+		return tongo.BlockIDExt{}, nil, err
+	}
+	cached.V1 = extID
+	cached.V2 = &block
+	s.blockCache.Store(id, cached)
+	return cached.V1, cached.V2, nil
+}
+
+func (s *LiteStorage) getCachedAccountCode(ctx context.Context, id tongo.AccountID) ([]byte, error) {
+	if code, ok := s.accountCodeCache.Load(id); ok {
+		return code, nil
+	}
+	account, err := s.GetRawAccount(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+	s.accountCodeCache.Store(id, account.Code)
+	return account.Code, nil
+}
+
+func (s *LiteStorage) preloadBlock(id tongo.BlockID) error {
+	ctx := context.Background()
+	extID, block, err := s.getCachedBlock(ctx, id)
 	if err != nil {
 		return err
 	}
-	s.blockCache.Store(extID, &block)
 	for _, tx := range block.AllTransactions() {
 		accountID := tongo.AccountID{
 			Workchain: extID.Workchain,
 			Address:   tx.AccountAddr,
 		}
 		inspector := abi.NewContractInspector(abi.InspectWithLibraryResolver(s))
-		account, err := s.GetRawAccount(ctx, accountID)
+		accountCode, err := s.getCachedAccountCode(ctx, accountID)
 		if err != nil {
 			return err
 		}
-		cd, err := inspector.InspectContract(ctx, account.Code, s.executor, accountID)
+		cd, err := inspector.InspectContract(ctx, accountCode, s.cachedExecutor, accountID)
 		t, err := core.ConvertTransaction(extID.Workchain, tongo.Transaction{Transaction: *tx, BlockID: extID}, cd)
 		if err != nil {
 			return err
@@ -346,17 +406,11 @@ func (s *LiteStorage) GetBlockHeader(ctx context.Context, id tongo.BlockID) (*co
 		storageTimeHistogramVec.WithLabelValues("get_block_header").Observe(v)
 	}))
 	defer timer.ObserveDuration()
-	blockID, _, err := s.client.LookupBlock(ctx, id, 1, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-	block, err := s.client.GetBlock(ctx, blockID)
+	blockID, block, err := s.getCachedBlock(ctx, id)
 	if err != nil {
 		return nil, err
 	}
-
-	s.blockCache.Store(blockID, &block)
-	header, err := core.ConvertToBlockHeader(blockID, &block)
+	header, err := core.ConvertToBlockHeader(blockID, block)
 	if err != nil {
 		return nil, err
 	}

pkg/bath/cache/accountcode.db

@@ -0,0 +1,167 @@
+package litestorage
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"go.etcd.io/bbolt"
+
+	"github.com/tonkeeper/tongo/abi"
+	"github.com/tonkeeper/tongo/boc"
+	"github.com/tonkeeper/tongo/tlb"
+	"github.com/tonkeeper/tongo/ton"
+)
+
+type Cache[K, V any] interface {
+	Store(key K, value V)
+	Load(key K) (value V, ok bool)
+}
+
+var boltBucket = []byte("cache")
+
+// BoltCache is a bbolt-backed key/value cache with [string, []byte] types.
+// bbolt is a pure-Go embedded database; no CGO or shared libraries are required.
+type BoltCache struct {
+	db *bbolt.DB
+}
+
+// NewBoltCache opens (or creates) a bbolt database at the given file path.
+func NewBoltCache(path string) (*BoltCache, error) {
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return nil, err
+	}
+	db, err := bbolt.Open(path, 0600, nil)
+	if err != nil {
+		return nil, err
+	}
+	if err := db.Update(func(tx *bbolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists(boltBucket)
+		return err
+	}); err != nil {
+		db.Close()
+		return nil, err
+	}
+	return &BoltCache{db: db}, nil
+}
+
+func (c *BoltCache) Store(key string, value []byte) {
+	if err := c.db.Update(func(tx *bbolt.Tx) error {
+		return tx.Bucket(boltBucket).Put([]byte(key), value)
+	}); err != nil {
+		panic(fmt.Sprintf("BoltCache: Put %s: %v", key, err))
+	}
+}
+
+func (c *BoltCache) Load(key string) ([]byte, bool) {
+	var result []byte
+	if err := c.db.View(func(tx *bbolt.Tx) error {
+		v := tx.Bucket(boltBucket).Get([]byte(key))
+		if v != nil {
+			result = make([]byte, len(v))
+			copy(result, v)
+		}
+		return nil
+	}); err != nil {
+		return nil, false
+	}
+	return result, result != nil
+}
+
+// BytesDiskCache is a Cache[K, []byte] backed by a BoltCache.
+type BytesDiskCache[K fmt.Stringer] struct {
+	cache *BoltCache
+}
+
+func (dc *BytesDiskCache[K]) Store(key K, value []byte) {
+	dc.cache.Store(key.String(), value)
+}
+
+func (dc *BytesDiskCache[K]) Load(key K) ([]byte, bool) {
+	return dc.cache.Load(key.String())
+}
+
+// JsonTlbDiskCache is a Cache[K, V] that serialises values as JSON-encoded TLB cells,
+// backed by a BoltCache.
+type JsonTlbDiskCache[K fmt.Stringer, V any] struct {
+	cache *BoltCache
+}
+
+func (dc *JsonTlbDiskCache[K, V]) Store(key K, value V) {
+	cell := boc.NewCell()
+	if err := tlb.Marshal(cell, value); err != nil {
+		panic(err)
+	}
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(cell); err != nil {
+		panic(err)
+	}
+	dc.cache.Store(key.String(), buf.Bytes())
+}
+
+func (dc *JsonTlbDiskCache[K, V]) Load(key K) (V, bool) {
+	var value V
+	data, ok := dc.cache.Load(key.String())
+	if !ok {
+		return value, false
+	}
+	cell := boc.NewCell()
+	if err := json.NewDecoder(bytes.NewReader(data)).Decode(cell); err != nil {
+		panic(err)
+	}
+	if err := tlb.Unmarshal(cell, &value); err != nil {
+		panic(err)
+	}
+	return value, true
+}
+
+type CachedExecResult struct {
+	Code  uint32
+	Stack tlb.VmStack
+}
+
+type wrappedString struct {
+	value string
+}
+
+func (w wrappedString) String() string {
+	return w.value
+}
+
+type CachedExecutor struct {
+	executor abi.Executor
+	cache    *JsonTlbDiskCache[wrappedString, CachedExecResult]
+}
+
+func (ce *CachedExecutor) RunSmcMethodByID(ctx context.Context, accountID ton.AccountID, methodID int, params tlb.VmStack) (uint32, tlb.VmStack, error) {
+	argHash := sha256.New()
+	argHash.Write([]byte(accountID.String()))
+	argHash.Write([]byte(strconv.Itoa(methodID)))
+	paramsBytes, err := params.MarshalTL()
+	if err != nil {
+		return 0, tlb.VmStack{}, err
+	}
+	argHash.Write(paramsBytes)
+	argHashStr := hex.EncodeToString(argHash.Sum(nil))
+	result, found := ce.cache.Load(wrappedString{argHashStr})
+	if found {
+		return result.Code, result.Stack, nil
+	}
+	code, stack, err := ce.executor.RunSmcMethodByID(ctx, accountID, methodID, params)
+	if err != nil {
+		return code, stack, err
+	}
+	ce.cache.Store(wrappedString{argHashStr}, CachedExecResult{Code: code, Stack: stack})
+	return code, stack, err
+}
+
+type Tuple2[V1, V2 any] struct {
+	V1 V1
+	V2 V2
+}