Skip to content

Commit eaba17c

Browse files
committed
feat(toolkit): exclude historical balance DBs from lite snapshot
**Exclude historical balance data from lite snapshots** ### What Changed - Lite history cleanup now removes balance trace data and the related account trace entries for trimmed blocks. - When rebuilding a lite database, account trace data is copied from the start, so historical balance lookups still work after the split. - Snapshot tests now enable historical balance lookup to cover this behavior. - Added `.claude` to `.gitignore`. ### Impact `✅ Smaller lite snapshots` `✅ Complete balance history after trimming` `✅ Fewer missing trace records in rebuilt databases`
1 parent 29f475e commit eaba17c

6 files changed

Lines changed: 87 additions & 16 deletions

File tree

.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,3 +57,6 @@ Wallet
5757

5858
/framework/propPath
5959
.cache
60+
61+
#AI
62+
.claude

plugins/src/main/java/common/org/tron/plugins/DbLite.java

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import com.google.common.primitives.Bytes;
77
import com.google.common.primitives.Ints;
88
import com.google.common.primitives.Longs;
9+
import com.google.protobuf.ByteString;
910
import java.io.File;
1011
import java.io.FileNotFoundException;
1112
import java.io.IOException;
@@ -30,6 +31,7 @@
3031
import org.tron.plugins.utils.db.DBIterator;
3132
import org.tron.plugins.utils.db.DbTool;
3233
import org.tron.protos.Protocol;
34+
import org.tron.protos.contract.BalanceContract;
3335
import picocli.CommandLine;
3436

3537
@Slf4j(topic = "lite")
@@ -57,13 +59,17 @@ public class DbLite implements Callable<Integer> {
5759
private static final String TRANSACTION_HISTORY_DB_NAME = "transactionHistoryStore";
5860
private static final String PROPERTIES_DB_NAME = "properties";
5961
private static final String TRANS_CACHE_DB_NAME = "trans-cache";
62+
private static final String BALANCE_TRACE_DB_NAME = "balance-trace";
63+
private static final String ACCOUNT_TRACE_DB_NAME = "account-trace";
6064

6165
private static final List<String> archiveDbs = Arrays.asList(
6266
BLOCK_DB_NAME,
6367
BLOCK_INDEX_DB_NAME,
6468
TRANS_DB_NAME,
6569
TRANSACTION_RET_DB_NAME,
66-
TRANSACTION_HISTORY_DB_NAME);
70+
TRANSACTION_HISTORY_DB_NAME,
71+
BALANCE_TRACE_DB_NAME,
72+
ACCOUNT_TRACE_DB_NAME);
6773

6874
enum Operate { split, merge }
6975

@@ -522,24 +528,42 @@ private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo)
522528
DBInterface blockDb = DbTool.getDB(liteDir, BLOCK_DB_NAME);
523529
DBInterface transDb = DbTool.getDB(liteDir, TRANS_DB_NAME);
524530
DBInterface tranRetDb = DbTool.getDB(liteDir, TRANSACTION_RET_DB_NAME);
525-
531+
DBInterface balanceTraceDb = DbTool.getDB(liteDir, BALANCE_TRACE_DB_NAME);
532+
DBInterface accountTraceDb = DbTool.getDB(liteDir, ACCOUNT_TRACE_DB_NAME);
526533

527534
ProgressBar.wrap(LongStream.rangeClosed(start, end)
528535
.boxed()
529-
.sorted((a, b) -> Long.compare(b, a)), "trimHistory").forEach(n -> {
536+
.sorted((a, b) -> Long.compare(b, a)), "trimHistory")
537+
.map(ByteArray::fromLong).forEach(n -> {
530538
try {
531-
byte[] blockIdHash = blockIndexDb.get(ByteArray.fromLong(n));
539+
byte[] blockIdHash = blockIndexDb.get(n);
532540
Protocol.Block block = Protocol.Block.parseFrom(blockDb.get(blockIdHash));
533541
// delete transactions
534542
for (Protocol.Transaction e : block.getTransactionsList()) {
535543
transDb.delete(DBUtils.getTransactionId(e).getBytes());
536544
}
537545
// delete transaction result
538-
tranRetDb.delete(ByteArray.fromLong(n));
546+
tranRetDb.delete(n);
539547
// delete block
540548
blockDb.delete(blockIdHash);
541549
// delete block index
542-
blockIndexDb.delete(ByteArray.fromLong(n));
550+
blockIndexDb.delete(n);
551+
byte[] balanceTrace = balanceTraceDb.get(n);
552+
if (balanceTrace != null) {
553+
// delete account trace
554+
long blockNum = ByteArray.toLong(n);
555+
BalanceContract.BlockBalanceTrace.parseFrom(balanceTrace)
556+
.getTransactionBalanceTraceList().stream()
557+
.flatMap(tx -> tx.getOperationList().stream())
558+
.map(BalanceContract.TransactionBalanceTrace.Operation::getAddress)
559+
.distinct()
560+
.map(ByteString::toByteArray)
561+
.map(address -> Bytes.concat(address, Longs.toByteArray(
562+
blockNum ^ Long.MAX_VALUE)))
563+
.forEach(accountTraceDb::delete);
564+
// delete balance trace
565+
balanceTraceDb.delete(n);
566+
}
543567
} catch (IOException e) {
544568
throw new RuntimeException(e);
545569
}
@@ -570,7 +594,8 @@ private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws
570594
DBInterface bakDb = DbTool.getDB(bakDir.toString(), dbName);
571595
DBInterface destDb = DbTool.getDB(liteDir, dbName);
572596
try (DBIterator iterator = bakDb.iterator()) {
573-
if (TRANS_DB_NAME.equals(dbName) || TRANSACTION_HISTORY_DB_NAME.equals(dbName)) {
597+
if (TRANS_DB_NAME.equals(dbName) || TRANSACTION_HISTORY_DB_NAME.equals(dbName)
598+
|| ACCOUNT_TRACE_DB_NAME.equals(dbName)) {
574599
iterator.seekToFirst();
575600
} else {
576601
iterator.seek(head);

plugins/src/test/java/org/tron/plugins/DbLiteTest.java

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,20 @@ public void clear() {
8989

9090
public void testTools(String dbType, int checkpointVersion)
9191
throws InterruptedException, IOException {
92-
logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion);
92+
testTools(dbType, checkpointVersion, false, false);
93+
}
94+
95+
/**
96+
* @param advanceSnapshot when true, produce extra blocks on the snapshot before merge so that
97+
* snapshotMaxNum > historyMaxNum (exercises mergeBak2Database). When false, merge
98+
* immediately so that historyMaxNum > snapshotMaxNum (exercises trimExtraHistory).
99+
* @param historyBalanceLookup when true, enable historical balance lookup so that balance-trace
100+
* and account-trace data are written and can be exercised during trim.
101+
*/
102+
public void testTools(String dbType, int checkpointVersion, boolean advanceSnapshot,
103+
boolean historyBalanceLookup) throws InterruptedException, IOException {
104+
logger.info("dbType {}, checkpointVersion {}, advanceSnapshot {}, historyBalanceLookup {}",
105+
dbType, checkpointVersion, advanceSnapshot, historyBalanceLookup);
93106
dbPath = String.format("%s_%s_%d", dbPath, dbType, System.currentTimeMillis());
94107
init(dbType);
95108
final String[] argsForSnapshot =
@@ -104,6 +117,7 @@ public void testTools(String dbType, int checkpointVersion)
104117
new String[] {"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir,
105118
"--dataset-path", dbPath + File.separator + "history"};
106119
Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion);
120+
Args.getInstance().setHistoryBalanceLookup(historyBalanceLookup);
107121
DbLite.setRecentBlks(3);
108122
// start fullNode
109123
startApp();
@@ -117,8 +131,9 @@ public void testTools(String dbType, int checkpointVersion)
117131
cli.execute(argsForSnapshot);
118132
// start fullNode
119133
startApp();
120-
// produce transactions
121-
generateSomeTransactions(checkpointVersion == 1 ? 6 : 18);
134+
// produce transactions; when trimming history we need more blocks beyond the snapshot
135+
// so that trimExtraHistory has enough blocks with balance-trace data to exercise deletion
136+
generateSomeTransactions(advanceSnapshot ? (checkpointVersion == 1 ? 6 : 18) : 18);
122137
// stop the node
123138
shutdown();
124139
// generate history
@@ -137,11 +152,13 @@ public void testTools(String dbType, int checkpointVersion)
137152
String.format("rename snapshot to %s failed",
138153
Paths.get(dbPath, databaseDir)));
139154
}
140-
// start and validate the snapshot
141-
startApp();
142-
generateSomeTransactions(checkpointVersion == 1 ? 18 : 6);
143-
// stop the node
144-
shutdown();
155+
if (advanceSnapshot) {
156+
// start and validate the snapshot, producing blocks beyond history
157+
startApp();
158+
generateSomeTransactions(checkpointVersion == 1 ? 18 : 6);
159+
// stop the node
160+
shutdown();
161+
}
145162
// merge history
146163
cli.execute(argsForMerge);
147164
// start and validate

plugins/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,6 @@ public class DbLiteRocksDbV2Test extends DbLiteTest {
88

99
@Test
1010
public void testToolsWithRocksDB() throws InterruptedException, IOException {
11-
testTools("ROCKSDB", 2);
11+
testTools("ROCKSDB", 2, true, false);
1212
}
1313
}
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package org.tron.plugins.rocksdb;
2+
3+
import java.io.IOException;
4+
import org.junit.Test;
5+
import org.tron.plugins.DbLiteTest;
6+
7+
public class DbLiteWithHistoryRocksDbTest extends DbLiteTest {
8+
9+
@Test
10+
public void testToolsWithTrimHistory() throws InterruptedException, IOException {
11+
testTools("ROCKSDB", 1, true, true);
12+
}
13+
}
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package org.tron.plugins.rocksdb;
2+
3+
import java.io.IOException;
4+
import org.junit.Test;
5+
import org.tron.plugins.DbLiteTest;
6+
7+
public class DbLiteWithHistoryRocksDbV2Test extends DbLiteTest {
8+
9+
@Test
10+
public void testToolsWithTrimHistory() throws InterruptedException, IOException {
11+
testTools("ROCKSDB", 2, false, true);
12+
}
13+
}

0 commit comments

Comments
 (0)