Skip to content

Optimized the overall performance of IoTDB #17664

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Caideyipi wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Optimized the overall performance of IoTDB #17664

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
508c7e5
Opt
Caideyipi May 14, 2026
f28b5e1
Update UnclosedFileScanHandleImpl.java
Caideyipi May 14, 2026
3bf8da0
Update StorageEngine.java
Caideyipi May 14, 2026
828d3e8
Update ClosedFileScanHandleImpl.java
Caideyipi May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 31 additions & 16 deletions iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,20 +127,15 @@ public IoTDBJDBCDataSet(

// deduplicate and map
if (columnNameIndex != null) {
int deduplicatedColumnSize = (int) columnNameIndex.values().stream().distinct().count();
this.columnTypeDeduplicatedList = new ArrayList<>(deduplicatedColumnSize);
for (int i = 0; i < deduplicatedColumnSize; i++) {
columnTypeDeduplicatedList.add(null);
}
this.columnTypeDeduplicatedList =
initDeduplicatedColumnTypes(getDeduplicatedColumnSize(columnNameIndex));
for (int i = 0; i < columnNameList.size(); i++) {
String name = columnNameList.get(i);
this.columnNameList.add(name);
this.columnTypeList.add(columnTypeList.get(i));
if (!columnOrdinalMap.containsKey(name)) {
int index = columnNameIndex.get(name);
if (!columnOrdinalMap.containsValue(index + START_INDEX)) {
columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnTypeList.get(i)));
}
setColumnTypeIfAbsent(columnTypeDeduplicatedList, index, columnTypeList.get(i));
columnOrdinalMap.put(name, index + START_INDEX);
}
}
Expand Down Expand Up @@ -242,11 +237,8 @@ public IoTDBJDBCDataSet(

// deduplicate and map
if (columnNameIndex != null) {
int deduplicatedColumnSize = (int) columnNameIndex.values().stream().distinct().count();
this.columnTypeDeduplicatedList = new ArrayList<>(deduplicatedColumnSize);
for (int i = 0; i < deduplicatedColumnSize; i++) {
columnTypeDeduplicatedList.add(null);
}
this.columnTypeDeduplicatedList =
initDeduplicatedColumnTypes(getDeduplicatedColumnSize(columnNameIndex));
for (int i = 0; i < columnNameList.size(); i++) {
String name = "";
if (sgList != null
Expand All @@ -262,9 +254,7 @@ public IoTDBJDBCDataSet(
// "Time".equals(name) -> to allow the Time column appear in value columns
if (!columnOrdinalMap.containsKey(name) || "Time".equals(name)) {
int index = columnNameIndex.get(name);
if (!columnOrdinalMap.containsValue(index + START_INDEX)) {
columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnTypeList.get(i)));
}
setColumnTypeIfAbsent(columnTypeDeduplicatedList, index, columnTypeList.get(i));
columnOrdinalMap.put(name, index + START_INDEX);
}
}
Expand Down Expand Up @@ -320,6 +310,31 @@ public IoTDBJDBCDataSet(
this.emptyResultSet = (queryDataSet == null || !queryDataSet.time.hasRemaining());
}

private static int getDeduplicatedColumnSize(Map<String, Integer> columnNameIndex) {
int deduplicatedColumnSize = 0;
for (Integer index : columnNameIndex.values()) {
if (index != null && index + 1 > deduplicatedColumnSize) {
deduplicatedColumnSize = index + 1;
}
}
return deduplicatedColumnSize;
}

private static List<TSDataType> initDeduplicatedColumnTypes(int deduplicatedColumnSize) {
List<TSDataType> columnTypes = new ArrayList<>(deduplicatedColumnSize);
for (int i = 0; i < deduplicatedColumnSize; i++) {
columnTypes.add(null);
}
return columnTypes;
}

private static void setColumnTypeIfAbsent(
List<TSDataType> columnTypeDeduplicatedList, int index, String columnType) {
if (columnTypeDeduplicatedList.get(index) == null) {
columnTypeDeduplicatedList.set(index, TSDataType.valueOf(columnType));
}
}

public void close() throws StatementExecutionException, TException {
if (isClosed) {
return;
Expand Down
13 changes: 9 additions & 4 deletions iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/stmt/PreparedParameterSerde.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
/** Serializer for PreparedStatement parameters. */
public class PreparedParameterSerde {

private static final char[] HEX_DIGITS = "0123456789ABCDEF".toCharArray();

public static class DeserializedParam {
public final TSDataType type;
public final Object value;
Expand Down Expand Up @@ -166,10 +168,13 @@ private static Object deserializeValue(ByteBuffer buffer, TSDataType type) {

/** Convert byte array to hexadecimal string representation. */
public static String bytesToHex(byte[] bytes) {
StringBuilder sb = new StringBuilder(bytes.length * 2);
for (byte b : bytes) {
sb.append(String.format("%02X", b));
char[] chars = new char[bytes.length * 2];
for (int i = 0; i < bytes.length; i++) {
int value = bytes[i] & 0xFF;
int index = i * 2;
chars[index] = HEX_DIGITS[value >>> 4];
chars[index + 1] = HEX_DIGITS[value & 0x0F];
}
return sb.toString();
return new String(chars);
}
}
186 changes: 96 additions & 90 deletions iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -852,31 +852,12 @@
TSCreateAlignedTimeseriesReq request = new TSCreateAlignedTimeseriesReq();
request.setPrefixPath(prefixPath);
request.setMeasurements(measurements);
request.setDataTypes(dataTypes.stream().map(TSDataType::ordinal).collect(Collectors.toList()));
request.setEncodings(encodings.stream().map(TSEncoding::ordinal).collect(Collectors.toList()));
request.setCompressors(
compressors.stream().map(i -> (int) i.serialize()).collect(Collectors.toList()));
if (measurementAliasList != null) {
measurementAliasList =
measurementAliasList.stream()
.map(value -> value != null ? value : "")
.collect(Collectors.toList());
}
request.setMeasurementAlias(measurementAliasList);
if (tagsList != null) {
tagsList =
tagsList.stream()
.map(value -> value != null ? value : new HashMap<String, String>())
.collect(Collectors.toList());
}
request.setTagsList(tagsList);
if (attributesList != null) {
attributesList =
attributesList.stream()
.map(value -> value != null ? value : new HashMap<String, String>())
.collect(Collectors.toList());
}
request.setAttributesList(attributesList);
request.setDataTypes(toDataTypeOrdinals(dataTypes));
request.setEncodings(toEncodingOrdinals(encodings));
request.setCompressors(toCompressionOrdinals(compressors));
request.setMeasurementAlias(replaceNullStrings(measurementAliasList));
request.setTagsList(replaceNullMaps(tagsList));
request.setAttributesList(replaceNullMaps(attributesList));
return request;
}

Expand Down Expand Up @@ -916,47 +897,14 @@
TSCreateMultiTimeseriesReq request = new TSCreateMultiTimeseriesReq();

request.setPaths(paths);

List<Integer> dataTypeOrdinals = new ArrayList<>(dataTypes.size());
for (TSDataType dataType : dataTypes) {
dataTypeOrdinals.add(dataType.ordinal());
}
request.setDataTypes(dataTypeOrdinals);

List<Integer> encodingOrdinals = new ArrayList<>(dataTypes.size());
for (TSEncoding encoding : encodings) {
encodingOrdinals.add(encoding.ordinal());
}
request.setEncodings(encodingOrdinals);

List<Integer> compressionOrdinals = new ArrayList<>(paths.size());
for (CompressionType compression : compressors) {
compressionOrdinals.add((int) compression.serialize());
}
request.setCompressors(compressionOrdinals);
request.setDataTypes(toDataTypeOrdinals(dataTypes));
request.setEncodings(toEncodingOrdinals(encodings));
request.setCompressors(toCompressionOrdinals(compressors));

request.setPropsList(propsList);
if (tagsList != null) {
tagsList =
tagsList.stream()
.map(value -> value != null ? value : new HashMap<String, String>())
.collect(Collectors.toList());
}
request.setTagsList(tagsList);
if (attributesList != null) {
attributesList =
attributesList.stream()
.map(value -> value != null ? value : new HashMap<String, String>())
.collect(Collectors.toList());
}
request.setAttributesList(attributesList);
if (measurementAliasList != null) {
measurementAliasList =
measurementAliasList.stream()
.map(value -> value != null ? value : "")
.collect(Collectors.toList());
}
request.setMeasurementAliasList(measurementAliasList);
request.setTagsList(replaceNullMaps(tagsList));
request.setAttributesList(replaceNullMaps(attributesList));
request.setMeasurementAliasList(replaceNullStrings(measurementAliasList));

return request;
}
Expand Down Expand Up @@ -1819,19 +1767,23 @@
List<String> measurementsList,
List<TSDataType> types,
List<Object> valuesList) {
Map<String, Object> nullMap = new HashMap<>();
Map<String, Object> nullMap = logger.isInfoEnabled() ? new HashMap<>() : null;
for (int i = valuesList.size() - 1; i >= 0; i--) {
if (valuesList.get(i) == null) {
nullMap.put(measurementsList.get(i), valuesList.get(i));
if (nullMap != null) {
nullMap.put(measurementsList.get(i), valuesList.get(i));
}
valuesList.remove(i);
measurementsList.remove(i);
types.remove(i);
}
}
if (valuesList.isEmpty()) {
logger.info("All values of the {} are null,null values are {}", deviceId, nullMap);
if (nullMap != null) {
logger.info("All values of the {} are null,null values are {}", deviceId, nullMap);
}
return true;
} else {
} else if (nullMap != null) {
logger.info("Some values of {} are null,null values are {}", deviceId, nullMap);
}
return false;
Expand Down Expand Up @@ -1867,18 +1819,22 @@
*/
private boolean filterNullValueAndMeasurementWithStringType(
List<String> valuesList, String deviceId, List<String> measurementsList) {
Map<String, Object> nullMap = new HashMap<>();
Map<String, Object> nullMap = logger.isInfoEnabled() ? new HashMap<>() : null;
for (int i = valuesList.size() - 1; i >= 0; i--) {
if (valuesList.get(i) == null) {
nullMap.put(measurementsList.get(i), valuesList.get(i));
if (nullMap != null) {
nullMap.put(measurementsList.get(i), valuesList.get(i));
}
valuesList.remove(i);
measurementsList.remove(i);
}
}
if (valuesList.isEmpty()) {
logger.info("All values of the {} are null,null values are {}", deviceId, nullMap);
if (nullMap != null) {
logger.info("All values of the {} are null,null values are {}", deviceId, nullMap);
}
return true;
} else {
} else if (nullMap != null) {
logger.info("Some values of {} are null,null values are {}", deviceId, nullMap);
}
return false;
Expand Down Expand Up @@ -2622,7 +2578,65 @@
* @return ordered list
*/
private static <T> List<T> sortList(List<T> source, Integer[] index) {
return Arrays.stream(index).map(source::get).collect(Collectors.toList());
List<T> sortedList = new ArrayList<>(index.length);
for (int position : index) {
sortedList.add(source.get(position));
}
return sortedList;
}

private static List<Integer> toDataTypeOrdinals(List<TSDataType> dataTypes) {
List<Integer> ordinals = new ArrayList<>(dataTypes.size());
for (TSDataType dataType : dataTypes) {
ordinals.add(dataType.ordinal());
}
return ordinals;
}

private static List<Integer> toEncodingOrdinals(List<TSEncoding> encodings) {
List<Integer> ordinals = new ArrayList<>(encodings.size());
for (TSEncoding encoding : encodings) {
ordinals.add(encoding.ordinal());
}
return ordinals;
}

private static List<Integer> toCompressionOrdinals(List<CompressionType> compressors) {
List<Integer> ordinals = new ArrayList<>(compressors.size());
for (CompressionType compression : compressors) {
ordinals.add((int) compression.serialize());
}
return ordinals;
}

private static List<Byte> toEnumOrdinalsAsBytes(List<? extends Enum<?>> enumValues) {
List<Byte> ordinals = new ArrayList<>(enumValues.size());
for (Enum<?> enumValue : enumValues) {
ordinals.add((byte) enumValue.ordinal());
}
return ordinals;
}

private static List<String> replaceNullStrings(List<String> values) {
if (values == null) {
return null;

Check warning on line 2622 in iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Return an empty collection instead of null. 


See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AZ4kZNfayRwYTrR2Bsn6&open=AZ4kZNfayRwYTrR2Bsn6&pullRequest=17664
}
List<String> replacedValues = new ArrayList<>(values.size());
for (String value : values) {
replacedValues.add(value != null ? value : "");
}
return replacedValues;
}

private static List<Map<String, String>> replaceNullMaps(List<Map<String, String>> values) {
if (values == null) {
return null;

Check warning on line 2633 in iotdb-client/session/src/main/java/org/apache/iotdb/session/Session.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Return an empty collection instead of null. 


See more on https://sonarcloud.io/project/issues?id=apache_iotdb&issues=AZ4kZNfayRwYTrR2Bsn7&open=AZ4kZNfayRwYTrR2Bsn7&pullRequest=17664
}
List<Map<String, String>> replacedValues = new ArrayList<>(values.size());
for (Map<String, String> value : values) {
replacedValues.add(value != null ? value : new HashMap<>());
}
return replacedValues;
}

private List<ByteBuffer> objectValuesListToByteBufferList(
Expand Down Expand Up @@ -2820,10 +2834,7 @@
} else {
TSInsertTabletReq request = genTSInsertTabletReq(tablet, false, false);
request.setWriteToTable(true);
request.setColumnCategories(
tablet.getColumnTypes().stream()
.map(t -> (byte) t.ordinal())
.collect(Collectors.toList()));
request.setColumnCategories(toEnumOrdinalsAsBytes(tablet.getColumnTypes()));
try {
getDefaultSessionConnection().insertTablet(request);
} catch (RedirectException ignored) {
Expand Down Expand Up @@ -2889,8 +2900,7 @@
Tablet tablet = entry.getValue();
TSInsertTabletReq request = genTSInsertTabletReq(tablet, false, false);
request.setWriteToTable(true);
request.setColumnCategories(
tablet.getColumnTypes().stream().map(t -> (byte) t.ordinal()).collect(Collectors.toList()));
request.setColumnCategories(toEnumOrdinalsAsBytes(tablet.getColumnTypes()));
try {
connection.insertTablet(request);
} catch (RedirectException e) {
Expand Down Expand Up @@ -2932,9 +2942,7 @@
TSInsertTabletReq request = genTSInsertTabletReq(subTablet, false, false);
request.setWriteToTable(true);
request.setColumnCategories(
subTablet.getColumnTypes().stream()
.map(t -> (byte) t.ordinal())
.collect(Collectors.toList()));
toEnumOrdinalsAsBytes(subTablet.getColumnTypes()));
InsertConsumer<TSInsertTabletReq> insertConsumer =
SessionConnection::insertTablet;
try {
Expand Down Expand Up @@ -3904,10 +3912,9 @@
TSAppendSchemaTemplateReq req = new TSAppendSchemaTemplateReq();
req.setName(templateName);
req.setMeasurements(measurementsPath);
req.setDataTypes(dataTypes.stream().map(TSDataType::ordinal).collect(Collectors.toList()));
req.setEncodings(encodings.stream().map(TSEncoding::ordinal).collect(Collectors.toList()));
req.setCompressors(
compressors.stream().map(i -> (int) i.serialize()).collect(Collectors.toList()));
req.setDataTypes(toDataTypeOrdinals(dataTypes));
req.setEncodings(toEncodingOrdinals(encodings));
req.setCompressors(toCompressionOrdinals(compressors));
req.setIsAligned(true);
getDefaultSessionConnection().appendSchemaTemplate(req);
}
Expand Down Expand Up @@ -3950,10 +3957,9 @@
TSAppendSchemaTemplateReq req = new TSAppendSchemaTemplateReq();
req.setName(templateName);
req.setMeasurements(measurementsPath);
req.setDataTypes(dataTypes.stream().map(TSDataType::ordinal).collect(Collectors.toList()));
req.setEncodings(encodings.stream().map(TSEncoding::ordinal).collect(Collectors.toList()));
req.setCompressors(
compressors.stream().map(i -> (int) i.serialize()).collect(Collectors.toList()));
req.setDataTypes(toDataTypeOrdinals(dataTypes));
req.setEncodings(toEncodingOrdinals(encodings));
req.setCompressors(toCompressionOrdinals(compressors));
req.setIsAligned(false);
getDefaultSessionConnection().appendSchemaTemplate(req);
}
Expand Down
Loading
Loading