From 2e32bc7f26a2f5a75df6901670465527f43c85af Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 14 May 2026 20:18:49 +0800 Subject: [PATCH 1/2] flag- --- .../impl/mem/MTreeBelowSGMemoryImpl.java | 68 ++++- .../mtree/impl/mem/mnode/IMemMNode.java | 8 + .../impl/mem/mnode/basic/BasicMNode.java | 16 +- .../mem/mnode/impl/AboveDatabaseMNode.java | 10 + .../impl/mem/mnode/impl/DatabaseMNode.java | 10 + .../impl/mem/mnode/impl/MeasurementMNode.java | 10 + .../mtree/traverser/Traverser.java | 56 +++++ .../impl/mem/MTreeBelowSGMemoryImplTest.java | 234 ++++++++++++++++++ .../schema/tree/AbstractTreeVisitor.java | 4 + 9 files changed, 407 insertions(+), 9 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImplTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java index d872a1130a4ee..57108c045f6c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImpl.java @@ -207,6 +207,46 @@ private void applySubtreeMeasurementDelta(IMemMNode startNode, final long delta) } } + private IDeviceMNode setToEntityAndUpdateFlags(final IMemMNode node) { + final boolean wasDevice = node.isDevice(); + final IDeviceMNode deviceMNode = store.setToEntity(node); + if (!wasDevice) { + markAncestorsHavingDeviceDescendant(node); + } + return deviceMNode; + } + + private void markAncestorsHavingDeviceDescendant(final IMemMNode deviceNode) { + IMemMNode current = deviceNode.getParent(); + while (current != null && !current.hasDeviceDescendant()) { + current.setHasDeviceDescendant(true); + current = current.getParent(); + } + } + + private boolean hasDeviceDescendantInChildren(final IMemMNode node) { + final IMNodeIterator iterator = store.getChildrenIterator(node); + try { + while (iterator.hasNext()) { + final IMemMNode child = iterator.next(); + if (child.isDevice() || child.hasDeviceDescendant()) { + return true; + } + } + return false; + } finally { + iterator.close(); + } + } + + private void refreshAncestorsHavingDeviceDescendant(IMemMNode startNode) { + IMemMNode current = startNode; + while (current != null) { + current.setHasDeviceDescendant(hasDeviceDescendantInChildren(current)); + current = current.getParent(); + } + } + private long getTemplateMeasurementCount(final int templateId) { final Template template = ClusterTemplateManager.getInstance().getTemplate(templateId); return template == null ? 0L : template.getMeasurementNumber(); @@ -315,7 +355,7 @@ public IMeasurementMNode createTimeSeries( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); } // create a non-aligned time series @@ -409,7 +449,7 @@ public List> createAlignedTimeSeries( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); entityMNode.setAligned(true); } @@ -673,6 +713,7 @@ public void deleteEmptyInternalMNode(final IDeviceMNode entityMNode) synchronized (this) { curNode = store.setToInternal(entityMNode); } + refreshAncestorsHavingDeviceDescendant(curNode.getParent()); } else if (!hasNonViewMeasurement) { // has some measurement but they are all logical view entityMNode.setAligned(null); @@ -1033,7 +1074,7 @@ public void activateTemplate(final PartialPath activatePath, final Template temp if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } } @@ -1141,7 +1182,7 @@ public void activateTemplateWithoutCheck( if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } if (!entityMNode.isAligned()) { @@ -1196,14 +1237,24 @@ public void rebuildSubtreeMeasurementCount() { private long rebuildSubtreeMeasurementCountFromNode(final IMemMNode node) { long count = node.isMeasurement() ? 1L : 0L; + boolean hasDeviceDescendant = false; final IMNodeIterator iterator = store.getChildrenIterator(node); - while (iterator.hasNext()) { - count += rebuildSubtreeMeasurementCountFromNode(iterator.next()); + try { + while (iterator.hasNext()) { + final IMemMNode child = iterator.next(); + count += rebuildSubtreeMeasurementCountFromNode(child); + if (child.isDevice() || child.hasDeviceDescendant()) { + hasDeviceDescendant = true; + } + } + } finally { + iterator.close(); } if (node.isDevice() && node.getAsDeviceMNode().isUseTemplate()) { count += getTemplateMeasurementCount(node.getAsDeviceMNode().getSchemaTemplateId()); } node.setSubtreeMeasurementCount(count); + node.setHasDeviceDescendant(hasDeviceDescendant); return count; } @@ -1786,7 +1837,7 @@ public IMeasurementMNode createLogicalView( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); // this parent has no measurement before. The leafName is his first child who is a logical // view. entityMNode.setAligned(null); @@ -1923,7 +1974,7 @@ public void createOrUpdateTableDevice( (TableDeviceInfo) entityMNode.getDeviceInfo(); attributeUpdater.accept(deviceInfo.getAttributePointer()); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); final TableDeviceInfo deviceInfo = new TableDeviceInfo<>(); deviceInfo.setAttributePointer(attributePointerGetter.getAsInt()); entityMNode.getAsInternalMNode().setDeviceInfo(deviceInfo); @@ -2040,6 +2091,7 @@ protected Void collectMNode(final IMemMNode node) { collector.traverse(); } databaseMNode.deleteChild(tableName); + refreshAncestorsHavingDeviceDescendant(databaseMNode); regionStatistics.resetTableDevice(tableName); store.releaseMemory(memoryReleased.get()); return true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java index 6cd14800f3e3f..33e50b7864d0d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/IMemMNode.java @@ -29,4 +29,12 @@ public interface IMemMNode extends IMNode { long getSubtreeMeasurementCount(); void setSubtreeMeasurementCount(long subtreeMeasurementCount); + + /** + * Whether there is any device node in the subtree rooted at this node, excluding the node itself. + * This flag is maintained in memory only. + */ + boolean hasDeviceDescendant(); + + void setHasDeviceDescendant(boolean hasDeviceDescendant); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java index 2eab69749fd79..54d01087b9507 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/basic/BasicMNode.java @@ -49,6 +49,9 @@ public class BasicMNode implements IMemMNode { /** Cached count of measurements in this node's subtree, rebuilt on restart. */ private long subtreeMeasurementCount = 0L; + /** Cached flag showing whether there is any device in the subtree below this node. */ + private boolean hasDeviceDescendant = false; + /** from root to this node, only be set when used once for InternalMNode */ private String fullPath; @@ -112,6 +115,16 @@ public void setSubtreeMeasurementCount(final long subtreeMeasurementCount) { this.subtreeMeasurementCount = subtreeMeasurementCount; } + @Override + public boolean hasDeviceDescendant() { + return hasDeviceDescendant; + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + this.hasDeviceDescendant = hasDeviceDescendant; + } + @Override public PartialPath getPartialPath() { final List detachedPath = new ArrayList<>(); @@ -239,6 +252,7 @@ public R accept(final MNodeVisitor visitor, final C context) { *
  • parent reference, 8B *
  • fullPath reference, 8B *
  • subtreeMeasurementCount, 8B + *
  • hasDeviceDescendant, 1B * *
  • MapEntry in parent *
      @@ -250,7 +264,7 @@ public R accept(final MNodeVisitor visitor, final C context) { */ @Override public int estimateSize() { - return 8 + 8 + 8 + 8 + 8 + 8 + 8 + 28 + basicMNodeInfo.estimateSize(); + return 8 + 8 + 8 + 8 + 8 + 1 + 8 + 8 + 28 + basicMNodeInfo.estimateSize(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/AboveDatabaseMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/AboveDatabaseMNode.java index 87144d4954a5d..59951d5c3e84d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/AboveDatabaseMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/AboveDatabaseMNode.java @@ -43,4 +43,14 @@ public long getSubtreeMeasurementCount() { public void setSubtreeMeasurementCount(long subtreeMeasurementCount) { basicMNode.setSubtreeMeasurementCount(subtreeMeasurementCount); } + + @Override + public boolean hasDeviceDescendant() { + return basicMNode.hasDeviceDescendant(); + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + basicMNode.setHasDeviceDescendant(hasDeviceDescendant); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/DatabaseMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/DatabaseMNode.java index 290cf4273601b..4089bfae008d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/DatabaseMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/DatabaseMNode.java @@ -55,4 +55,14 @@ public long getSubtreeMeasurementCount() { public void setSubtreeMeasurementCount(long subtreeMeasurementCount) { basicMNode.setSubtreeMeasurementCount(subtreeMeasurementCount); } + + @Override + public boolean hasDeviceDescendant() { + return basicMNode.hasDeviceDescendant(); + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + basicMNode.setHasDeviceDescendant(hasDeviceDescendant); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/MeasurementMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/MeasurementMNode.java index d2a2cbd80c90c..40f803882c516 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/MeasurementMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/mnode/impl/MeasurementMNode.java @@ -62,4 +62,14 @@ public long getSubtreeMeasurementCount() { public void setSubtreeMeasurementCount(long subtreeMeasurementCount) { basicMNode.setSubtreeMeasurementCount(subtreeMeasurementCount); } + + @Override + public boolean hasDeviceDescendant() { + return basicMNode.hasDeviceDescendant(); + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + basicMNode.setHasDeviceDescendant(hasDeviceDescendant); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java index 24b7e93939e03..4482c6b0221ce 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PathPatternTree; import org.apache.iotdb.commons.path.fa.IFAState; import org.apache.iotdb.commons.path.fa.IFATransition; +import org.apache.iotdb.commons.path.fa.match.IStateMatchInfo; import org.apache.iotdb.commons.schema.node.IMNode; import org.apache.iotdb.commons.schema.node.role.IDeviceMNode; import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; @@ -32,6 +33,8 @@ import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.tree.AbstractTreeVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.MemMTreeStore; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator.MNodeIterator; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.ReentrantReadOnlyCachedMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; @@ -41,9 +44,11 @@ import org.slf4j.LoggerFactory; import java.util.Iterator; +import java.util.LinkedHashSet; import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; +import java.util.Set; import static org.apache.iotdb.commons.schema.SchemaConstant.NON_TEMPLATE; @@ -247,10 +252,61 @@ protected Iterator getChildrenIterator(N parent) throws MetadataException { if (parent.isAboveDatabase()) { return new MNodeIterator<>(parent.getChildren().values().iterator()); } else { + final Iterator optimizedIterator = getOptimizedChildrenIterator(parent); + if (optimizedIterator != null) { + return optimizedIterator; + } return store.getTraverserIterator(parent, templateMap, skipPreDeletedSchema); } } + @SuppressWarnings("unchecked") + private Iterator getOptimizedChildrenIterator(final N parent) throws MetadataException { + if (!(store instanceof MemMTreeStore) || !parent.isDevice()) { + return null; + } + + final IMemMNode memNode = (IMemMNode) parent; + if (memNode.hasDeviceDescendant()) { + return null; + } + + final IStateMatchInfo stateMatchInfo = getCurrentStateMatchInfo(); + final Set candidateNames = new LinkedHashSet<>(); + + for (int i = 0; i < stateMatchInfo.getMatchedStateSize(); i++) { + final IFAState matchedState = stateMatchInfo.getMatchedState(i); + + for (final IFATransition transition : + patternFA.getPreciseMatchTransition(matchedState).values()) { + if (patternFA.getNextState(matchedState, transition).isFinal()) { + candidateNames.add(transition.getAcceptEvent()); + } + } + + final Iterator fuzzyTransitionIterator = + patternFA.getFuzzyMatchTransitionIterator(matchedState); + while (fuzzyTransitionIterator.hasNext()) { + if (patternFA.getNextState(matchedState, fuzzyTransitionIterator.next()).isFinal()) { + return null; + } + } + } + + if (candidateNames.isEmpty()) { + return null; + } + + // For leaf devices, `**.measurement` does not need to enumerate every measurement child. + try { + return getChildrenIterator(parent, candidateNames.iterator()); + } catch (final MetadataException e) { + throw e; + } catch (final Exception e) { + throw new MetadataException(e.getMessage(), e); + } + } + @Override protected void releaseNodeIterator(Iterator nodeIterator) { if (nodeIterator instanceof IMNodeIterator) { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImplTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImplTest.java new file mode 100644 index 0000000000000..3226e8c7f4ecb --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/MTreeBelowSGMemoryImplTest.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; +import org.apache.iotdb.commons.schema.node.utils.IMNodeFactory; +import org.apache.iotdb.commons.schema.node.utils.IMNodeIterator; +import org.apache.iotdb.commons.schema.template.Template; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.schemaengine.metric.SchemaRegionMemMetric; +import org.apache.iotdb.db.schemaengine.rescon.MemSchemaEngineStatistics; +import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MeasurementCollector; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public class MTreeBelowSGMemoryImplTest { + + @Test + public void testDeviceDescendantFlagIsMaintainedAcrossCreateDeleteAndRebuild() throws Exception { + final MTreeBelowSGMemoryImpl mtree = newMTree(); + + mtree.createTimeSeries( + new MeasurementPath("root.sg.a.b.s1"), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null, + false, + null); + + IMemMNode database = mtree.getNodeByPath(new PartialPath("root.sg")); + IMemMNode aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + IMemMNode bNode = mtree.getNodeByPath(new PartialPath("root.sg.a.b")); + + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(bNode.isDevice()); + Assert.assertFalse(bNode.hasDeviceDescendant()); + + mtree.createTimeSeries( + new MeasurementPath("root.sg.a.s0"), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null, + false, + null); + + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertTrue(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + + database.setHasDeviceDescendant(false); + aNode.setHasDeviceDescendant(false); + bNode.setHasDeviceDescendant(true); + mtree.rebuildSubtreeMeasurementCount(); + + database = mtree.getNodeByPath(new PartialPath("root.sg")); + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + bNode = mtree.getNodeByPath(new PartialPath("root.sg.a.b")); + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertFalse(bNode.hasDeviceDescendant()); + + mtree.deleteTimeSeries(new PartialPath("root.sg.a.s0")); + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + + mtree.deleteTimeSeries(new PartialPath("root.sg.a.b.s1")); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertFalse(database.hasDeviceDescendant()); + assertPathNotExist(mtree, new PartialPath("root.sg.a")); + } + + @Test + public void testLeafDeviceWildcardSuffixUsesDirectMeasurementLookup() throws Exception { + final PartialPath databasePath = PartialPath.getQualifiedDatabasePartialPath("root.sg"); + final MemSchemaRegionStatistics regionStatistics = newRegionStatistics(); + final CountingMemMTreeStore store = + new CountingMemMTreeStore( + databasePath, regionStatistics, new SchemaRegionMemMetric(regionStatistics, "root.sg")); + final IMNodeFactory nodeFactory = + MNodeFactoryLoader.getInstance().getMemMNodeIMNodeFactory(); + + final IMemMNode databaseMNode = store.getRoot(); + final IMemMNode rootNode = store.generatePrefix(databasePath); + final IMemMNode deviceNode = + store.addChild(databaseMNode, "d1", nodeFactory.createInternalMNode(databaseMNode, "d1")); + store.setToEntity(deviceNode); + + for (int i = 0; i < 64; i++) { + final String measurement = i == 0 ? "target" : "s" + i; + final IMeasurementMNode measurementMNode = + nodeFactory.createMeasurementMNode( + deviceNode.getAsDeviceMNode(), + measurement, + new MeasurementSchema( + measurement, TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY), + null); + store.addChild(deviceNode, measurement, measurementMNode.getAsMNode()); + } + + store.watchParent(deviceNode); + final List matchedPaths = new ArrayList<>(); + try (final MeasurementCollector collector = + new MeasurementCollector( + rootNode, + new PartialPath("root.sg.**.target"), + store, + false, + SchemaConstant.ALL_MATCH_SCOPE) { + @Override + protected Void collectMeasurement(final IMeasurementMNode node) { + matchedPaths.add(getCurrentMeasurementPathInTraverse(node).getFullPath()); + return null; + } + }) { + collector.traverse(); + } + + Assert.assertEquals(Collections.singletonList("root.sg.d1.target"), matchedPaths); + Assert.assertEquals(0, store.getWatchedTraverserIteratorCount()); + Assert.assertEquals(1, store.getWatchedChildLookupCount()); + } + + private static MTreeBelowSGMemoryImpl newMTree() throws Exception { + final MemSchemaRegionStatistics regionStatistics = newRegionStatistics(); + return new MTreeBelowSGMemoryImpl( + PartialPath.getQualifiedDatabasePartialPath("root.sg"), + node -> Collections.emptyMap(), + node -> Collections.emptyMap(), + regionStatistics, + new SchemaRegionMemMetric(regionStatistics, "root.sg")); + } + + private static MemSchemaRegionStatistics newRegionStatistics() { + return new MemSchemaRegionStatistics(0, new MemSchemaEngineStatistics()); + } + + private static void assertPathNotExist(final MTreeBelowSGMemoryImpl mtree, final PartialPath path) + throws MetadataException { + try { + mtree.getNodeByPath(path); + Assert.fail("Expected path not exist: " + path.getFullPath()); + } catch (final PathNotExistException ignored) { + // expected + } + } + + private static class CountingMemMTreeStore extends MemMTreeStore { + private IMemMNode watchedParent; + private int watchedTraverserIteratorCount; + private int watchedChildLookupCount; + + private CountingMemMTreeStore( + final PartialPath rootPath, + final MemSchemaRegionStatistics regionStatistics, + final SchemaRegionMemMetric metric) { + super(rootPath, regionStatistics, metric); + } + + private void watchParent(final IMemMNode parent) { + this.watchedParent = parent; + this.watchedTraverserIteratorCount = 0; + this.watchedChildLookupCount = 0; + } + + private int getWatchedTraverserIteratorCount() { + return watchedTraverserIteratorCount; + } + + private int getWatchedChildLookupCount() { + return watchedChildLookupCount; + } + + @Override + public IMemMNode getChild(final IMemMNode parent, final String name) { + if (parent == watchedParent) { + watchedChildLookupCount++; + } + return super.getChild(parent, name); + } + + @Override + public IMNodeIterator getTraverserIterator( + final IMemMNode parent, + final Map templateMap, + final boolean skipPreDeletedSchema) + throws MetadataException { + if (parent == watchedParent) { + watchedTraverserIteratorCount++; + } + return super.getTraverserIterator(parent, templateMap, skipPreDeletedSchema); + } + } +} diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java index a60a4ff121ad7..512270c379a87 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/schema/tree/AbstractTreeVisitor.java @@ -415,6 +415,10 @@ protected Iterator getCurrentChildrenIterator() { } } + protected final IStateMatchInfo getCurrentStateMatchInfo() { + return currentStateMatchInfo; + } + // Release a child node. protected void releaseNode(N node) {} From 49071fd20373306dba2ae3f4ad64756d40660aa6 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Thu, 14 May 2026 22:02:56 +0800 Subject: [PATCH 2/2] Fix --- .../impl/pbtree/MTreeBelowSGCachedImpl.java | 76 ++++++- .../mtree/impl/pbtree/mnode/ICachedMNode.java | 8 + .../pbtree/mnode/basic/CachedBasicMNode.java | 35 ++- .../mnode/impl/CachedAboveDatabaseMNode.java | 20 ++ .../mnode/impl/CachedDatabaseMNode.java | 20 ++ .../mnode/impl/CachedMeasurementMNode.java | 20 ++ .../mtree/traverser/Traverser.java | 59 +++++- .../traverser/basic/MeasurementTraverser.java | 5 + .../schemaRegion/SchemaRegionBasicTest.java | 22 ++ .../pbtree/MTreeBelowSGCachedImplTest.java | 200 ++++++++++++++++++ 10 files changed, 454 insertions(+), 11 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImplTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java index 8ca06b83f7dcf..7598d8687b896 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImpl.java @@ -137,6 +137,71 @@ public class MTreeBelowSGCachedImpl { private final int levelOfDB; private final CachedSchemaRegionStatistics regionStatistics; + private IDeviceMNode setToEntityAndUpdateFlags(final ICachedMNode node) + throws MetadataException { + final boolean wasDevice = node.isDevice(); + if (!node.isDeviceDescendantComputed()) { + node.setHasDeviceDescendant(hasDeviceDescendantInChildren(node)); + node.setDeviceDescendantComputed(true); + } + final IDeviceMNode deviceMNode = store.setToEntity(node); + if (!wasDevice) { + markAncestorsHavingDeviceDescendant(node); + } + return deviceMNode; + } + + private void markAncestorsHavingDeviceDescendant(final ICachedMNode deviceNode) { + ICachedMNode current = deviceNode.getParent(); + while (current != null && !current.hasDeviceDescendant()) { + current.setHasDeviceDescendant(true); + current.setDeviceDescendantComputed(true); + current = current.getParent(); + } + } + + private boolean hasDeviceDescendantInChildren(final ICachedMNode node) throws MetadataException { + final IMNodeIterator iterator = store.getChildrenIterator(node); + try { + while (iterator.hasNext()) { + final ICachedMNode child = iterator.next(); + try { + if (child.isDevice() || hasDeviceDescendant(child)) { + return true; + } + } finally { + unPinMNode(child); + } + } + return false; + } finally { + iterator.close(); + } + } + + private boolean hasDeviceDescendant(final ICachedMNode node) throws MetadataException { + if (node.isMeasurement()) { + node.setHasDeviceDescendant(false); + node.setDeviceDescendantComputed(true); + return false; + } + if (!node.isDeviceDescendantComputed()) { + node.setHasDeviceDescendant(hasDeviceDescendantInChildren(node)); + node.setDeviceDescendantComputed(true); + } + return node.hasDeviceDescendant(); + } + + private void refreshAncestorsHavingDeviceDescendant(ICachedMNode startNode) + throws MetadataException { + ICachedMNode current = startNode; + while (current != null) { + current.setHasDeviceDescendant(hasDeviceDescendantInChildren(current)); + current.setDeviceDescendantComputed(true); + current = current.getParent(); + } + } + // region MTree initialization, clear and serialization public MTreeBelowSGCachedImpl( PartialPath storageGroupPath, @@ -353,7 +418,7 @@ public IMeasurementMNode createTimeSeriesWithPinnedReturn( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); device = entityMNode.getAsMNode(); } @@ -458,7 +523,7 @@ public List> createAlignedTimeSeries( if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); entityMNode.setAligned(true); device = entityMNode.getAsMNode(); } @@ -703,6 +768,7 @@ private void deleteAndUnpinEmptyInternalMNode(IDeviceMNode entityM if (!hasMeasurement) { curNode = store.setToInternal(entityMNode); + refreshAncestorsHavingDeviceDescendant(curNode.getParent()); } else if (!hasNonViewMeasurement) { // has some measurement but they are all logical view store.updateMNode(entityMNode.getAsMNode(), o -> o.getAsDeviceMNode().setAligned(null)); @@ -1075,7 +1141,7 @@ public void createLogicalView(PartialPath path, ViewExpression viewExpression) if (device.isDevice()) { entityMNode = device.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(device); + entityMNode = setToEntityAndUpdateFlags(device); // this parent has no measurement before. The leafName is his first child who is a // logical // view. @@ -1197,7 +1263,7 @@ public void activateTemplate(PartialPath activatePath, Template template) if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } if (entityMNode.isUseTemplate()) { @@ -1243,7 +1309,7 @@ public void activateTemplateWithoutCheck( if (cur.isDevice()) { entityMNode = cur.getAsDeviceMNode(); } else { - entityMNode = store.setToEntity(cur); + entityMNode = setToEntityAndUpdateFlags(cur); } store.updateMNode( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java index 489b1a642aad9..85e01c70b5cca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/ICachedMNode.java @@ -24,6 +24,14 @@ import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.cache.CacheEntry; public interface ICachedMNode extends IMNode { + boolean hasDeviceDescendant(); + + void setHasDeviceDescendant(boolean hasDeviceDescendant); + + boolean isDeviceDescendantComputed(); + + void setDeviceDescendantComputed(boolean deviceDescendantComputed); + CacheEntry getCacheEntry(); void setCacheEntry(CacheEntry cacheEntry); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java index d34fde474abee..f099b238438f3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/basic/CachedBasicMNode.java @@ -47,6 +47,17 @@ public class CachedBasicMNode implements ICachedMNode { private ICachedMNode parent; private final CacheMNodeInfo cacheMNodeInfo; + /** Cached flag showing whether there is any device in the subtree below this node. */ + private boolean hasDeviceDescendant = false; + + /** + * Whether {@link #hasDeviceDescendant} is trusted for the current in-memory node instance. + * + *

      This state is intentionally not persisted. A node reloaded from PBTree can lazily recompute + * it when the wildcard-suffix optimization needs it. + */ + private boolean deviceDescendantComputed = false; + /** from root to this node, only be set when used once for InternalMNode */ private String fullPath; @@ -99,6 +110,26 @@ public void setFullPath(String fullPath) { this.fullPath = fullPath; } + @Override + public boolean hasDeviceDescendant() { + return hasDeviceDescendant; + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + this.hasDeviceDescendant = hasDeviceDescendant; + } + + @Override + public boolean isDeviceDescendantComputed() { + return deviceDescendantComputed; + } + + @Override + public void setDeviceDescendantComputed(final boolean deviceDescendantComputed) { + this.deviceDescendantComputed = deviceDescendantComputed; + } + @Override public PartialPath getPartialPath() { List detachedPath = new ArrayList<>(); @@ -245,6 +276,8 @@ public void setLockEntry(LockEntry lockEntry) { *

    1. basicMNodeInfo reference, 8B *
    2. parent reference, 8B *
    3. fullPath reference, 8B + *
    4. hasDeviceDescendant, 1B + *
    5. deviceDescendantComputed, 1B *
    *
  • MapEntry in parent *
      @@ -256,7 +289,7 @@ public void setLockEntry(LockEntry lockEntry) { */ @Override public int estimateSize() { - return 8 + 8 + 8 + 8 + 8 + 8 + 28 + cacheMNodeInfo.estimateSize(); + return 8 + 8 + 8 + 8 + 1 + 1 + 8 + 8 + 28 + cacheMNodeInfo.estimateSize(); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedAboveDatabaseMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedAboveDatabaseMNode.java index e5ed185630ffb..fd5be94305d30 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedAboveDatabaseMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedAboveDatabaseMNode.java @@ -31,6 +31,26 @@ public CachedAboveDatabaseMNode(ICachedMNode parent, String name) { super(new CachedBasicInternalMNode(parent, name)); } + @Override + public boolean hasDeviceDescendant() { + return basicMNode.hasDeviceDescendant(); + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + basicMNode.setHasDeviceDescendant(hasDeviceDescendant); + } + + @Override + public boolean isDeviceDescendantComputed() { + return basicMNode.isDeviceDescendantComputed(); + } + + @Override + public void setDeviceDescendantComputed(final boolean deviceDescendantComputed) { + basicMNode.setDeviceDescendantComputed(deviceDescendantComputed); + } + @Override public CacheEntry getCacheEntry() { return basicMNode.getCacheEntry(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedDatabaseMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedDatabaseMNode.java index 79def6816d714..8ef44d00a1674 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedDatabaseMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedDatabaseMNode.java @@ -34,6 +34,26 @@ public CachedDatabaseMNode(ICachedMNode parent, String name) { super(new CachedBasicInternalMNode(parent, name), new DatabaseInfo<>()); } + @Override + public boolean hasDeviceDescendant() { + return basicMNode.hasDeviceDescendant(); + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + basicMNode.setHasDeviceDescendant(hasDeviceDescendant); + } + + @Override + public boolean isDeviceDescendantComputed() { + return basicMNode.isDeviceDescendantComputed(); + } + + @Override + public void setDeviceDescendantComputed(final boolean deviceDescendantComputed) { + basicMNode.setDeviceDescendantComputed(deviceDescendantComputed); + } + @Override public CacheEntry getCacheEntry() { return basicMNode.getCacheEntry(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedMeasurementMNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedMeasurementMNode.java index fff9e6150b317..337c3b7e41546 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedMeasurementMNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/mnode/impl/CachedMeasurementMNode.java @@ -40,6 +40,26 @@ public CachedMeasurementMNode( new MeasurementInfo(schema, alias)); } + @Override + public boolean hasDeviceDescendant() { + return basicMNode.hasDeviceDescendant(); + } + + @Override + public void setHasDeviceDescendant(final boolean hasDeviceDescendant) { + basicMNode.setHasDeviceDescendant(hasDeviceDescendant); + } + + @Override + public boolean isDeviceDescendantComputed() { + return basicMNode.isDeviceDescendantComputed(); + } + + @Override + public void setDeviceDescendantComputed(final boolean deviceDescendantComputed) { + basicMNode.setDeviceDescendantComputed(deviceDescendantComputed); + } + @Override public CacheEntry getCacheEntry() { return basicMNode.getCacheEntry(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java index 4482c6b0221ce..790754be8ec58 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/Traverser.java @@ -33,11 +33,11 @@ import org.apache.iotdb.commons.schema.template.Template; import org.apache.iotdb.commons.schema.tree.AbstractTreeVisitor; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.IMTreeStore; -import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.MemMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.iterator.MNodeIterator; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.ReentrantReadOnlyCachedMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.ICachedMNode; import org.apache.iotdb.db.schemaengine.schemaregion.utils.MNodeUtils; import org.slf4j.Logger; @@ -252,7 +252,10 @@ protected Iterator getChildrenIterator(N parent) throws MetadataException { if (parent.isAboveDatabase()) { return new MNodeIterator<>(parent.getChildren().values().iterator()); } else { - final Iterator optimizedIterator = getOptimizedChildrenIterator(parent); + final Iterator optimizedIterator = + shouldUseLeafDeviceMeasurementOptimization() + ? getOptimizedChildrenIterator(parent) + : null; if (optimizedIterator != null) { return optimizedIterator; } @@ -260,14 +263,25 @@ protected Iterator getChildrenIterator(N parent) throws MetadataException { } } + protected boolean shouldUseLeafDeviceMeasurementOptimization() { + return false; + } + @SuppressWarnings("unchecked") private Iterator getOptimizedChildrenIterator(final N parent) throws MetadataException { - if (!(store instanceof MemMTreeStore) || !parent.isDevice()) { + if (!parent.isDevice()) { return null; } - final IMemMNode memNode = (IMemMNode) parent; - if (memNode.hasDeviceDescendant()) { + if (parent instanceof IMemMNode) { + if (((IMemMNode) parent).hasDeviceDescendant()) { + return null; + } + } else if (parent instanceof ICachedMNode) { + if (hasDeviceDescendant((ICachedMNode) parent)) { + return null; + } + } else { return null; } @@ -307,6 +321,41 @@ private Iterator getOptimizedChildrenIterator(final N parent) throws Metadata } } + @SuppressWarnings("unchecked") + private boolean hasDeviceDescendant(final ICachedMNode node) throws MetadataException { + if (node.isMeasurement()) { + node.setHasDeviceDescendant(false); + node.setDeviceDescendantComputed(true); + return false; + } + if (!node.isDeviceDescendantComputed()) { + node.setHasDeviceDescendant(hasDeviceDescendantInChildren(node)); + node.setDeviceDescendantComputed(true); + } + return node.hasDeviceDescendant(); + } + + @SuppressWarnings("unchecked") + private boolean hasDeviceDescendantInChildren(final ICachedMNode node) throws MetadataException { + final IMTreeStore cachedStore = (IMTreeStore) store; + final IMNodeIterator iterator = cachedStore.getChildrenIterator(node); + try { + while (iterator.hasNext()) { + final ICachedMNode child = iterator.next(); + try { + if (child.isDevice() || hasDeviceDescendant(child)) { + return true; + } + } finally { + cachedStore.unPin(child); + } + } + return false; + } finally { + iterator.close(); + } + } + @Override protected void releaseNodeIterator(Iterator nodeIterator) { if (nodeIterator instanceof IMNodeIterator) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java index 4996e3f0c8b36..490145d5371fe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/traverser/basic/MeasurementTraverser.java @@ -84,4 +84,9 @@ protected boolean shouldVisitSubtreeOfFullMatchedNode(N node) { protected boolean shouldVisitSubtreeOfInternalMatchedNode(N node) { return !node.isMeasurement(); } + + @Override + protected boolean shouldUseLeafDeviceMeasurementOptimization() { + return true; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java index 601233d9d6094..3de059127d3c0 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/metadata/schemaRegion/SchemaRegionBasicTest.java @@ -1185,6 +1185,28 @@ public void testGetMatchedDevicesWithSpecialPattern() throws Exception { Assert.assertEquals(expectedPathList, actualPathList); } + @Test + public void testShowTimeseriesWildcardSuffixWithNestedAndLeafDevices() throws Exception { + final ISchemaRegion schemaRegion = getSchemaRegion("root.test", 0); + + SchemaRegionTestUtil.createSimpleTimeSeriesByList( + schemaRegion, + Arrays.asList( + "root.test.d1.s1", "root.test.d1.a.d2.s1", "root.test.d3.s1", "root.test.d4.s2")); + + final List result = + SchemaRegionTestUtil.showTimeseries(schemaRegion, new PartialPath("root.test.**.s1")); + final Set expectedPathList = + new HashSet<>(Arrays.asList("root.test.d1.s1", "root.test.d1.a.d2.s1", "root.test.d3.s1")); + Assert.assertEquals(expectedPathList.size(), result.size()); + + final Set actualPathList = new HashSet<>(); + for (final ITimeSeriesSchemaInfo timeSeriesSchemaInfo : result) { + actualPathList.add(timeSeriesSchemaInfo.getFullPath()); + } + Assert.assertEquals(expectedPathList, actualPathList); + } + @Test public void testGetMatchedDevicesWithSpecialPattern2() throws Exception { final ISchemaRegion schemaRegion = getSchemaRegion("root.test", 0); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImplTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImplTest.java new file mode 100644 index 0000000000000..64fa4b46dfa9a --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/MTreeBelowSGCachedImplTest.java @@ -0,0 +1,200 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree; + +import org.apache.iotdb.commons.exception.MetadataException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.schema.SchemaConstant; +import org.apache.iotdb.commons.schema.node.role.IMeasurementMNode; +import org.apache.iotdb.db.conf.IoTDBConfig; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.metadata.PathNotExistException; +import org.apache.iotdb.db.schemaengine.metric.SchemaRegionCachedMetric; +import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaEngineStatistics; +import org.apache.iotdb.db.schemaengine.rescon.CachedSchemaRegionStatistics; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.ReleaseFlushMonitor; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.ICachedMNode; +import org.apache.iotdb.db.schemaengine.schemaregion.mtree.traverser.collector.MeasurementCollector; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.external.commons.io.FileUtils; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class MTreeBelowSGCachedImplTest { + + private static final Field STORE_FIELD; + private static final Field ROOT_NODE_FIELD; + + static { + try { + STORE_FIELD = MTreeBelowSGCachedImpl.class.getDeclaredField("store"); + STORE_FIELD.setAccessible(true); + ROOT_NODE_FIELD = MTreeBelowSGCachedImpl.class.getDeclaredField("rootNode"); + ROOT_NODE_FIELD.setAccessible(true); + } catch (final NoSuchFieldException e) { + throw new ExceptionInInitializerError(e); + } + } + + private final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig(); + private int rawCachedMNodeSize; + private MTreeBelowSGCachedImpl mtree; + + @Before + public void setUp() { + rawCachedMNodeSize = config.getCachedMNodeSizeInPBTreeMode(); + config.setCachedMNodeSizeInPBTreeMode(10000); + ReleaseFlushMonitor.getInstance().clear(); + ReleaseFlushMonitor.getInstance().init(new CachedSchemaEngineStatistics()); + } + + @After + public void tearDown() throws Exception { + if (mtree != null) { + mtree.clear(); + mtree = null; + } + ReleaseFlushMonitor.getInstance().clear(); + FileUtils.deleteDirectory(new File(config.getSchemaDir())); + config.setCachedMNodeSizeInPBTreeMode(rawCachedMNodeSize); + } + + @Test + public void testDeviceDescendantFlagIsMaintainedAcrossCreateDeleteAndLazyRecompute() + throws Exception { + mtree = newMTree(); + + createBooleanTimeSeries("root.sg.a.b.s1"); + + ICachedMNode database = mtree.getNodeByPath(new PartialPath("root.sg")); + ICachedMNode aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + ICachedMNode bNode = mtree.getNodeByPath(new PartialPath("root.sg.a.b")); + + Assert.assertTrue(database.hasDeviceDescendant()); + Assert.assertTrue(database.isDeviceDescendantComputed()); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + Assert.assertTrue(bNode.isDevice()); + Assert.assertFalse(bNode.hasDeviceDescendant()); + Assert.assertTrue(bNode.isDeviceDescendantComputed()); + mtree.unPinMNode(database); + mtree.unPinMNode(aNode); + mtree.unPinMNode(bNode); + + createBooleanTimeSeries("root.sg.a.s0"); + + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertTrue(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + + aNode.setHasDeviceDescendant(false); + aNode.setDeviceDescendantComputed(false); + mtree.unPinMNode(aNode); + + Assert.assertEquals( + Collections.singletonList("root.sg.a.b.s1"), + collectMeasurementPaths(new PartialPath("root.sg.a.**.s1"))); + + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + mtree.unPinMNode(aNode); + + mtree.deleteTimeseries(new PartialPath("root.sg.a.s0")); + aNode = mtree.getNodeByPath(new PartialPath("root.sg.a")); + Assert.assertFalse(aNode.isDevice()); + Assert.assertTrue(aNode.hasDeviceDescendant()); + Assert.assertTrue(aNode.isDeviceDescendantComputed()); + mtree.unPinMNode(aNode); + + mtree.deleteTimeseries(new PartialPath("root.sg.a.b.s1")); + database = mtree.getNodeByPath(new PartialPath("root.sg")); + Assert.assertFalse(database.hasDeviceDescendant()); + Assert.assertTrue(database.isDeviceDescendantComputed()); + mtree.unPinMNode(database); + assertPathNotExist(new PartialPath("root.sg.a")); + } + + private MTreeBelowSGCachedImpl newMTree() throws Exception { + final CachedSchemaRegionStatistics regionStatistics = + new CachedSchemaRegionStatistics(0, new CachedSchemaEngineStatistics()); + return new MTreeBelowSGCachedImpl( + PartialPath.getQualifiedDatabasePartialPath("root.sg"), + node -> Collections.emptyMap(), + node -> Collections.emptyMap(), + () -> {}, + node -> {}, + node -> {}, + 0, + regionStatistics, + new SchemaRegionCachedMetric(regionStatistics, "root.sg")); + } + + private void createBooleanTimeSeries(final String path) throws MetadataException { + mtree.createTimeSeries( + new MeasurementPath(path), + TSDataType.BOOLEAN, + TSEncoding.PLAIN, + CompressionType.SNAPPY, + null, + null); + } + + private List collectMeasurementPaths(final PartialPath pattern) throws Exception { + final ICachedMNode rootNode = (ICachedMNode) ROOT_NODE_FIELD.get(mtree); + final CachedMTreeStore store = (CachedMTreeStore) STORE_FIELD.get(mtree); + final List matchedPaths = new ArrayList<>(); + try (MeasurementCollector collector = + new MeasurementCollector( + rootNode, pattern, store, false, SchemaConstant.ALL_MATCH_SCOPE) { + @Override + protected Void collectMeasurement(final IMeasurementMNode node) { + matchedPaths.add(getCurrentMeasurementPathInTraverse(node).getFullPath()); + return null; + } + }) { + collector.traverse(); + } + return matchedPaths; + } + + private void assertPathNotExist(final PartialPath path) throws MetadataException { + try { + mtree.getNodeByPath(path); + Assert.fail("Expected path not exist: " + path.getFullPath()); + } catch (final PathNotExistException ignored) { + // expected + } + } +}