From 3e936ddc97253e244b8c5458a70da740143c1d9c Mon Sep 17 00:00:00 2001 From: morrySnow Date: Wed, 14 Jan 2026 17:44:41 +0800 Subject: [PATCH] [opt](memo) reuse GroupPlan in Group to reduce memory usage --- .../nereids/jobs/joinorder/hypergraph/HyperGraph.java | 5 +++-- .../jobs/joinorder/hypergraph/receiver/PlanReceiver.java | 6 +++--- .../main/java/org/apache/doris/nereids/memo/Group.java | 8 ++++++++ .../src/main/java/org/apache/doris/nereids/memo/Memo.java | 2 +- .../doris/nereids/pattern/GroupExpressionMatching.java | 3 +-- .../org/apache/doris/nereids/pattern/GroupMatching.java | 2 +- .../apache/doris/nereids/properties/DistributionSpec.java | 2 +- .../org/apache/doris/nereids/properties/OrderSpec.java | 5 ++--- .../properties/ChildrenPropertiesRegulatorTest.java | 4 ++++ 9 files changed, 24 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java index 99d3d9d9282bad..54822650c24d49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/HyperGraph.java @@ -400,8 +400,9 @@ private Pair buildForMv(Plan plan) { Group group = ((GroupPlan) plan).getGroup(); GroupExpression groupExpression = group.getLogicalExpressions().get(0); return buildForMv(groupExpression.getPlan() - .withChildren( - groupExpression.children().stream().map(GroupPlan::new).collect(Collectors.toList()))); + .withChildren(groupExpression.children().stream() + .map(Group::getGroupPlan) + .collect(Collectors.toList()))); } // process Project if (isValidProject(plan)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java index 9047a1763220f4..c28cfd2ce46112 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/receiver/PlanReceiver.java @@ -102,8 +102,8 @@ public boolean emitCsgCmp(long left, long right, List edges) { } Memo memo = jobContext.getCascadesContext().getMemo(); - GroupPlan leftPlan = new GroupPlan(planTable.get(left)); - GroupPlan rightPlan = new GroupPlan(planTable.get(right)); + GroupPlan leftPlan = planTable.get(left).getGroupPlan(); + GroupPlan rightPlan = planTable.get(right).getGroupPlan(); // First, we implement all possible physical plans // In this step, we don't generate logical expression because they are useless in DPhyp. @@ -205,7 +205,7 @@ private LogicalPlan proposeJoin(JoinType joinType, Plan left, Plan right, List(), bitmap, bitmap); + Plan plan = proposeProject(group.getGroupPlan(), new ArrayList<>(), bitmap, bitmap); if (!(plan instanceof GroupPlan)) { CopyInResult copyInResult = jobContext.getCascadesContext().getMemo().copyIn(plan, null, false, planTable); group = copyInResult.correspondingExpression.getOwnerGroup(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java index 0b9c78bdec6a06..3958c5075cecee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.JoinType; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -63,6 +64,7 @@ public class Group { private final List physicalExpressions = Lists.newArrayList(); private final Map enforcers = Maps.newHashMap(); private final Map enforcerSpecs = Maps.newHashMap(); + private final GroupPlan groupPlan; private boolean isStatsReliable = true; private LogicalProperties logicalProperties; @@ -92,6 +94,7 @@ public Group(GroupId groupId, GroupExpression groupExpression, LogicalProperties this.groupId = groupId; addGroupExpression(groupExpression); this.logicalProperties = logicalProperties; + this.groupPlan = new GroupPlan(this); } /** @@ -102,6 +105,7 @@ public Group(GroupId groupId, GroupExpression groupExpression, LogicalProperties public Group(GroupId groupId, LogicalProperties logicalProperties) { this.groupId = groupId; this.logicalProperties = logicalProperties; + this.groupPlan = new GroupPlan(this); } public GroupId getGroupId() { @@ -176,6 +180,10 @@ public List getPhysicalExpressions() { return physicalExpressions; } + public GroupPlan getGroupPlan() { + return groupPlan; + } + /** * Remove groupExpression from this group. * diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index de1a2cba892ab6..0ab94348f27013 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -746,7 +746,7 @@ private Plan replaceChildrenToGroupPlan(Plan plan, List childrenGroups) { ImmutableList.Builder groupPlanChildren = ImmutableList.builderWithExpectedSize(childrenGroups.size()); for (Group childrenGroup : childrenGroups) { - groupPlanChildren.add(new GroupPlan(childrenGroup)); + groupPlanChildren.add(childrenGroup.getGroupPlan()); } return plan.withChildren(groupPlanChildren.build()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java index 292e3e5ccf8e7f..1461e9ef2a960d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java @@ -20,7 +20,6 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.Plan; import com.google.common.collect.ImmutableList; @@ -111,7 +110,7 @@ public GroupExpressionIterator(Pattern pattern, GroupExpression groupExpre if (childrenPlan.isEmpty()) { if (pattern instanceof SubTreePattern) { - childrenPlan = ImmutableList.of(new GroupPlan(childGroup)); + childrenPlan = ImmutableList.of(childGroup.getGroupPlan()); } else { // current pattern is match but children patterns not match return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java index b3e59590d37a96..dd0bdf3e349ba2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupMatching.java @@ -35,7 +35,7 @@ public class GroupMatching { public static List getAllMatchingPlans(Pattern pattern, Group group) { List matchingPlans = new ArrayList<>(); if (pattern.isGroup() || pattern.isMultiGroup()) { - GroupPlan groupPlan = new GroupPlan(group); + GroupPlan groupPlan = group.getGroupPlan(); if (((Pattern) pattern).matchPredicates(groupPlan)) { matchingPlans.add(groupPlan); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java index cda4b82b76a876..f5e0e4f29934e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/DistributionSpec.java @@ -44,7 +44,7 @@ public GroupExpression addEnforcer(Group child) { // If we don't set LogicalProperties explicitly, node will compute a applicable LogicalProperties for itself. PhysicalDistribute distribution = new PhysicalDistribute<>( this, - new GroupPlan(child)); + child.getGroupPlan()); return new GroupExpression(distribution, Lists.newArrayList(child)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderSpec.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderSpec.java index 1f85bbb3b7a964..e8787cc681b59b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/OrderSpec.java @@ -19,7 +19,6 @@ import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.SortPhase; import org.apache.doris.nereids.trees.plans.physical.PhysicalQuickSort; @@ -67,7 +66,7 @@ public boolean satisfy(OrderSpec other) { public GroupExpression addLocalQuickSortEnforcer(Group child) { return new GroupExpression( new PhysicalQuickSort<>(orderKeys, SortPhase.LOCAL_SORT, child.getLogicalProperties(), - new GroupPlan(child)), + child.getGroupPlan()), Lists.newArrayList(child) ); } @@ -78,7 +77,7 @@ public GroupExpression addLocalQuickSortEnforcer(Group child) { public GroupExpression addGlobalQuickSortEnforcer(Group child) { return new GroupExpression( new PhysicalQuickSort<>(orderKeys, SortPhase.MERGE_SORT, child.getLogicalProperties(), - new GroupPlan(child)), + child.getGroupPlan()), Lists.newArrayList(child) ); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulatorTest.java index f63cc5d8d45396..cb2534765cc1b5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/ChildrenPropertiesRegulatorTest.java @@ -100,6 +100,8 @@ public void testMustShuffleProject(Class childClazz, List children; Group childGroup = Mockito.mock(Group.class); Mockito.when(childGroup.getLogicalProperties()).thenReturn(Mockito.mock(LogicalProperties.class)); + GroupPlan childGroupPlan = new GroupPlan(childGroup); + Mockito.when(childGroup.getGroupPlan()).thenReturn(childGroupPlan); GroupExpression child = Mockito.mock(GroupExpression.class); Mockito.when(child.getOutputProperties(Mockito.any())).thenReturn(PhysicalProperties.MUST_SHUFFLE); Mockito.when(child.getOwnerGroup()).thenReturn(childGroup); @@ -159,6 +161,8 @@ private void testMustShuffleFilter(Class childClazz) { List children; Group childGroup = Mockito.mock(Group.class); Mockito.when(childGroup.getLogicalProperties()).thenReturn(Mockito.mock(LogicalProperties.class)); + GroupPlan childGroupPlan = new GroupPlan(childGroup); + Mockito.when(childGroup.getGroupPlan()).thenReturn(childGroupPlan); GroupExpression child = Mockito.mock(GroupExpression.class); Mockito.when(child.getOutputProperties(Mockito.any())).thenReturn(PhysicalProperties.MUST_SHUFFLE); Mockito.when(child.getOwnerGroup()).thenReturn(childGroup);