PHOENIX-7876 Improve EXPLAIN

phoenix-core-client/src/main/antlr3/PhoenixSQL.g

@@ -203,6 +203,7 @@ import java.lang.Boolean;
 import java.math.BigDecimal;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.Stack;
@@ -233,7 +234,7 @@ import org.apache.phoenix.util.SchemaUtil;
 import org.apache.phoenix.parse.LikeParseNode.LikeType;
 import org.apache.phoenix.trace.util.Tracing;
 import org.apache.phoenix.parse.AddJarsStatement;
-import org.apache.phoenix.parse.ExplainType;
+import org.apache.phoenix.parse.ExplainOptions;
 }
 
 @lexer::header {
@@ -318,6 +319,69 @@ package org.apache.phoenix.parse;
          }
     }
 
+    /**
+     * Closed-set keys for the EXPLAIN option list, used to reject duplicate options as they are
+     * encountered while parsing.
+     */
+    private enum ExplainOpt { REGIONS, VERBOSE, FORMAT }
+
+    /**
+     * Mutable accumulator for the EXPLAIN option list. Lifetime is one explain_node parse.
+     */
+    private static final class ExplainOptsAcc {
+        boolean regions;
+        boolean verbose;
+        ExplainOptions.Format format;
+        final EnumSet<ExplainOpt> seen = EnumSet.noneOf(ExplainOpt.class);
+
+        void mark(ExplainOpt opt) {
+            if (!seen.add(opt)) {
+                throw new RuntimeException("Duplicate EXPLAIN option: " + opt);
+            }
+        }
+
+        ExplainOptions build() {
+            return new ExplainOptions(regions, verbose,
+                format == null ? ExplainOptions.Format.TEXT : format);
+        }
+    }
+
+    /**
+     * Parse a single EXPLAIN option into the given accumulator. The option name is matched
+     * case-insensitively against the closed set {REGIONS, VERBOSE, FORMAT}. REGIONS and VERBOSE are
+     * flags and must not carry a value; FORMAT requires a value of TEXT or JSON. Duplicate options
+     * are rejected.
+     */
+    private void parseExplainOption(ExplainOptsAcc opts, String name, String value) {
+        if ("REGIONS".equalsIgnoreCase(name)) {
+            if (value != null) {
+                throw new RuntimeException("EXPLAIN option REGIONS does not take a value");
+            }
+            opts.mark(ExplainOpt.REGIONS);
+            opts.regions = true;
+        } else if ("VERBOSE".equalsIgnoreCase(name)) {
+            if (value != null) {
+                throw new RuntimeException("EXPLAIN option VERBOSE does not take a value");
+            }
+            opts.mark(ExplainOpt.VERBOSE);
+            opts.verbose = true;
+        } else if ("FORMAT".equalsIgnoreCase(name)) {
+            if (value == null) {
+                throw new RuntimeException("EXPLAIN option FORMAT requires a value: TEXT or JSON");
+            }
+            opts.mark(ExplainOpt.FORMAT);
+            if ("TEXT".equalsIgnoreCase(value)) {
+                opts.format = ExplainOptions.Format.TEXT;
+            } else if ("JSON".equalsIgnoreCase(value)) {
+                opts.format = ExplainOptions.Format.JSON;
+            } else {
+                throw new RuntimeException("Unknown EXPLAIN FORMAT: " + value);
+            }
+        } else {
+            throw new RuntimeException("Unknown EXPLAIN option: " + name);
+        }
+    }
+
     protected Object recoverFromMismatchedToken(IntStream input, int ttype, BitSet follow)
         throws RecognitionException {
         RecognitionException e = null;
@@ -477,16 +541,25 @@ oneStatement returns [BindableStatement ret]
 finally{ contextStack.pop(); }
 
 explain_node returns [BindableStatement ret]
-    :   EXPLAIN (w=WITH)? (r=REGIONS)? q=oneStatement
+@init { ExplainOptsAcc opts = new ExplainOptsAcc(); }
+    :   EXPLAIN
+        (
+            (LPAREN explain_option[opts] (COMMA explain_option[opts])* RPAREN)
+          | (WITH REGIONS                   { opts.mark(ExplainOpt.REGIONS); opts.regions = true; })
+        )?
+        q=oneStatement
      {
-        if ((w==null && r!=null) || (w!=null && r==null)) {
-            throw new RuntimeException("Valid usage: EXPLAIN {query} OR EXPLAIN WITH REGIONS {query}");
-        }
-        ret = (w==null && r==null) ? factory.explain(q, ExplainType.DEFAULT)
-         : factory.explain(q, ExplainType.WITH_REGIONS);
+        ret = factory.explain(q, opts.build());
      }
     ;
 
+// A single EXPLAIN option. REGIONS is a global keyword token. The remaining option keywords
+// (VERBOSE, FORMAT, TEXT, JSON) are not reserved and arrive as NAME tokens, validated in the action.
+explain_option[ExplainOptsAcc opts]
+    :   REGIONS                 { opts.mark(ExplainOpt.REGIONS); opts.regions = true; }
+    |   k=NAME (v=NAME)?        { parseExplainOption(opts, k.getText(), v == null ? null : v.getText()); }
+    ;
+
 // Parse a create table statement.
 create_table_node returns [CreateTableStatement ret]
     :   CREATE (im=IMMUTABLE)? TABLE (IF NOT ex=EXISTS)? t=from_table_name 

phoenix-core-client/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java

@@ -59,6 +59,7 @@
 import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
 import org.apache.phoenix.index.IndexMaintainer;
 import org.apache.phoenix.index.PhoenixIndexCodec;
+import org.apache.phoenix.iterate.ExplainTable;
 import org.apache.phoenix.iterate.ResultIterator;
 import org.apache.phoenix.jdbc.PhoenixConnection;
 import org.apache.phoenix.jdbc.PhoenixResultSet;
@@ -595,9 +596,11 @@ public MutationPlan compile(DeleteStatement delete, MutationState.ReturnResult r
       delete.getLimit(), null, delete.getBindCount(), false, false,
       Collections.<SelectStatement> emptyList(), delete.getUdfParseNodes());
     select = StatementNormalizer.normalize(select, resolverToBe);
-
+    // Pre-build a context so the early rewrite pass records top of plan breadcrumbs that are
+    // adopted by the DELETE data query plan's compilation context.
+    StatementContext rewriteContext = StatementContext.forRewrite(statement);
     SelectStatement transformedSelect =
-      SubqueryRewriter.transform(select, resolverToBe, connection);
+      SubqueryRewriter.transform(select, resolverToBe, connection, rewriteContext);
     boolean hasPreProcessing = transformedSelect != select;
     if (transformedSelect != select) {
       resolverToBe = FromCompiler.getResolverForQuery(transformedSelect, connection, false,
@@ -624,7 +627,8 @@ public MutationPlan compile(DeleteStatement delete, MutationState.ReturnResult r
     QueryOptimizer optimizer = new QueryOptimizer(services);
     QueryCompiler compiler =
       new QueryCompiler(statement, select, resolverToBe, Collections.<PColumn> emptyList(),
-        parallelIteratorFactoryToBe, new SequenceManager(statement));
+        parallelIteratorFactoryToBe, new SequenceManager(statement))
+          .withRewriteContext(rewriteContext);
     final QueryPlan dataPlan = compiler.compile();
     // TODO: the select clause should know that there's a sub query, but doesn't seem to currently
     queryPlans = Lists.newArrayList(!clientSideIndexes.isEmpty()
@@ -667,7 +671,8 @@ Collections.<PColumn> emptyList(), parallelIteratorFactoryToBe)
       // from the data table, while the others will be for deleting rows from immutable indexes.
       List<MutationPlan> mutationPlans = Lists.newArrayListWithExpectedSize(queryPlans.size());
       for (final QueryPlan plan : queryPlans) {
-        mutationPlans.add(new SingleRowDeleteMutationPlan(plan, connection, maxSize, maxSizeBytes));
+        mutationPlans.add(new SingleRowDeleteMutationPlan(plan, connection, maxSize, maxSizeBytes,
+          delete.isReturningRow()));
       }
       return new MultiRowDeleteMutationPlan(dataPlan, mutationPlans);
     } else if (runOnServer) {
@@ -700,7 +705,7 @@ Collections.<PColumn> emptyList(), parallelIteratorFactoryToBe)
         new AggregatePlan(context, select, dataPlan.getTableRef(), projector, null, null,
           OrderBy.EMPTY_ORDER_BY, null, GroupBy.EMPTY_GROUP_BY, null, dataPlan);
       return new ServerSelectDeleteMutationPlan(dataPlan, connection, aggPlan, projector, maxSize,
-        maxSizeBytes);
+        maxSizeBytes, delete.isReturningRow());
     } else {
       final DeletingParallelIteratorFactory parallelIteratorFactory = parallelIteratorFactoryToBe;
       List<PColumn> adjustedProjectedColumns =
@@ -745,7 +750,7 @@ public int getPosition() {
       }
       return new ClientSelectDeleteMutationPlan(targetTableRef, dataPlan, bestPlan,
         hasPreOrPostProcessing, parallelIteratorFactory, otherTableRefs, projectedTableRef, maxSize,
-        maxSizeBytes, connection);
+        maxSizeBytes, connection, delete.isReturningRow());
     }
   }
 
@@ -759,14 +764,16 @@ private class SingleRowDeleteMutationPlan implements MutationPlan {
     private final int maxSize;
     private final StatementContext context;
     private final long maxSizeBytes;
+    private final boolean returningRow;
 
     public SingleRowDeleteMutationPlan(QueryPlan dataPlan, PhoenixConnection connection,
-      int maxSize, long maxSizeBytes) {
+      int maxSize, long maxSizeBytes, boolean returningRow) {
       this.dataPlan = dataPlan;
       this.connection = connection;
       this.maxSize = maxSize;
       this.context = dataPlan.getContext();
       this.maxSizeBytes = maxSizeBytes;
+      this.returningRow = returningRow;
     }
 
     @Override
@@ -793,7 +800,19 @@ public MutationState execute() throws SQLException {
 
     @Override
     public ExplainPlan getExplainPlan() throws SQLException {
-      return new ExplainPlan(Collections.singletonList("DELETE SINGLE ROW"));
+      ExplainPlanAttributesBuilder builder =
+        new ExplainPlanAttributesBuilder().setAbstractExplainPlan("DELETE SINGLE ROW");
+      builder.setReturningRow(returningRow);
+      if (getContext().isRoot()) {
+        ExplainTable.populateTopOfPlanAttributes(builder, getContext(), getTargetRef());
+        ExplainTable.populateTopOfPlanEstimates(builder, this);
+      }
+      List<String> planSteps = Lists.newArrayListWithExpectedSize(2);
+      planSteps.add("DELETE SINGLE ROW");
+      if (returningRow) {
+        planSteps.add("    RETURNING *");
+      }
+      return new ExplainPlan(planSteps, builder.build());
     }
 
     @Override
@@ -852,16 +871,19 @@ public class ServerSelectDeleteMutationPlan implements MutationPlan {
     private final RowProjector projector;
     private final int maxSize;
     private final long maxSizeBytes;
+    private final boolean returningRow;
 
     public ServerSelectDeleteMutationPlan(QueryPlan dataPlan, PhoenixConnection connection,
-      QueryPlan aggPlan, RowProjector projector, int maxSize, long maxSizeBytes) {
+      QueryPlan aggPlan, RowProjector projector, int maxSize, long maxSizeBytes,
+      boolean returningRow) {
       this.context = dataPlan.getContext();
       this.dataPlan = dataPlan;
       this.connection = connection;
       this.aggPlan = aggPlan;
       this.projector = projector;
       this.maxSize = maxSize;
       this.maxSizeBytes = maxSizeBytes;
+      this.returningRow = returningRow;
     }
 
     @Override
@@ -970,12 +992,24 @@ public ExplainPlan getExplainPlan() throws SQLException {
       ExplainPlan explainPlan = aggPlan.getExplainPlan();
       List<String> queryPlanSteps = explainPlan.getPlanSteps();
       ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes();
-      List<String> planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 1);
+      List<String> planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 2);
       ExplainPlanAttributesBuilder newBuilder =
         new ExplainPlanAttributesBuilder(explainPlanAttributes);
       newBuilder.setAbstractExplainPlan("DELETE ROWS SERVER SELECT");
+      newBuilder.setReturningRow(returningRow);
       planSteps.add("DELETE ROWS SERVER SELECT");
+      if (returningRow) {
+        planSteps.add("    RETURNING *");
+      }
       planSteps.addAll(queryPlanSteps);
+      // Surface the row-identity projection the scan actually reads so VERBOSE explain describes
+      // the delete rather than the count.
+      ExplainTable.overrideMutationProject(planSteps, explainPlanAttributes, newBuilder,
+        dataPlan.getProjector());
+      if (getContext().isRoot()) {
+        ExplainTable.populateTopOfPlanAttributes(newBuilder, getContext(), getTargetRef());
+        ExplainTable.populateTopOfPlanEstimates(newBuilder, this);
+      }
       return new ExplainPlan(planSteps, newBuilder.build());
     }
 
@@ -1016,11 +1050,13 @@ public class ClientSelectDeleteMutationPlan implements MutationPlan {
     private final int maxSize;
     private final long maxSizeBytes;
     private final PhoenixConnection connection;
+    private final boolean returningRow;
 
     public ClientSelectDeleteMutationPlan(TableRef targetTableRef, QueryPlan dataPlan,
       QueryPlan bestPlan, boolean hasPreOrPostProcessing,
       DeletingParallelIteratorFactory parallelIteratorFactory, List<TableRef> otherTableRefs,
-      TableRef projectedTableRef, int maxSize, long maxSizeBytes, PhoenixConnection connection) {
+      TableRef projectedTableRef, int maxSize, long maxSizeBytes, PhoenixConnection connection,
+      boolean returningRow) {
       this.context = bestPlan.getContext();
       this.targetTableRef = targetTableRef;
       this.dataPlan = dataPlan;
@@ -1032,6 +1068,7 @@ public ClientSelectDeleteMutationPlan(TableRef targetTableRef, QueryPlan dataPla
       this.maxSize = maxSize;
       this.maxSizeBytes = maxSizeBytes;
       this.connection = connection;
+      this.returningRow = returningRow;
     }
 
     @Override
@@ -1105,12 +1142,20 @@ public ExplainPlan getExplainPlan() throws SQLException {
       ExplainPlan explainPlan = bestPlan.getExplainPlan();
       List<String> queryPlanSteps = explainPlan.getPlanSteps();
       ExplainPlanAttributes explainPlanAttributes = explainPlan.getPlanStepsAsAttributes();
-      List<String> planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 1);
+      List<String> planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 2);
       ExplainPlanAttributesBuilder newBuilder =
         new ExplainPlanAttributesBuilder(explainPlanAttributes);
       newBuilder.setAbstractExplainPlan("DELETE ROWS CLIENT SELECT");
+      newBuilder.setReturningRow(returningRow);
       planSteps.add("DELETE ROWS CLIENT SELECT");
+      if (returningRow) {
+        planSteps.add("    RETURNING *");
+      }
       planSteps.addAll(queryPlanSteps);
+      if (getContext().isRoot()) {
+        ExplainTable.populateTopOfPlanAttributes(newBuilder, getContext(), getTargetRef());
+        ExplainTable.populateTopOfPlanEstimates(newBuilder, this);
+      }
       return new ExplainPlan(planSteps, newBuilder.build());
     }
 

phoenix-core-client/src/main/java/org/apache/phoenix/compile/ExplainJsonRenderer.java

@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.compile;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.util.DefaultIndenter;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import java.sql.SQLException;
+
+/**
+ * Serializes an {@link ExplainPlanAttributes} tree to a JSON document for the
+ * {@code EXPLAIN (FORMAT JSON) <stmt>} statement.
+ * <p>
+ * The output is pretty-printed with two space indentation for both objects and arrays.
+ * <p>
+ * The JSON layout tracks the Java field names and structure of {@link ExplainPlanAttributes}. It is
+ * deliberately not a stable contract and carries no version field. It is an opt-in view onto an
+ * internal structure, useful for tooling and assertions.
+ * <p>
+ * This class intentionally does not reuse the shared {@link org.apache.phoenix.util.JacksonUtil}
+ * mapper so that the general-purpose mapper configuration can change without affecting the EXPLAIN
+ * JSON contract.
+ */
+public final class ExplainJsonRenderer {
+
+  private static final ObjectWriter WRITER = buildWriter();
+
+  private static ObjectWriter buildWriter() {
+    ObjectMapper mapper = new ObjectMapper();
+    // Emit every field, with an explicit null for any unset value, so the JSON view is a faithful
+    // projection of the attributes tree.
+    mapper.setSerializationInclusion(JsonInclude.Include.ALWAYS);
+    mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
+    mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
+    // Jackson's stock DefaultPrettyPrinter indents object fields but uses a single space
+    // FixedSpaceIndenter for array elements. Set the same two space indenter on both so nested
+    // objects and array elements each start on their own indented line.
+    DefaultIndenter indenter = new DefaultIndenter("  ", "\n");
+    DefaultPrettyPrinter printer =
+      new DefaultPrettyPrinter().withObjectIndenter(indenter).withArrayIndenter(indenter);
+    return mapper.writer(printer);
+  }
+
+  private ExplainJsonRenderer() {
+  }
+
+  /**
+   * Serialize the given attributes to a pretty-printed JSON document.
+   * @param attributes the plan attributes to serialize
+   * @return the JSON document
+   * @throws SQLException if serialization fails
+   */
+  public static String render(ExplainPlanAttributes attributes) throws SQLException {
+    try {
+      return WRITER.writeValueAsString(attributes);
+    } catch (JsonProcessingException e) {
+      throw new SQLException("Failed to serialize EXPLAIN attributes as JSON", e);
+    }
+  }
+}