diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/CheckQueryPropertiesHook.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/CheckQueryPropertiesHook.java index 916a8daa1d52..b1eaa739d81f 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/CheckQueryPropertiesHook.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/CheckQueryPropertiesHook.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.hooks; import org.apache.hadoop.hive.ql.QueryProperties; +import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; @@ -41,13 +42,14 @@ public void run(HookContext hookContext) { if (queryProps != null) { console.printError("Has Join: " + queryProps.hasJoin()); - console.printError("Has Group By: " + queryProps.hasGroupBy()); - console.printError("Has Sort By: " + queryProps.hasSortBy()); - console.printError("Has Order By: " + queryProps.hasOrderBy()); - console.printError("Has Group By After Join: " + queryProps.hasJoinFollowedByGroupBy()); - console.printError("Uses Script: " + queryProps.usesScript()); - console.printError("Has Distribute By: " + queryProps.hasDistributeBy()); - console.printError("Has Cluster By: " + queryProps.hasClusterBy()); + console.printError("Has Group By: " + queryProps.hasFeature(QueryFeature.GROUP_BY)); + console.printError("Has Sort By: " + queryProps.hasFeature(QueryFeature.SORT_BY)); + console.printError("Has Order By: " + queryProps.hasFeature(QueryFeature.ORDER_BY)); + console.printError("Has Group By After Join: " + + queryProps.hasFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY)); + console.printError("Uses Script: " + queryProps.hasFeature(QueryFeature.USES_SCRIPT)); + console.printError("Has Distribute By: " + queryProps.hasFeature(QueryFeature.DISTRIBUTE_BY)); + console.printError("Has Cluster By: " + queryProps.hasFeature(QueryFeature.CLUSTER_BY)); } } } \ No newline at end of file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java index 8bbc38661b61..9ed0c53046ef 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql; +import java.util.EnumSet; import java.util.HashSet; import java.util.Set; @@ -56,6 +57,24 @@ public String getName() { } } + public enum QueryFeature { + GROUP_BY, + ORDER_BY, + OUTER_ORDER_BY, + SORT_BY, + LIMIT, + JOIN_FOLLOWED_BY_GROUP_BY, + PTF, + WINDOWING, + QUALIFY, + EXCEPT, + INTERSECT, + USES_SCRIPT, + DISTRIBUTE_BY, + CLUSTER_BY, + MAP_GROUP_BY + } + boolean query; boolean analyzeCommand; boolean noScanAnalyzeCommand; @@ -64,25 +83,8 @@ public String getName() { int outerQueryLimit; boolean hasJoin = false; - boolean hasGroupBy = false; - boolean hasOrderBy = false; - boolean hasOuterOrderBy = false; - boolean hasSortBy = false; - boolean hasLimit = false; - boolean hasJoinFollowedByGroupBy = false; - boolean hasPTF = false; - boolean hasWindowing = false; - boolean hasQualify = false; - boolean hasExcept = false; - boolean hasIntersect = false; - - // does the query have a using clause - boolean usesScript = false; - - boolean hasDistributeBy = false; - boolean hasClusterBy = false; + private final EnumSet features = EnumSet.noneOf(QueryFeature.class); boolean mapJoinRemoved = false; - boolean hasMapGroupBy = false; private boolean hasLateralViews = false; private boolean cboSupportedLateralViews = true; @@ -196,116 +198,124 @@ public boolean isCBOSupportedLateralViews() { return cboSupportedLateralViews; } + public void addFeature(QueryFeature feature) { + features.add(feature); + } + + public boolean hasFeature(QueryFeature feature) { + return features.contains(feature); + } + public boolean hasGroupBy() { - return hasGroupBy; + return hasFeature(QueryFeature.GROUP_BY); } public void setHasGroupBy(boolean hasGroupBy) { - this.hasGroupBy = hasGroupBy; + setFeature(QueryFeature.GROUP_BY, hasGroupBy); } public boolean hasOrderBy() { - return hasOrderBy; + return hasFeature(QueryFeature.ORDER_BY); } public void setHasOrderBy(boolean hasOrderBy) { - this.hasOrderBy = hasOrderBy; + setFeature(QueryFeature.ORDER_BY, hasOrderBy); } public boolean hasOuterOrderBy() { - return hasOuterOrderBy; + return hasFeature(QueryFeature.OUTER_ORDER_BY); } public void setHasOuterOrderBy(boolean hasOuterOrderBy) { - this.hasOuterOrderBy = hasOuterOrderBy; + setFeature(QueryFeature.OUTER_ORDER_BY, hasOuterOrderBy); } public boolean hasSortBy() { - return hasSortBy; + return hasFeature(QueryFeature.SORT_BY); } public void setHasSortBy(boolean hasSortBy) { - this.hasSortBy = hasSortBy; + setFeature(QueryFeature.SORT_BY, hasSortBy); } public void setHasLimit(boolean hasLimit) { - this.hasLimit = hasLimit; + setFeature(QueryFeature.LIMIT, hasLimit); } public boolean hasLimit() { - return hasLimit; + return hasFeature(QueryFeature.LIMIT); } public boolean hasJoinFollowedByGroupBy() { - return hasJoinFollowedByGroupBy; + return hasFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY); } public void setHasJoinFollowedByGroupBy(boolean hasJoinFollowedByGroupBy) { - this.hasJoinFollowedByGroupBy = hasJoinFollowedByGroupBy; + setFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY, hasJoinFollowedByGroupBy); } public boolean usesScript() { - return usesScript; + return hasFeature(QueryFeature.USES_SCRIPT); } public void setUsesScript(boolean usesScript) { - this.usesScript = usesScript; + setFeature(QueryFeature.USES_SCRIPT, usesScript); } public boolean hasDistributeBy() { - return hasDistributeBy; + return hasFeature(QueryFeature.DISTRIBUTE_BY); } public void setHasDistributeBy(boolean hasDistributeBy) { - this.hasDistributeBy = hasDistributeBy; + setFeature(QueryFeature.DISTRIBUTE_BY, hasDistributeBy); } public boolean hasClusterBy() { - return hasClusterBy; + return hasFeature(QueryFeature.CLUSTER_BY); } public void setHasClusterBy(boolean hasClusterBy) { - this.hasClusterBy = hasClusterBy; + setFeature(QueryFeature.CLUSTER_BY, hasClusterBy); } public boolean hasPTF() { - return hasPTF; + return hasFeature(QueryFeature.PTF); } public void setHasPTF(boolean hasPTF) { - this.hasPTF = hasPTF; + setFeature(QueryFeature.PTF, hasPTF); } public boolean hasWindowing() { - return hasWindowing; + return hasFeature(QueryFeature.WINDOWING); } public void setHasWindowing(boolean hasWindowing) { - this.hasWindowing = hasWindowing; + setFeature(QueryFeature.WINDOWING, hasWindowing); } public boolean hasQualify() { - return hasQualify; + return hasFeature(QueryFeature.QUALIFY); } public void setHasQualify(boolean hasQualify) { - this.hasQualify = hasQualify; + setFeature(QueryFeature.QUALIFY, hasQualify); } public boolean hasExcept() { - return hasExcept; + return hasFeature(QueryFeature.EXCEPT); } public void setHasExcept(boolean hasExcept) { - this.hasExcept = hasExcept; + setFeature(QueryFeature.EXCEPT, hasExcept); } public boolean hasIntersect() { - return hasIntersect; + return hasFeature(QueryFeature.INTERSECT); } public void setHasIntersect(boolean hasIntersect) { - this.hasIntersect = hasIntersect; + setFeature(QueryFeature.INTERSECT, hasIntersect); } public boolean isMapJoinRemoved() { @@ -317,11 +327,11 @@ public void setMapJoinRemoved(boolean mapJoinRemoved) { } public boolean isHasMapGroupBy() { - return hasMapGroupBy; + return hasFeature(QueryFeature.MAP_GROUP_BY); } public void setHasMapGroupBy(boolean hasMapGroupBy) { - this.hasMapGroupBy = hasMapGroupBy; + setFeature(QueryFeature.MAP_GROUP_BY, hasMapGroupBy); } public boolean hasMultiDestQuery() { @@ -386,24 +396,8 @@ public void clear() { isMaterializedView = false; hasJoin = false; - hasGroupBy = false; - hasOrderBy = false; - hasOuterOrderBy = false; - hasSortBy = false; - hasJoinFollowedByGroupBy = false; - hasPTF = false; - hasWindowing = false; - hasQualify = false; - hasExcept = false; - hasIntersect = false; - - // does the query have a using clause - usesScript = false; - - hasDistributeBy = false; - hasClusterBy = false; + features.clear(); mapJoinRemoved = false; - hasMapGroupBy = false; noOfJoins = 0; noOfOuterJoins = 0; @@ -413,4 +407,12 @@ public void clear() { usedTables.clear(); } + + private void setFeature(QueryFeature feature, boolean enabled) { + if (enabled) { + addFeature(feature); + } else { + features.remove(feature); + } + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java index ec2a6ccb818c..7ab5d7e89c61 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java @@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.GroupByOperator; import org.apache.hadoop.hive.ql.exec.Operator; @@ -212,7 +213,7 @@ else if (!HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_GROUPBY_SKEW)) { convertGroupByMapSideSortedGroupBy(hiveConf, groupByOp, depth); } else if (optimizeDistincts && !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) { - pGraphContext.getQueryProperties().setHasMapGroupBy(true); + pGraphContext.getQueryProperties().addFeature(QueryFeature.MAP_GROUP_BY); ReduceSinkOperator reduceSinkOp = (ReduceSinkOperator)groupByOp.getChildOperators().get(0); GroupByDesc childGroupByDesc = @@ -514,7 +515,7 @@ private GroupByOptimizerSortMatch matchBucketSortCols( // The operators specified by depth and removed from the tree. protected void convertGroupByMapSideSortedGroupBy( HiveConf conf, GroupByOperator groupByOp, int depth) { - pGraphContext.getQueryProperties().setHasMapGroupBy(true); + pGraphContext.getQueryProperties().addFeature(QueryFeature.MAP_GROUP_BY); if (removeChildren(groupByOp, depth)) { // Use bucketized hive input format - that makes sure that one mapper reads the entire file diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java index f5431fa34934..ec3917e56d80 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java @@ -38,6 +38,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.Order; +import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; @@ -732,7 +733,7 @@ public ReduceSinkOperator getReduceSinkOp(List partitionPositions, List // should honor the ordering of records provided by ORDER BY in SELECT statement ReduceSinkOperator parentRSOp = OperatorUtils.findSingleOperatorUpstream(parent, ReduceSinkOperator.class); - if (parentRSOp != null && parseCtx.getQueryProperties().hasOuterOrderBy()) { + if (parentRSOp != null && parseCtx.getQueryProperties().hasFeature(QueryFeature.OUTER_ORDER_BY)) { String parentRSOpOrder = parentRSOp.getConf().getOrder(); String parentRSOpNullOrder = parentRSOp.getConf().getNullOrder(); if (parentRSOpOrder != null && !parentRSOpOrder.isEmpty() && sortPositions.isEmpty()) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index ecc226d42a2e..f0e1811cf1ee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -141,6 +141,7 @@ import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; +import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FunctionInfo; @@ -689,9 +690,9 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept this.ctx.setCboInfo(cboMsg); // Determine if we should re-throw the exception OR if we try to mark the query to retry as non-CBO. - final boolean requiresCBO = queryProperties.hasQualify() - || queryProperties.hasExcept() - || queryProperties.hasIntersect(); + final boolean requiresCBO = queryProperties.hasFeature(QueryFeature.QUALIFY) + || queryProperties.hasFeature(QueryFeature.EXCEPT) + || queryProperties.hasFeature(QueryFeature.INTERSECT); if (requiresCBO || fallbackStrategy.isFatal(e)) { if (e instanceof RuntimeException || e instanceof SemanticException) { // These types of exceptions do not need wrapped @@ -1000,13 +1001,13 @@ private static String canHandleQbForCbo(QueryProperties queryProperties, HiveConf conf, boolean topLevelQB) { List reasons = new ArrayList<>(); // Not ok to run CBO, build error message. - if (queryProperties.hasSortBy() && queryProperties.hasLimit()) { + if (queryProperties.hasFeature(QueryFeature.SORT_BY) && queryProperties.hasFeature(QueryFeature.LIMIT)) { reasons.add("has sort by with limit"); } - if (queryProperties.hasPTF()) { + if (queryProperties.hasFeature(QueryFeature.PTF)) { reasons.add("has PTF"); } - if (queryProperties.usesScript()) { + if (queryProperties.hasFeature(QueryFeature.USES_SCRIPT)) { reasons.add("uses scripts"); } if (!queryProperties.isCBOSupportedLateralViews()) { @@ -1024,7 +1025,7 @@ private static EnumSet obtainCBOProfiles(QueryProperties que profilesCBO.add(ExtendedCBOProfile.JOIN_REORDERING); } // If the query contains windowing processing - if (queryProperties.hasWindowing()) { + if (queryProperties.hasFeature(QueryFeature.WINDOWING)) { profilesCBO.add(ExtendedCBOProfile.WINDOWING_POSTPROCESSING); } return profilesCBO; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 7781ded7bfa7..28cf7eef9079 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -111,6 +111,7 @@ import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.QueryProperties; +import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.cache.results.CacheUsage; import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache; @@ -669,19 +670,19 @@ void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String id, String alias, boolean qbexpr.setOpcode(QBExpr.Opcode.UNION); break; case HiveParser.TOK_INTERSECTALL: - queryProperties.setHasIntersect(true); + queryProperties.addFeature(QueryFeature.INTERSECT); qbexpr.setOpcode(QBExpr.Opcode.INTERSECTALL); break; case HiveParser.TOK_INTERSECTDISTINCT: - queryProperties.setHasIntersect(true); + queryProperties.addFeature(QueryFeature.INTERSECT); qbexpr.setOpcode(QBExpr.Opcode.INTERSECT); break; case HiveParser.TOK_EXCEPTALL: - queryProperties.setHasExcept(true); + queryProperties.addFeature(QueryFeature.EXCEPT); qbexpr.setOpcode(QBExpr.Opcode.EXCEPTALL); break; case HiveParser.TOK_EXCEPTDISTINCT: - queryProperties.setHasExcept(true); + queryProperties.addFeature(QueryFeature.EXCEPT); qbexpr.setOpcode(QBExpr.Opcode.EXCEPT); break; default: @@ -724,7 +725,7 @@ private Map doPhase1GetAggregationsFromSelect( for (ASTNode wdwFn : wdwFns) { WindowingSpec spec = qb.getWindowingSpec(dest); if(spec == null) { - queryProperties.setHasWindowing(true); + queryProperties.addFeature(QueryFeature.WINDOWING); spec = new WindowingSpec(); qb.addDestToWindowingSpec(dest, spec); } @@ -1674,7 +1675,7 @@ private void processJoin(QB qb, ASTNode join) throws SemanticException { } else if (child.getToken().getType() == HiveParser.TOK_SUBQUERY) { processSubQuery(qb, child); } else if (child.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) { - queryProperties.setHasPTF(true); + queryProperties.addFeature(QueryFeature.PTF); processPTF(qb, child); PTFInvocationSpec ptfInvocationSpec = qb.getPTFInvocationSpec(child); String inputAlias = ptfInvocationSpec == null ? null : @@ -1785,7 +1786,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx, } if ((ast.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM)) { - queryProperties.setUsesScript(true); + queryProperties.addFeature(QueryFeature.USES_SCRIPT); } Map aggregations = doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest); @@ -1894,7 +1895,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx, processJoin(qb, frm); qbp.setJoinExpr(frm); }else if(frm.getToken().getType() == HiveParser.TOK_PTBLFUNCTION){ - queryProperties.setHasPTF(true); + queryProperties.addFeature(QueryFeature.PTF); processPTF(qb, frm); } break; @@ -1902,14 +1903,14 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx, case HiveParser.TOK_CLUSTERBY: // Get the clusterby aliases - these are aliased to the entries in the // select list - queryProperties.setHasClusterBy(true); + queryProperties.addFeature(QueryFeature.CLUSTER_BY); qbp.setClusterByExprForClause(ctx_1.dest, ast); break; case HiveParser.TOK_DISTRIBUTEBY: // Get the distribute by aliases - these are aliased to the entries in // the select list - queryProperties.setHasDistributeBy(true); + queryProperties.addFeature(QueryFeature.DISTRIBUTE_BY); qbp.setDistributeByExprForClause(ctx_1.dest, ast); if (qbp.getClusterByForClause(ctx_1.dest) != null) { throw new SemanticException(generateErrorMessage(ast, @@ -1923,7 +1924,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx, case HiveParser.TOK_SORTBY: // Get the sort by aliases - these are aliased to the entries in the // select list - queryProperties.setHasSortBy(true); + queryProperties.addFeature(QueryFeature.SORT_BY); qbp.setSortByExprForClause(ctx_1.dest, ast); if (qbp.getClusterByForClause(ctx_1.dest) != null) { throw new SemanticException(generateErrorMessage(ast, @@ -1938,7 +1939,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx, case HiveParser.TOK_ORDERBY: // Get the order by aliases - these are aliased to the entries in the // select list - queryProperties.setHasOrderBy(true); + queryProperties.addFeature(QueryFeature.ORDER_BY); qbp.setOrderByExprForClause(ctx_1.dest, ast); if (qbp.getClusterByForClause(ctx_1.dest) != null) { throw new SemanticException(generateErrorMessage(ast, @@ -1955,9 +1956,9 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx, case HiveParser.TOK_GROUPING_SETS: // Get the groupby aliases - these are aliased to the entries in the // select list - queryProperties.setHasGroupBy(true); + queryProperties.addFeature(QueryFeature.GROUP_BY); if (qbp.getJoinExpr() != null) { - queryProperties.setHasJoinFollowedByGroupBy(true); + queryProperties.addFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY); } qbp.setGroupByExprForClause(ctx_1.dest, ast); skipRecursion = true; @@ -1982,7 +1983,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx, break; case HiveParser.TOK_QUALIFY: - queryProperties.setHasQualify(true); + queryProperties.addFeature(QueryFeature.QUALIFY); qbp.setQualifyExprForClause(ctx_1.dest, ast); qbp.addAggregationExprsForClause(ctx_1.dest, doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest)); @@ -1997,7 +1998,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1, PlannerContext plannerCtx, break; case HiveParser.TOK_LIMIT: - queryProperties.setHasLimit(true); + queryProperties.addFeature(QueryFeature.LIMIT); if (ast.getChildCount() == 2) { qbp.setDestLimit(ctx_1.dest, Integer.valueOf(ast.getChild(0).getText()), Integer.valueOf(ast.getChild(1).getText())); @@ -4767,7 +4768,7 @@ private Operator genSelectPlan(String dest, ASTNode selExprList, QB qb, Opera boolean isInTransform = (selExprList.getChild(posn).getChild(0).getType() == HiveParser.TOK_TRANSFORM); if (isInTransform) { - queryProperties.setUsesScript(true); + queryProperties.addFeature(QueryFeature.USES_SCRIPT); globalLimitCtx.setHasTransformOrUDTF(true); trfm = (ASTNode) selExprList.getChild(posn).getChild(0); } @@ -11594,7 +11595,7 @@ private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb, curr = genHavingPlan(dest, qb, curr, aliasToOpInfo); } - if(queryProperties.hasWindowing() && qb.getWindowingSpec(dest) != null) { + if(queryProperties.hasFeature(QueryFeature.WINDOWING) && qb.getWindowingSpec(dest) != null) { curr = genWindowingPlan(qb, qb.getWindowingSpec(dest), curr); // GBy for DISTINCT after windowing if ((qbp.getAggregationExprsForClause(dest).size() != 0 @@ -12419,7 +12420,7 @@ private Operator genPlan(QB qb, boolean skipAmbiguityCheck) Operator srcOpInfo = null; Operator lastPTFOp = null; - if(queryProperties.hasPTF()){ + if(queryProperties.hasFeature(QueryFeature.PTF)){ //After processing subqueries and source tables, process // partitioned table functions @@ -15118,8 +15119,9 @@ private void copyInfoToQueryProperties(QueryProperties queryProperties) { if (qb.getParseInfo().hasInsertTables()) { queryProperties.setQueryType(QueryProperties.QueryType.DML); } - queryProperties.setHasOuterOrderBy(!qb.getParseInfo().getIsSubQ() && - !qb.getParseInfo().getDestToOrderBy().isEmpty()); + if (!qb.getParseInfo().getIsSubQ() && !qb.getParseInfo().getDestToOrderBy().isEmpty()) { + queryProperties.addFeature(QueryFeature.OUTER_ORDER_BY); + } queryProperties.setOuterQueryLimit(qb.getParseInfo().getOuterQueryLimit()); queryProperties.setView(forViewCreation); queryProperties.setMaterializedView(qb.isMaterializedView()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/DisallowTransformHook.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/DisallowTransformHook.java index 59a57ca9d30b..b0571c47dd56 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/DisallowTransformHook.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/DisallowTransformHook.java @@ -21,6 +21,7 @@ import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext; import org.apache.hadoop.hive.ql.hooks.HookContext; import org.apache.hadoop.hive.ql.QueryProperties; +import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; public class DisallowTransformHook implements ExecuteWithHookContext { @@ -30,7 +31,7 @@ public void run(HookContext hookContext) throws Exception { if (null == qProps) { return; // its a ddl query. } - if (qProps.usesScript()) { + if (qProps.hasFeature(QueryFeature.USES_SCRIPT)) { throw new HiveAccessControlException("Query with transform clause is disallowed in" + " current configuration."); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java index f41ed705afb2..fa062620bb91 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.QueryProperties; +import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature; import org.apache.hadoop.hive.ql.QueryState; import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager; @@ -129,7 +130,7 @@ public void testExceptAll() throws Exception { } QueryProperties properties = analyze( "SELECT t1 FROM test_db.test_table EXCEPT ALL SELECT t2 FROM test_db.test_table", false); - Assert.assertTrue(properties.hasExcept()); + Assert.assertTrue(properties.hasFeature(QueryFeature.EXCEPT)); } @Test @@ -140,7 +141,7 @@ public void testExceptDistinct() throws Exception { } QueryProperties properties = analyze( "SELECT t1 FROM test_db.test_table EXCEPT DISTINCT SELECT t2 FROM test_db.test_table", false); - Assert.assertTrue(properties.hasExcept()); + Assert.assertTrue(properties.hasFeature(QueryFeature.EXCEPT)); } @Test @@ -151,7 +152,7 @@ public void testIntersectAll() throws Exception { } QueryProperties properties = analyze( "SELECT t1 FROM test_db.test_table INTERSECT ALL SELECT t2 FROM test_db.test_table", false); - Assert.assertTrue(properties.hasIntersect()); + Assert.assertTrue(properties.hasFeature(QueryFeature.INTERSECT)); } @Test @@ -162,7 +163,7 @@ public void testIntersectDistinct() throws Exception { } QueryProperties properties = analyze( "SELECT t1 FROM test_db.test_table INTERSECT DISTINCT SELECT t2 FROM test_db.test_table", false); - Assert.assertTrue(properties.hasIntersect()); + Assert.assertTrue(properties.hasFeature(QueryFeature.INTERSECT)); } @Test @@ -173,7 +174,7 @@ public void testQualify() throws Exception { } QueryProperties properties = analyze( "SELECT t1 FROM test_db.test_table QUALIFY row_number() OVER (PARTITION BY t1 ORDER BY t2) = 1", false); - Assert.assertTrue(properties.hasQualify()); + Assert.assertTrue(properties.hasFeature(QueryFeature.QUALIFY)); } @Test