Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.hooks;

import org.apache.hadoop.hive.ql.QueryProperties;
import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;

Expand All @@ -41,13 +42,14 @@ public void run(HookContext hookContext) {

if (queryProps != null) {
console.printError("Has Join: " + queryProps.hasJoin());
console.printError("Has Group By: " + queryProps.hasGroupBy());
console.printError("Has Sort By: " + queryProps.hasSortBy());
console.printError("Has Order By: " + queryProps.hasOrderBy());
console.printError("Has Group By After Join: " + queryProps.hasJoinFollowedByGroupBy());
console.printError("Uses Script: " + queryProps.usesScript());
console.printError("Has Distribute By: " + queryProps.hasDistributeBy());
console.printError("Has Cluster By: " + queryProps.hasClusterBy());
console.printError("Has Group By: " + queryProps.hasFeature(QueryFeature.GROUP_BY));
console.printError("Has Sort By: " + queryProps.hasFeature(QueryFeature.SORT_BY));
console.printError("Has Order By: " + queryProps.hasFeature(QueryFeature.ORDER_BY));
console.printError("Has Group By After Join: "
+ queryProps.hasFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY));
console.printError("Uses Script: " + queryProps.hasFeature(QueryFeature.USES_SCRIPT));
console.printError("Has Distribute By: " + queryProps.hasFeature(QueryFeature.DISTRIBUTE_BY));
console.printError("Has Cluster By: " + queryProps.hasFeature(QueryFeature.CLUSTER_BY));
}
}
}
132 changes: 67 additions & 65 deletions ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql;


import java.util.EnumSet;
import java.util.HashSet;
import java.util.Set;

Expand Down Expand Up @@ -56,6 +57,24 @@ public String getName() {
}
}

public enum QueryFeature {
GROUP_BY,
ORDER_BY,
OUTER_ORDER_BY,
SORT_BY,
LIMIT,
JOIN_FOLLOWED_BY_GROUP_BY,
PTF,
WINDOWING,
QUALIFY,
EXCEPT,
INTERSECT,
USES_SCRIPT,
DISTRIBUTE_BY,
CLUSTER_BY,
MAP_GROUP_BY
}

boolean query;
boolean analyzeCommand;
boolean noScanAnalyzeCommand;
Expand All @@ -64,25 +83,8 @@ public String getName() {
int outerQueryLimit;

boolean hasJoin = false;
boolean hasGroupBy = false;
boolean hasOrderBy = false;
boolean hasOuterOrderBy = false;
boolean hasSortBy = false;
boolean hasLimit = false;
boolean hasJoinFollowedByGroupBy = false;
boolean hasPTF = false;
boolean hasWindowing = false;
boolean hasQualify = false;
boolean hasExcept = false;
boolean hasIntersect = false;

// does the query have a using clause
boolean usesScript = false;

boolean hasDistributeBy = false;
boolean hasClusterBy = false;
private final EnumSet<QueryFeature> features = EnumSet.noneOf(QueryFeature.class);
boolean mapJoinRemoved = false;
boolean hasMapGroupBy = false;

private boolean hasLateralViews = false;
private boolean cboSupportedLateralViews = true;
Expand Down Expand Up @@ -196,116 +198,124 @@ public boolean isCBOSupportedLateralViews() {
return cboSupportedLateralViews;
}

public void addFeature(QueryFeature feature) {
features.add(feature);
}

public boolean hasFeature(QueryFeature feature) {
return features.contains(feature);
}

public boolean hasGroupBy() {
return hasGroupBy;
return hasFeature(QueryFeature.GROUP_BY);
}

public void setHasGroupBy(boolean hasGroupBy) {
this.hasGroupBy = hasGroupBy;
setFeature(QueryFeature.GROUP_BY, hasGroupBy);
}

public boolean hasOrderBy() {
return hasOrderBy;
return hasFeature(QueryFeature.ORDER_BY);
}

public void setHasOrderBy(boolean hasOrderBy) {
this.hasOrderBy = hasOrderBy;
setFeature(QueryFeature.ORDER_BY, hasOrderBy);
}

public boolean hasOuterOrderBy() {
return hasOuterOrderBy;
return hasFeature(QueryFeature.OUTER_ORDER_BY);
}

public void setHasOuterOrderBy(boolean hasOuterOrderBy) {
this.hasOuterOrderBy = hasOuterOrderBy;
setFeature(QueryFeature.OUTER_ORDER_BY, hasOuterOrderBy);
}

public boolean hasSortBy() {
return hasSortBy;
return hasFeature(QueryFeature.SORT_BY);
}

public void setHasSortBy(boolean hasSortBy) {
this.hasSortBy = hasSortBy;
setFeature(QueryFeature.SORT_BY, hasSortBy);
}

public void setHasLimit(boolean hasLimit) {
this.hasLimit = hasLimit;
setFeature(QueryFeature.LIMIT, hasLimit);
}

public boolean hasLimit() {
return hasLimit;
return hasFeature(QueryFeature.LIMIT);
}

public boolean hasJoinFollowedByGroupBy() {
return hasJoinFollowedByGroupBy;
return hasFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY);
}

public void setHasJoinFollowedByGroupBy(boolean hasJoinFollowedByGroupBy) {
this.hasJoinFollowedByGroupBy = hasJoinFollowedByGroupBy;
setFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY, hasJoinFollowedByGroupBy);
}

public boolean usesScript() {
return usesScript;
return hasFeature(QueryFeature.USES_SCRIPT);
}

public void setUsesScript(boolean usesScript) {
this.usesScript = usesScript;
setFeature(QueryFeature.USES_SCRIPT, usesScript);
}

public boolean hasDistributeBy() {
return hasDistributeBy;
return hasFeature(QueryFeature.DISTRIBUTE_BY);
}

public void setHasDistributeBy(boolean hasDistributeBy) {
this.hasDistributeBy = hasDistributeBy;
setFeature(QueryFeature.DISTRIBUTE_BY, hasDistributeBy);
}

public boolean hasClusterBy() {
return hasClusterBy;
return hasFeature(QueryFeature.CLUSTER_BY);
}

public void setHasClusterBy(boolean hasClusterBy) {
this.hasClusterBy = hasClusterBy;
setFeature(QueryFeature.CLUSTER_BY, hasClusterBy);
}

public boolean hasPTF() {
return hasPTF;
return hasFeature(QueryFeature.PTF);
}

public void setHasPTF(boolean hasPTF) {
this.hasPTF = hasPTF;
setFeature(QueryFeature.PTF, hasPTF);
}

public boolean hasWindowing() {
return hasWindowing;
return hasFeature(QueryFeature.WINDOWING);
}

public void setHasWindowing(boolean hasWindowing) {
this.hasWindowing = hasWindowing;
setFeature(QueryFeature.WINDOWING, hasWindowing);
}

public boolean hasQualify() {
return hasQualify;
return hasFeature(QueryFeature.QUALIFY);
}

public void setHasQualify(boolean hasQualify) {
this.hasQualify = hasQualify;
setFeature(QueryFeature.QUALIFY, hasQualify);
}

public boolean hasExcept() {
return hasExcept;
return hasFeature(QueryFeature.EXCEPT);
}

public void setHasExcept(boolean hasExcept) {
this.hasExcept = hasExcept;
setFeature(QueryFeature.EXCEPT, hasExcept);
}

public boolean hasIntersect() {
return hasIntersect;
return hasFeature(QueryFeature.INTERSECT);
}

public void setHasIntersect(boolean hasIntersect) {
this.hasIntersect = hasIntersect;
setFeature(QueryFeature.INTERSECT, hasIntersect);
}

public boolean isMapJoinRemoved() {
Expand All @@ -317,11 +327,11 @@ public void setMapJoinRemoved(boolean mapJoinRemoved) {
}

public boolean isHasMapGroupBy() {
return hasMapGroupBy;
return hasFeature(QueryFeature.MAP_GROUP_BY);
}

public void setHasMapGroupBy(boolean hasMapGroupBy) {
this.hasMapGroupBy = hasMapGroupBy;
setFeature(QueryFeature.MAP_GROUP_BY, hasMapGroupBy);
}

public boolean hasMultiDestQuery() {
Expand Down Expand Up @@ -386,24 +396,8 @@ public void clear() {
isMaterializedView = false;

hasJoin = false;
hasGroupBy = false;
hasOrderBy = false;
hasOuterOrderBy = false;
hasSortBy = false;
hasJoinFollowedByGroupBy = false;
hasPTF = false;
hasWindowing = false;
hasQualify = false;
hasExcept = false;
hasIntersect = false;

// does the query have a using clause
usesScript = false;

hasDistributeBy = false;
hasClusterBy = false;
features.clear();
mapJoinRemoved = false;
hasMapGroupBy = false;

noOfJoins = 0;
noOfOuterJoins = 0;
Expand All @@ -413,4 +407,12 @@ public void clear() {

usedTables.clear();
}

private void setFeature(QueryFeature feature, boolean enabled) {
if (enabled) {
addFeature(feature);
} else {
features.remove(feature);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
Expand Down Expand Up @@ -212,7 +213,7 @@ else if (!HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_GROUPBY_SKEW)) {
convertGroupByMapSideSortedGroupBy(hiveConf, groupByOp, depth);
}
else if (optimizeDistincts && !HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) {
pGraphContext.getQueryProperties().setHasMapGroupBy(true);
pGraphContext.getQueryProperties().addFeature(QueryFeature.MAP_GROUP_BY);
ReduceSinkOperator reduceSinkOp =
(ReduceSinkOperator)groupByOp.getChildOperators().get(0);
GroupByDesc childGroupByDesc =
Expand Down Expand Up @@ -514,7 +515,7 @@ private GroupByOptimizerSortMatch matchBucketSortCols(
// The operators specified by depth and removed from the tree.
protected void convertGroupByMapSideSortedGroupBy(
HiveConf conf, GroupByOperator groupByOp, int depth) {
pGraphContext.getQueryProperties().setHasMapGroupBy(true);
pGraphContext.getQueryProperties().addFeature(QueryFeature.MAP_GROUP_BY);

if (removeChildren(groupByOp, depth)) {
// Use bucketized hive input format - that makes sure that one mapper reads the entire file
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
Expand Down Expand Up @@ -732,7 +733,7 @@ public ReduceSinkOperator getReduceSinkOp(List<Integer> partitionPositions, List
// should honor the ordering of records provided by ORDER BY in SELECT statement
ReduceSinkOperator parentRSOp = OperatorUtils.findSingleOperatorUpstream(parent,
ReduceSinkOperator.class);
if (parentRSOp != null && parseCtx.getQueryProperties().hasOuterOrderBy()) {
if (parentRSOp != null && parseCtx.getQueryProperties().hasFeature(QueryFeature.OUTER_ORDER_BY)) {
String parentRSOpOrder = parentRSOp.getConf().getOrder();
String parentRSOpNullOrder = parentRSOp.getConf().getNullOrder();
if (parentRSOpOrder != null && !parentRSOpOrder.isEmpty() && sortPositions.isEmpty()) {
Expand Down
15 changes: 8 additions & 7 deletions ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
Expand Down Expand Up @@ -689,9 +690,9 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticExcept
this.ctx.setCboInfo(cboMsg);

// Determine if we should re-throw the exception OR if we try to mark the query to retry as non-CBO.
final boolean requiresCBO = queryProperties.hasQualify()
|| queryProperties.hasExcept()
|| queryProperties.hasIntersect();
final boolean requiresCBO = queryProperties.hasFeature(QueryFeature.QUALIFY)
|| queryProperties.hasFeature(QueryFeature.EXCEPT)
|| queryProperties.hasFeature(QueryFeature.INTERSECT);
if (requiresCBO || fallbackStrategy.isFatal(e)) {
if (e instanceof RuntimeException || e instanceof SemanticException) {
// These types of exceptions do not need wrapped
Expand Down Expand Up @@ -1000,13 +1001,13 @@ private static String canHandleQbForCbo(QueryProperties queryProperties,
HiveConf conf, boolean topLevelQB) {
List<String> reasons = new ArrayList<>();
// Not ok to run CBO, build error message.
if (queryProperties.hasSortBy() && queryProperties.hasLimit()) {
if (queryProperties.hasFeature(QueryFeature.SORT_BY) && queryProperties.hasFeature(QueryFeature.LIMIT)) {
reasons.add("has sort by with limit");
}
if (queryProperties.hasPTF()) {
if (queryProperties.hasFeature(QueryFeature.PTF)) {
reasons.add("has PTF");
}
if (queryProperties.usesScript()) {
if (queryProperties.hasFeature(QueryFeature.USES_SCRIPT)) {
reasons.add("uses scripts");
}
if (!queryProperties.isCBOSupportedLateralViews()) {
Expand All @@ -1024,7 +1025,7 @@ private static EnumSet<ExtendedCBOProfile> obtainCBOProfiles(QueryProperties que
profilesCBO.add(ExtendedCBOProfile.JOIN_REORDERING);
}
// If the query contains windowing processing
if (queryProperties.hasWindowing()) {
if (queryProperties.hasFeature(QueryFeature.WINDOWING)) {
profilesCBO.add(ExtendedCBOProfile.WINDOWING_POSTPROCESSING);
}
return profilesCBO;
Expand Down
Loading
Loading