From 91cddde2562ae27e457459612615109ed6ae9bfa Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 24 Mar 2026 10:47:33 +0800 Subject: [PATCH 1/2] address comments --- .../spark/sql/execution/basicPhysicalOperators.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala index 49da71e2b0afe..680fecbcf9b4c 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala @@ -259,8 +259,8 @@ case class FilterExec(condition: Expression, child: SparkPlan) val boundOtherPreds = otherPreds.map( BindReferences.bindReference(_, output)) // Pre-evaluate input variables before CSE analysis: CSE clears - // ctx.currentVars[i].code as a side effect, which causes "is not an rvalue" - // errors if notNullPreds reference the same inputs. + // ctx.currentVars[i].code as a side effect; without this pre-evaluation, Janino fails + // with "Unknown variable or type" when notNullPreds reference the same input columns. val otherPredInputAttrs = AttributeSet(otherPreds.flatMap(_.references)) val inputVarsEvalCode = evaluateRequiredVariables( child.output, input, otherPredInputAttrs) @@ -275,6 +275,13 @@ case class FilterExec(condition: Expression, child: SparkPlan) } code } + // Note: subExprs.exprCodesNeedEvaluate is intentionally not used here, unlike ProjectExec. + // evaluateRequiredVariables above cleared input[i].code = EmptyBlock for all + // otherPredInputAttrs before CSE analysis ran, so getLocalInputVariableValues never + // adds them to exprCodesNeedEvaluate — it is always empty. Do NOT replace the + // pre-evaluation above with evaluateVariables(subExprs.exprCodesNeedEvaluate): + // that would leave notNullPreds referencing undeclared variables (Janino: "Unknown + // variable or type") for any input column shared between notNullPreds and otherPreds. (inputVarsEvalCode, ctx.evaluateSubExprEliminationState(subExprs.states.values), predCode) } else { ("", "", generatePredicateCode( From dada1b476d402b91524e32b0853e27b33c0c18d2 Mon Sep 17 00:00:00 2001 From: yangjie01 Date: Tue, 24 Mar 2026 13:11:43 +0800 Subject: [PATCH 2/2] fix style --- .../org/apache/spark/sql/execution/basicPhysicalOperators.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala index 680fecbcf9b4c..5e73cdd6da8f4 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala @@ -278,7 +278,7 @@ case class FilterExec(condition: Expression, child: SparkPlan) // Note: subExprs.exprCodesNeedEvaluate is intentionally not used here, unlike ProjectExec. // evaluateRequiredVariables above cleared input[i].code = EmptyBlock for all // otherPredInputAttrs before CSE analysis ran, so getLocalInputVariableValues never - // adds them to exprCodesNeedEvaluate — it is always empty. Do NOT replace the + // adds them to exprCodesNeedEvaluate -- it is always empty. Do NOT replace the // pre-evaluation above with evaluateVariables(subExprs.exprCodesNeedEvaluate): // that would leave notNullPreds referencing undeclared variables (Janino: "Unknown // variable or type") for any input column shared between notNullPreds and otherPreds.