Skip to content

[spark] Allow specific plugin impl can use catalog config to overwrite default table prop #3364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
luoyuxia merged 2 commits into from
May 22, 2026
Merged

[spark] Allow specific plugin impl can use catalog config to overwrite default table prop #3364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6e8cb81
allow specific plugin impl can use catalog config to overwrite defaul…
Yohahaha May 22, 2026
f2cff55
Potential fix for pull request finding
Yohahaha May 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 7 additions & 5 deletions ...park/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/FlussScanBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,14 +93,16 @@ trait FlussLakeSupportsPushDownV2Filters extends FlussSupportsPushDownV2Filters

def tablePath: TablePath

def flussConfig: FlussConfiguration

override def pushPredicates(predicates: Array[Predicate]): Array[Predicate] = {
val pairs =
SparkPredicateConverter.convertPerPredicate(tableInfo.getRowType, predicates.toSeq)
val (acceptedSpark, acceptedFluss) = if (pairs.isEmpty) {
(Seq.empty[Predicate], Seq.empty[FlussPredicate])
} else {
val lakeSource =
FlussLakeUtils.createLakeSource(tableInfo.getProperties.toMap, tablePath)
FlussLakeUtils.createLakeSource(flussConfig.toMap, tableInfo.getProperties.toMap, tablePath)
val result = FlussLakeBatch.applyLakeFilters(lakeSource, pairs.map(_._2).asJava)
// Identity-match: lake sources are expected to return the same instances they received.
val acceptedSet: JSet[FlussPredicate] =
Expand All @@ -119,7 +121,7 @@ class FlussAppendScanBuilder(
tablePath: TablePath,
val tableInfo: TableInfo,
options: CaseInsensitiveStringMap,
flussConfig: FlussConfiguration)
val flussConfig: FlussConfiguration)
extends FlussSupportsPushDownV2Filters {

override def build(): Scan = {
Expand All @@ -140,7 +142,7 @@ class FlussLakeAppendScanBuilder(
val tablePath: TablePath,
val tableInfo: TableInfo,
options: CaseInsensitiveStringMap,
flussConfig: FlussConfiguration)
val flussConfig: FlussConfiguration)
extends FlussLakeSupportsPushDownV2Filters {

override def build(): Scan = {
Expand All @@ -160,7 +162,7 @@ class FlussUpsertScanBuilder(
tablePath: TablePath,
val tableInfo: TableInfo,
options: CaseInsensitiveStringMap,
flussConfig: FlussConfiguration)
val flussConfig: FlussConfiguration)
extends FlussSupportsPushDownPartitionFilters {

override def build(): Scan = {
Expand All @@ -173,7 +175,7 @@ class FlussLakeUpsertScanBuilder(
val tablePath: TablePath,
val tableInfo: TableInfo,
options: CaseInsensitiveStringMap,
flussConfig: FlussConfiguration)
val flussConfig: FlussConfiguration)
extends FlussLakeSupportsPushDownV2Filters {

override def build(): Scan = {
Expand Down
3 changes: 2 additions & 1 deletion ...s-spark-common/src/main/scala/org/apache/fluss/spark/read/lake/FlussLakeAppendBatch.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ class FlussLakeAppendBatch(
throw e
}

val lakeSource = FlussLakeUtils.createLakeSource(tableInfo.getProperties.toMap, tablePath)
val lakeSource =
FlussLakeUtils.createLakeSource(flussConfig.toMap, tableInfo.getProperties.toMap, tablePath)
lakeSource.withProject(FlussLakeUtils.lakeProjection(projection))
pushedPredicate.foreach(FlussLakeBatch.applyLakeFilters(lakeSource, _))

Expand Down
2 changes: 1 addition & 1 deletion ...mon/src/main/scala/org/apache/fluss/spark/read/lake/FlussLakePartitionReaderFactory.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ class FlussLakePartitionReaderFactory(
extends PartitionReaderFactory {

@transient private lazy val lakeSource: LakeSource[LakeSplit] = {
val source = FlussLakeUtils.createLakeSource(tableProperties, tablePath)
val source = FlussLakeUtils.createLakeSource(flussConfig.toMap, tableProperties, tablePath)
source.withProject(FlussLakeUtils.lakeProjection(projection))
flussPredicate.foreach(FlussLakeBatch.applyLakeFilters(source, _))
source
Expand Down
3 changes: 2 additions & 1 deletion ...s-spark-common/src/main/scala/org/apache/fluss/spark/read/lake/FlussLakeUpsertBatch.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ class FlussLakeUpsertBatch(
throw e
}

val lakeSource = FlussLakeUtils.createLakeSource(tableInfo.getProperties.toMap, tablePath)
val lakeSource =
FlussLakeUtils.createLakeSource(flussConfig.toMap, tableInfo.getProperties.toMap, tablePath)
lakeSource.withProject(FlussLakeUtils.lakeProjection(projection))
pushedPredicate.foreach(FlussLakeBatch.applyLakeFilters(lakeSource, _))

Expand Down
12 changes: 10 additions & 2 deletions ...k/fluss-spark-common/src/main/scala/org/apache/fluss/spark/read/lake/FlussLakeUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,17 +25,25 @@ import org.apache.fluss.utils.PropertiesUtils

import java.util

import scala.collection.JavaConverters._

object FlussLakeUtils {

private val SPARK_CATALOG_PREFIX = "spark.sql.catalog."

def createLakeSource(
catalogProperties: util.Map[String, String],
tableProperties: util.Map[String, String],
tablePath: TablePath): LakeSource[LakeSplit] = {
val tableConfig = Configuration.fromMap(tableProperties)
val datalakeFormat = tableConfig.get(ConfigOptions.TABLE_DATALAKE_FORMAT)
val dataLakePrefix = "table.datalake." + datalakeFormat + "."

val catalogProperties = PropertiesUtils.extractAndRemovePrefix(tableProperties, dataLakePrefix)
val lakeConfig = Configuration.fromMap(catalogProperties)
val lakeConfig =
Configuration.fromMap(PropertiesUtils.extractAndRemovePrefix(tableProperties, dataLakePrefix))
catalogProperties.asScala.foreach {
case (k, v) => lakeConfig.setString(s"$SPARK_CATALOG_PREFIX$k", v)
}
Comment on lines +44 to +46
val lakeStoragePlugin =
LakeStoragePluginSetUp.fromDataLakeFormat(datalakeFormat.toString, null)
val lakeStorage = lakeStoragePlugin.createLakeStorage(lakeConfig)
Expand Down
Loading