Skip to content

Commit 8236b58

Browse files
viiryaclaude
viirya
and
claude
committed
[SPARK] Implement createTableLike in InMemoryTableCatalog with tests
InMemoryTableCatalog overrides createTableLike to demonstrate connector-specific copy semantics: source properties are merged into the target (user overrides win), and source constraints are copied from sourceTable.constraints() directly. BasicInMemoryTableCatalog does not override createTableLike and uses the default fallback, which copies only schema, partitioning, and user-specified overrides. Tests added to CatalogSuite covering: - User-specified properties in tableInfo are applied to the target - Source properties are copied by the connector implementation - User-specified properties override source properties - Source constraints are copied by the connector implementation - Default fallback does not copy source properties CreateTableLikeSuite updated to reflect that InMemoryTableCatalog's createTableLike copies source properties, and adds a test for user override precedence. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent ae4516f commit 8236b58

3 files changed

Lines changed: 158 additions & 4 deletions

File tree

‎sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/CatalogSuite.scala‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1377,4 +1377,132 @@ class CatalogSuite extends SparkFunSuite {
13771377
catalog.alterTable(testIdent, TableChange.addConstraint(constraints.apply(0), "3"))
13781378
assert(catalog.loadTable(testIdent).version() == "4")
13791379
}
1380+
1381+
// ---- createTableLike tests ----
1382+
1383+
test("createTableLike: user-specified properties in tableInfo are applied to target") {
1384+
val catalog = newCatalog()
1385+
val srcIdent = Identifier.of(Array("ns"), "src")
1386+
val dstIdent = Identifier.of(Array("ns"), "dst")
1387+
1388+
val srcProps = Map("source.key" -> "source.value").asJava
1389+
catalog.createTable(srcIdent, columns, emptyTrans, srcProps)
1390+
val sourceTable = catalog.loadTable(srcIdent)
1391+
1392+
// tableInfo contains only user overrides
1393+
val overrides = Map("user.key" -> "user.value").asJava
1394+
val tableInfo = new TableInfo.Builder()
1395+
.withColumns(columns)
1396+
.withPartitions(emptyTrans)
1397+
.withProperties(overrides)
1398+
.build()
1399+
catalog.createTableLike(dstIdent, tableInfo, sourceTable)
1400+
1401+
val dst = catalog.loadTable(dstIdent)
1402+
assert(dst.properties.asScala("user.key") == "user.value",
1403+
"user-specified properties should be applied to the target")
1404+
}
1405+
1406+
test("createTableLike: source properties are copied to target by connector implementation") {
1407+
val catalog = newCatalog()
1408+
val srcIdent = Identifier.of(Array("ns"), "src")
1409+
val dstIdent = Identifier.of(Array("ns"), "dst")
1410+
1411+
val srcProps = Map("format.version" -> "2", "format.feature" -> "deletion-vectors").asJava
1412+
catalog.createTable(srcIdent, columns, emptyTrans, srcProps)
1413+
val sourceTable = catalog.loadTable(srcIdent)
1414+
1415+
// tableInfo contains no overrides; connector should copy from sourceTable
1416+
val tableInfo = new TableInfo.Builder()
1417+
.withColumns(columns)
1418+
.withPartitions(emptyTrans)
1419+
.withProperties(emptyProps)
1420+
.build()
1421+
catalog.createTableLike(dstIdent, tableInfo, sourceTable)
1422+
1423+
val dst = catalog.loadTable(dstIdent)
1424+
assert(dst.properties.asScala("format.version") == "2",
1425+
"connector should copy source properties from sourceTable")
1426+
assert(dst.properties.asScala("format.feature") == "deletion-vectors",
1427+
"connector should copy source properties from sourceTable")
1428+
}
1429+
1430+
test("createTableLike: user-specified properties override source properties") {
1431+
val catalog = newCatalog()
1432+
val srcIdent = Identifier.of(Array("ns"), "src")
1433+
val dstIdent = Identifier.of(Array("ns"), "dst")
1434+
1435+
val srcProps = Map("format.version" -> "1", "source.only" -> "yes").asJava
1436+
catalog.createTable(srcIdent, columns, emptyTrans, srcProps)
1437+
val sourceTable = catalog.loadTable(srcIdent)
1438+
1439+
// user explicitly overrides format.version
1440+
val overrides = Map("format.version" -> "2").asJava
1441+
val tableInfo = new TableInfo.Builder()
1442+
.withColumns(columns)
1443+
.withPartitions(emptyTrans)
1444+
.withProperties(overrides)
1445+
.build()
1446+
catalog.createTableLike(dstIdent, tableInfo, sourceTable)
1447+
1448+
val dst = catalog.loadTable(dstIdent)
1449+
assert(dst.properties.asScala("format.version") == "2",
1450+
"user-specified properties should override source properties")
1451+
assert(dst.properties.asScala("source.only") == "yes",
1452+
"non-overridden source properties should still be copied")
1453+
}
1454+
1455+
test("createTableLike: source constraints are copied to target by connector implementation") {
1456+
val catalog = newCatalog()
1457+
val srcIdent = Identifier.of(Array("ns"), "src")
1458+
val dstIdent = Identifier.of(Array("ns"), "dst")
1459+
1460+
val srcTableInfo = new TableInfo.Builder()
1461+
.withColumns(columns)
1462+
.withPartitions(emptyTrans)
1463+
.withProperties(emptyProps)
1464+
.withConstraints(constraints)
1465+
.build()
1466+
catalog.createTable(srcIdent, srcTableInfo)
1467+
val sourceTable = catalog.loadTable(srcIdent)
1468+
1469+
val tableInfo = new TableInfo.Builder()
1470+
.withColumns(columns)
1471+
.withPartitions(emptyTrans)
1472+
.withProperties(emptyProps)
1473+
.build()
1474+
catalog.createTableLike(dstIdent, tableInfo, sourceTable)
1475+
1476+
val dst = catalog.loadTable(dstIdent)
1477+
assert(dst.constraints().toSet == constraints.toSet,
1478+
"connector should copy source constraints from sourceTable.constraints()")
1479+
}
1480+
1481+
test("createTableLike: default implementation falls back to createTable with tableInfo only") {
1482+
// BasicInMemoryTableCatalog does not override createTableLike, so it uses the default
1483+
// implementation which calls createTable(ident, tableInfo); source properties are NOT copied.
1484+
val catalog = new BasicInMemoryTableCatalog
1485+
catalog.initialize("basic", CaseInsensitiveStringMap.empty())
1486+
1487+
val srcIdent = Identifier.of(Array("ns"), "src")
1488+
val dstIdent = Identifier.of(Array("ns"), "dst")
1489+
1490+
val srcProps = Map("source.key" -> "source.value").asJava
1491+
catalog.createTable(srcIdent, columns, emptyTrans, srcProps)
1492+
val sourceTable = catalog.loadTable(srcIdent)
1493+
1494+
val overrides = Map("user.key" -> "user.value").asJava
1495+
val tableInfo = new TableInfo.Builder()
1496+
.withColumns(columns)
1497+
.withPartitions(emptyTrans)
1498+
.withProperties(overrides)
1499+
.build()
1500+
catalog.createTableLike(dstIdent, tableInfo, sourceTable)
1501+
1502+
val dst = catalog.loadTable(dstIdent)
1503+
assert(dst.properties.asScala("user.key") == "user.value",
1504+
"user-specified properties should be present")
1505+
assert(!dst.properties.containsKey("source.key"),
1506+
"default createTableLike does not copy source properties; connector must override to do so")
1507+
}
13801508
}

‎sql/catalyst/src/test/scala/org/apache/spark/sql/connector/catalog/InMemoryTableCatalog.scala‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,18 @@ class BasicInMemoryTableCatalog extends TableCatalog {
238238
class InMemoryTableCatalog extends BasicInMemoryTableCatalog with SupportsNamespaces
239239
with ProcedureCatalog {
240240

241+
override def createTableLike(
242+
ident: Identifier,
243+
tableInfo: TableInfo,
244+
sourceTable: Table): Table = {
245+
// Format-specific behavior: merge source properties with user overrides, with user overrides
246+
// taking precedence. Copy source constraints from sourceTable directly. This demonstrates
247+
// how a connector uses sourceTable to access source-format-specific metadata.
248+
val mergedProps = (sourceTable.properties().asScala ++ tableInfo.properties().asScala).asJava
249+
createTable(ident, tableInfo.columns(), tableInfo.partitions(), mergedProps,
250+
Distributions.unspecified(), Array.empty, None, None, sourceTable.constraints())
251+
}
252+
241253
override def capabilities: java.util.Set[TableCatalogCapability] = {
242254
Set(
243255
TableCatalogCapability.SUPPORT_COLUMN_DEFAULT_VALUE,

‎sql/core/src/test/scala/org/apache/spark/sql/connector/CreateTableLikeSuite.scala‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,14 +142,28 @@ class CreateTableLikeSuite extends DatasourceV2SQLBase {
142142
// Property / provider behavior
143143
// -------------------------------------------------------------------------
144144

145-
test("source TBLPROPERTIES are NOT copied to target") {
145+
test("source TBLPROPERTIES are copied to target when connector implements createTableLike") {
146+
// InMemoryTableCatalog overrides createTableLike to merge source properties into the target,
147+
// demonstrating connector-specific copy semantics. Connectors that do not override
148+
// createTableLike fall back to createTable and do not copy source properties.
146149
withTable("src", "testcat.dst") {
147-
sql("CREATE TABLE src (id bigint) USING parquet TBLPROPERTIES ('secret_key' = 'secret')")
150+
sql("CREATE TABLE src (id bigint) USING parquet TBLPROPERTIES ('source_key' = 'source')")
148151
sql("CREATE TABLE testcat.dst LIKE src")
149152

150153
val dst = testCatalog.loadTable(Identifier.of(Array(), "dst"))
151-
assert(!dst.properties.containsKey("secret_key"),
152-
"Source TBLPROPERTIES should not be copied to target")
154+
assert(dst.properties.containsKey("source_key"),
155+
"Connector-implemented createTableLike copies source TBLPROPERTIES to target")
156+
}
157+
}
158+
159+
test("user-specified TBLPROPERTIES override source TBLPROPERTIES in createTableLike") {
160+
withTable("src", "testcat.dst") {
161+
sql("CREATE TABLE src (id bigint) USING parquet TBLPROPERTIES ('key' = 'source_value')")
162+
sql("CREATE TABLE testcat.dst LIKE src TBLPROPERTIES ('key' = 'user_value')")
163+
164+
val dst = testCatalog.loadTable(Identifier.of(Array(), "dst"))
165+
assert(dst.properties.get("key") == "user_value",
166+
"User-specified TBLPROPERTIES should override source TBLPROPERTIES")
153167
}
154168
}
155169

0 commit comments

Comments
 (0)