diff --git a/docs/source/user_guide/data_types.rst b/docs/source/user_guide/data_types.rst index f5326428..3d529332 100644 --- a/docs/source/user_guide/data_types.rst +++ b/docs/source/user_guide/data_types.rst @@ -188,15 +188,14 @@ and `Arrow DataTypes `` - Map - - Data type of an associative array that maps keys to values (including NULL). A map cannot contain duplicate keys; each key can map to at most one value. + - Data type of an associative array that maps keys (including NULL) to values (including NULL). A map cannot contain duplicate keys; each key can map to at most one value. There is no restriction of element types; it is the responsibility of the user to ensure uniqueness. The type can be declared using ``MAP`` where kt is the data type of the key elements and vt is the data type of the value elements. - **Note:** In Paimon C++, map keys must be explicitly marked as ``NOT NULL``. - Apache Arrow does not support nullable map keys. If the key type is not - marked as ``NOT NULL`` in the schema, parsing will fail with an error. + **Note:** Paimon C++ accepts nullable map key declarations for schema compatibility. + A query fails only if the data actually contains a NULL map key. * - ``MULTISET`` - Not Supported diff --git a/docs/source/user_guide/schema.rst b/docs/source/user_guide/schema.rst index 0fe8e720..5db69ef9 100644 --- a/docs/source/user_guide/schema.rst +++ b/docs/source/user_guide/schema.rst @@ -84,35 +84,11 @@ DataField represents a column of the table. 3. ``type``: data type, very similar to SQL type string. 4. ``description``: string. -Limitations ------------ +Nullable MAP Keys +----------------- -MAP Key Must Be NOT NULL -^^^^^^^^^^^^^^^^^^^^^^^^ - -Apache Arrow does not support nullable map keys. When defining a ``MAP`` type in the schema, -the key must be explicitly marked as ``NOT NULL``. If the key is not marked as ``NOT NULL``, -schema parsing will fail with an error. - -For example, the following is **valid**: - -.. code-block:: json - - { - "type": "MAP", - "key": "TINYINT NOT NULL", - "value": "SMALLINT" - } - -The following is **invalid** and will be rejected: - -.. code-block:: json - - { - "type": "MAP", - "key": "TINYINT", - "value": "SMALLINT" - } +Paimon C++ accepts nullable ``MAP`` key declarations for compatibility with existing schemas. +A query fails only when the data actually contains a NULL map key. Update Schema ------------- diff --git a/src/paimon/common/types/data_type_json_parser.cpp b/src/paimon/common/types/data_type_json_parser.cpp index 950308a5..33308ab6 100644 --- a/src/paimon/common/types/data_type_json_parser.cpp +++ b/src/paimon/common/types/data_type_json_parser.cpp @@ -689,17 +689,13 @@ Result> DataTypeJsonParser::ParseMapType( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr key, ParseType("key", type_json_value["key"])); + // NOTE: Unlike Java Paimon, this C++ implementation does not support nullable keys in // MapType. This is a limitation of Apache Arrow, which does not allow null keys in its - // MapType. As a result, we validate `nullable = false` for the map key. - if (key->nullable()) { - return Status::Invalid(fmt::format( - "Map field '{}' has a nullable key." - "Map keys must be explicitly marked as NOT NULL in the schema for paimon-cpp " - "because Apache Arrow does not support nullable map keys. " - "Please add 'NOT NULL' to the key type definition.", - name)); - } + // MapType. As a result, we explicitly set `nullable = false` for the map key, regardless of + // the original schema. Please be aware of this behavioral difference when migrating or + // interoperating with Java Paimon. + key = key->WithNullable(false); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr value, ParseType("value", type_json_value["value"])); return arrow::field(name, std::make_shared(key, value), nullable); diff --git a/src/paimon/common/types/data_type_json_parser_test.cpp b/src/paimon/common/types/data_type_json_parser_test.cpp index bbb4fb52..bfa69c32 100644 --- a/src/paimon/common/types/data_type_json_parser_test.cpp +++ b/src/paimon/common/types/data_type_json_parser_test.cpp @@ -51,7 +51,7 @@ TEST(DataTypeJsonParserTest, ParseTypeMapTypeSuccess) { const std::string name = "map_field"; const char* json = R"({ "type": "MAP", - "key": "STRING NOT NULL", + "key": "STRING", "value": "INT" })"; rapidjson::Document doc; @@ -60,6 +60,8 @@ TEST(DataTypeJsonParserTest, ParseTypeMapTypeSuccess) { ASSERT_OK_AND_ASSIGN(std::shared_ptr field, DataTypeJsonParser::ParseType(name, doc)); ASSERT_NE(field, nullptr); + auto map_type = std::static_pointer_cast(field->type()); + ASSERT_FALSE(map_type->key_field()->nullable()); } TEST(DataTypeJsonParserTest, ParseTypeRowTypeSuccess) { diff --git a/src/paimon/core/schema/arrow_schema_validator.cpp b/src/paimon/core/schema/arrow_schema_validator.cpp index b9ea5da2..e01afe01 100644 --- a/src/paimon/core/schema/arrow_schema_validator.cpp +++ b/src/paimon/core/schema/arrow_schema_validator.cpp @@ -231,10 +231,6 @@ Status ArrowSchemaValidator::ValidateField(const std::shared_ptr& arrow::internal::checked_cast(*field->type()).key_field(); const auto& item_field = arrow::internal::checked_cast(*field->type()).item_field(); - if (key_field->nullable()) { - return Status::Invalid( - fmt::format("Map field '{}' has a nullable key.", field->name())); - } PAIMON_RETURN_NOT_OK(ValidateField(key_field, /*allow_blob=*/false)); PAIMON_RETURN_NOT_OK(ValidateField(item_field, /*allow_blob=*/false)); break; diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 57179e8c..3c3054ac 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -938,7 +938,7 @@ TEST(SchemaValidationTest, TestMapStorageLayout) { } } -TEST(SchemaValidationTest, TestMapRequiresNonNullableKey) { +TEST(SchemaValidationTest, TestMapSharedShreddingRequiresNonNullableKey) { auto nullable_key_map = std::make_shared(arrow::field("key", arrow::utf8(), /*nullable=*/true), arrow::field("value", arrow::int64())); @@ -951,9 +951,11 @@ TEST(SchemaValidationTest, TestMapRequiresNonNullableKey) { {Options::BUCKET_KEY, "f0"}, {"fields.f1.map.storage-layout", "shared-shredding"}, }; - ASSERT_NOK_WITH_MSG(TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, - /*primary_keys=*/{}, options), - "Map field 'f1' has a nullable key."); + ASSERT_OK_AND_ASSIGN(std::shared_ptr table_schema, + TableSchema::Create(/*schema_id=*/0, schema, /*partition_keys=*/{}, + /*primary_keys=*/{}, options)); + ASSERT_NOK_WITH_MSG(SchemaValidation::ValidateTableSchema(*table_schema), + "map key type is nullable"); } TEST(SchemaValidationTest, TestMapSharedShreddingRejectsBlobValue) { diff --git a/src/paimon/core/schema/table_schema_test.cpp b/src/paimon/core/schema/table_schema_test.cpp index fd0ac422..1a46eaa6 100644 --- a/src/paimon/core/schema/table_schema_test.cpp +++ b/src/paimon/core/schema/table_schema_test.cpp @@ -1297,7 +1297,7 @@ TEST_F(TableSchemaTest, SetFieldIdNestedListInStruct) { } } -TEST_F(TableSchemaTest, MapKeyMustBeNotNull) { +TEST_F(TableSchemaTest, NullableMapKeySchemaIsSupported) { std::string table_schema_str = R"({ "version" : 3, "id" : 0, @@ -1316,16 +1316,21 @@ TEST_F(TableSchemaTest, MapKeyMustBeNotNull) { "options" : {}, "timeMillis" : 1721614341162 })"; - ASSERT_NOK_WITH_MSG(TableSchema::CreateFromJson(table_schema_str), - "Map field 'f0' has a nullable key."); + ASSERT_OK_AND_ASSIGN(std::unique_ptr table_schema, + TableSchema::CreateFromJson(table_schema_str)); + auto json_map_type = std::static_pointer_cast(table_schema->Fields()[0].Type()); + ASSERT_FALSE(json_map_type->key_field()->nullable()); auto nullable_key_map = std::make_shared(arrow::field("key", arrow::int8(), /*nullable=*/true), arrow::field("value", arrow::int16())); - ASSERT_NOK_WITH_MSG( + ASSERT_OK_AND_ASSIGN( + std::shared_ptr direct_table_schema, TableSchema::Create(/*schema_id=*/0, arrow::schema({arrow::field("f0", nullable_key_map)}), - /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{}), - "Map field 'f0' has a nullable key."); + /*partition_keys=*/{}, /*primary_keys=*/{}, /*options=*/{})); + auto direct_map_type = + std::static_pointer_cast(direct_table_schema->Fields()[0].Type()); + ASSERT_TRUE(direct_map_type->key_field()->nullable()); } TEST_F(TableSchemaTest, MapKeysSortedIsNormalized) { diff --git a/src/paimon/format/avro/avro_file_batch_reader.cpp b/src/paimon/format/avro/avro_file_batch_reader.cpp index 92ac769c..f48ec4cc 100644 --- a/src/paimon/format/avro/avro_file_batch_reader.cpp +++ b/src/paimon/format/avro/avro_file_batch_reader.cpp @@ -123,6 +123,7 @@ Result AvroFileBatchReader::NextBatch() { } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, array_builder_->Finish()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); std::unique_ptr c_array = std::make_unique(); std::unique_ptr c_schema = std::make_unique(); PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportArray(*array, c_array.get(), c_schema.get())); diff --git a/src/paimon/format/orc/orc_adapter.cpp b/src/paimon/format/orc/orc_adapter.cpp index 992387d1..4773f2ed 100644 --- a/src/paimon/format/orc/orc_adapter.cpp +++ b/src/paimon/format/orc/orc_adapter.cpp @@ -945,6 +945,7 @@ Result> OrcAdapter::AppendBatch( MakeArrowBuilder(type, batch, pool)); std::shared_ptr array; PAIMON_RETURN_NOT_OK_FROM_ARROW(builder->Finish(&array)); + PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); return array; } diff --git a/src/paimon/format/parquet/parquet_file_batch_reader.cpp b/src/paimon/format/parquet/parquet_file_batch_reader.cpp index 4f1d013f..fcc47c18 100644 --- a/src/paimon/format/parquet/parquet_file_batch_reader.cpp +++ b/src/paimon/format/parquet/parquet_file_batch_reader.cpp @@ -537,6 +537,7 @@ Result ParquetFileBatchReader::NextBatch() { } PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, batch->ToStructArray()); + PAIMON_RETURN_NOT_OK_FROM_ARROW(array->Validate()); PAIMON_ASSIGN_OR_RAISE(bool need_cast, ParquetTimestampConverter::NeedCastArrayForTimestamp( array->type(), read_data_type_)); if (need_cast) { diff --git a/test/inte/scan_and_read_inte_test.cpp b/test/inte/scan_and_read_inte_test.cpp index a9294ef0..23384632 100644 --- a/test/inte/scan_and_read_inte_test.cpp +++ b/test/inte/scan_and_read_inte_test.cpp @@ -2819,6 +2819,50 @@ TEST_P(ScanAndReadInteTest, TestWithPKBucketSelectByPredicate) { ASSERT_TRUE(expected->Equals(read_result)) << read_result->ToString(); } +TEST_P(ScanAndReadInteTest, TestReadNullableMapKey) { + auto file_format = FileFormat(); + if (file_format != "orc") { + return; + } + // Java Parquet does not support writing null MAP keys; see + // ParquetRowDataWriter.MapWriter#writeMapData in Apache Paimon. Therefore, this test uses an + // ORC table. The table contains three rows: the first two rows have non-null MAP keys, while + // the third row has a null MAP key and is expected to fail during reading. + const std::string table_path = + paimon::test::GetDataDir() + "orc/nullable_map_key.db/nullable_map_key"; + + ScanContextBuilder scan_context_builder(table_path); + ASSERT_OK_AND_ASSIGN(auto scan_context, scan_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_scan, TableScan::Create(std::move(scan_context))); + ASSERT_OK_AND_ASSIGN(auto result_plan, table_scan->CreatePlan()); + ASSERT_TRUE(result_plan->SnapshotId()); + ASSERT_EQ(result_plan->SnapshotId().value(), 1); + ASSERT_EQ(result_plan->Splits().size(), 1); + + ReadContextBuilder read_context_builder(table_path); + read_context_builder.AddOption(Options::READ_BATCH_SIZE, "1"); + ASSERT_OK_AND_ASSIGN(auto read_context, read_context_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto table_read, TableRead::Create(std::move(read_context))); + ASSERT_OK_AND_ASSIGN(auto batch_reader, table_read->CreateReader(result_plan->Splits())); + + const std::vector expected_rows = {R"([[0, 1, [["one", 10]]]])", + R"([[0, 2, [["two", 20]]]])"}; + for (int32_t i = 0; i < 2; ++i) { + ASSERT_OK_AND_ASSIGN(BatchReader::ReadBatch batch, batch_reader->NextBatch()); + ASSERT_FALSE(BatchReader::IsEofBatch(batch)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr array, + ReadResultCollector::GetArray(std::move(batch))); + ASSERT_EQ(array->length(), 1); + + std::shared_ptr expected_array = + arrow::ipc::internal::json::ArrayFromJSON(array->type(), expected_rows[i]).ValueOrDie(); + ASSERT_TRUE(array->Equals(expected_array)) + << "actual: " << array->ToString() << ", expected: " << expected_array->ToString(); + } + ASSERT_NOK_WITH_MSG(batch_reader->NextBatch(), "Map array keys array should have no nulls"); + batch_reader->Close(); +} + TEST_P(ScanAndReadInteTest, TestCountRowsEmptySplits) { auto file_format = FileFormat(); std::string table_path = paimon::test::GetDataDir() + file_format + diff --git a/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0 b/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0 index e3b8f71a..c0d6d755 100644 --- a/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0 +++ b/test/test_data/avro/append_multiple.db/append_multiple/schema/schema-0 @@ -93,7 +93,7 @@ "name": "f0", "type": { "type": "MAP", - "key": "STRING NOT NULL", + "key": "STRING", "value": "INT" } }, diff --git a/test/test_data/avro/append_simple.db/append_simple/schema/schema-0 b/test/test_data/avro/append_simple.db/append_simple/schema/schema-0 index c8b4be27..be12e26c 100644 --- a/test/test_data/avro/append_simple.db/append_simple/schema/schema-0 +++ b/test/test_data/avro/append_simple.db/append_simple/schema/schema-0 @@ -23,7 +23,7 @@ "name" : "f0", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : "INT" } }, { diff --git a/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0 b/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0 index 72376320..b118e224 100644 --- a/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0 +++ b/test/test_data/avro/append_with_multiple_map.db/append_with_multiple_map/schema/schema-0 @@ -6,7 +6,7 @@ "name" : "f0", "type" : { "type" : "MAP", - "key" : "INT NOT NULL", + "key" : "INT", "value" : "INT" } }, { @@ -14,7 +14,7 @@ "name" : "f1", "type" : { "type" : "MAP", - "key" : "DOUBLE NOT NULL", + "key" : "DOUBLE", "value" : "DOUBLE" } }, { @@ -22,7 +22,7 @@ "name" : "f2", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : "STRING" } }, { @@ -30,7 +30,7 @@ "name" : "f3", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : "BINARY(6)" } }, { @@ -38,7 +38,7 @@ "name" : "f4", "type" : { "type" : "MAP", - "key" : "TIMESTAMP(6) NOT NULL", + "key" : "TIMESTAMP(6)", "value" : "TIMESTAMP(6)" } }, { @@ -46,7 +46,7 @@ "name" : "f5", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : { "type" : "ARRAY", "element" : "DOUBLE" @@ -57,10 +57,10 @@ "name" : "f6", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : { "type" : "MAP", - "key" : "DOUBLE NOT NULL", + "key" : "DOUBLE", "value" : "STRING" } } @@ -69,7 +69,7 @@ "name" : "f7", "type" : { "type" : "MAP", - "key" : "BIGINT NOT NULL", + "key" : "BIGINT", "value" : { "type" : "ROW", "fields" : [ { diff --git a/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0 b/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0 index 5533e14b..608d0df8 100644 --- a/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0 +++ b/test/test_data/avro/pk_with_multiple_type.db/pk_with_multiple_type/schema/schema-0 @@ -55,7 +55,7 @@ "name" : "f0", "type" : { "type" : "MAP", - "key" : "STRING NOT NULL", + "key" : "STRING", "value" : "INT" } }, { diff --git a/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 b/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 index 699682f2..f973a64c 100644 --- a/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 +++ b/test/test_data/orc/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 @@ -6,7 +6,7 @@ "name" : "f1", "type" : { "type" : "MAP", - "key" : "TINYINT NOT NULL", + "key" : "TINYINT", "value" : "SMALLINT" } }, { diff --git a/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0 b/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0 index c754fec8..57cd38af 100644 --- a/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0 +++ b/test/test_data/orc/append_table_with_nested_type.db/append_table_with_nested_type/schema/schema-0 @@ -64,7 +64,7 @@ "type" : { "type" : "MAP", "key" : { - "type" : "ROW NOT NULL", + "type" : "ROW", "fields" : [ { "id" : 13, "name" : "sub1", diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/README b/test/test_data/orc/nullable_map_key.db/nullable_map_key/README new file mode 100644 index 00000000..02b0d4e9 --- /dev/null +++ b/test/test_data/orc/nullable_map_key.db/nullable_map_key/README @@ -0,0 +1,13 @@ +id:int attrs:map(string, int) (all can be null) +no partition key +no primary key +bucket count: -1 +file format: orc + +Rows (snapshot-1): +Add: [1, {"one": 10}] +Add: [2, {"two": 20}] +Add: [3, {null: 30}] +NoCompact + +The third row contains a null MAP key and is expected to fail during reading. diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/bucket-0/data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc b/test/test_data/orc/nullable_map_key.db/nullable_map_key/bucket-0/data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc new file mode 100644 index 00000000..89c6f60a Binary files /dev/null and b/test/test_data/orc/nullable_map_key.db/nullable_map_key/bucket-0/data-7e6899e9-f3cd-4d2e-86bc-7a5f383d5922-0.orc differ diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0 b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0 new file mode 100644 index 00000000..e96a4bbe Binary files /dev/null and b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-a956b9da-04c5-41b6-b6f0-20efa3974bd4-0 differ diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0 b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0 new file mode 100644 index 00000000..0b805178 Binary files /dev/null and b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0 differ diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1 b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1 new file mode 100644 index 00000000..47fc5e98 Binary files /dev/null and b/test/test_data/orc/nullable_map_key.db/nullable_map_key/manifest/manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1 differ diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/schema/schema-0 b/test/test_data/orc/nullable_map_key.db/nullable_map_key/schema/schema-0 new file mode 100644 index 00000000..e1c711c2 --- /dev/null +++ b/test/test_data/orc/nullable_map_key.db/nullable_map_key/schema/schema-0 @@ -0,0 +1,25 @@ +{ + "version" : 3, + "id" : 0, + "fields" : [ { + "id" : 0, + "name" : "id", + "type" : "INT" + }, { + "id" : 1, + "name" : "attrs", + "type" : { + "type" : "MAP", + "key" : "STRING", + "value" : "INT" + } + } ], + "highestFieldId" : 1, + "partitionKeys" : [ ], + "primaryKeys" : [ ], + "options" : { + "bucket" : "-1", + "file.format" : "orc" + }, + "timeMillis" : 1785815466343 +} \ No newline at end of file diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/EARLIEST b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/EARLIEST new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/EARLIEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/LATEST b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/LATEST new file mode 100644 index 00000000..56a6051c --- /dev/null +++ b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/LATEST @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/snapshot-1 b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/snapshot-1 new file mode 100644 index 00000000..78c34677 --- /dev/null +++ b/test/test_data/orc/nullable_map_key.db/nullable_map_key/snapshot/snapshot-1 @@ -0,0 +1,17 @@ +{ + "version" : 3, + "uuid" : "67b9daa5-c055-45f9-ac6a-d87d017490b1", + "id" : 1, + "schemaId" : 0, + "baseManifestList" : "manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-0", + "baseManifestListSize" : 1006, + "deltaManifestList" : "manifest-list-67c61c9b-ab3c-4564-a5f2-612aa684a8f2-1", + "deltaManifestListSize" : 1110, + "commitUser" : "8bd3a648-0532-4a3e-8815-9e1da2e6d7f9", + "commitIdentifier" : 0, + "commitKind" : "APPEND", + "timeMillis" : 1785815467254, + "totalRecordCount" : 3, + "deltaRecordCount" : 3, + "nextRowId" : 0 +} \ No newline at end of file diff --git a/test/test_data/orc/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0 b/test/test_data/orc/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0 index 7a3fdc5c..7f38243c 100644 --- a/test/test_data/orc/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0 +++ b/test/test_data/orc/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0 @@ -44,7 +44,7 @@ "name" : "col2", "type" : { "type" : "MAP", - "key" : "INT NOT NULL", + "key" : "INT", "value" : "INT" } } ], diff --git a/test/test_data/parquet/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 b/test/test_data/parquet/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 index 4bf9bca7..296bbca9 100644 --- a/test/test_data/parquet/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 +++ b/test/test_data/parquet/append_complex_build_in_fieldid.db/append_complex_build_in_fieldid/schema/schema-0 @@ -6,7 +6,7 @@ "name" : "f1", "type" : { "type" : "MAP", - "key" : "TINYINT NOT NULL", + "key" : "TINYINT", "value" : "SMALLINT" } }, { diff --git a/test/test_data/parquet/parquet_append_table.db/parquet_append_table/schema/schema-0 b/test/test_data/parquet/parquet_append_table.db/parquet_append_table/schema/schema-0 index 524117ca..ae19f8a6 100644 --- a/test/test_data/parquet/parquet_append_table.db/parquet_append_table/schema/schema-0 +++ b/test/test_data/parquet/parquet_append_table.db/parquet_append_table/schema/schema-0 @@ -43,7 +43,7 @@ "type" : { "type" : "MAP", "key" : { - "type" : "ARRAY NOT NULL", + "type" : "ARRAY", "element" : "FLOAT" }, "value" : { diff --git a/test/test_data/parquet/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0 b/test/test_data/parquet/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0 index 22968d4c..db050929 100644 --- a/test/test_data/parquet/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0 +++ b/test/test_data/parquet/pk_table_nested_type.db/pk_table_nested_type/schema/schema-0 @@ -44,7 +44,7 @@ "name" : "col2", "type" : { "type" : "MAP", - "key" : "INT NOT NULL", + "key" : "INT", "value" : "INT" } } ],