support VARIANT for pypaimon

Author: chenghuichen

docs/content/pypaimon/python-api.md

@@ -736,11 +736,11 @@ from pypaimon.data.generic_variant import GenericVariant
 read_builder = table.new_read_builder()
 result = read_builder.new_read().to_arrow(read_builder.new_scan().plan().splits())
 
-for row in result.column("payload").to_pylist():
-    if row is not None:
-        gv = GenericVariant.from_dict(row)        # wrap raw bytes
-        print(gv.to_python())                     # decode to Python object
-        print(gv.variant_get("$.city", "string")) # path extraction
+for record in result.to_pylist():
+    if (payload := record["payload"]) is not None:
+        gv = GenericVariant.from_arrow_struct(payload)
+        print(gv.to_python())                      # decode to Python object
+        print(gv.variant_get("$.city", "string"))  # path extraction
 ```
 
 **Writing a VARIANT column:**
@@ -773,7 +773,7 @@ table_commit.close()
 |:-------|:------------|
 | `GenericVariant.from_json(json_str)` | Build from a JSON string |
 | `GenericVariant.from_python(obj)` | Build from a Python object (`dict`, `list`, `int`, `str`, …) |
-| `GenericVariant.from_dict({"value": b"...", "metadata": b"..."})` | Wrap raw bytes from an Arrow VARIANT struct row |
+| `GenericVariant.from_arrow_struct({"value": b"...", "metadata": b"..."})` | Wrap raw bytes from an Arrow VARIANT struct row (read path) |
 | `GenericVariant.to_arrow_array([gv1, gv2, None, ...])` | Convert a list of `GenericVariant` (or `None`) to a `pa.StructArray` for writing |
 | `gv.to_python()` | Decode to native Python (`dict`, `list`, `int`, `str`, `None`, …) |
 | `gv.to_json()` | Decode to a JSON string |
@@ -785,7 +785,7 @@ table_commit.close()
 - `VARIANT` is only supported with Parquet file format. Writing to ORC or Avro raises `NotImplementedError`.
 - `VARIANT` cannot be used as a primary key or partition key.
 - Shredded VARIANT files (written by Paimon Java with `typed_value` sub-fields) are readable
-  via the raw `from_dict` path, but the extra fields are not automatically interpreted.
+  via the raw `from_arrow_struct` path, but the extra fields are not automatically interpreted.
 
 ## Predicate
 

paimon-python/pypaimon/data/generic_variant.py

@@ -709,7 +709,7 @@ class GenericVariant:
 
         # Construct from raw bytes (e.g. what to_arrow() returns for a VARIANT column)
         row = result.column('payload')[0].as_py()   # {'value': bytes, 'metadata': bytes}
-        v = GenericVariant.from_dict(row)
+        v = GenericVariant.from_arrow_struct(row)
         print(v.to_python())                        # {'age': 30, 'city': 'Beijing'}
     """
 
@@ -741,8 +741,15 @@ def from_python(cls, obj) -> 'GenericVariant':
         return builder.result()
 
     @classmethod
-    def from_dict(cls, d: dict) -> 'GenericVariant':
-        """Wrap raw bytes from a PyArrow VARIANT struct: {'value': bytes, 'metadata': bytes}."""
+    def from_arrow_struct(cls, d: dict) -> 'GenericVariant':
+        """Wrap raw bytes from a PyArrow VARIANT struct: {'value': bytes, 'metadata': bytes}.
+
+        Use this on the read path after calling ``column.to_pylist()`` on a VARIANT column::
+
+            for row in result.column("payload").to_pylist():
+                if row is not None:
+                    gv = GenericVariant.from_arrow_struct(row)
+        """
         return cls(bytes(d['value']), bytes(d['metadata']))
 
     @classmethod

paimon-python/pypaimon/tests/e2e/java_py_read_write_test.py

@@ -710,17 +710,17 @@ def test_py_read_variant_table(self):
         payload_list = result_sorted.column('payload').to_pylist()
 
         # Row 1: Alice, {"age":30,"city":"Beijing"}
-        gv_alice = GenericVariant.from_dict(payload_list[id_list.index(1)])
+        gv_alice = GenericVariant.from_arrow_struct(payload_list[id_list.index(1)])
         self.assertEqual(gv_alice.variant_get('$.age', 'int'), 30)
         self.assertEqual(gv_alice.variant_get('$.city', 'string'), 'Beijing')
 
         # Row 2: Bob, {"age":25,"city":"Shanghai"}
-        gv_bob = GenericVariant.from_dict(payload_list[id_list.index(2)])
+        gv_bob = GenericVariant.from_arrow_struct(payload_list[id_list.index(2)])
         self.assertEqual(gv_bob.variant_get('$.age', 'int'), 25)
         self.assertEqual(gv_bob.variant_get('$.city', 'string'), 'Shanghai')
 
         # Row 3: Carol, [1,2,3]
-        gv_carol = GenericVariant.from_dict(payload_list[id_list.index(3)])
+        gv_carol = GenericVariant.from_arrow_struct(payload_list[id_list.index(3)])
         self.assertEqual(gv_carol.to_python(), [1, 2, 3])
 
         print(f"test_py_read_variant_table: verified {result.num_rows} VARIANT rows")

paimon-python/pypaimon/tests/variant_test.py

@@ -538,14 +538,14 @@ def test_string_cast_on_int(self):
 
 class TestConstructors(unittest.TestCase):
 
-    def test_from_dict_roundtrip(self):
+    def test_from_arrow_struct_roundtrip(self):
         original = GenericVariant.from_json('{"x":1,"y":2}')
-        restored = GenericVariant.from_dict({'value': original.value(), 'metadata': original.metadata()})
+        restored = GenericVariant.from_arrow_struct({'value': original.value(), 'metadata': original.metadata()})
         self.assertEqual(restored.to_json(), original.to_json())
 
-    def test_from_dict_array(self):
+    def test_from_arrow_struct_array(self):
         original = GenericVariant.from_json('[1,2,3]')
-        restored = GenericVariant.from_dict({'value': original.value(), 'metadata': original.metadata()})
+        restored = GenericVariant.from_arrow_struct({'value': original.value(), 'metadata': original.metadata()})
         self.assertEqual(restored.get_type(), Type.ARRAY)
         self.assertEqual(restored.to_python(), [1, 2, 3])
 
@@ -578,7 +578,7 @@ def test_basic(self):
         arr = GenericVariant.to_arrow_array([gv1, gv2])
         self.assertIsInstance(arr, pa.StructArray)
         self.assertEqual(len(arr), 2)
-        restored = GenericVariant.from_dict(arr[0].as_py())
+        restored = GenericVariant.from_arrow_struct(arr[0].as_py())
         self.assertEqual(restored.variant_get('$.a', 'int'), 1)
 
     def test_with_nulls(self):