Added custom sqlite types support

dataframe-jdbc/api/dataframe-jdbc.api

@@ -261,13 +261,23 @@ public final class org/jetbrains/kotlinx/dataframe/io/db/PostgreSql : org/jetbra
 }
 
 public final class org/jetbrains/kotlinx/dataframe/io/db/Sqlite : org/jetbrains/kotlinx/dataframe/io/db/DbType {
-	public static final field INSTANCE Lorg/jetbrains/kotlinx/dataframe/io/db/Sqlite;
+	public static final field Companion Lorg/jetbrains/kotlinx/dataframe/io/db/Sqlite$Companion;
+	public fun <init> ()V
+	public fun <init> (Ljava/util/Map;)V
+	public synthetic fun <init> (Ljava/util/Map;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
 	public fun buildTableMetadata (Ljava/sql/ResultSet;)Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;
 	public fun createConnection (Lorg/jetbrains/kotlinx/dataframe/io/DbConnectionConfig;)Ljava/sql/Connection;
+	public final fun getCustomTypesMap ()Ljava/util/Map;
 	public fun getDriverClassName ()Ljava/lang/String;
+	public fun getExpectedJdbcType (Lorg/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata;)Lkotlin/reflect/KType;
 	public fun isSystemTable (Lorg/jetbrains/kotlinx/dataframe/io/db/TableMetadata;)Z
 }
 
+public final class org/jetbrains/kotlinx/dataframe/io/db/Sqlite$Companion {
+	public final fun getDefault ()Lorg/jetbrains/kotlinx/dataframe/io/db/Sqlite;
+	public final fun withCustomTypes (Ljava/util/Map;)Lorg/jetbrains/kotlinx/dataframe/io/db/Sqlite;
+}
+
 public final class org/jetbrains/kotlinx/dataframe/io/db/TableColumnMetadata {
 	public fun <init> (Ljava/lang/String;Ljava/lang/String;IILjava/lang/String;Z)V
 	public synthetic fun <init> (Ljava/lang/String;Ljava/lang/String;IILjava/lang/String;ZILkotlin/jvm/internal/DefaultConstructorMarker;)V

dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Sqlite.kt

@@ -1,25 +1,32 @@
 package org.jetbrains.kotlinx.dataframe.io.db
 
 import org.jetbrains.kotlinx.dataframe.io.DbConnectionConfig
-import org.jetbrains.kotlinx.dataframe.io.db.TableColumnMetadata
-import org.jetbrains.kotlinx.dataframe.io.db.TableMetadata
-import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema
 import org.sqlite.SQLiteConfig
 import java.sql.Connection
 import java.sql.DriverManager
 import java.sql.ResultSet
 import kotlin.reflect.KType
+import kotlin.reflect.full.withNullability
 
 /**
  * Represents the Sqlite database type.
  *
  * This class provides methods to convert data from a ResultSet to the appropriate type for Sqlite,
  * and to generate the corresponding column schema.
+ *
+ * Use [customTypesMap] to register custom types and provide the corresponding [KType] for each one.
+ * [KType] must correspond to JDBC actual type.
+ * Even for default Sqlite types, you can override the actual [KType] after reading the column
+ * by providing a custom type in [customTypesMap].
  */
-public object Sqlite : DbType("sqlite") {
+public class Sqlite(public val customTypesMap: Map<String, KType> = mapOf()) : DbType("sqlite") {
     override val driverClassName: String
         get() = "org.sqlite.JDBC"
 
+    override fun getExpectedJdbcType(tableColumnMetadata: TableColumnMetadata): KType =
+        customTypesMap[tableColumnMetadata.sqlTypeName]?.withNullability(tableColumnMetadata.isNullable)
+            ?: super.getExpectedJdbcType(tableColumnMetadata)
+
     override fun isSystemTable(tableMetadata: TableMetadata): Boolean = tableMetadata.name.startsWith("sqlite_")
 
     override fun buildTableMetadata(tables: ResultSet): TableMetadata =
@@ -37,4 +44,10 @@ public object Sqlite : DbType("sqlite") {
         } else {
             DriverManager.getConnection(dbConfig.url, dbConfig.user, dbConfig.password)
         }
+
+    public companion object {
+        public val default: Sqlite = Sqlite()
+
+        public fun withCustomTypes(customTypesMap: Map<String, KType>): Sqlite = Sqlite(customTypesMap)
+    }
 }

dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/util.kt

@@ -99,7 +99,7 @@ public fun extractDBTypeFromUrl(url: String?): DbType {
 
         MySql.dbTypeInJdbcUrl in url -> MySql
 
-        Sqlite.dbTypeInJdbcUrl in url -> Sqlite
+        Sqlite.default.dbTypeInJdbcUrl in url -> Sqlite.default
 
         PostgreSql.dbTypeInJdbcUrl in url -> PostgreSql
 

dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/db/jdbcTypesTest.kt

@@ -3,6 +3,7 @@
 package org.jetbrains.kotlinx.dataframe.io.db
 
 import io.kotest.matchers.shouldBe
+import org.jetbrains.kotlinx.dataframe.io.db.JdbcTypesTest.MySqlDBTypes.BIGINT_UNSIGNED
 import org.junit.Test
 import org.junit.experimental.runners.Enclosed
 import org.junit.runner.RunWith
@@ -18,6 +19,7 @@ class JdbcTypesTest {
         val sqlTypeName: String,
         val jdbcType: Int,
         val javaClassName: String,
+        val isNullable: Boolean,
         val expectedKotlinType: KType,
     ) {
         fun mockkColMetaData() =
@@ -27,7 +29,7 @@ class JdbcTypesTest {
                 jdbcType,
                 10,
                 javaClassName,
-                false,
+                isNullable,
             )
     }
 
@@ -37,6 +39,7 @@ class JdbcTypesTest {
             "BIGINT UNSIGNED",
             20,
             "java.math.BigInteger",
+            false,
             typeOf<BigInteger>(),
         )
 
@@ -58,6 +61,7 @@ class JdbcTypesTest {
             "BIGINT UNSIGNED",
             20,
             "java.math.BigInteger",
+            false,
             typeOf<BigInteger>(),
         )
 
@@ -72,4 +76,40 @@ class JdbcTypesTest {
             }
         }
     }
+
+    class SqliteTypes {
+
+        // Taken from #964
+
+        object LONGVARCHAR_1 : ColumnType(
+            "LONGVARCHAR",
+            -2,
+            "java.lang.Object",
+            false,
+            typeOf<String>(),
+        )
+
+        object LONGVARCHAR_2 : ColumnType(
+            "LONGVARCHAR",
+            12,
+            "java.lang.String",
+            true,
+            typeOf<String?>(),
+        )
+
+        val customTypes: List<ColumnType> = listOf(
+            LONGVARCHAR_1,
+            LONGVARCHAR_2,
+        )
+
+        @Test
+        fun `SQLite custom types`() {
+            val sqliteCustom = Sqlite(
+                mapOf("LONGVARCHAR" to typeOf<String>()),
+            )
+            customTypes.forEach { type ->
+                sqliteCustom.getExpectedJdbcType(type.mockkColMetaData()) shouldBe type.expectedKotlinType
+            }
+        }
+    }
 }

dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteCustomTypesTest.kt

@@ -0,0 +1,54 @@
+package org.jetbrains.kotlinx.dataframe.io
+
+import io.kotest.matchers.shouldBe
+import org.jetbrains.kotlinx.dataframe.DataFrame
+import org.jetbrains.kotlinx.dataframe.io.db.Sqlite
+import org.jetbrains.kotlinx.dataframe.type
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Test
+import java.sql.Connection
+import java.sql.DriverManager
+import kotlin.reflect.typeOf
+
+class SqliteTestCustomTypes {
+
+    companion object {
+        private lateinit var connection: Connection
+
+        private val dbUrl =
+            "jdbc:sqlite:${(this::class as Any).javaClass.classLoader
+                .getResource("safe_moz_places_sample.sqlite").path}"
+
+        @BeforeClass
+        @JvmStatic
+        fun setUpClass() {
+            connection = DriverManager.getConnection(dbUrl)
+        }
+
+        @AfterClass
+        @JvmStatic
+        fun tearDownClass() {
+            try {
+                connection.close()
+            } catch (e: Exception) {
+                // Log, but not fail
+                println("Warning: Could not clean up test database file: ${e.message}")
+            }
+        }
+    }
+
+    private val sqliteCustomTypes = Sqlite.withCustomTypes(mapOf("LONGVARCHAR" to typeOf<String>()))
+
+    private val df = DataFrame.readSqlTable(connection, "moz_places", dbType = sqliteCustomTypes)
+
+    @Test
+    fun `LONGVARCHAR columns should be read as String`() {
+        df["url"].type shouldBe typeOf<String>()
+        df["title"].type shouldBe typeOf<String?>()
+
+        df["url"][0] shouldBe "https://support.example.org/products/browser"
+        df["title"][0] shouldBe null
+        df["title"][4] shouldBe "Column selectors | Sample Docs"
+    }
+}

dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/sqliteDynamicTypes.kt

@@ -0,0 +1,55 @@
+package org.jetbrains.kotlinx.dataframe.io
+
+import io.kotest.matchers.shouldBe
+import org.jetbrains.kotlinx.dataframe.DataFrame
+import org.jetbrains.kotlinx.dataframe.type
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Ignore
+import org.junit.Test
+import java.sql.Connection
+import java.sql.DriverManager
+import kotlin.reflect.typeOf
+
+/**
+ * TODO:
+ * Xerial SQLite JDBC driver seems to give identical metadata for `Int?` and `Long?` columns,
+ * so we have to solve it #1747.
+ */
+class SqliteTestDynamicTypes {
+
+    companion object {
+        private lateinit var connection: Connection
+
+        private val dbUrl =
+            "jdbc:sqlite:${(this::class as Any).javaClass.classLoader
+                .getResource("simple_int_long_nullable.sqlite").path}"
+
+        @BeforeClass
+        @JvmStatic
+        fun setUpClass() {
+            connection = DriverManager.getConnection(dbUrl)
+        }
+
+        @AfterClass
+        @JvmStatic
+        fun tearDownClass() {
+            try {
+                connection.close()
+            } catch (e: Exception) {
+                // Log, but not fail
+                println("Warning: Could not clean up test database file: ${e.message}")
+            }
+        }
+    }
+
+    private val df = DataFrame.readSqlTable(connection, "numbers")
+
+    @Ignore
+    @Test
+    fun `INTEGER column with big values should be read as Long`() {
+        df["int_col"].type shouldBe typeOf<Int?>()
+        // Fails! #1747
+        df["long_col"].type shouldBe typeOf<Long?>()
+    }
+}