PoC: New dataframe read source interface

core/api/core.api

@@ -663,6 +663,7 @@ public final class org/jetbrains/kotlinx/dataframe/api/ChunkedKt {
 }
 
 public final class org/jetbrains/kotlinx/dataframe/api/CodeString {
+	public static final field Companion Lorg/jetbrains/kotlinx/dataframe/api/CodeString$Companion;
 	public static final synthetic fun box-impl (Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/api/CodeString;
 	public static fun constructor-impl (Ljava/lang/String;)Ljava/lang/String;
 	public fun equals (Ljava/lang/Object;)Z
@@ -676,6 +677,9 @@ public final class org/jetbrains/kotlinx/dataframe/api/CodeString {
 	public final synthetic fun unbox-impl ()Ljava/lang/String;
 }
 
+public final class org/jetbrains/kotlinx/dataframe/api/CodeString$Companion {
+}
+
 public abstract interface class org/jetbrains/kotlinx/dataframe/api/ColColumnsSelectionDsl {
 	public fun col (Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/columns/ColumnAccessor;
 	public fun col (Ljava/lang/String;I)Lorg/jetbrains/kotlinx/dataframe/columns/SingleColumn;
@@ -2471,13 +2475,15 @@ public abstract interface class org/jetbrains/kotlinx/dataframe/api/GlobalParser
 	public abstract fun getNulls ()Ljava/util/Set;
 	public abstract fun getParseExperimentalInstant ()Z
 	public abstract fun getParseExperimentalUuid ()Z
+	public abstract fun getParseToDataFrameReadSource ()Z
 	public abstract fun getSkipTypes ()Ljava/util/Set;
 	public abstract fun getUseFastDoubleParser ()Z
 	public abstract fun resetToDefault ()V
 	public abstract fun setDateTimeLibrary (Lorg/jetbrains/kotlinx/dataframe/api/ParseDateTimeLibrary;)V
 	public abstract fun setLocale (Ljava/util/Locale;)V
 	public abstract fun setParseExperimentalInstant (Z)V
 	public abstract fun setParseExperimentalUuid (Z)V
+	public abstract fun setParseToDataFrameReadSource (Z)V
 	public abstract fun setUseFastDoubleParser (Z)V
 }
 
@@ -3618,18 +3624,19 @@ public final class org/jetbrains/kotlinx/dataframe/api/ParserOptions {
 	public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
 	public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;)V
 	public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
-	public fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)V
-	public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
-	public fun <init> (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)V
-	public synthetic fun <init> (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
-	public final fun copy (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/lang/Iterable;Ljava/lang/Iterable;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
-	public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/lang/Iterable;Ljava/lang/Iterable;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
+	public fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)V
+	public synthetic fun <init> (Ljava/util/Locale;Ljava/time/format/DateTimeFormatter;Ljava/lang/String;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public fun <init> (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)V
+	public synthetic fun <init> (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/util/Set;Ljava/util/Set;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public final fun copy (Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/lang/Iterable;Ljava/lang/Iterable;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
+	public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;Ljava/util/Locale;Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;Ljava/lang/Iterable;Ljava/lang/Iterable;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;Ljava/lang/Boolean;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/api/ParserOptions;
 	public fun equals (Ljava/lang/Object;)Z
 	public final fun getDateTime ()Lorg/jetbrains/kotlinx/dataframe/api/DateTimeParserOptions;
 	public final fun getLocale ()Ljava/util/Locale;
 	public final fun getNullStrings ()Ljava/util/Set;
 	public final fun getParseExperimentalInstant ()Ljava/lang/Boolean;
 	public final fun getParseExperimentalUuid ()Ljava/lang/Boolean;
+	public final fun getParseToDataFrameReadSource ()Ljava/lang/Boolean;
 	public final fun getSkipTypes ()Ljava/util/Set;
 	public final fun getUseFastDoubleParser ()Ljava/lang/Boolean;
 	public fun hashCode ()I
@@ -6028,6 +6035,37 @@ public final class org/jetbrains/kotlinx/dataframe/io/DataFrameHtmlData$Companio
 	public static synthetic fun tableDefinitions$default (Lorg/jetbrains/kotlinx/dataframe/io/DataFrameHtmlData$Companion;ZZILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/DataFrameHtmlData;
 }
 
+public abstract interface class org/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions {
+}
+
+public abstract interface class org/jetbrains/kotlinx/dataframe/io/DataFrameReadSource {
+	public abstract fun acceptsSource (Lorg/jetbrains/kotlinx/dataframe/io/DataSourceInfo;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;)Z
+	public abstract fun getSupportedTypes ()Ljava/util/Set;
+	public abstract fun getTestOrder ()I
+	public abstract fun readDataFrameOrNull (Ljava/lang/Object;Lorg/jetbrains/kotlinx/dataframe/io/DataSourceInfo;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
+	public static synthetic fun readDataFrameOrNull$default (Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadSource;Ljava/lang/Object;Lorg/jetbrains/kotlinx/dataframe/io/DataSourceInfo;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
+	public fun readDataFrameSchemaOrNull (Ljava/lang/Object;Lorg/jetbrains/kotlinx/dataframe/io/DataSourceInfo;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
+	public static synthetic fun readDataFrameSchemaOrNull$default (Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadSource;Ljava/lang/Object;Lorg/jetbrains/kotlinx/dataframe/io/DataSourceInfo;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
+	public fun readDataSchemaCodeOrNull-myXLQ2E (Ljava/lang/Object;Lorg/jetbrains/kotlinx/dataframe/io/DataSourceInfo;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;)Ljava/lang/String;
+	public static synthetic fun readDataSchemaCodeOrNull-myXLQ2E$default (Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadSource;Ljava/lang/Object;Lorg/jetbrains/kotlinx/dataframe/io/DataSourceInfo;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;ILjava/lang/Object;)Ljava/lang/String;
+}
+
+public final class org/jetbrains/kotlinx/dataframe/io/DataSourceInfo {
+	public fun <init> (Lkotlin/reflect/KType;Ljava/lang/String;Ljava/lang/String;)V
+	public synthetic fun <init> (Lkotlin/reflect/KType;Ljava/lang/String;Ljava/lang/String;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public final fun component1 ()Lkotlin/reflect/KType;
+	public final fun component2 ()Ljava/lang/String;
+	public final fun component3 ()Ljava/lang/String;
+	public final fun copy (Lkotlin/reflect/KType;Ljava/lang/String;Ljava/lang/String;)Lorg/jetbrains/kotlinx/dataframe/io/DataSourceInfo;
+	public static synthetic fun copy$default (Lorg/jetbrains/kotlinx/dataframe/io/DataSourceInfo;Lkotlin/reflect/KType;Ljava/lang/String;Ljava/lang/String;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/io/DataSourceInfo;
+	public fun equals (Ljava/lang/Object;)Z
+	public final fun getExtension ()Ljava/lang/String;
+	public final fun getKType ()Lkotlin/reflect/KType;
+	public final fun getMimeType ()Ljava/lang/String;
+	public fun hashCode ()I
+	public fun toString ()Ljava/lang/String;
+}
+
 public final class org/jetbrains/kotlinx/dataframe/io/DisplayConfiguration {
 	public static final field Companion Lorg/jetbrains/kotlinx/dataframe/io/DisplayConfiguration$Companion;
 	public synthetic fun <init> (Ljava/lang/Integer;Ljava/lang/Integer;ILkotlin/jvm/functions/Function3;Ljava/lang/String;ZZZZZILkotlin/jvm/internal/DefaultConstructorMarker;)V
@@ -6078,6 +6116,18 @@ public final class org/jetbrains/kotlinx/dataframe/io/DisplayConfiguration$Compa
 	public final fun getDEFAULT ()Lorg/jetbrains/kotlinx/dataframe/io/DisplayConfiguration;
 }
 
+public final class org/jetbrains/kotlinx/dataframe/io/Guess2Kt {
+	public static final fun getNewSupportedFormats ()Ljava/util/List;
+	public static final fun readSource (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/lang/Object;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
+	public static final fun readSource (Lorg/jetbrains/kotlinx/dataframe/DataRow$Companion;Ljava/lang/Object;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
+	public static final fun readSource (Lorg/jetbrains/kotlinx/dataframe/api/CodeString$Companion;Ljava/lang/Object;Lkotlin/reflect/KType;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;Ljava/util/List;)Ljava/lang/String;
+	public static final fun readSource (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljava/lang/Object;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
+	public static synthetic fun readSource$default (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/lang/Object;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
+	public static synthetic fun readSource$default (Lorg/jetbrains/kotlinx/dataframe/DataRow$Companion;Ljava/lang/Object;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/DataRow;
+	public static synthetic fun readSource$default (Lorg/jetbrains/kotlinx/dataframe/api/CodeString$Companion;Ljava/lang/Object;Lkotlin/reflect/KType;Ljava/lang/String;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;Ljava/util/List;ILjava/lang/Object;)Ljava/lang/String;
+	public static synthetic fun readSource$default (Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema$Companion;Ljava/lang/Object;Lkotlin/reflect/KType;Lorg/jetbrains/kotlinx/dataframe/io/DataFrameReadOptions;Ljava/util/List;ILjava/lang/Object;)Lorg/jetbrains/kotlinx/dataframe/schema/DataFrameSchema;
+}
+
 public final class org/jetbrains/kotlinx/dataframe/io/GuessKt {
 	public static final fun read (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/io/File;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;
 	public static final fun read (Lorg/jetbrains/kotlinx/dataframe/DataFrame$Companion;Ljava/lang/String;Ljava/util/List;)Lorg/jetbrains/kotlinx/dataframe/DataFrame;

core/build.gradle.kts

@@ -59,6 +59,7 @@ dependencies {
 
     implementation(libs.commonsIo)
     implementation(libs.fastDoubleParser)
+    implementation(libs.tika)
 
     api(libs.kotlin.datetimeJvm)
     implementation(libs.kotlinpoet)
@@ -76,7 +77,6 @@ dependencies {
     testImplementation(libs.kotlin.scriptingJvm)
     testImplementation(libs.jsoup)
     testImplementation(libs.sl4jsimple)
-    testImplementation(projects.dataframeJson)
     testImplementation(libs.serialization.core)
     testImplementation(libs.serialization.json)
 
@@ -85,6 +85,13 @@ dependencies {
 
     // for samples.api
     testImplementation(projects.dataframeCsv)
+    testImplementation(projects.dataframeJson)
+    testImplementation(projects.dataframeArrow)
+    testImplementation(projects.dataframeExcel)
+//    testImplementation(projects.dataframeGeo)
+    testImplementation(projects.dataframeJdbc)
+    testImplementation(libs.h2db)
+    testImplementation(projects.dataframeOpenapiGenerator)
 }
 
 // Configure testJava16 dependencies to extend from test
@@ -308,6 +315,8 @@ tasks.withType<KotlinCompile> {
 
 tasks.test {
     maxHeapSize = "1g"
+    // Arrow's off-heap allocator needs deep reflection into java.nio.
+    jvmArgs("--add-opens", "java.base/java.nio=ALL-UNNAMED")
 }
 
 // Test task for Java 16+ language-specific tests

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/ParserOptions.kt

@@ -291,6 +291,8 @@ public interface GlobalParserOptions {
      * @see [addJavaDateTimePattern]
      */
     public var dateTimeLibrary: ParseDateTimeLibrary?
+
+    public var parseToDataFrameReadSource: Boolean
 }
 
 /** @include [GlobalParserOptions] */
@@ -410,6 +412,7 @@ public class ParserOptions(
     public val useFastDoubleParser: Boolean? = null,
     public val parseExperimentalUuid: Boolean? = null,
     public val parseExperimentalInstant: Boolean? = null,
+    public val parseToDataFrameReadSource: Boolean? = null,
 ) {
     public fun copy(
         locale: Locale? = this.locale,
@@ -419,6 +422,7 @@ public class ParserOptions(
         useFastDoubleParser: Boolean? = this.useFastDoubleParser,
         parseExperimentalUuid: Boolean? = this.parseExperimentalUuid,
         parseExperimentalInstant: Boolean? = this.parseExperimentalInstant,
+        parseToDataFrameReadSource: Boolean? = this.parseToDataFrameReadSource,
     ): ParserOptions =
         ParserOptions(
             locale = locale,
@@ -428,6 +432,7 @@ public class ParserOptions(
             useFastDoubleParser = useFastDoubleParser,
             parseExperimentalUuid = parseExperimentalUuid,
             parseExperimentalInstant = parseExperimentalInstant,
+            parseToDataFrameReadSource = parseToDataFrameReadSource,
         )
 
     override fun equals(other: Any?): Boolean {
@@ -439,6 +444,7 @@ public class ParserOptions(
         if (useFastDoubleParser != other.useFastDoubleParser) return false
         if (parseExperimentalUuid != other.parseExperimentalUuid) return false
         if (parseExperimentalInstant != other.parseExperimentalInstant) return false
+        if (parseToDataFrameReadSource != other.parseToDataFrameReadSource) return false
         if (locale != other.locale) return false
         if (dateTime != other.dateTime) return false
         if (nullStrings != other.nullStrings) return false
@@ -451,6 +457,8 @@ public class ParserOptions(
         var result = useFastDoubleParser?.hashCode() ?: 0
         result = 31 * result + (parseExperimentalUuid?.hashCode() ?: 0)
         result = 31 * result + (parseExperimentalInstant?.hashCode() ?: 0)
+        result = 31 * result + (parseToDataFrameReadSource?.hashCode() ?: 0)
+        result = 31 * result + (parseExperimentalInstant?.hashCode() ?: 0)
         result = 31 * result + (locale?.hashCode() ?: 0)
         result = 31 * result + (dateTime?.hashCode() ?: 0)
         result = 31 * result + (nullStrings?.hashCode() ?: 0)
@@ -459,7 +467,7 @@ public class ParserOptions(
     }
 
     override fun toString(): String =
-        "ParserOptions(locale=$locale, dateTimeParserOptions=$dateTime, nullStrings=$nullStrings, skipTypes=$skipTypes, useFastDoubleParser=$useFastDoubleParser, parseExperimentalUuid=$parseExperimentalUuid, parseExperimentalInstant=$parseExperimentalInstant)"
+        "ParserOptions(locale=$locale, dateTimeParserOptions=$dateTime, nullStrings=$nullStrings, skipTypes=$skipTypes, useFastDoubleParser=$useFastDoubleParser, parseExperimentalUuid=$parseExperimentalUuid, parseExperimentalInstant=$parseExperimentalInstant, parseToDataFrameReadSource=$parseToDataFrameReadSource)"
 
     // region deprecated constructors
 
@@ -518,6 +526,7 @@ public class ParserOptions(
         useFastDoubleParser: Boolean? = null,
         parseExperimentalUuid: Boolean? = null,
         parseExperimentalInstant: Boolean? = null,
+        parseToDataFrameReadSource: Boolean? = null,
     ) : this(
         locale = locale,
         dateTime = 0.run {
@@ -535,6 +544,7 @@ public class ParserOptions(
         useFastDoubleParser = useFastDoubleParser,
         parseExperimentalUuid = parseExperimentalUuid,
         parseExperimentalInstant = parseExperimentalInstant,
+        parseToDataFrameReadSource = parseToDataFrameReadSource,
     )
     // endregion
 }

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/api/generateCode.kt

@@ -297,6 +297,8 @@ public val NameNormalizer.Companion.default: NameNormalizer get() = NameNormaliz
 @RequiredByIntellijPlugin
 public value class CodeString(public val value: String) {
     override fun toString(): String = value
+
+    public companion object
 }
 
 @PublishedApi

core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/impl/Utils.kt

@@ -511,3 +511,19 @@ internal val KCallable<*>.columnName: String
         is KProperty<*> -> columnName
         else -> findAnnotation<ColumnName>()?.name ?: getterName
     }
+
+/**
+ * Similar to [Result.map], but allows a new [Result] to be returned if [this] was successful.
+ * This result will be unpacked (so you won't get `Result<Result<U>>`).
+ */
+@PublishedApi
+internal inline fun <T, U> Result<T>.flatMap(function: (T) -> Result<U>): Result<U> =
+    fold(
+        onSuccess = { function(it) },
+        onFailure = { Result.failure(it) },
+    )
+
+internal fun <T> Result<Result<T>>.flatten(): Result<T> = this.flatMap { it }
+
+internal fun <T : Any> T?.toResult(exception: Throwable = NullPointerException()) =
+    this?.let { Result.success(it) } ?: Result.failure(exception)