diff --git a/benchmarks/datafusion-bench/src/lib.rs b/benchmarks/datafusion-bench/src/lib.rs index fdda58188e7..1100d38d24e 100644 --- a/benchmarks/datafusion-bench/src/lib.rs +++ b/benchmarks/datafusion-bench/src/lib.rs @@ -122,9 +122,10 @@ pub fn format_to_df_format(format: Format) -> Arc { } fn vortex_table_options() -> VortexTableOptions { - VortexTableOptions { - projection_pushdown: true, - predicate_pushdown: true, - ..Default::default() - } + let mut opts = VortexTableOptions::default(); + + opts.predicate_pushdown = true; + opts.predicate_pushdown = true; + + opts } diff --git a/vortex-datafusion/src/persistent/format.rs b/vortex-datafusion/src/persistent/format.rs index a0d49e6105a..c139fe2767f 100644 --- a/vortex-datafusion/src/persistent/format.rs +++ b/vortex-datafusion/src/persistent/format.rs @@ -15,8 +15,9 @@ use datafusion_common::GetExt; use datafusion_common::Result as DFResult; use datafusion_common::ScalarValue as DFScalarValue; use datafusion_common::Statistics; +use datafusion_common::config::ConfigExtension; use datafusion_common::config::ConfigField; -use datafusion_common::config_namespace; +use datafusion_common::extensions_options; use datafusion_common::internal_datafusion_err; use datafusion_common::not_impl_err; use datafusion_common::parsers::CompressionTypeVariant; @@ -132,28 +133,62 @@ impl Debug for VortexFormat { } } -config_namespace! { +extensions_options! { /// Options to configure [`VortexFormat`] and [`VortexSource`]. /// - /// These options are usually set on a [`VortexFormatFactory`] and inherited - /// by the `VortexFormat` / `VortexSource` instances created for individual - /// tables. + /// The API follows DataFusion's built-in Parquet and JSON format factories: + /// a format factory may carry customized defaults, the session may carry + /// format defaults, and `CREATE EXTERNAL TABLE ... OPTIONS(...)` can + /// override individual fields for one table. + /// + /// [`FileFormatFactory::create`] builds the `VortexTableOptions` copied into + /// each [`VortexFormat`] as follows: + /// + /// 1. If the factory has explicit options from + /// [`VortexFormatFactory::with_options`] or + /// [`VortexFormatFactory::new_with_options`], start from that complete + /// `VortexTableOptions` value. This matches + /// [`ParquetFormatFactory::new_with_options`] and + /// [`JsonFormatFactory::new_with_options`]: factory options replace + /// session defaults; they are not merged with them field-by-field. + /// 2. If the factory does not have explicit options, read the session's + /// `vortex` extension at the time `create` is called. This is the value + /// changed by `SET vortex.