DuckDB MultiFile Reader

benchmarks/duckdb-bench/src/lib.rs

@@ -90,7 +90,7 @@ impl DuckClient {
         let connection = db.connect()?;
         vortex_duckdb::initialize(&db)?;
 
-        // Enable Parquet metadata cache for all benchmark runs.
+        // Enable metadata caches for all benchmark runs.
         //
         // `parquet_metadata_cache` is an extension-specific option that's
         // only available after the Parquet extension is loaded. The Parquet
@@ -100,6 +100,7 @@ impl DuckClient {
         // "Invalid Input Error: The following options were not recognized:
         // parquet_metadata_cache" when running DuckDB in debug mode.
         connection.query("SET parquet_metadata_cache = true")?;
+        connection.query("SET vortex_metadata_cache = true")?;
 
         Ok((db, connection))
     }

vortex-duckdb/build.rs

@@ -20,7 +20,7 @@ const DUCKDB_SOURCE_COMMIT_URL: &str = "https://github.com/duckdb/duckdb/archive
 
 const BUILD_ARTIFACTS: [&str; 3] = ["libduckdb.dylib", "libduckdb.so", "libduckdb_static.a"];
 
-const SOURCE_FILES: [&str; 17] = [
+const SOURCE_FILES: [&str; 19] = [
     "cpp/client_context.cpp",
     "cpp/config.cpp",
     "cpp/copy_function.cpp",
@@ -30,6 +30,8 @@ const SOURCE_FILES: [&str; 17] = [
     "cpp/expr.cpp",
     "cpp/file_system.cpp",
     "cpp/logical_type.cpp",
+    "cpp/multi_file_function.cpp",
+    "cpp/object_cache.cpp",
     "cpp/replacement_scan.cpp",
     "cpp/reusable_dict.cpp",
     "cpp/scalar_function.cpp",

vortex-duckdb/cpp/file_system.cpp

@@ -12,11 +12,21 @@ DUCKDB_INCLUDES_BEGIN
 #include <duckdb/main/client_context.hpp>
 DUCKDB_INCLUDES_END
 
+#include <memory>
 #include <utility>
 
 using namespace duckdb;
 using vortex::SetError;
 
+struct duckdb_vx_file_handle_ {
+    explicit duckdb_vx_file_handle_(shared_ptr<ClientContext> context, unique_ptr<FileHandle> handle)
+        : context(std::move(context)), handle(std::move(handle)) {
+    }
+
+    shared_ptr<ClientContext> context;
+    unique_ptr<FileHandle> handle;
+};
+
 extern "C" duckdb_vx_file_handle
 duckdb_vx_fs_open(duckdb_client_context ctx, const char *path, duckdb_vx_error *error_out) {
     if (!ctx || !path) {
@@ -29,7 +39,7 @@ duckdb_vx_fs_open(duckdb_client_context ctx, const char *path, duckdb_vx_error *
     try {
         auto &fs = FileSystem::GetFileSystem(*client_context);
         auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_PARALLEL_ACCESS);
-        return reinterpret_cast<duckdb_vx_file_handle>(handle.release());
+        return new duckdb_vx_file_handle_(client_context->shared_from_this(), std::move(handle));
     } catch (const std::exception &e) {
         SetError(error_out, e.what());
         return nullptr;
@@ -50,7 +60,7 @@ duckdb_vx_fs_create(duckdb_client_context ctx, const char *path, duckdb_vx_error
     try {
         auto &fs = FileSystem::GetFileSystem(*client_context);
         auto handle = fs.OpenFile(path, flags);
-        return reinterpret_cast<duckdb_vx_file_handle>(handle.release());
+        return new duckdb_vx_file_handle_(client_context->shared_from_this(), std::move(handle));
     } catch (const std::exception &e) {
         SetError(error_out, e.what());
         return nullptr;
@@ -59,7 +69,7 @@ duckdb_vx_fs_create(duckdb_client_context ctx, const char *path, duckdb_vx_error
 
 extern "C" void duckdb_vx_fs_close(duckdb_vx_file_handle *handle) {
     if (handle && *handle) {
-        delete reinterpret_cast<FileHandle *>(std::exchange(*handle, nullptr));
+        delete std::exchange(*handle, nullptr);
     }
 }
 
@@ -70,7 +80,7 @@ duckdb_vx_fs_get_size(duckdb_vx_file_handle handle, idx_t *size_out, duckdb_vx_e
     }
 
     try {
-        *size_out = reinterpret_cast<FileHandle *>(handle)->GetFileSize();
+        *size_out = handle->handle->GetFileSize();
     } catch (const std::exception &e) {
         return SetError(error_out, e.what());
     }
@@ -88,7 +98,7 @@ extern "C" duckdb_state duckdb_vx_fs_read(duckdb_vx_file_handle handle,
     }
 
     try {
-        reinterpret_cast<FileHandle *>(handle)->Read(buffer, len, offset);
+        handle->handle->Read(buffer, len, offset);
         *out_len = len;
     } catch (const std::exception &e) {
         return SetError(error_out, e.what());
@@ -107,7 +117,7 @@ extern "C" duckdb_state duckdb_vx_fs_write(duckdb_vx_file_handle handle,
     }
 
     try {
-        reinterpret_cast<FileHandle *>(handle)->Write(QueryContext(), buffer, len, offset);
+        handle->handle->Write(QueryContext(), buffer, len, offset);
         *out_len = len;
     } catch (const std::exception &e) {
         return SetError(error_out, e.what());
@@ -144,7 +154,7 @@ extern "C" duckdb_state duckdb_vx_fs_sync(duckdb_vx_file_handle handle, duckdb_v
     }
 
     try {
-        reinterpret_cast<FileHandle *>(handle)->Sync();
+        handle->handle->Sync();
     } catch (const std::exception &e) {
         return SetError(error_out, e.what());
     }

vortex-duckdb/cpp/include/duckdb_vx.h

@@ -12,6 +12,8 @@
 #include "duckdb_vx/expr.h"
 #include "duckdb_vx/file_system.h"
 #include "duckdb_vx/logical_type.h"
+#include "duckdb_vx/multi_file_function.h"
+#include "duckdb_vx/object_cache.h"
 #include "duckdb_vx/reusable_dict.h"
 #include "duckdb_vx/replacement_scan.h"
 #include "duckdb_vx/scalar_function.h"