feat(kernel): wire ABI-version check, mTLS, and CloudFetch toggle options

connector.go

@@ -45,14 +45,16 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
 	if c.cfg.UseKernel {
 		be, err = newKernelBackend(ctx, c.cfg)
 	} else {
-		// The experimental WithKernel* TLS options have no Thrift-path equivalent —
-		// reject them loudly rather than silently ignore, so a caller who sets a
-		// trusted-CA bundle / an independent hostname skip and forgets WithUseKernel
-		// learns the option had no effect instead of connecting with a
-		// weaker-than-intended (or unconfigured) TLS trust store.
+		// The experimental WithKernel* options have no Thrift-path equivalent —
+		// reject them loudly rather than silently ignore, so a caller who sets one
+		// (a trusted-CA bundle, an independent hostname skip, an mTLS client
+		// certificate, or a CloudFetch toggle) and forgets WithUseKernel learns the
+		// option had no effect instead of connecting with a weaker-than-intended
+		// (or unconfigured) setup.
 		if c.cfg.KernelExperimental != nil {
 			return nil, fmt.Errorf("databricks: a WithKernel* option "+
-				"(WithKernelTrustedCerts / WithKernelSkipHostnameVerify) %w; "+
+				"(WithKernelTrustedCerts / WithKernelSkipHostnameVerify / "+
+				"WithKernelClientCertificate / WithKernelCloudFetch) %w; "+
 				"add WithUseKernel(true) or remove it", dbsqlerr.ErrRequiresKernelBackend)
 		}
 		be, err = thrift.New(ctx, c.cfg, c.client)
@@ -603,3 +605,44 @@ func WithKernelSkipHostnameVerify() ConnOption {
 		kernelExperimental(c).TLSSkipHostnameVerify = true
 	}
 }
+
+// WithKernelClientCertificate configures a client certificate + private key for
+// mutual TLS (mTLS) on the kernel backend. cert is a PEM leaf certificate,
+// optionally followed by its intermediate chain; key is the matching PEM private
+// key (use PKCS#8 for portability). Both halves are required together — they map
+// to the single paired kernel_session_config_set_tls_client_certificate, so
+// cert-without-key is unrepresentable. The private key is never logged.
+//
+// EXPERIMENTAL, kernel-only: the default (Thrift) backend rejects this at connect.
+func WithKernelClientCertificate(cert, key []byte) ConnOption {
+	return func(c *config.Config) {
+		// Copy defensively (matching KernelExperimentalConfig.DeepCopy) so a
+		// caller mutating the buffers between NewConnector and Connect can't change
+		// the identity out from under us.
+		k := kernelExperimental(c)
+		if len(cert) > 0 {
+			k.TLSClientCertPEM = append([]byte(nil), cert...)
+		} else {
+			k.TLSClientCertPEM = cert
+		}
+		if len(key) > 0 {
+			k.TLSClientKeyPEM = append([]byte(nil), key...)
+		} else {
+			k.TLSClientKeyPEM = key
+		}
+	}
+}
+
+// WithKernelCloudFetch toggles CloudFetch (cloud-storage-backed download of large
+// result sets) on the kernel backend. Passing false forces the inline-Arrow path
+// for all result sizes. Unlike the backend-neutral WithCloudFetch (a plain bool
+// whose unset state is indistinguishable from false), this is tri-state: not
+// calling it leaves the kernel default (CloudFetch on), and it maps to
+// kernel_session_config_set_cloudfetch_enabled only when set.
+//
+// EXPERIMENTAL, kernel-only: the default (Thrift) backend rejects this at connect.
+func WithKernelCloudFetch(enabled bool) ConnOption {
+	return func(c *config.Config) {
+		kernelExperimental(c).CloudFetchEnabled = &enabled
+	}
+}

doc.go

@@ -228,15 +228,22 @@ the C ABI can't interrupt session open mid-call, a connection-context deadline i
 honored during that window. Use PAT or OAuth M2M for headless / deadline-bound
 connects.
 
-Experimental kernel-only TLS options (rejected by the default backend; the
-WithKernel* prefix marks them experimental):
+Experimental kernel-only options (rejected by the default backend; the WithKernel*
+prefix marks them experimental):
   - WithKernelTrustedCerts(pem) adds a PEM CA bundle on top of the system roots (for
     a re-signing proxy or on-prem CA). Required because the kernel's TLS stack does
     not read SSL_CERT_FILE.
   - WithKernelSkipHostnameVerify() skips only the hostname check while keeping chain
     validation (finer-grained than WithSkipTLSHostVerify).
-
-Setting either without WithUseKernel fails Connect with an error wrapping the
+  - WithKernelClientCertificate(cert, key) configures a client certificate + private
+    key for mutual TLS (mTLS). Both PEM halves are required together; the key is
+    never logged.
+  - WithKernelCloudFetch(enabled) toggles CloudFetch. Tri-state: leaving it unset
+    keeps the kernel default (on); false forces the inline-Arrow path for all result
+    sizes. Distinct from the backend-neutral WithCloudFetch, whose plain-bool unset
+    state can't be told apart from false.
+
+Setting any of these without WithUseKernel fails Connect with an error wrapping the
 sentinel ErrRequiresKernelBackend, detectable with errors.Is.
 
 Features above the backend seam are inherited unchanged: the database/sql connection

internal/backend/kernel/backend.go

@@ -54,6 +54,19 @@ type Config struct {
 	TLSTrustedCertsPEM    []byte
 	TLSSkipHostnameVerify bool
 
+	// TLSClientCertPEM / TLSClientKeyPEM are the mTLS client cert (+ optional
+	// chain) and matching private key (from WithKernelClientCertificate). They
+	// travel as a pair — both set or both empty — and are forwarded via the single
+	// paired kernel_session_config_set_tls_client_certificate. The key is never
+	// logged.
+	TLSClientCertPEM []byte
+	TLSClientKeyPEM  []byte
+
+	// CloudFetchEnabled toggles CloudFetch (from WithKernelCloudFetch). Tri-state:
+	// nil keeps the kernel default (on); a non-nil value is forwarded via
+	// kernel_session_config_set_cloudfetch_enabled.
+	CloudFetchEnabled *bool
+
 	// ProxyURL configures an HTTP proxy, already resolved for this endpoint from
 	// the same HTTP(S)_PROXY / NO_PROXY environment the Thrift path uses (NO_PROXY
 	// is applied during resolution). Empty leaves the kernel on a direct
@@ -105,6 +118,13 @@ func (k *KernelBackend) OpenSession(ctx context.Context) error {
 	if err := ctx.Err(); err != nil {
 		return err
 	}
+	// Verify the linked kernel library's C-ABI version matches the header the
+	// driver compiled against before making any other kernel call — a mismatch
+	// means every status code / error struct we read afterward could be
+	// misinterpreted. Runs once per process; cheap and cached thereafter.
+	if err := checkABIVersion(); err != nil {
+		return err
+	}
 	initKernelLogging()
 	klogCtx(ctx, "OpenSession host=%s httpPath=%s warehouse=%s", k.cfg.Host, k.cfg.HTTPPath, k.cfg.WarehouseID)
 
@@ -189,6 +209,19 @@ func (k *KernelBackend) OpenSession(ctx context.Context) error {
 		}
 	}
 
+	// Experimental kernel-only CloudFetch toggle (WithKernelCloudFetch). Tri-state:
+	// only set it when the caller did, so an unset value leaves the kernel default
+	// (CloudFetch on). Do NOT route this through set_session_conf — the kernel owns
+	// the can_cloud_download conf and the server rejects it as not user-settable.
+	if k.cfg.CloudFetchEnabled != nil {
+		enabled := C.bool(*k.cfg.CloudFetchEnabled)
+		if err := call(func() C.KernelStatusCode {
+			return C.kernel_session_config_set_cloudfetch_enabled(cfg, enabled)
+		}); err != nil {
+			return fmt.Errorf("kernel: set_cloudfetch_enabled: %w", toConnError(err))
+		}
+	}
+
 	// Session confs (STATEMENT_TIMEOUT, QUERY_TAGS, TIMEZONE, …) — the same map
 	// the Thrift backend forwards, applied one key at a time.
 	for key, val := range k.cfg.SessionConf {
@@ -261,10 +294,9 @@ func (k *KernelBackend) OpenSession(ctx context.Context) error {
 }
 
 // applyKernelTLS forwards the experimental kernel-only TLS knobs to the session
-// config: a custom CA bundle and an independent hostname-skip. Each is a no-op
-// when its field is unset, so this is safe to call unconditionally. (mTLS client
-// cert/key is a separate follow-up — it needs a kernel C-ABI setter that is not
-// yet on kernel main.)
+// config: a custom CA bundle, an independent hostname-skip, and an mTLS client
+// certificate + key. Each is a no-op when its field is unset, so this is safe to
+// call unconditionally.
 func (k *KernelBackend) applyKernelTLS(cfg *C.KernelSessionConfig) error {
 	if len(k.cfg.TLSTrustedCertsPEM) > 0 {
 		ca := newCBytes(k.cfg.TLSTrustedCertsPEM)
@@ -282,6 +314,20 @@ func (k *KernelBackend) applyKernelTLS(cfg *C.KernelSessionConfig) error {
 			return fmt.Errorf("kernel: set_tls_skip_hostname_verification: %w", toConnError(err))
 		}
 	}
+	// mTLS client identity. The cert and key travel as a pair (WithKernelClientCertificate
+	// sets both or neither), forwarded via the single paired setter; checking the
+	// cert is enough. The key bytes go to owned Rust memory and are never logged.
+	if len(k.cfg.TLSClientCertPEM) > 0 {
+		cert := newCBytes(k.cfg.TLSClientCertPEM)
+		defer cert.free()
+		key := newCBytes(k.cfg.TLSClientKeyPEM)
+		defer key.free()
+		if err := call(func() C.KernelStatusCode {
+			return C.kernel_session_config_set_tls_client_certificate(cfg, cert.ptr, cert.len, key.ptr, key.len)
+		}); err != nil {
+			return fmt.Errorf("kernel: set_tls_client_certificate: %w", toConnError(err))
+		}
+	}
 	return nil
 }
 
@@ -393,14 +439,17 @@ func (k *KernelBackend) runNamespaceStmt(ctx context.Context, sql string) error
 	return closeErr
 }
 
-// CloseSession tears down the server-side session. Best-effort: the kernel's
-// close is async (see the C header), so an error is logged, not hard-failed.
+// CloseSession tears down the server-side session. Best-effort: kernel_session
+// _close initiates the delete without waiting (see the C header), so it does not
+// block and an error is logged, not hard-failed.
 //
-// Deferred (tracked): this ignores ctx and blocks in the synchronous call() until
-// kernel_session_close returns, with no deadline — a stalled kernel-side close
-// (e.g. a shutdown-time network partition) can block database/sql pool cleanup.
-// A bounded close needs either a kernel_session_close_blocking with a deadline or
-// a Go-side watchdog; grouped with the kernel C-ABI follow-ups.
+// Stays fire-and-forget deliberately. The C ABI also offers
+// kernel_session_close_blocking (awaits DeleteSession, for Python/Node parity),
+// but adopting it here would make close a blocking network round-trip with no
+// deadline honored — a stalled close (shutdown-time network partition) would then
+// block database/sql pool cleanup. Swapping to it is grouped with the
+// cancellable-close follow-up so the blocking close ships with a ctx bridge; until
+// then fire-and-forget is the safer default.
 func (k *KernelBackend) CloseSession(ctx context.Context) error {
 	if k.session == nil {
 		return nil

internal/backend/kernel/cgo.go

@@ -153,6 +153,44 @@ func initKernelLogging() {
 	})
 }
 
+// abiCheckOnce ensures the ABI-version handshake runs at most once per process
+// (the linked library can't change under us mid-run), and abiCheckErr caches its
+// outcome so every OpenSession after the first returns the same verdict cheaply.
+var (
+	abiCheckOnce sync.Once
+	abiCheckErr  error
+)
+
+// checkABIVersion verifies the C-ABI version compiled into the linked kernel
+// library matches the version the driver's header declares. The build-time
+// status-code assertions in this file catch header-vs-source drift at compile
+// time; this catches the runtime hazard those can't see — a driver built against
+// one header linked against a differently-built prebuilt .a (the eventual
+// download-a-prebuilt-lib distribution path). A mismatch means the
+// KernelStatusCode enum / KernelError struct layout the driver reads may not
+// match what the library writes, so we refuse to open rather than silently
+// misread status codes / error fields. Runs once; the result is cached.
+func checkABIVersion() error {
+	abiCheckOnce.Do(func() {
+		got, want := abiVersions()
+		klog("checkABIVersion got=%d want=%d", got, want)
+		if got != want {
+			abiCheckErr = fmt.Errorf("databricks: kernel ABI version mismatch: linked library reports %d, "+
+				"driver header expects %d; rebuild the kernel static lib and header together (make kernel-lib)", got, want)
+		}
+	})
+	return abiCheckErr
+}
+
+// abiVersions returns (library version, header version): the C-ABI version
+// compiled into the linked kernel library and the version the driver's header
+// declares. A production-side seam so tests can assert the handshake without
+// putting cgo in a _test.go file (which Go forbids); not used in production
+// beyond checkABIVersion.
+func abiVersions() (got, want uint32) {
+	return uint32(C.kernel_abi_version()), uint32(C.DATABRICKS_KERNEL_ABI_VERSION)
+}
+
 // call runs a fallible kernel entry point and, on a non-Success status, reads
 // the kernel's thread-local last error into a Go error.
 //

internal/backend/kernel/kernel_test.go

@@ -58,13 +58,20 @@ func TestSetAuthByMode(t *testing.T) {
 // drifted.
 func TestSetKernelTLS(t *testing.T) {
 	ca := []byte("-----BEGIN CERTIFICATE-----\nca\n-----END CERTIFICATE-----\n")
+	cert := []byte("-----BEGIN CERTIFICATE-----\nleaf\n-----END CERTIFICATE-----\n")
+	// Assemble the key's PEM marker at runtime so the repo's secret scanner
+	// doesn't flag a literal BEGIN-PRIVATE-KEY (the bytes are opaque to the
+	// marshalling path under test).
+	key := []byte("-----BEGIN " + "PRIVATE" + " KEY-----\nkey\n-----END PRIVATE KEY-----\n")
 	cases := []struct {
 		name string
 		cfg  Config
 	}{
 		{"trusted certs only", Config{TLSTrustedCertsPEM: ca}},
 		{"skip hostname only", Config{TLSSkipHostnameVerify: true}},
 		{"both together", Config{TLSTrustedCertsPEM: ca, TLSSkipHostnameVerify: true}},
+		{"client certificate (mTLS)", Config{TLSClientCertPEM: cert, TLSClientKeyPEM: key}},
+		{"all together", Config{TLSTrustedCertsPEM: ca, TLSSkipHostnameVerify: true, TLSClientCertPEM: cert, TLSClientKeyPEM: key}},
 		{"none (no-op)", Config{}},
 	}
 	for _, c := range cases {
@@ -76,6 +83,23 @@ func TestSetKernelTLS(t *testing.T) {
 	}
 }
 
+// TestABIVersionMatches asserts the linked kernel library's C-ABI version
+// matches the header the driver compiled against — the same handshake
+// checkABIVersion runs at connect. It exercises the real cgo symbols
+// (kernel_abi_version + the DATABRICKS_KERNEL_ABI_VERSION macro), so a stale
+// .a-vs-header pairing fails here at test time rather than misreading status
+// codes at runtime. It also asserts checkABIVersion() returns nil for the
+// matched pair the test binary links.
+func TestABIVersionMatches(t *testing.T) {
+	got, want := abiVersions()
+	if got != want {
+		t.Fatalf("kernel_abi_version() = %d, header DATABRICKS_KERNEL_ABI_VERSION = %d", got, want)
+	}
+	if err := checkABIVersion(); err != nil {
+		t.Errorf("checkABIVersion() = %v, want nil for the linked (matching) library", err)
+	}
+}
+
 // TestKernelLogLevel and TestResolveKernelLogArg — the pure level-resolution tests —
 // live in the untagged logging_level_test.go so they run under CGO_ENABLED=0. The
 // tests below exercise klog/klogCtx and so need the cgo build.

internal/config/config.go

@@ -76,6 +76,19 @@ type KernelExperimentalConfig struct {
 	// of the blanket InsecureSkipVerify (which relaxes both chain and hostname).
 	// Maps to kernel_session_config_set_tls_skip_hostname_verification.
 	TLSSkipHostnameVerify bool
+	// TLSClientCertPEM / TLSClientKeyPEM are the client leaf cert (+ optional
+	// chain) and matching private key for mutual TLS (mTLS). They travel as a
+	// pair — both set or both nil (WithKernelClientCertificate is the only setter
+	// and takes both) — and map to the single paired
+	// kernel_session_config_set_tls_client_certificate. The key is never logged.
+	TLSClientCertPEM []byte
+	TLSClientKeyPEM  []byte
+	// CloudFetchEnabled toggles CloudFetch on the kernel path. Tri-state: nil
+	// keeps the kernel default (on); a non-nil *false forces the inline-Arrow
+	// path. A *bool (not a plain bool) so "unset" is distinguishable from
+	// "explicitly false" — maps to kernel_session_config_set_cloudfetch_enabled.
+	// Deliberately NOT the plain-bool WithCloudFetch (which can't express unset).
+	CloudFetchEnabled *bool
 }
 
 // DeepCopy returns a deep copy of the experimental config, or nil for a nil
@@ -89,6 +102,16 @@ func (k *KernelExperimentalConfig) DeepCopy() *KernelExperimentalConfig {
 	if k.TLSTrustedCertsPEM != nil {
 		cp.TLSTrustedCertsPEM = append([]byte(nil), k.TLSTrustedCertsPEM...)
 	}
+	if k.TLSClientCertPEM != nil {
+		cp.TLSClientCertPEM = append([]byte(nil), k.TLSClientCertPEM...)
+	}
+	if k.TLSClientKeyPEM != nil {
+		cp.TLSClientKeyPEM = append([]byte(nil), k.TLSClientKeyPEM...)
+	}
+	if k.CloudFetchEnabled != nil {
+		v := *k.CloudFetchEnabled
+		cp.CloudFetchEnabled = &v
+	}
 	return cp
 }
 

kernel_backend.go

@@ -47,13 +47,17 @@ func newKernelBackend(_ context.Context, cfg *config.Config) (backend.Backend, e
 	if cfg.TLSConfig != nil && cfg.TLSConfig.InsecureSkipVerify {
 		kc.TLSSkipVerify = true
 	}
-	// Experimental kernel-only TLS knobs (WithKernelTrustedCerts /
-	// WithKernelSkipHostnameVerify), if any. These have no Thrift-path equivalent
-	// (the connector rejects them on that path) and are forwarded verbatim to the
+	// Experimental kernel-only knobs (WithKernelTrustedCerts /
+	// WithKernelSkipHostnameVerify / WithKernelClientCertificate /
+	// WithKernelCloudFetch), if any. These have no Thrift-path equivalent (the
+	// connector rejects them on that path) and are forwarded verbatim to the
 	// kernel C ABI in OpenSession.
 	if ke := cfg.KernelExperimental; ke != nil {
 		kc.TLSTrustedCertsPEM = ke.TLSTrustedCertsPEM
 		kc.TLSSkipHostnameVerify = ke.TLSSkipHostnameVerify
+		kc.TLSClientCertPEM = ke.TLSClientCertPEM
+		kc.TLSClientKeyPEM = ke.TLSClientKeyPEM
+		kc.CloudFetchEnabled = ke.CloudFetchEnabled
 	}
 	// Proxy: the Thrift path uses http.ProxyFromEnvironment; mirror it by reading
 	// the same HTTP(S)_PROXY / NO_PROXY environment for the kernel.

kernel_config.go

@@ -86,6 +86,20 @@ func validateKernelConfig(cfg *config.Config) (kernel.Auth, error) {
 		return kernel.Auth{}, fmt.Errorf("databricks: disabling retries via WithRetries is %w "+
 			"(the kernel retries internally); omit it or use the default (Thrift) backend", dbsqlerr.ErrNotSupportedByKernel)
 	}
+	// mTLS client identity (WithKernelClientCertificate) must be a complete pair:
+	// mTLS needs both a client cert and its private key. Reject an unpaired
+	// credential loudly at connect — otherwise applyKernelTLS (which gates the mTLS
+	// forward on the cert being present) would silently drop a lone key and connect
+	// with no client identity, contradicting the "both halves required together /
+	// never silently dropped" contract. The cert-without-key case would fail at the
+	// kernel setter (it rejects a null/empty key), but validating both directions
+	// here surfaces the error earlier and with a clearer message.
+	if ke := cfg.KernelExperimental; ke != nil {
+		if (len(ke.TLSClientCertPEM) == 0) != (len(ke.TLSClientKeyPEM) == 0) {
+			return kernel.Auth{}, fmt.Errorf("databricks: WithKernelClientCertificate requires " +
+				"both a client certificate and a private key; one was empty")
+		}
+	}
 	return kauth, nil
 }