diff --git a/web-common/src/features/add-data/form/ConnectorForm.svelte b/web-common/src/features/add-data/form/ConnectorForm.svelte
index 0f02f31d4144..d3572261fbb4 100644
--- a/web-common/src/features/add-data/form/ConnectorForm.svelte
+++ b/web-common/src/features/add-data/form/ConnectorForm.svelte
@@ -1,7 +1,7 @@
-
-{#if connectorName != null && cachedEnvBlob != null && cachedFormValues != null}
-
-{/if}
diff --git a/web-common/src/features/add-data/form/ImportTableForm.svelte b/web-common/src/features/add-data/form/ImportTableForm.svelte
index 54eb3f360e20..1ace8ef1cbc9 100644
--- a/web-common/src/features/add-data/form/ImportTableForm.svelte
+++ b/web-common/src/features/add-data/form/ImportTableForm.svelte
@@ -30,6 +30,9 @@
} from "@rilldata/web-common/features/add-data/manager/steps/utils.ts";
import DatabaseExplorer from "@rilldata/web-common/features/connectors/explorer/DatabaseExplorer.svelte";
import { ConnectorExplorerStore } from "@rilldata/web-common/features/connectors/explorer/connector-explorer-store.ts";
+ import { getEnvFileStore } from "@rilldata/web-common/features/env-management/env-file-store.ts";
+ import { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
+ import { getConnectorSchema } from "@rilldata/web-common/features/sources/modal/connector-schemas.ts";
export let config: AddDataConfig;
export let step: ExploreConnectorStep;
@@ -40,6 +43,13 @@
const runtimeClient = useRuntimeClient();
+ const envStore = getEnvFileStore();
+ const envEditSession = new EnvEditSession(
+ envStore,
+ step.connector,
+ getConnectorSchema(step.schema),
+ );
+
$: connectorDriverQuery = getAnalyzedConnectorByName(
runtimeClient,
step.connector,
@@ -124,7 +134,7 @@
connector: step.connector,
importFrom,
importTo: generateImportToConfig(importFrom),
- envBlob: null,
+ envEditSession,
} satisfies ImportStepConfig);
},
validationMethod: "onsubmit",
diff --git a/web-common/src/features/add-data/form/SourceForm.svelte b/web-common/src/features/add-data/form/SourceForm.svelte
index 0f16b7467004..ba6684f877ef 100644
--- a/web-common/src/features/add-data/form/SourceForm.svelte
+++ b/web-common/src/features/add-data/form/SourceForm.svelte
@@ -6,12 +6,10 @@
} from "@rilldata/web-common/runtime-client";
import { getConnectorSchema } from "@rilldata/web-common/features/sources/modal/connector-schemas.ts";
import { onMount } from "svelte";
- import { getSourceYamlPreview } from "./yaml-preview.ts";
+ import { getSourceYAML } from "./connector-source-yaml-generator.ts";
import AddDataFormStructure from "@rilldata/web-common/features/add-data/form/AddDataFormStructure.svelte";
import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryClient.ts";
import { prepareSourceFormData } from "@rilldata/web-common/features/sources/sourceUtils.ts";
- import { getSchemaSecretKeys } from "@rilldata/web-common/features/templates/schema-utils.ts";
- import { updateDotEnvWithSecrets } from "@rilldata/web-common/features/connectors/code-utils.ts";
import {
type AddDataConfig,
type CreateModelStep,
@@ -30,9 +28,10 @@
import {
getConnectorDriverForSchema,
getImportStepsForSource,
- maybeGetEnvContent,
} from "@rilldata/web-common/features/add-data/manager/steps/utils.ts";
import { maybeInitProject } from "@rilldata/web-common/features/add-data/manager/steps/connector.ts";
+ import { getEnvFileStore } from "@rilldata/web-common/features/env-management/env-file-store.ts";
+ import { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
import { setSubmitError } from "@rilldata/web-common/features/add-data/form/errors.ts";
import type { AddDataStateManager } from "@rilldata/web-common/features/add-data/manager/AddDataStateManager.svelte.ts";
import { setError, type SuperValidated } from "sveltekit-superforms";
@@ -52,15 +51,20 @@
$connectorDriverQuery.data?.driver ??
getConnectorDriverForSchema(step.schema);
+ const envStore = getEnvFileStore();
+ const envEditSession = new EnvEditSession(
+ envStore,
+ step.connector,
+ getConnectorSchema(step.schema),
+ );
+
const importSteps = getImportStepsForSource(config);
// Capture .env blob ONCE on mount for consistent conflict detection in YAML preview.
// This prevents the preview from updating when Test and Connect writes to .env.
// Use null to indicate "not yet loaded" vs "" for "loaded but empty"
- let existingEnvBlob: string | null = null;
let defaultOLAP = "duckdb";
onMount(async () => {
- existingEnvBlob = await maybeGetEnvContent();
try {
const runtimeInstance = await queryClient.fetchQuery({
queryKey: getRuntimeServiceGetInstanceQueryKey(
@@ -95,12 +99,12 @@
$: schema = getConnectorSchema(step.schema);
$: yamlPreview = connectorDriver
- ? getSourceYamlPreview({
+ ? getSourceYAML({
connectorName: step.connector,
connector: connectorDriver,
formValues: $form,
schema,
- existingEnvBlob,
+ envEditSession,
outputConnector: defaultOLAP,
})
: "";
@@ -117,27 +121,6 @@
await maybeInitProject(runtimeClient);
- const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData(
- connectorDriver,
- formValues,
- { connectorInstanceName: step.connector },
- );
- const schema = getConnectorSchema(rewrittenConnector.name ?? "");
- const schemaSecretKeys = schema
- ? getSchemaSecretKeys(schema, { step: "source" })
- : [];
-
- // Create or update the `.env` file
- const { newBlob } = await updateDotEnvWithSecrets(
- runtimeClient,
- queryClient,
- rewrittenConnector,
- rewrittenFormValues,
- {
- secretKeys: schemaSecretKeys,
- },
- );
-
if (formValues.file) {
// TODO: support multiple files upload
const firstFile = formValues.file[0];
@@ -154,12 +137,20 @@
throw e; // rethrow so that global error handler is triggered
}
}
- const yaml = getSourceYamlPreview({
+
+ const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData(
+ connectorDriver,
+ formValues,
+ { connectorInstanceName: step.connector },
+ );
+ const schema = getConnectorSchema(rewrittenConnector.name ?? "");
+
+ const yaml = getSourceYAML({
connectorName: step.connector,
connector: connectorDriver,
- formValues,
+ formValues: rewrittenFormValues,
schema,
- existingEnvBlob,
+ envEditSession,
outputConnector: defaultOLAP,
});
@@ -176,7 +167,7 @@
importFrom,
formValues.name as string | undefined,
),
- envBlob: newBlob,
+ envEditSession,
} satisfies ImportStepConfig;
onSubmit(importConfig);
diff --git a/web-common/src/features/add-data/form/connector-source-yaml-generator.spec.ts b/web-common/src/features/add-data/form/connector-source-yaml-generator.spec.ts
new file mode 100644
index 000000000000..39d1a12fed69
--- /dev/null
+++ b/web-common/src/features/add-data/form/connector-source-yaml-generator.spec.ts
@@ -0,0 +1,701 @@
+import { describe, expect, it } from "vitest";
+import { makeTestEnvEditSession } from "@rilldata/web-common/features/env-management/test/test-env-store.ts";
+import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client";
+import { getConnectorYAML } from "@rilldata/web-common/features/add-data/form/connector-source-yaml-generator.ts";
+import { clickhouseSchema } from "@rilldata/web-common/features/templates/schemas/clickhouse.ts";
+import { ducklakeSchema } from "@rilldata/web-common/features/templates/schemas/ducklake.ts";
+import { httpsSchema } from "@rilldata/web-common/features/templates/schemas/https.ts";
+import { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
+
+describe("getConnectorYAML", () => {
+ describe("clickhouse", () => {
+ const connector: V1ConnectorDriver = { name: "clickhouse" };
+ const schema = clickhouseSchema;
+ const formValuesWithoutPassword = {
+ host: "ch.example.com",
+ username: "user",
+ };
+ const formValuesWithPassword = {
+ ...formValuesWithoutPassword,
+ password: "pass",
+ };
+
+ it("should retain same value across edit commits", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `password: "{{ .env.CLICKHOUSE_PASSWORD }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass",
+ });
+
+ // New changes arrived but value didnt change
+ testEnvs["CLICKHOUSE_PASSWORD"] = "pass";
+ await envStore.pull();
+
+ const yamlAfterPull = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ ...formValuesWithPassword,
+ password: "pass_1",
+ },
+ envEditSession,
+ });
+ expect(yamlAfterPull).toContain(
+ `password: "{{ .env.CLICKHOUSE_PASSWORD }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass_1",
+ });
+
+ // New changes arrived with new values.
+ testEnvs["CLICKHOUSE_PASSWORD"] = "pass_source";
+ await envStore.pull();
+
+ const yamlAfterSourceUpdate = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ ...formValuesWithPassword,
+ password: "pass_2",
+ },
+ envEditSession,
+ });
+ expect(yamlAfterSourceUpdate).toContain(
+ `password: "{{ .env.CLICKHOUSE_PASSWORD_1 }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass_source",
+ CLICKHOUSE_PASSWORD_1: "pass_2",
+ });
+ });
+
+ it("should delete unused vars if not updated from outside", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `password: "{{ .env.CLICKHOUSE_PASSWORD }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass",
+ });
+
+ // New changes arrived but value didnt change
+ testEnvs["CLICKHOUSE_PASSWORD"] = "pass";
+ await envStore.pull();
+
+ const yamlWithoutPassword = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithoutPassword,
+ envEditSession,
+ });
+ expect(yamlWithoutPassword).not.toContain("password:");
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({});
+ });
+
+ it("should delete vars on rollback if not updated from outside", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `password: "{{ .env.CLICKHOUSE_PASSWORD }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass",
+ });
+
+ // New changes arrived but value didnt change
+ testEnvs["CLICKHOUSE_PASSWORD"] = "pass";
+ await envStore.pull();
+
+ await envEditSession.rollback();
+ expect(testEnvs).toEqual({});
+ });
+
+ it("should retain unused vars if updated from outside", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `password: "{{ .env.CLICKHOUSE_PASSWORD }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass",
+ });
+
+ // New changes arrived with new values.
+ testEnvs["CLICKHOUSE_PASSWORD"] = "pass_source";
+ await envStore.pull();
+
+ const yamlWithoutPassword = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithoutPassword,
+ envEditSession,
+ });
+ expect(yamlWithoutPassword).not.toContain("password:");
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass_source",
+ });
+ });
+
+ it("should retain vars on rollback if updated from outside", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `password: "{{ .env.CLICKHOUSE_PASSWORD }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass",
+ });
+
+ // New changes arrived with new values.
+ testEnvs["CLICKHOUSE_PASSWORD"] = "pass_source";
+ await envStore.pull();
+
+ await envEditSession.rollback();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass_source",
+ });
+ });
+
+ it("should delete vars on rollback when unrelated changes to envs happened just before commit", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+
+ // Initial yaml compilation
+ getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+
+ // An unrelated pull fires before commit
+ await envStore.pull();
+
+ // Commit happens after a pull
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass",
+ });
+
+ // Rollback removes the vars.
+ await envEditSession.rollback();
+ expect(testEnvs).toEqual({});
+ });
+
+ it("should delete vars on rollback when the env is updated just before commit", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+
+ // Initial yaml compilation
+ getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+
+ // New changes arrived with new values.
+ testEnvs["CLICKHOUSE_PASSWORD"] = "pass_source";
+ await envStore.pull();
+
+ // Commit happens after a pull
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass",
+ });
+
+ // Rollback removes the vars. This is a known race condition.
+ await envEditSession.rollback();
+ expect(testEnvs).toEqual({});
+ });
+
+ it("should not reuse vars for new connectors", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema, {});
+ getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass",
+ });
+
+ const newEnvEditSession = new EnvEditSession(
+ envStore,
+ connector.name + "_1",
+ schema,
+ );
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ ...formValuesWithPassword,
+ password: "new_pass",
+ },
+ envEditSession: newEnvEditSession,
+ });
+ // Since clickhouse has `x-secret-value` name of the connector doesnt affect the variable name
+ expect(yamlInitial).toContain(
+ `password: "{{ .env.CLICKHOUSE_PASSWORD_1 }}"`,
+ );
+ await newEnvEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "pass",
+ CLICKHOUSE_PASSWORD_1: "new_pass",
+ });
+ });
+ });
+
+ describe("ducklake", () => {
+ const connector: V1ConnectorDriver = { name: "ducklake" };
+ const schema = ducklakeSchema;
+
+ describe("direct attach field", () => {
+ it("should retain same value across edit commits", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ connection_mode: "sql",
+ attach:
+ "'ducklake:postgres:dbname=mydb host=localhost user=postgres password=pass'",
+ },
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `attach: "'ducklake:postgres:{{ .env.DUCKLAKE_POSTGRES }}'"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ DUCKLAKE_POSTGRES:
+ "dbname=mydb host=localhost user=postgres password=pass",
+ });
+
+ // New changes arrived but value didnt change
+ testEnvs["DUCKLAKE_POSTGRES"] =
+ "dbname=mydb host=localhost user=postgres password=pass";
+ await envStore.pull();
+
+ const yamlAfterPull = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ connection_mode: "sql",
+ attach:
+ "'ducklake:postgres:dbname=mydb host=localhost user=postgres password=pass_1'",
+ },
+ envEditSession,
+ });
+ expect(yamlAfterPull).toContain(
+ `attach: "'ducklake:postgres:{{ .env.DUCKLAKE_POSTGRES }}'"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ DUCKLAKE_POSTGRES:
+ "dbname=mydb host=localhost user=postgres password=pass_1",
+ });
+
+ // New changes arrived with new values.
+ testEnvs["DUCKLAKE_POSTGRES"] =
+ "dbname=mydb host=localhost user=postgres password=pass_source";
+ await envStore.pull();
+
+ const yamlAfterSourceUpdate = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ connection_mode: "sql",
+ attach:
+ "'ducklake:postgres:dbname=mydb host=localhost user=postgres password=pass_2'",
+ },
+ envEditSession,
+ });
+ expect(yamlAfterSourceUpdate).toContain(
+ `attach: "'ducklake:postgres:{{ .env.DUCKLAKE_POSTGRES_1 }}'"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ DUCKLAKE_POSTGRES:
+ "dbname=mydb host=localhost user=postgres password=pass_source",
+ DUCKLAKE_POSTGRES_1:
+ "dbname=mydb host=localhost user=postgres password=pass_2",
+ });
+ });
+ });
+
+ describe("build attach from params", () => {
+ const formValuesWithoutPassword = {
+ connection_mode: "parameters",
+ catalog_type: "postgres",
+ catalog_postgres_dbname: "mydb",
+ catalog_postgres_host: "localhost",
+ catalog_postgres_user: "postgres",
+ };
+ const formValuesWithPassword = {
+ ...formValuesWithoutPassword,
+ catalog_postgres_password: "pass",
+ };
+
+ it("should retain same value across edit commits for separate fields", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `attach: "'ducklake:postgres:dbname=mydb host=localhost user=postgres password={{ .env.DUCKLAKE_CATALOG_POSTGRES_PASSWORD }}'"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ DUCKLAKE_CATALOG_POSTGRES_PASSWORD: "pass",
+ });
+
+ // New changes arrived but value didnt change
+ testEnvs["DUCKLAKE_CATALOG_POSTGRES_PASSWORD"] = "pass";
+ await envStore.pull();
+
+ const yamlAfterPull = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ ...formValuesWithoutPassword,
+ catalog_postgres_password: "pass_1",
+ },
+ envEditSession,
+ });
+ expect(yamlAfterPull).toContain(
+ `attach: "'ducklake:postgres:dbname=mydb host=localhost user=postgres password={{ .env.DUCKLAKE_CATALOG_POSTGRES_PASSWORD }}'"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ DUCKLAKE_CATALOG_POSTGRES_PASSWORD: "pass_1",
+ });
+
+ // New changes arrived with new values.
+ testEnvs["DUCKLAKE_CATALOG_POSTGRES_PASSWORD"] = "pass_source";
+ await envStore.pull();
+
+ const yamlAfterSourceUpdate = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ ...formValuesWithoutPassword,
+ catalog_postgres_password: "pass_2",
+ },
+ envEditSession,
+ });
+ expect(yamlAfterSourceUpdate).toContain(
+ `attach: "'ducklake:postgres:dbname=mydb host=localhost user=postgres password={{ .env.DUCKLAKE_CATALOG_POSTGRES_PASSWORD_1 }}'"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ DUCKLAKE_CATALOG_POSTGRES_PASSWORD: "pass_source",
+ DUCKLAKE_CATALOG_POSTGRES_PASSWORD_1: "pass_2",
+ });
+ });
+
+ it("should delete unused vars if not updated from outside", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `attach: "'ducklake:postgres:dbname=mydb host=localhost user=postgres password={{ .env.DUCKLAKE_CATALOG_POSTGRES_PASSWORD }}'"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ DUCKLAKE_CATALOG_POSTGRES_PASSWORD: "pass",
+ });
+
+ // New changes arrived but value didnt change
+ testEnvs["DUCKLAKE_CATALOG_POSTGRES_PASSWORD"] = "pass";
+ await envStore.pull();
+
+ const yamlWithoutPassword = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithoutPassword,
+ envEditSession,
+ });
+ expect(yamlWithoutPassword).toContain(
+ `attach: "'ducklake:postgres:dbname=mydb host=localhost user=postgres'"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({});
+ });
+
+ it("should retain unused vars if updated from outside", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithPassword,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `attach: "'ducklake:postgres:dbname=mydb host=localhost user=postgres password={{ .env.DUCKLAKE_CATALOG_POSTGRES_PASSWORD }}'"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ DUCKLAKE_CATALOG_POSTGRES_PASSWORD: "pass",
+ });
+
+ // New changes arrived with new values.
+ testEnvs["DUCKLAKE_CATALOG_POSTGRES_PASSWORD"] = "pass_source";
+ await envStore.pull();
+
+ const yamlWithoutPassword = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithoutPassword,
+ envEditSession,
+ });
+ expect(yamlWithoutPassword).toContain(
+ `attach: "'ducklake:postgres:dbname=mydb host=localhost user=postgres'"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ DUCKLAKE_CATALOG_POSTGRES_PASSWORD: "pass_source",
+ });
+ });
+ });
+ });
+
+ describe("https", () => {
+ const connector: V1ConnectorDriver = { name: "https" };
+ const schema = httpsSchema;
+ const formValuesWithoutAuth = {
+ auth_method: "with_headers",
+ headers: [{ key: "Content-Type", value: "application/json" }],
+ };
+ const formValuesWithAuth = {
+ auth_method: "with_headers",
+ headers: [
+ { key: "Content-Type", value: "application/json" },
+ { key: "Authorization", value: "Bearer my_token" },
+ ],
+ };
+
+ it("should retain same value across edit commits", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithAuth,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `"Authorization": "Bearer {{ .env.HTTPS_AUTHORIZATION }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "my_token",
+ });
+
+ // New changes arrived but value didnt change
+ testEnvs["HTTPS_AUTHORIZATION"] = "my_token";
+ await envStore.pull();
+
+ const yamlAfterPull = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ ...formValuesWithAuth,
+ headers: [
+ { key: "Content-Type", value: "application/json" },
+ { key: "Authorization", value: "Bearer my_token_1" },
+ ],
+ },
+ envEditSession,
+ });
+ expect(yamlAfterPull).toContain(
+ `"Authorization": "Bearer {{ .env.HTTPS_AUTHORIZATION }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "my_token_1",
+ });
+
+ // New changes arrived with new values.
+ testEnvs["HTTPS_AUTHORIZATION"] = "my_token_source";
+ await envStore.pull();
+
+ const yamlAfterSourceUpdate = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ ...formValuesWithAuth,
+ headers: [
+ { key: "Content-Type", value: "application/json" },
+ { key: "Authorization", value: "Bearer my_token_2" },
+ ],
+ },
+ envEditSession,
+ });
+ expect(yamlAfterSourceUpdate).toContain(
+ `"Authorization": "Bearer {{ .env.HTTPS_AUTHORIZATION_1 }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "my_token_source",
+ HTTPS_AUTHORIZATION_1: "my_token_2",
+ });
+ });
+
+ it("should delete unused vars if not updated from outside", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithAuth,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `"Authorization": "Bearer {{ .env.HTTPS_AUTHORIZATION }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "my_token",
+ });
+
+ // New changes arrived but value didnt change
+ testEnvs["HTTPS_AUTHORIZATION"] = "my_token";
+ await envStore.pull();
+
+ const yamlWithoutAuth = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithoutAuth,
+ envEditSession,
+ });
+ expect(yamlWithoutAuth).not.toContain("Authorization");
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({});
+ });
+
+ it("should delete vars on rollback if not updated from outside", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema);
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithAuth,
+ envEditSession,
+ });
+ expect(yamlInitial).toContain(
+ `"Authorization": "Bearer {{ .env.HTTPS_AUTHORIZATION }}"`,
+ );
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "my_token",
+ });
+
+ // New changes arrived but value didnt change
+ testEnvs["HTTPS_AUTHORIZATION"] = "my_token";
+ await envStore.pull();
+
+ await envEditSession.rollback();
+ expect(testEnvs).toEqual({});
+ });
+
+ it("should not reuse vars for new connectors", async () => {
+ const { envEditSession, testEnvs, envStore } =
+ await makeTestEnvEditSession(connector.name, schema, {});
+ getConnectorYAML({
+ connector,
+ schema,
+ formValues: formValuesWithAuth,
+ envEditSession,
+ });
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "my_token",
+ });
+
+ const newEnvEditSession = new EnvEditSession(
+ envStore,
+ connector.name + "_1",
+ schema,
+ );
+ const yamlInitial = getConnectorYAML({
+ connector,
+ schema,
+ formValues: {
+ ...formValuesWithAuth,
+ headers: [
+ { key: "Content-Type", value: "application/json" },
+ { key: "Authorization", value: "Bearer my_token_2" },
+ ],
+ },
+ envEditSession: newEnvEditSession,
+ });
+ // Since clickhouse has `x-secret-value` name of the connector doesnt affect the variable name
+ expect(yamlInitial).toContain(
+ `"Authorization": "Bearer {{ .env.HTTPS_1_AUTHORIZATION }}"`,
+ );
+ await newEnvEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "my_token",
+ HTTPS_1_AUTHORIZATION: "my_token_2",
+ });
+ });
+ });
+});
diff --git a/web-common/src/features/add-data/form/yaml-preview.ts b/web-common/src/features/add-data/form/connector-source-yaml-generator.ts
similarity index 60%
rename from web-common/src/features/add-data/form/yaml-preview.ts
rename to web-common/src/features/add-data/form/connector-source-yaml-generator.ts
index a07b216de9f6..d730f59dbd6b 100644
--- a/web-common/src/features/add-data/form/yaml-preview.ts
+++ b/web-common/src/features/add-data/form/connector-source-yaml-generator.ts
@@ -1,32 +1,28 @@
import {
- filterSchemaValuesForSubmit,
getSchemaFieldMetaList,
getSchemaSecretKeys,
getSchemaStringKeys,
} from "@rilldata/web-common/features/templates/schema-utils.ts";
import type { MultiStepFormSchema } from "@rilldata/web-common/features/templates/schemas/types.ts";
-import {
- applyDuckLakeFormPipeline,
- injectDuckLakeAttach,
-} from "@rilldata/web-common/features/templates/schemas/ducklake-utils.ts";
-import { compileConnectorYAML } from "@rilldata/web-common/features/connectors/code-utils.ts";
+import { generateYAML } from "@rilldata/web-common/features/connectors/code-utils.ts";
import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client";
import {
- compileSourceYAML,
+ generateSourceYAML,
prepareSourceFormData,
} from "@rilldata/web-common/features/sources/sourceUtils.ts";
import { getConnectorSchema } from "@rilldata/web-common/features/sources/modal/connector-schemas.ts";
+import type { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
-export function getConnectorYamlPreview({
+export function getConnectorYAML({
connector,
schema,
formValues,
- existingEnvBlob,
+ envEditSession,
}: {
connector: V1ConnectorDriver;
schema: MultiStepFormSchema | null;
formValues: Record;
- existingEnvBlob: string | null;
+ envEditSession: EnvEditSession;
}) {
const schemaFields = schema
? getSchemaFieldMetaList(schema, { step: "connector" })
@@ -37,23 +33,7 @@ export function getConnectorYamlPreview({
const schemaStringKeys = schema
? getSchemaStringKeys(schema, { step: "connector" })
: [];
- // DuckLake: mirror the submit path so the preview reflects the composed
- // ATTACH clause and rewritten env-var refs. Discards extractedSecrets
- // since previews don't write to `.env`.
- const { transformedValues } = applyDuckLakeFormPipeline(schema, formValues, {
- connectorName: connector.name ?? "",
- existingEnvBlob: existingEnvBlob ?? "",
- });
- const filteredValues = schema
- ? injectDuckLakeAttach(
- schema,
- filterSchemaValuesForSubmit(schema, transformedValues, {
- step: "connector",
- }),
- transformedValues,
- )
- : transformedValues;
- const yamlPreview = compileConnectorYAML(connector, filteredValues, {
+ const yaml = generateYAML(connector, formValues, envEditSession, {
fieldFilter: (property) => {
if ("internal" in property && property.internal) return false;
return !("noPrompt" in property && property.noPrompt);
@@ -62,25 +42,24 @@ export function getConnectorYamlPreview({
secretKeys: schemaSecretKeys,
stringKeys: schemaStringKeys,
schema: schema ?? undefined,
- existingEnvBlob: existingEnvBlob ?? "",
});
- return yamlPreview;
+ return yaml;
}
-export function getSourceYamlPreview({
+export function getSourceYAML({
connectorName,
connector,
schema,
formValues,
- existingEnvBlob,
+ envEditSession,
outputConnector,
}: {
connectorName: string;
connector: V1ConnectorDriver;
schema: MultiStepFormSchema | null;
formValues: Record;
- existingEnvBlob: string | null;
+ envEditSession: EnvEditSession;
outputConnector?: string;
}) {
const isPublicAuth = formValues.auth_method === "public";
@@ -101,17 +80,22 @@ export function getSourceYamlPreview({
? getSchemaStringKeys(rewrittenSchema, { step: "source" })
: undefined;
if (isRewrittenToDuckDb) {
- return compileSourceYAML(rewrittenConnector, rewrittenFormValues, {
- secretKeys: rewrittenSecretKeys,
- stringKeys: rewrittenStringKeys,
- originalDriverName: connector.name || undefined,
- outputConnector,
- });
+ return generateSourceYAML(
+ rewrittenConnector,
+ rewrittenFormValues,
+ envEditSession,
+ {
+ secretKeys: rewrittenSecretKeys,
+ stringKeys: rewrittenStringKeys,
+ originalDriverName: connector.name || undefined,
+ outputConnector,
+ },
+ );
}
- return getConnectorYamlPreview({
+ return getConnectorYAML({
connector,
schema,
formValues: rewrittenFormValues,
- existingEnvBlob,
+ envEditSession,
});
}
diff --git a/web-common/src/features/add-data/manager/AddDataManager.svelte b/web-common/src/features/add-data/manager/AddDataManager.svelte
index e489d15783b9..d84e0dd4f157 100644
--- a/web-common/src/features/add-data/manager/AddDataManager.svelte
+++ b/web-common/src/features/add-data/manager/AddDataManager.svelte
@@ -23,9 +23,9 @@
getConnectorDriverForSchema,
} from "@rilldata/web-common/features/add-data/manager/steps/utils.ts";
import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client";
- import ConnectorFormWrapper from "@rilldata/web-common/features/add-data/form/ConnectorFormWrapper.svelte";
import { getAddDataClass } from "@rilldata/web-common/features/add-data/class-utils.ts";
import { inferSchemaForConnector } from "@rilldata/web-common/features/entity-management/add/selectors.ts";
+ import ConnectorForm from "@rilldata/web-common/features/add-data/form/ConnectorForm.svelte";
const {
config,
@@ -177,7 +177,7 @@
{#if stepState.step === AddDataStep.SelectConnector}
{:else if stepState.step === AddDataStep.CreateConnector}
-
diff --git a/web-common/src/features/add-data/manager/AddDataStateManager.svelte.spec.ts b/web-common/src/features/add-data/manager/AddDataStateManager.svelte.spec.ts
index b3b400f1bf87..d8cf1379cc1b 100644
--- a/web-common/src/features/add-data/manager/AddDataStateManager.svelte.spec.ts
+++ b/web-common/src/features/add-data/manager/AddDataStateManager.svelte.spec.ts
@@ -16,7 +16,7 @@ import { connectorFormCache } from "@rilldata/web-common/features/add-data/manag
const ClickhouseSchema = "clickhouse";
const ClickhouseConnector = "clickhouse_conn";
const ClickhouseDriver = getConnectorDriverForSchema(ClickhouseSchema)!;
-const ClickhouseImportConfig: ImportStepConfig = {
+const ClickhouseImportConfig = {
importSteps: [
ImportDataStep.CreateMetricsView,
ImportDataStep.CreateDashboard,
@@ -29,8 +29,7 @@ const ClickhouseImportConfig: ImportStepConfig = {
database: "public",
},
importTo: {},
- envBlob: null,
-};
+} as ImportStepConfig;
describe("AddDataStateManager", () => {
const TestCases: {
diff --git a/web-common/src/features/add-data/manager/GenerateDashboardStatus.svelte b/web-common/src/features/add-data/manager/GenerateDashboardStatus.svelte
index 5cd29472097a..06dc6b59b29c 100644
--- a/web-common/src/features/add-data/manager/GenerateDashboardStatus.svelte
+++ b/web-common/src/features/add-data/manager/GenerateDashboardStatus.svelte
@@ -19,7 +19,6 @@
import { onMount } from "svelte";
import { useRuntimeClient } from "@rilldata/web-common/runtime-client/v2";
import { addLeadingSlash } from "@rilldata/web-common/features/entity-management/entity-mappers.ts";
- import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryClient.ts";
import {
getFileHref,
withEditorPrefix,
@@ -113,11 +112,7 @@
}
async function cleanupAndBack() {
- await cleanupImportStep(
- runtimeClient,
- queryClient,
- importAddDataStep.config,
- );
+ await cleanupImportStep(runtimeClient, importAddDataStep.config);
onBack();
}
diff --git a/web-common/src/features/add-data/manager/steps/connector.ts b/web-common/src/features/add-data/manager/steps/connector.ts
index 98bf5ed1221d..77ca6e9b4ace 100644
--- a/web-common/src/features/add-data/manager/steps/connector.ts
+++ b/web-common/src/features/add-data/manager/steps/connector.ts
@@ -5,7 +5,6 @@ import {
getRuntimeServiceGetResourceQueryKey,
runtimeServiceDeleteFile,
runtimeServiceGetFile,
- runtimeServicePushEnv,
runtimeServicePutFile,
runtimeServiceUnpackEmpty,
type V1ConnectorDriver,
@@ -24,11 +23,7 @@ import {
getConnectorSchema,
isMultiStepConnector,
} from "@rilldata/web-common/features/sources/modal/connector-schemas.ts";
-import {
- findRadioEnumKey,
- getSchemaSecretKeys,
-} from "@rilldata/web-common/features/templates/schema-utils.ts";
-import { applyDuckLakeFormPipeline } from "@rilldata/web-common/features/templates/schemas/ducklake-utils.ts";
+import { findRadioEnumKey } from "@rilldata/web-common/features/templates/schema-utils.ts";
import type { MultiStepFormSchema } from "@rilldata/web-common/features/templates/schemas/types.ts";
import {
addLeadingSlash,
@@ -37,22 +32,19 @@ import {
import { EntityType } from "@rilldata/web-common/features/entity-management/types.ts";
import {
maybeUnsetOlapConnectorInYaml,
- replaceOrAddEnvVariable,
- unsetResourceEnvVars,
- updateDotEnvWithSecrets,
updateRillYAMLWithOlapConnector,
} from "@rilldata/web-common/features/connectors/code-utils.ts";
import type { QueryClient } from "@tanstack/svelte-query";
import { fileArtifacts } from "@rilldata/web-common/features/entity-management/file-artifacts.ts";
import { ResourceKind } from "@rilldata/web-common/features/entity-management/resource-selectors.ts";
-import { getConnectorYamlPreview } from "@rilldata/web-common/features/add-data/form/yaml-preview.ts";
+import { getConnectorYAML } from "@rilldata/web-common/features/add-data/form/connector-source-yaml-generator.ts";
import { getName } from "@rilldata/web-common/features/entity-management/name-utils.ts";
import {
getProjectParserVersion,
waitForProjectParserVersion,
} from "@rilldata/web-common/features/entity-management/project-parser.ts";
-import { isCloudRuntimeEditEnvironment } from "@rilldata/web-common/features/entity-management/edit-environment.ts";
-import { maybeGetEnvContent } from "@rilldata/web-common/features/add-data/manager/steps/utils.ts";
+import { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
+import type { EnvStore } from "@rilldata/web-common/features/env-management/env-store.ts";
export async function createConnector({
runtimeClient,
@@ -62,7 +54,7 @@ export async function createConnector({
schemaName,
formValues,
validate,
- existingEnvBlob,
+ envEditSession,
}: {
runtimeClient: RuntimeClient;
queryClient: QueryClient;
@@ -71,7 +63,7 @@ export async function createConnector({
schemaName?: string;
formValues: Record;
validate: boolean;
- existingEnvBlob: string | null;
+ envEditSession: EnvEditSession;
}) {
await maybeInitProject(runtimeClient);
@@ -80,59 +72,17 @@ export async function createConnector({
// their own schema fields.
const schema = getConnectorSchema(schemaName ?? connectorDriver.name ?? "");
- // DuckLake: compose Parameters tab into `attach` (with password fields
- // stored in `.env` and referenced via template) and route raw-ATTACH
- // catalog URIs through `.env`. Done before updateDotEnvWithSecrets so the
- // same baseline blob drives env-var name conflict detection for all
- // ducklake-derived secrets and any form-field secrets.
- const duckLakeResult = applyDuckLakeFormPipeline(schema, formValues, {
- connectorName: connectorDriver.name ?? "",
- existingEnvBlob: existingEnvBlob ?? "",
- });
- formValues = duckLakeResult.transformedValues;
- const duckLakeAttachSecrets = duckLakeResult.extractedSecrets;
-
// Fast-path: public auth skips validation/test and advances directly
if (isMultiStepConnector(schema) && isPublicAuth(schema, formValues)) {
return connectorDriver.name!;
}
- const schemaSecretKeys = schema
- ? getSchemaSecretKeys(schema, { step: "connector" })
- : [];
-
// Create connector file path outside try block for cleanup
const newConnectorFilePath = addLeadingSlash(
getFileAPIPathFromNameAndType(connectorName, EntityType.Connector),
);
try {
- // Capture original .env and compute updated contents up front
- // Use originalBlob from updateDotEnvWithSecrets for consistent conflict detection
- const { newBlob: initialEnvBlob, originalBlob: originalEnvBlob } =
- await updateDotEnvWithSecrets(
- runtimeClient,
- queryClient,
- connectorDriver,
- formValues,
- {
- secretKeys: schemaSecretKeys,
- schema: schema ?? undefined,
- existingEnvBlob: existingEnvBlob ?? undefined,
- },
- );
- let newEnvBlob = initialEnvBlob;
-
- // Persist DuckLake catalog URIs extracted from the raw ATTACH clause.
- // These are not tied to a form field, so updateDotEnvWithSecrets cannot
- // write them; append directly using the same env blob so write ordering
- // matches the rest of the secret handling.
- for (const [envVarName, rawValue] of Object.entries(
- duckLakeAttachSecrets,
- )) {
- newEnvBlob = replaceOrAddEnvVariable(newEnvBlob, envVarName, rawValue);
- }
-
/**
* Optimistic updates (Test and Connect):
* 1. Write the `.env` file so secrets exist before connector reconciliation
@@ -153,27 +103,17 @@ export async function createConnector({
}),
)?.resource?.meta?.stateVersion;
- if (existingEnvBlob !== newEnvBlob) {
- await runtimeServicePutFile(runtimeClient, {
- path: ".env",
- blob: newEnvBlob,
- create: true,
- createOnly: false,
- });
- if (isCloudRuntimeEditEnvironment()) {
- // Only push env on cloud for now. We will revisit this for rill-dev.
- await runtimeServicePushEnv(runtimeClient, {});
- }
- }
+ const connectorYaml = getConnectorYAML({
+ connector: connectorDriver,
+ formValues,
+ schema,
+ envEditSession,
+ });
+ await envEditSession.commit();
await runtimeServicePutFile(runtimeClient, {
path: newConnectorFilePath,
- blob: getConnectorYamlPreview({
- connector: connectorDriver,
- formValues,
- schema,
- existingEnvBlob: originalEnvBlob,
- }),
+ blob: connectorYaml,
create: true,
createOnly: false,
});
@@ -230,40 +170,20 @@ export async function maybeDeleteConnector(
runtimeClient: RuntimeClient,
queryClient: QueryClient,
connectorName: string,
+ envEditSession: EnvEditSession,
) {
const connectorFilePath = addLeadingSlash(
getFileAPIPathFromNameAndType(connectorName, EntityType.Connector),
);
if (!fileArtifacts.hasFileArtifact(connectorFilePath)) return;
- const connectorFileArtifact =
- fileArtifacts.getFileArtifact(connectorFilePath);
- const connectorYaml = await connectorFileArtifact.fetchContent();
- if (!connectorYaml) return;
-
- // Get the existing env and remove the connector's env vars
- const [envBlob, envBlobChanged] = await unsetResourceEnvVars(
- runtimeClient,
- queryClient,
- connectorYaml,
- );
-
// Delete the connector file
await runtimeServiceDeleteFile(runtimeClient, {
path: connectorFilePath,
});
- if (envBlobChanged) {
- // Update the .env file with the removed env vars
- await runtimeServicePutFile(runtimeClient, {
- path: ".env",
- blob: envBlob,
- });
- if (isCloudRuntimeEditEnvironment()) {
- // Only push env on cloud for now. We will revisit this for rill-dev.
- await runtimeServicePushEnv(runtimeClient, {});
- }
- }
+ // Update the .env file with the removed env vars
+ await envEditSession.rollback();
// Update the rill.yaml file to remove the connector as the OLAP connector.
await unsetOlapConnectorInRillYAML(runtimeClient, queryClient, connectorName);
@@ -377,7 +297,7 @@ export function isPublicAuth(
type CacheEntry = {
name: string;
formValues: Record;
- existingEnvBlob: string;
+ envEditSession: EnvEditSession;
};
export class ConnectorFormCache {
@@ -390,7 +310,11 @@ export class ConnectorFormCache {
return id.toString();
}
- public async getOrCreate(schema: string, id: string): Promise {
+ public getOrCreate(
+ schema: string,
+ id: string,
+ envStore: EnvStore,
+ ): CacheEntry {
if (this.cache.has(id)) {
return this.cache.get(id)!;
}
@@ -400,12 +324,16 @@ export class ConnectorFormCache {
fileArtifacts.getNamesForKind(ResourceKind.Connector),
);
- const envBlob = await maybeGetEnvContent();
+ const envEditSession = new EnvEditSession(
+ envStore,
+ name, // use generated connector name as prefix
+ getConnectorSchema(schema) ?? undefined,
+ );
const entry = {
name,
formValues: {},
- existingEnvBlob: envBlob,
+ envEditSession,
};
this.cache.set(id, entry);
return entry;
@@ -418,6 +346,10 @@ export class ConnectorFormCache {
}
}
+ public delete(id: string) {
+ this.cache.delete(id);
+ }
+
public clear() {
this.cache.clear();
this.id = 0;
diff --git a/web-common/src/features/add-data/manager/steps/import.ts b/web-common/src/features/add-data/manager/steps/import.ts
index f4f931a90fb1..d2fe4073612d 100644
--- a/web-common/src/features/add-data/manager/steps/import.ts
+++ b/web-common/src/features/add-data/manager/steps/import.ts
@@ -12,13 +12,10 @@ import {
runtimeServiceGenerateCanvasFile,
runtimeServiceGenerateMetricsViewFile,
runtimeServiceGetInstance,
- runtimeServicePushEnv,
runtimeServicePutFile,
} from "@rilldata/web-common/runtime-client";
import {
- deleteFileArtifact,
maybeDeleteFileArtifact,
- runtimeServicePutFileAndWaitForReconciliation,
waitForResourceReconciliation,
} from "@rilldata/web-common/features/entity-management/actions/actions.ts";
import {
@@ -30,18 +27,15 @@ import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryCl
import { get } from "svelte/store";
import { RuntimeClient } from "@rilldata/web-common/runtime-client/v2";
import {
- compileSourceYAML,
+ generateSourceYAML,
inferModelNameFromSQL,
} from "@rilldata/web-common/features/sources/sourceUtils.ts";
import { featureFlags } from "@rilldata/web-common/features/feature-flags.ts";
import { generateBlobForNewResourceFile } from "@rilldata/web-common/features/entity-management/add/new-files.ts";
import { getName } from "@rilldata/web-common/features/entity-management/name-utils.ts";
-import type { QueryClient } from "@tanstack/svelte-query";
-import { unsetResourceEnvVars } from "@rilldata/web-common/features/connectors/code-utils.ts";
import { maybeGetConnectorDriver } from "@rilldata/web-common/features/add-data/manager/steps/utils.ts";
import { behaviourEvent } from "@rilldata/web-common/metrics/initMetrics.ts";
import { BehaviourEventAction } from "@rilldata/web-common/metrics/service/BehaviourEventTypes.ts";
-import { isCloudRuntimeEditEnvironment } from "@rilldata/web-common/features/entity-management/edit-environment.ts";
export async function runImportSteps(
runtimeClient: RuntimeClient,
@@ -122,35 +116,14 @@ export function generateImportToConfig(
export async function cleanupImportStep(
runtimeClient: RuntimeClient,
- queryClient: QueryClient,
config: ImportStepConfig,
) {
const importToConfig = config.importTo;
- let envBlob: string | null = null;
- let envBlobChanged = false;
- if (
- importToConfig.modelPath &&
- fileArtifacts.hasFileArtifact(importToConfig.modelPath)
- ) {
- const modelArtifact = fileArtifacts.getFileArtifact(
- importToConfig.modelPath,
- );
- const modelYaml = await modelArtifact.fetchContent();
-
- // Get the existing env and remove the connector's env vars
- [envBlob, envBlobChanged] = await unsetResourceEnvVars(
- runtimeClient,
- queryClient,
- modelYaml ?? "",
- );
-
- await deleteFileArtifact(runtimeClient, importToConfig.modelPath);
- }
-
// Cleanup any generated files.
await Promise.all(
[
+ importToConfig.modelPath,
importToConfig.metricsViewPath,
importToConfig.explorePath,
importToConfig.canvasPath,
@@ -160,17 +133,7 @@ export async function cleanupImportStep(
}),
);
- if (envBlob && envBlobChanged) {
- // Update the .env file with the removed env vars
- await runtimeServicePutFile(runtimeClient, {
- path: ".env",
- blob: envBlob,
- });
- if (isCloudRuntimeEditEnvironment()) {
- // Only push env on cloud for now. We will revisit this for rill-dev.
- await runtimeServicePushEnv(runtimeClient, {});
- }
- }
+ await config.envEditSession.rollback();
}
async function runCreateModelStep(
@@ -215,12 +178,13 @@ async function runCreateModelStep(
// User provided a SQL query to generate the model.
case "sql":
- yaml = compileSourceYAML(
+ yaml = generateSourceYAML(
connectorDriver,
{
name: importToConfig.modelName,
sql: importFromConfig.sql,
},
+ config.envEditSession,
{
connectorInstanceName: config.connector,
outputConnector: defaultOLAP,
@@ -235,12 +199,13 @@ async function runCreateModelStep(
(importFromConfig.schema ? importFromConfig.schema + "." : "") +
importFromConfig.table;
const sql = `SELECT * FROM ${fromTableName}`;
- yaml = compileSourceYAML(
+ yaml = generateSourceYAML(
connectorDriver,
{
name: importToConfig.modelName,
sql: sql,
},
+ config.envEditSession,
{
connectorInstanceName: config.connector,
outputConnector: defaultOLAP,
@@ -250,19 +215,7 @@ async function runCreateModelStep(
}
}
- if (config.envBlob) {
- // Make sure the file has reconciled before testing the connection
- await runtimeServicePutFileAndWaitForReconciliation(runtimeClient, {
- path: ".env",
- blob: config.envBlob,
- create: true,
- createOnly: false,
- });
- if (isCloudRuntimeEditEnvironment()) {
- // Only push env on cloud for now. We will revisit this for rill-dev.
- await runtimeServicePushEnv(runtimeClient, {});
- }
- }
+ await config.envEditSession.commit();
let putFile = true;
// Determine if the model file already exists and has the same content as the generated YAML.
diff --git a/web-common/src/features/add-data/manager/steps/types.ts b/web-common/src/features/add-data/manager/steps/types.ts
index 848dda5c9430..bfc0902d88e9 100644
--- a/web-common/src/features/add-data/manager/steps/types.ts
+++ b/web-common/src/features/add-data/manager/steps/types.ts
@@ -4,6 +4,7 @@ import {
} from "@rilldata/web-common/metrics/service/MetricsTypes.ts";
import { BehaviourEventMedium } from "@rilldata/web-common/metrics/service/BehaviourEventTypes.ts";
import type { ResourceKind } from "@rilldata/web-common/features/entity-management/resource-selectors.ts";
+import type { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
export enum AddDataStep {
// Used purely to transition from Init to one of the other steps.
@@ -104,7 +105,7 @@ export type ImportStepConfig = {
connector: string;
importFrom: ImportFromConfig;
importTo: ImportToConfig;
- envBlob: string | null;
+ envEditSession: EnvEditSession;
};
export type ImportFromConfig =
diff --git a/web-common/src/features/connectors/ai/AddAiConnectorDialog.svelte b/web-common/src/features/connectors/ai/AddAiConnectorDialog.svelte
index ac590a093508..6293af6a05f8 100644
--- a/web-common/src/features/connectors/ai/AddAiConnectorDialog.svelte
+++ b/web-common/src/features/connectors/ai/AddAiConnectorDialog.svelte
@@ -28,11 +28,14 @@
} from "../../../metrics/service/BehaviourEventTypes";
import { MetricsEventSpace } from "../../../metrics/service/MetricsTypes";
import { getScreenNameFromPage } from "../../file-explorer/telemetry";
+ import { getEnvFileStore } from "@rilldata/web-common/features/env-management/env-file-store.ts";
+ import { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
export let open = false;
const queryClient = useQueryClient();
const runtimeClient = useRuntimeClient();
+ const envStore = getEnvFileStore();
/** Expected API key prefixes per provider, used for soft validation. */
const API_KEY_PREFIXES: Record = {
@@ -66,6 +69,8 @@
? `https://docs.rilldata.com/developers/build/connectors/services/${getBackendConnectorName(schemaName)}`
: "";
+ $: envEditSession = new EnvEditSession(envStore, schemaName, schema);
+
// Soft validation: warn when the API key doesn't match the expected prefix
$: apiKeyWarning = getApiKeyWarning(schemaName, apiKey);
@@ -136,7 +141,13 @@
try {
const formValues: Record = { api_key: apiKey };
if (model) formValues.model = model;
- await saveAiConnector(runtimeClient, queryClient, schemaName, formValues);
+ await saveAiConnector(
+ runtimeClient,
+ queryClient,
+ schemaName,
+ formValues,
+ envEditSession,
+ );
behaviourEvent?.fireSourceTriggerEvent(
BehaviourEventAction.SourceAdd,
BehaviourEventMedium.Button,
diff --git a/web-common/src/features/connectors/ai/saveAiConnector.ts b/web-common/src/features/connectors/ai/saveAiConnector.ts
index fcb094b7f98d..f9c993e15362 100644
--- a/web-common/src/features/connectors/ai/saveAiConnector.ts
+++ b/web-common/src/features/connectors/ai/saveAiConnector.ts
@@ -1,11 +1,7 @@
import type { QueryClient } from "@tanstack/query-core";
import { runtimeServicePutFile } from "../../../runtime-client";
import type { RuntimeClient } from "../../../runtime-client/v2";
-import {
- compileConnectorYAML,
- updateDotEnvWithSecrets,
- updateRillYAMLWithAiConnector,
-} from "../code-utils";
+import { generateYAML, updateRillYAMLWithAiConnector } from "../code-utils";
import { getFileAPIPathFromNameAndType } from "../../entity-management/entity-mappers";
import { fileArtifacts } from "../../entity-management/file-artifacts";
import { getName } from "../../entity-management/name-utils";
@@ -22,6 +18,7 @@ import {
getSchemaStringKeys,
} from "../../templates/schema-utils";
import { maybeInitProject } from "@rilldata/web-common/features/add-data/manager/steps/connector.ts";
+import type { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
async function setAiConnectorInRillYAML(
queryClient: QueryClient,
@@ -49,6 +46,7 @@ export async function saveAiConnector(
queryClient: QueryClient,
schemaName: string,
formValues: Record,
+ envEditSession: EnvEditSession,
): Promise {
const connector = toConnectorDriver(schemaName);
if (!connector) throw new Error(`Unknown AI connector: ${schemaName}`);
@@ -76,34 +74,22 @@ export async function saveAiConnector(
EntityType.Connector,
);
- // Write secrets to .env
- const { newBlob: newEnvBlob, originalBlob: envBlobForYaml } =
- await updateDotEnvWithSecrets(client, queryClient, connector, formValues, {
- secretKeys: schemaSecretKeys,
- schema: schema ?? undefined,
- });
-
- await runtimeServicePutFile(client, {
- path: ".env",
- blob: newEnvBlob,
- create: true,
- createOnly: false,
+ const connectorYAML = generateYAML(connector, formValues, envEditSession, {
+ connectorInstanceName: newConnectorName,
+ orderedProperties: schemaFields,
+ secretKeys: schemaSecretKeys,
+ stringKeys: schemaStringKeys,
+ schema: schema ?? undefined,
+ fieldFilter: schemaFields
+ ? (property) => !("internal" in property && property.internal)
+ : undefined,
});
+ await envEditSession.commit();
// Write connector YAML
await runtimeServicePutFile(client, {
path: newConnectorFilePath,
- blob: compileConnectorYAML(connector, formValues, {
- connectorInstanceName: newConnectorName,
- orderedProperties: schemaFields,
- secretKeys: schemaSecretKeys,
- stringKeys: schemaStringKeys,
- schema: schema ?? undefined,
- existingEnvBlob: envBlobForYaml,
- fieldFilter: schemaFields
- ? (property) => !("internal" in property && property.internal)
- : undefined,
- }),
+ blob: connectorYAML,
create: true,
createOnly: false,
});
diff --git a/web-common/src/features/connectors/code-utils.spec.ts b/web-common/src/features/connectors/code-utils.spec.ts
index 7b26983a114d..0f613cef582b 100644
--- a/web-common/src/features/connectors/code-utils.spec.ts
+++ b/web-common/src/features/connectors/code-utils.spec.ts
@@ -1,20 +1,18 @@
-import { describe, expect, it, vi, beforeEach } from "vitest";
+import { describe, expect, it } from "vitest";
import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client";
-import type { RuntimeClient } from "@rilldata/web-common/runtime-client/v2";
import {
replaceAiConnectorInYAML,
replaceOlapConnectorInYAML,
- replaceOrAddEnvVariable,
- getGenericEnvVarName,
- envVarExists,
- findAvailableEnvVarName,
- makeEnvVarKey,
- compileConnectorYAML,
+ generateYAML,
formatHeadersAsYamlMap,
- updateDotEnvWithSecrets,
- getEnvVarsFromConnectorYAML,
maybeUnsetOlapConnectorInYaml,
} from "./code-utils";
+import {
+ envMappedVarsAndValuesToObject,
+ makeTestEnvEditSession,
+} from "@rilldata/web-common/features/env-management/test/test-env-store.ts";
+import { getGenericEnvVarName } from "@rilldata/web-common/features/connectors/env-utils.ts";
+import type { JSONSchemaObject } from "@rilldata/web-common/features/templates/schemas/types.ts";
// Import the template for testing
const YAML_MODEL_TEMPLATE = `type: model
@@ -66,43 +64,6 @@ describe("YAML Model Template", () => {
});
});
-describe("replaceOrAddEnvVariable", () => {
- it("should create a new env file", () => {
- const updatedEnvBlob = replaceOrAddEnvVariable("", "KEY1", "VALUE1");
- expect(updatedEnvBlob).toBe("KEY1=VALUE1");
- });
-
- const existingEnvBlob = `# This is a comment
-# This is another comment
-KEY1=VALUE1
-KEY2=VALUE2`;
-
- it("should update an existing key in the env file", () => {
- const updatedEnvBlob = replaceOrAddEnvVariable(
- existingEnvBlob,
- "KEY1",
- "NEW_VALUE1",
- );
- expect(updatedEnvBlob).toBe(`# This is a comment
-# This is another comment
-KEY1=NEW_VALUE1
-KEY2=VALUE2`);
- });
-
- it("should add a new key to the env file", () => {
- const updatedEnvBlob = replaceOrAddEnvVariable(
- existingEnvBlob,
- "KEY3",
- "VALUE3",
- );
- expect(updatedEnvBlob).toBe(`# This is a comment
-# This is another comment
-KEY1=VALUE1
-KEY2=VALUE2
-KEY3=VALUE3`);
- });
-});
-
describe("replaceOlapConnectorInYAML", () => {
it("should add a new `olap_connector` key to a blank file", () => {
const updatedBlob = replaceOlapConnectorInYAML("", "clickhouse");
@@ -303,400 +264,173 @@ describe("getGenericEnvVarName", () => {
});
});
-describe("envVarExists", () => {
- it("should return true when variable exists", () => {
- const envBlob = `KEY1=VALUE1\nKEY2=VALUE2\nKEY3=VALUE3`;
- expect(envVarExists(envBlob, "KEY2")).toBe(true);
- });
-
- it("should return false when variable does not exist", () => {
- const envBlob = `KEY1=VALUE1\nKEY2=VALUE2`;
- expect(envVarExists(envBlob, "KEY3")).toBe(false);
- });
-
- it("should return true for variable at start of file", () => {
- const envBlob = `FIRST_KEY=VALUE\nKEY2=VALUE2`;
- expect(envVarExists(envBlob, "FIRST_KEY")).toBe(true);
- });
-
- it("should return true for variable at end of file", () => {
- const envBlob = `KEY1=VALUE1\nLAST_KEY=VALUE`;
- expect(envVarExists(envBlob, "LAST_KEY")).toBe(true);
- });
-
- it("should not match partial variable names", () => {
- const envBlob = `MY_VARIABLE_1=VALUE\nMY_VARIABLE_2=VALUE2`;
- expect(envVarExists(envBlob, "MY_VARIABLE")).toBe(false);
- });
-
- it("should return false for empty blob", () => {
- expect(envVarExists("", "KEY1")).toBe(false);
- });
-
- it("should handle variables with complex values", () => {
- const envBlob = `JSON_VAR={"key":"value","nested":{"data":"here"}}\nSIMPLE=value`;
- expect(envVarExists(envBlob, "JSON_VAR")).toBe(true);
- });
-
- it("should handle multiline values (only checks line start)", () => {
- const envBlob = `KEY1=line1\nline2\nKEY2=VALUE2`;
- expect(envVarExists(envBlob, "KEY1")).toBe(true);
- });
-});
-
-describe("findAvailableEnvVarName", () => {
- it("should return base name when no conflicts", () => {
- const envBlob = `OTHER_VAR=value`;
- const result = findAvailableEnvVarName(envBlob, "MY_VAR");
- expect(result).toBe("MY_VAR");
- });
-
- it("should append _1 when base name exists", () => {
- const envBlob = `MY_VAR=value`;
- const result = findAvailableEnvVarName(envBlob, "MY_VAR");
- expect(result).toBe("MY_VAR_1");
- });
-
- it("should append _2 when _1 exists", () => {
- const envBlob = `MY_VAR=value\nMY_VAR_1=value`;
- const result = findAvailableEnvVarName(envBlob, "MY_VAR");
- expect(result).toBe("MY_VAR_2");
- });
-
- it("should skip gaps and find next available number", () => {
- const envBlob = `MY_VAR=value\nMY_VAR_1=value\nMY_VAR_2=value\nMY_VAR_3=value`;
- const result = findAvailableEnvVarName(envBlob, "MY_VAR");
- expect(result).toBe("MY_VAR_4");
- });
-
- it("should handle empty blob", () => {
- const result = findAvailableEnvVarName("", "NEW_VAR");
- expect(result).toBe("NEW_VAR");
- });
-
- it("should handle base name with underscores", () => {
- const envBlob = `GOOGLE_APPLICATION_CREDENTIALS=value\nGOOGLE_APPLICATION_CREDENTIALS_1=value`;
- const result = findAvailableEnvVarName(
- envBlob,
- "GOOGLE_APPLICATION_CREDENTIALS",
- );
- expect(result).toBe("GOOGLE_APPLICATION_CREDENTIALS_2");
- });
-
- it("should be case sensitive", () => {
- const envBlob = `my_var=value`;
- const result = findAvailableEnvVarName(envBlob, "MY_VAR");
- expect(result).toBe("MY_VAR");
- });
-});
-
-describe("makeEnvVarKey", () => {
- // Mock schemas matching production x-env-var-name definitions
- const bigquerySchema = {
- properties: {
- google_application_credentials: {
- "x-env-var-name": "GOOGLE_APPLICATION_CREDENTIALS",
- },
- },
- };
-
- const s3Schema = {
- properties: {
- aws_access_key_id: { "x-env-var-name": "AWS_ACCESS_KEY_ID" },
- aws_secret_access_key: { "x-env-var-name": "AWS_SECRET_ACCESS_KEY" },
- },
- };
-
- const motherduckSchema = {
- properties: {
- token: { "x-env-var-name": "MOTHERDUCK_TOKEN" },
- },
- };
-
- const postgresSchema = {
- properties: {
- password: { "x-env-var-name": "POSTGRES_PASSWORD" },
- },
- };
-
- describe("Without existing env blob - returns schema-defined name", () => {
- it("should return GOOGLE_APPLICATION_CREDENTIALS for bigquery", () => {
- const result = makeEnvVarKey(
- "bigquery",
- "google_application_credentials",
- undefined,
- bigquerySchema,
- );
- expect(result).toBe("GOOGLE_APPLICATION_CREDENTIALS");
- });
-
- it("should return MOTHERDUCK_TOKEN for motherduck", () => {
- const result = makeEnvVarKey(
- "motherduck",
- "token",
+describe("formatHeadersAsYamlMap", () => {
+ describe("array input", () => {
+ it("should format non-sensitive headers as plain text", async () => {
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
+ "https",
undefined,
- motherduckSchema,
- );
- expect(result).toBe("MOTHERDUCK_TOKEN");
- });
-
- it("should return POSTGRES_PASSWORD for postgres", () => {
- const result = makeEnvVarKey("postgres", "password", "", postgresSchema);
- expect(result).toBe("POSTGRES_PASSWORD");
- });
- });
-
- describe("With existing env blob - handles conflicts with _# suffix", () => {
- it("should append _1 when variable already exists", () => {
- const envBlob = `GOOGLE_APPLICATION_CREDENTIALS=existing_value`;
- const result = makeEnvVarKey(
- "bigquery",
- "google_application_credentials",
- envBlob,
- bigquerySchema,
- );
- expect(result).toBe("GOOGLE_APPLICATION_CREDENTIALS_1");
- });
-
- it("should return base name when no conflict exists", () => {
- const envBlob = `OTHER_VAR=value`;
- const result = makeEnvVarKey(
- "bigquery",
- "google_application_credentials",
- envBlob,
- bigquerySchema,
- );
- expect(result).toBe("GOOGLE_APPLICATION_CREDENTIALS");
- });
-
- it("should find next available number for multiple connectors of same type", () => {
- const envBlob = `GOOGLE_APPLICATION_CREDENTIALS=first_creds\nGOOGLE_APPLICATION_CREDENTIALS_1=second_creds\nGOOGLE_APPLICATION_CREDENTIALS_2=third_creds`;
- const result = makeEnvVarKey(
- "bigquery",
- "google_application_credentials",
- envBlob,
- bigquerySchema,
- );
- expect(result).toBe("GOOGLE_APPLICATION_CREDENTIALS_3");
- });
-
- it("should handle multiple different properties", () => {
- const envBlob = `AWS_ACCESS_KEY_ID=key1\nAWS_SECRET_ACCESS_KEY=secret1`;
- const result = makeEnvVarKey(
- "s3",
- "aws_access_key_id",
- envBlob,
- s3Schema,
- );
- expect(result).toBe("AWS_ACCESS_KEY_ID_1");
- });
-
- it("should handle driver-specific properties with conflicts", () => {
- const envBlob = `MOTHERDUCK_TOKEN=token1\nMOTHERDUCK_TOKEN_1=token2`;
- const result = makeEnvVarKey(
- "motherduck",
- "token",
- envBlob,
- motherduckSchema,
- );
- expect(result).toBe("MOTHERDUCK_TOKEN_2");
- });
-
- it("should handle complex env blobs with comments and multiple variables", () => {
- const envBlob = `# This is a comment
-SOME_OTHER_VAR=value
-MOTHERDUCK_TOKEN=token1
-MOTHERDUCK_TOKEN_1=token2
-# Another comment
-DATABASE_URL=something`;
- const result = makeEnvVarKey(
- "motherduck",
- "token",
- envBlob,
- motherduckSchema,
);
- expect(result).toBe("MOTHERDUCK_TOKEN_2");
- });
- });
-
- describe("Integration - full workflows with schemas", () => {
- it("should support adding first BigQuery connector", () => {
- const emptyEnv = "";
- const result = makeEnvVarKey(
- "bigquery",
- "google_application_credentials",
- emptyEnv,
- bigquerySchema,
- );
- expect(result).toBe("GOOGLE_APPLICATION_CREDENTIALS");
- });
-
- it("should support adding second BigQuery connector", () => {
- const envAfterFirst = `GOOGLE_APPLICATION_CREDENTIALS=first_creds`;
- const result = makeEnvVarKey(
- "bigquery",
- "google_application_credentials",
- envAfterFirst,
- bigquerySchema,
- );
- expect(result).toBe("GOOGLE_APPLICATION_CREDENTIALS_1");
- });
-
- it("should support adding AWS credentials to existing non-AWS variables", () => {
- const envBlob = `MOTHERDUCK_TOKEN=token1\nGOOGLE_APPLICATION_CREDENTIALS=creds1`;
- const result = makeEnvVarKey(
- "s3",
- "aws_access_key_id",
- envBlob,
- s3Schema,
- );
- expect(result).toBe("AWS_ACCESS_KEY_ID");
- });
-
- it("should support adding multiple AWS connectors", () => {
- const envBlob = `AWS_ACCESS_KEY_ID=key1\nAWS_SECRET_ACCESS_KEY=secret1`;
- const result1 = makeEnvVarKey(
- "s3",
- "aws_access_key_id",
- envBlob,
- s3Schema,
- );
- expect(result1).toBe("AWS_ACCESS_KEY_ID_1");
-
- const updatedEnv = `${envBlob}\nAWS_ACCESS_KEY_ID_1=key2`;
- const result2 = makeEnvVarKey(
- "s3",
- "aws_secret_access_key",
- updatedEnv,
- s3Schema,
+ const result = formatHeadersAsYamlMap(
+ [
+ { key: "Content-Type", value: "application/json" },
+ { key: "Accept", value: "text/html" },
+ ],
+ envEditSession,
);
- expect(result2).toBe("AWS_SECRET_ACCESS_KEY_1");
- });
- });
-});
-
-describe("formatHeadersAsYamlMap", () => {
- describe("array input", () => {
- it("should format non-sensitive headers as plain text", () => {
- const result = formatHeadersAsYamlMap([
- { key: "Content-Type", value: "application/json" },
- { key: "Accept", value: "text/html" },
- ]);
expect(result).toBe(
`headers:\n "Content-Type": "application/json"\n "Accept": "text/html"`,
);
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({});
});
- it("should replace sensitive header with env ref when driverName provided", () => {
+ it("should replace sensitive header with env ref when driverName provided", async () => {
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
+ "https",
+ undefined,
+ );
const result = formatHeadersAsYamlMap(
[{ key: "Authorization", value: "my_secret_token" }],
- "https",
+ envEditSession,
);
expect(result).toContain(
- '"Authorization": "{{ .env.connector.https.authorization }}"',
+ '"Authorization": "{{ .env.HTTPS_AUTHORIZATION }}"',
);
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "my_secret_token",
+ });
});
- it("should preserve Bearer scheme prefix", () => {
+ it("should preserve Bearer scheme prefix", async () => {
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
+ "https",
+ undefined,
+ );
const result = formatHeadersAsYamlMap(
[{ key: "Authorization", value: "Bearer my_token" }],
- "https",
+ envEditSession,
);
expect(result).toContain(
- '"Authorization": "Bearer {{ .env.connector.https.authorization }}"',
+ '"Authorization": "Bearer {{ .env.HTTPS_AUTHORIZATION }}"',
);
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "my_token",
+ });
});
- it("should preserve Basic scheme prefix", () => {
- const result = formatHeadersAsYamlMap(
- [{ key: "Authorization", value: "Basic dXNlcjpwYXNz" }],
+ it("should handle mixed sensitive and non-sensitive headers", async () => {
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
"https",
+ undefined,
);
- expect(result).toContain(
- '"Authorization": "Basic {{ .env.connector.https.authorization }}"',
- );
- });
-
- it("should handle mixed sensitive and non-sensitive headers", () => {
const result = formatHeadersAsYamlMap(
[
{ key: "Content-Type", value: "application/json" },
{ key: "Authorization", value: "Bearer token123" },
],
- "https",
+ envEditSession,
);
expect(result).toContain('"Content-Type": "application/json"');
expect(result).toContain(
- '"Authorization": "Bearer {{ .env.connector.https.authorization }}"',
+ '"Authorization": "Bearer {{ .env.HTTPS_AUTHORIZATION }}"',
);
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "token123",
+ });
});
- it("should filter entries with empty keys", () => {
- const result = formatHeadersAsYamlMap([
- { key: "", value: "ignored" },
- { key: "Accept", value: "text/html" },
- ]);
+ it("should filter entries with empty keys", async () => {
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
+ "https",
+ undefined,
+ );
+ const result = formatHeadersAsYamlMap(
+ [
+ { key: "", value: "ignored" },
+ { key: "Accept", value: "text/html" },
+ ],
+ envEditSession,
+ );
expect(result).toBe(`headers:\n "Accept": "text/html"`);
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({});
});
- it("should return empty string for empty array", () => {
- expect(formatHeadersAsYamlMap([])).toBe("");
- });
-
- it("should use connectorInstanceName for env refs when provided", () => {
- const result = formatHeadersAsYamlMap(
- [{ key: "X-API-Key", value: "secret" }],
+ it("should return empty string for empty array", async () => {
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
"https",
- "my_api",
+ undefined,
);
- expect(result).toContain("{{ .env.connector.my_api.x_api_key }}");
- });
-
- it("should not create env refs when no driverName", () => {
- const result = formatHeadersAsYamlMap([
- { key: "Authorization", value: "Bearer token" },
- ]);
- expect(result).toContain('"Authorization": "Bearer token"');
- expect(result).not.toContain(".env.");
+ expect(formatHeadersAsYamlMap([], envEditSession)).toBe("");
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({});
});
});
describe("string input (legacy)", () => {
- it("should parse Key: Value lines", () => {
+ it("should parse Key: Value lines", async () => {
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
+ "https",
+ undefined,
+ );
const result = formatHeadersAsYamlMap(
"Content-Type: application/json\nAccept: text/html",
+ envEditSession,
);
expect(result).toBe(
`headers:\n "Content-Type": "application/json"\n "Accept": "text/html"`,
);
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({});
});
- it("should replace sensitive headers with env refs", () => {
- const result = formatHeadersAsYamlMap(
- "Authorization: Bearer my_token",
+ it("should replace sensitive headers with env refs", async () => {
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
"https",
+ undefined,
);
- expect(result).toContain(
- "Bearer {{ .env.connector.https.authorization }}",
+ const result = formatHeadersAsYamlMap(
+ "Authorization: Bearer my_token",
+ envEditSession,
);
+ expect(result).toContain("Bearer {{ .env.HTTPS_AUTHORIZATION }}");
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ HTTPS_AUTHORIZATION: "my_token",
+ });
});
- it("should return empty string for empty input", () => {
- expect(formatHeadersAsYamlMap("")).toBe("");
+ it("should return empty string for empty input", async () => {
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
+ "https",
+ undefined,
+ );
+ expect(formatHeadersAsYamlMap("", envEditSession)).toBe("");
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({});
});
});
});
-describe("compileConnectorYAML", () => {
- it("should produce basic connector YAML", () => {
+describe("generateYAML", () => {
+ it("should produce basic connector YAML", async () => {
const connector: V1ConnectorDriver = {
name: "clickhouse",
docsUrl:
"https://docs.rilldata.com/developers/build/connectors/data-source/clickhouse",
};
- const result = compileConnectorYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
{ host: "ch.example.com" },
+ envEditSession,
{
orderedProperties: [{ key: "host" }],
},
@@ -707,11 +441,16 @@ describe("compileConnectorYAML", () => {
expect(result).toContain("host: ch.example.com");
});
- it("should preserve property ordering from orderedProperties", () => {
+ it("should preserve property ordering from orderedProperties", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileConnectorYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
{ host: "ch.example.com", port: 9000, database: "default" },
+ envEditSession,
{
orderedProperties: [
{ key: "database" },
@@ -727,16 +466,22 @@ describe("compileConnectorYAML", () => {
expect(hostIdx).toBeLessThan(portIdx);
});
- it("should replace secret properties with env var placeholders", () => {
+ it("should replace secret properties with env var placeholders", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
const schema = {
+ type: "object",
properties: {
password: { "x-env-var-name": "CLICKHOUSE_PASSWORD" },
},
- };
- const result = compileConnectorYAML(
+ } satisfies JSONSchemaObject;
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ schema,
+ );
+ const result = generateYAML(
connector,
{ password: "super_secret" },
+ envEditSession,
{
orderedProperties: [{ key: "password", secret: true }],
secretKeys: ["password"],
@@ -745,13 +490,90 @@ describe("compileConnectorYAML", () => {
);
expect(result).toContain("{{ .env.CLICKHOUSE_PASSWORD }}");
expect(result).not.toContain("super_secret");
+ expect(envEditSession.entries.get("password")?.mappedEnvVarName).toEqual(
+ "CLICKHOUSE_PASSWORD",
+ );
+
+ // Value is saved in env only after a flush
+ expect(testEnvs).toEqual({});
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "super_secret",
+ });
});
- it("should quote string properties", () => {
+ it("should handle env var conflict resolution with env edit session", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileConnectorYAML(
+ const schema = {
+ type: "object",
+ properties: {
+ password: { "x-env-var-name": "CLICKHOUSE_PASSWORD" },
+ },
+ } satisfies JSONSchemaObject;
+ const { testEnvs, envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ schema,
+ {},
+ {
+ CLICKHOUSE_PASSWORD: "abc",
+ },
+ );
+ const result = generateYAML(
+ connector,
+ { password: "secret" },
+ envEditSession,
+ {
+ orderedProperties: [{ key: "password", secret: true }],
+ secretKeys: ["password"],
+ schema,
+ },
+ );
+ expect(result).toContain("CLICKHOUSE_PASSWORD_1");
+
+ // Value is saved in env only after a flush
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "abc",
+ });
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({
+ CLICKHOUSE_PASSWORD_1: "secret",
+ });
+
+ // Calling compile again should not create new variable.
+ const newResult = generateYAML(
+ connector,
+ { password: "secret_new" },
+ envEditSession,
+ {
+ orderedProperties: [{ key: "password", secret: true }],
+ secretKeys: ["password"],
+ schema,
+ },
+ );
+ expect(newResult).toContain("CLICKHOUSE_PASSWORD_1");
+
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "abc",
+ });
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({
+ CLICKHOUSE_PASSWORD_1: "secret_new",
+ });
+ await envEditSession.commit();
+ expect(testEnvs).toEqual({
+ CLICKHOUSE_PASSWORD: "abc",
+ CLICKHOUSE_PASSWORD_1: "secret_new",
+ });
+ });
+
+ it("should quote string properties", async () => {
+ const connector: V1ConnectorDriver = { name: "clickhouse" };
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
{ host: "ch.example.com" },
+ envEditSession,
{
orderedProperties: [{ key: "host" }],
stringKeys: ["host"],
@@ -760,84 +582,121 @@ describe("compileConnectorYAML", () => {
expect(result).toContain('host: "ch.example.com"');
});
- it("should not quote non-string properties", () => {
+ it("should not quote non-string properties", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileConnectorYAML(
- connector,
- { port: 9000 },
- { orderedProperties: [{ key: "port" }] },
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
);
+ const result = generateYAML(connector, { port: 9000 }, envEditSession, {
+ orderedProperties: [{ key: "port" }],
+ });
expect(result).toContain("port: 9000");
expect(result).not.toContain('"9000"');
});
- it("should filter out empty string values", () => {
+ it("should filter out empty string values", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileConnectorYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
{ host: "ch.example.com", database: "" },
+ envEditSession,
{ orderedProperties: [{ key: "host" }, { key: "database" }] },
);
expect(result).toContain("host:");
expect(result).not.toContain("database:");
});
- it("should filter out undefined values", () => {
+ it("should filter out undefined values", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileConnectorYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
{ host: "ch.example.com", database: undefined },
+ envEditSession,
{ orderedProperties: [{ key: "host" }, { key: "database" }] },
);
expect(result).not.toContain("database:");
});
- it("should filter out empty arrays", () => {
+ it("should filter out empty arrays", async () => {
const connector: V1ConnectorDriver = { name: "https" };
- const result = compileConnectorYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
{ url: "https://example.com", headers: [] },
+ envEditSession,
{ orderedProperties: [{ key: "url" }, { key: "headers" }] },
);
expect(result).not.toContain("headers:");
});
- it("should exclude clickhouse managed: false", () => {
+ it("should exclude clickhouse managed: false", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileConnectorYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
{ host: "ch.example.com", managed: false },
+ envEditSession,
{ orderedProperties: [{ key: "host" }, { key: "managed" }] },
);
expect(result).not.toContain("managed");
});
- it("should include clickhouse managed: true", () => {
+ it("should include clickhouse managed: true", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileConnectorYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
{ host: "ch.example.com", managed: true },
+ envEditSession,
{ orderedProperties: [{ key: "host" }, { key: "managed" }] },
);
expect(result).toContain("managed: true");
});
- it("should output driver as duckdb for motherduck", () => {
+ it("should output driver as duckdb for motherduck", async () => {
const connector: V1ConnectorDriver = { name: "motherduck" };
- const result = compileConnectorYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
{ path: "md:my_db" },
+ envEditSession,
{ orderedProperties: [{ key: "path" }] },
);
expect(result).toContain("driver: duckdb");
expect(result).not.toContain("driver: motherduck");
});
- it("should apply fieldFilter to exclude internal properties", () => {
+ it("should apply fieldFilter to exclude internal properties", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileConnectorYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
{ host: "ch.example.com", managed: true },
+ envEditSession,
{
orderedProperties: [
{ key: "host" },
@@ -850,325 +709,19 @@ describe("compileConnectorYAML", () => {
expect(result).not.toContain("managed:");
});
- it("should handle env var conflict resolution with existingEnvBlob", () => {
+ it("should produce no property lines when orderedProperties is empty", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const schema = {
- properties: {
- password: { "x-env-var-name": "CLICKHOUSE_PASSWORD" },
- },
- };
- const result = compileConnectorYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateYAML(
connector,
- { password: "secret" },
- {
- orderedProperties: [{ key: "password", secret: true }],
- secretKeys: ["password"],
- schema,
- existingEnvBlob: "CLICKHOUSE_PASSWORD=old_value",
- },
+ { host: "ch.example.com" },
+ envEditSession,
);
- expect(result).toContain("CLICKHOUSE_PASSWORD_1");
- });
-
- it("should produce no property lines when orderedProperties is empty", () => {
- const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileConnectorYAML(connector, { host: "ch.example.com" });
expect(result).toContain("type: connector");
expect(result).toContain("driver: clickhouse");
expect(result).not.toContain("host:");
});
});
-
-describe("getEnvVarsFromConnectorYAML", () => {
- it("should extract env vars from connector YAML", () => {
- const yaml = `
-type: connector
-driver: clickhouse
-host: {{ .env.CLICKHOUSE_HOST }}
-password: {{ .env.CLICKHOUSE_PASSWORD }}
-`;
- const result = getEnvVarsFromConnectorYAML(yaml);
- expect(result).toEqual(["CLICKHOUSE_HOST", "CLICKHOUSE_PASSWORD"]);
- });
-});
-
-describe("updateDotEnvWithSecrets", () => {
- const mockClient = {
- instanceId: "test-instance-id",
- } as unknown as RuntimeClient;
-
- // Track fetchQuery calls so tests can inspect them
- let mockEnvBlob = "";
- const mockQueryClient = {
- invalidateQueries: vi.fn().mockResolvedValue(undefined),
- fetchQuery: vi
- .fn()
- .mockImplementation(() => Promise.resolve({ blob: mockEnvBlob })),
- };
-
- beforeEach(() => {
- vi.clearAllMocks();
- mockEnvBlob = "";
- mockQueryClient.fetchQuery.mockImplementation(() =>
- Promise.resolve({ blob: mockEnvBlob }),
- );
- });
-
- it("should add secret keys to empty .env", async () => {
- const connector: V1ConnectorDriver = { name: "clickhouse" };
- const formValues: Record = {
- password: "my_secret",
- sql: "SELECT 1",
- };
- const { newBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: ["password"] },
- );
- expect(newBlob).toContain("CLICKHOUSE_PASSWORD=my_secret");
- });
-
- it("should add multiple secret keys", async () => {
- const connector: V1ConnectorDriver = { name: "s3" };
- const formValues: Record = {
- aws_access_key_id: "AKID123",
- aws_secret_access_key: "SECRET456",
- };
- const schema = {
- properties: {
- aws_access_key_id: { "x-env-var-name": "AWS_ACCESS_KEY_ID" },
- aws_secret_access_key: {
- "x-env-var-name": "AWS_SECRET_ACCESS_KEY",
- },
- },
- };
- const { newBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: ["aws_access_key_id", "aws_secret_access_key"], schema },
- );
- expect(newBlob).toContain("AWS_ACCESS_KEY_ID=AKID123");
- expect(newBlob).toContain("AWS_SECRET_ACCESS_KEY=SECRET456");
- });
-
- it("should append to existing .env without overwriting", async () => {
- mockEnvBlob = "EXISTING_VAR=existing_value";
- mockQueryClient.fetchQuery.mockResolvedValue({ blob: mockEnvBlob });
-
- const connector: V1ConnectorDriver = { name: "clickhouse" };
- const formValues: Record = { password: "new_pw" };
- const { newBlob, originalBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: ["password"] },
- );
- expect(originalBlob).toBe("EXISTING_VAR=existing_value");
- expect(newBlob).toContain("EXISTING_VAR=existing_value");
- expect(newBlob).toContain("CLICKHOUSE_PASSWORD=new_pw");
- });
-
- it("should handle env var conflicts with _1 suffix", async () => {
- mockEnvBlob = "CLICKHOUSE_PASSWORD=old_value";
- mockQueryClient.fetchQuery.mockResolvedValue({ blob: mockEnvBlob });
-
- const connector: V1ConnectorDriver = { name: "clickhouse" };
- const formValues: Record = { password: "new_value" };
- const { newBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: ["password"] },
- );
- // Should use _1 suffix since base name already exists
- expect(newBlob).toContain("CLICKHOUSE_PASSWORD=old_value");
- expect(newBlob).toContain("CLICKHOUSE_PASSWORD_1=new_value");
- });
-
- it("should skip empty or missing secret values", async () => {
- const connector: V1ConnectorDriver = { name: "clickhouse" };
- const formValues: Record = {
- password: "",
- dsn: undefined,
- };
- const { newBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: ["password", "dsn"] },
- );
- expect(newBlob).toBe("");
- });
-
- it("should persist sensitive header values as env entries", async () => {
- const connector: V1ConnectorDriver = { name: "https" };
- const formValues: Record = {
- headers: [
- { key: "Authorization", value: "Bearer my_token" },
- { key: "Content-Type", value: "application/json" },
- ],
- };
- const { newBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: [] },
- );
- // Authorization is sensitive — secret part stored (without Bearer prefix)
- expect(newBlob).toContain("my_token");
- // Content-Type is not sensitive — should NOT be in .env
- expect(newBlob).not.toContain("application/json");
- });
-
- it("should extract secret from Bearer scheme for sensitive headers", async () => {
- const connector: V1ConnectorDriver = { name: "https" };
- const formValues: Record = {
- headers: [{ key: "Authorization", value: "Bearer abc123" }],
- };
- const { newBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: [] },
- );
- // Only the secret portion (after "Bearer ") is stored
- expect(newBlob).toContain("=abc123");
- expect(newBlob).not.toContain("Bearer");
- });
-
- it("should store full value when no auth scheme prefix", async () => {
- const connector: V1ConnectorDriver = { name: "https" };
- const formValues: Record = {
- headers: [{ key: "X-API-Key", value: "raw_api_key_value" }],
- };
- const { newBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: [] },
- );
- expect(newBlob).toContain("=raw_api_key_value");
- });
-
- it("should handle both secrets and sensitive headers together", async () => {
- const connector: V1ConnectorDriver = { name: "https" };
- const formValues: Record = {
- password: "http_pass",
- headers: [{ key: "Authorization", value: "Token secret_tok" }],
- };
- const { newBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: ["password"] },
- );
- expect(newBlob).toContain("HTTPS_PASSWORD=http_pass");
- expect(newBlob).toContain("secret_tok");
- });
-
- it("should skip headers with empty keys or values", async () => {
- const connector: V1ConnectorDriver = { name: "https" };
- const formValues: Record = {
- headers: [
- { key: "", value: "some_value" },
- { key: "Authorization", value: "" },
- { key: " ", value: "Bearer token" },
- ],
- };
- const { newBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: [] },
- );
- expect(newBlob).toBe("");
- });
-
- it("should invalidate cache before reading .env", async () => {
- const connector: V1ConnectorDriver = { name: "clickhouse" };
- await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- { password: "pw" },
- { secretKeys: ["password"] },
- );
- expect(mockQueryClient.invalidateQueries).toHaveBeenCalledTimes(1);
- // invalidateQueries should be called before fetchQuery
- const invalidateOrder =
- mockQueryClient.invalidateQueries.mock.invocationCallOrder[0];
- const fetchOrder = mockQueryClient.fetchQuery.mock.invocationCallOrder[0];
- expect(invalidateOrder).toBeLessThan(fetchOrder);
- });
-
- it("should handle missing .env file gracefully", async () => {
- mockQueryClient.fetchQuery.mockRejectedValue({
- response: { data: { message: "no such file" } },
- });
-
- const connector: V1ConnectorDriver = { name: "clickhouse" };
- const { newBlob, originalBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- { password: "pw" },
- { secretKeys: ["password"] },
- );
- expect(originalBlob).toBe("");
- expect(newBlob).toContain("CLICKHOUSE_PASSWORD=pw");
- });
-
- it("should rethrow non-file-not-found errors", async () => {
- mockQueryClient.fetchQuery.mockRejectedValue({
- response: { data: { message: "permission denied" } },
- });
-
- const connector: V1ConnectorDriver = { name: "clickhouse" };
- await expect(
- updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- { password: "pw" },
- { secretKeys: ["password"] },
- ),
- ).rejects.toEqual({
- response: { data: { message: "permission denied" } },
- });
- });
-
- it("should use originalBlob for conflict detection across all secrets", async () => {
- // When adding multiple secrets, conflict detection should use the original blob,
- // not the progressively updated one
- mockEnvBlob = "";
- mockQueryClient.fetchQuery.mockResolvedValue({ blob: mockEnvBlob });
-
- const connector: V1ConnectorDriver = { name: "clickhouse" };
- const formValues: Record = {
- password: "pw1",
- dsn: "clickhouse://...",
- };
- const { newBlob } = await updateDotEnvWithSecrets(
- mockClient,
- mockQueryClient as any,
- connector,
- formValues,
- { secretKeys: ["password", "dsn"] },
- );
- // Both should use base name since originalBlob is empty
- expect(newBlob).toContain("CLICKHOUSE_PASSWORD=pw1");
- expect(newBlob).toContain("CLICKHOUSE_DSN=clickhouse://...");
- });
-});
diff --git a/web-common/src/features/connectors/code-utils.ts b/web-common/src/features/connectors/code-utils.ts
index 53a723ac4f66..66538ecfacd7 100644
--- a/web-common/src/features/connectors/code-utils.ts
+++ b/web-common/src/features/connectors/code-utils.ts
@@ -1,32 +1,35 @@
import { QueryClient } from "@tanstack/svelte-query";
import {
- type V1ConnectorDriver,
type ConnectorDriverProperty,
+ getRuntimeServiceAnalyzeConnectorsQueryKey,
getRuntimeServiceGetFileQueryKey,
+ getRuntimeServiceGetInstanceQueryKey,
+ runtimeServiceAnalyzeConnectors,
runtimeServiceGetFile,
- type V1GetFileResponse,
+ runtimeServiceGetInstance,
+ runtimeServicePutFile,
+ type V1ConnectorDriver,
} from "../../runtime-client";
import type { RuntimeClient } from "../../runtime-client/v2";
import { fileArtifacts } from "@rilldata/web-common/features/entity-management/file-artifacts";
import { ResourceKind } from "@rilldata/web-common/features/entity-management/resource-selectors";
-import { extractErrorMessage } from "@rilldata/web-common/lib/errors";
import { eventBus } from "@rilldata/web-common/lib/event-bus/event-bus";
import {
getName,
isNonStandardIdentifier,
} from "@rilldata/web-common/features/entity-management/name-utils";
-import {
- getRuntimeServiceAnalyzeConnectorsQueryKey,
- getRuntimeServiceGetInstanceQueryKey,
- runtimeServiceAnalyzeConnectors,
- runtimeServiceGetInstance,
- runtimeServicePutFile,
-} from "../../runtime-client";
import {
getDriverNameForConnector,
makeSufficientlyQualifiedTableName,
} from "./connectors-utils";
import { getDocsCategory } from "../sources/modal/connector-schemas";
+import type { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
+import {
+ applyDuckLakeFormPipeline,
+ injectDuckLakeAttach,
+} from "@rilldata/web-common/features/templates/schemas/ducklake-utils.ts";
+import { filterSchemaValuesForSubmit } from "@rilldata/web-common/features/templates/schema-utils.ts";
+import type { MultiStepFormSchema } from "@rilldata/web-common/features/templates/schemas/types.ts";
function yamlModelTemplate(driverName: string) {
return `# Model YAML
@@ -110,11 +113,8 @@ function splitAuthSchemePrefix(
*/
export function formatHeadersAsYamlMap(
value: Array<{ key: string; value: string }> | string,
- driverName?: string,
- connectorInstanceName?: string,
+ envEditSession: EnvEditSession,
): string {
- const nameForEnv = connectorInstanceName || driverName;
-
if (typeof value === "string") {
// Legacy textarea format: parse "Key: Value" lines
const lines = value
@@ -130,9 +130,13 @@ export function formatHeadersAsYamlMap(
.trim()
.replace(/^"|"$/g, "");
let v: string;
- if (nameForEnv && isSensitiveHeaderKey(k)) {
- const envRef = `{{ .env.connector.${nameForEnv}.${headerKeyToEnvSegment(k)} }}`;
+ if (isSensitiveHeaderKey(k)) {
const split = splitAuthSchemePrefix(raw);
+ const entry = envEditSession.acquire(
+ headerKeyToEnvSegment(k),
+ split ? split.secret : raw,
+ );
+ const envRef = `{{ .env.${entry.mappedEnvVarName} }}`;
v = split ? `${split.scheme}${envRef}` : envRef;
} else {
v = raw;
@@ -148,21 +152,27 @@ export function formatHeadersAsYamlMap(
const entries = valid.map((e) => {
const k = e.key.trim();
let v: string;
- if (nameForEnv && isSensitiveHeaderKey(k)) {
- const envRef = `{{ .env.connector.${nameForEnv}.${headerKeyToEnvSegment(k)} }}`;
- const split = splitAuthSchemePrefix(e.value.trim());
+ const trimmedVal = e.value.trim();
+ if (isSensitiveHeaderKey(k)) {
+ const split = splitAuthSchemePrefix(trimmedVal);
+ const entry = envEditSession.acquire(
+ headerKeyToEnvSegment(k),
+ split ? split.secret : trimmedVal,
+ );
+ const envRef = `{{ .env.${entry.mappedEnvVarName} }}`;
v = split ? `${split.scheme}${envRef}` : envRef;
} else {
- v = e.value.trim();
+ v = trimmedVal;
}
return ` "${k}": "${v}"`;
});
return `headers:\n${entries.join("\n")}`;
}
-export function compileConnectorYAML(
+export function generateYAML(
connector: V1ConnectorDriver,
formValues: Record,
+ envEditSession: EnvEditSession,
options?: {
fieldFilter?: (
property:
@@ -176,26 +186,7 @@ export function compileConnectorYAML(
connectorInstanceName?: string;
secretKeys?: string[];
stringKeys?: string[];
- schema?: {
- properties?: Record<
- string,
- {
- "x-env-var-name"?: string;
- default?: string | number | boolean;
- type?: string;
- "x-yaml-value"?:
- | string
- | number
- | boolean
- | {
- true?: string | number | boolean;
- false?: string | number | boolean;
- };
- "x-advanced"?: boolean;
- }
- >;
- };
- existingEnvBlob?: string;
+ schema?: MultiStepFormSchema;
},
) {
// Add instructions to the top of the file
@@ -222,6 +213,22 @@ driver: ${driverName}`;
// Get the secret property keys
const secretPropertyKeys = options?.secretKeys ?? [];
+ envEditSession.startEdit();
+
+ // Apply ducklake transforms
+ formValues = applyDuckLakeFormPipeline(options?.schema, formValues, {
+ connectorName: connector.name ?? "",
+ envEditSession,
+ });
+ formValues = options?.schema
+ ? injectDuckLakeAttach(
+ options.schema,
+ filterSchemaValuesForSubmit(options.schema, formValues, {
+ step: "connector",
+ }),
+ formValues,
+ )
+ : formValues;
// Get the string property keys
const stringPropertyKeys = options?.stringKeys ?? [];
@@ -250,7 +257,8 @@ driver: ${driverName}`;
const typeDefault =
schemaProp.type === "boolean"
? false
- : schemaProp.type === "number" || schemaProp.type === "integer"
+ : schemaProp.type === "number" ||
+ (schemaProp.type as any) === "integer"
? 0
: schemaProp.type === "string"
? ""
@@ -269,20 +277,14 @@ driver: ${driverName}`;
if (key === "headers") {
return formatHeadersAsYamlMap(
value as Array<{ key: string; value: string }> | string,
- driverName,
- options?.connectorInstanceName,
+ envEditSession,
);
}
const isSecretProperty = secretPropertyKeys.includes(key);
if (isSecretProperty) {
- const envVarName = makeEnvVarKey(
- connector.name as string,
- key,
- options?.existingEnvBlob,
- options?.schema,
- );
- return `${key}: "{{ .env.${envVarName} }}"`; // uses standard Go template syntax
+ const entry = envEditSession.acquire(key, String(value));
+ return `${key}: "{{ .env.${entry.mappedEnvVarName} }}"`; // uses standard Go template syntax
}
// For boolean fields with x-yaml-value, emit the mapped value instead of true/false.
@@ -315,300 +317,6 @@ driver: ${driverName}`;
return `${topOfFile}\n` + compiledKeyValues;
}
-const EnvTemplateRegex = /{{\s*\.env\.([^.\s]+)\s*}}/g;
-
-export function getEnvVarsFromConnectorYAML(yaml: string) {
- const envVars: string[] = [];
- let match: RegExpExecArray | null;
- while ((match = EnvTemplateRegex.exec(yaml)) !== null) {
- envVars.push(match[1]);
- }
- return envVars;
-}
-
-export async function unsetResourceEnvVars(
- runtimeClient: RuntimeClient,
- queryClient: QueryClient,
- yaml: string,
-): Promise<[string, boolean]> {
- let envBlob: V1GetFileResponse | undefined = undefined;
- try {
- envBlob = await queryClient.fetchQuery({
- queryKey: getRuntimeServiceGetFileQueryKey(runtimeClient.instanceId, {
- path: ".env",
- }),
- queryFn: () => runtimeServiceGetFile(runtimeClient, { path: ".env" }),
- });
- } catch (error) {
- if (error.message?.includes("no such file or directory")) {
- return ["", false];
- }
- throw error;
- }
-
- // Get the existing env and remove the resource's env vars
- let blob = envBlob?.blob ?? "";
- const envVars = getEnvVarsFromConnectorYAML(yaml);
- envVars.forEach((envVar) => {
- blob = deleteEnvVariable(blob, envVar);
- });
-
- return [blob, envVars.length > 0];
-}
-
-export async function updateDotEnvWithSecrets(
- client: RuntimeClient,
- queryClient: QueryClient,
- connector: V1ConnectorDriver,
- formValues: Record,
- opts?: {
- secretKeys?: string[];
- schema?: { properties?: Record };
- // Existing .env blob to use instead of reading from disk.
- // Use this when a concurrent operation (e.g. Test and Connect) may have
- // already modified .env; passing the original blob avoids generating
- // duplicate suffixed env var names.
- existingEnvBlob?: string;
- },
-): Promise<{ newBlob: string; originalBlob: string }> {
- let blob: string;
- let originalBlob: string;
-
- if (opts?.existingEnvBlob !== undefined) {
- blob = opts.existingEnvBlob;
- originalBlob = opts.existingEnvBlob;
- } else {
- // Invalidate the cache to ensure we get fresh .env content
- // This prevents overwriting credentials added by a previous step
- await queryClient.invalidateQueries({
- queryKey: getRuntimeServiceGetFileQueryKey(client.instanceId, {
- path: ".env",
- }),
- });
-
- // Get the existing .env file with fresh data
- try {
- const file = await queryClient.fetchQuery({
- queryKey: getRuntimeServiceGetFileQueryKey(client.instanceId, {
- path: ".env",
- }),
- queryFn: () => runtimeServiceGetFile(client, { path: ".env" }),
- });
- blob = file.blob || "";
- originalBlob = blob; // Keep original for conflict detection
- } catch (error) {
- // Handle the case where the .env file does not exist
- if (extractErrorMessage(error).includes("no such file")) {
- blob = "";
- originalBlob = "";
- } else {
- throw error;
- }
- }
- }
-
- // Get the secret keys
- const secretKeys = opts?.secretKeys ?? [];
-
- // In reality, all connectors have secret keys, but this is a safeguard
- if (!secretKeys) {
- return { newBlob: blob, originalBlob };
- }
-
- // Update the blob with the new secrets
- // Use originalBlob for conflict detection so all secrets use consistent naming
- secretKeys.forEach((key) => {
- if (!key || !formValues[key]) {
- return;
- }
-
- const connectorSecretKey = makeEnvVarKey(
- connector.name as string,
- key,
- originalBlob,
- opts?.schema,
- );
-
- blob = replaceOrAddEnvVariable(
- blob,
- connectorSecretKey,
- formValues[key] as string,
- );
- });
-
- // Persist sensitive header values (e.g. Authorization, API keys) as
- // individual .env entries so tokens are not stored as raw text in YAML.
- const headers = formValues.headers;
- if (Array.isArray(headers)) {
- for (const entry of headers as Array<{ key: string; value: string }>) {
- if (!entry.key?.trim() || !entry.value?.trim()) continue;
- if (!isSensitiveHeaderKey(entry.key)) continue;
- const envSegment = headerKeyToEnvSegment(entry.key);
- const envKey = makeEnvVarKey(
- connector.name as string,
- envSegment,
- originalBlob,
- opts?.schema,
- );
- const raw = entry.value.trim();
- const split = splitAuthSchemePrefix(raw);
- blob = replaceOrAddEnvVariable(blob, envKey, split ? split.secret : raw);
- }
- }
-
- return { newBlob: blob, originalBlob };
-}
-
-export function replaceOrAddEnvVariable(
- existingEnvBlob: string,
- key: string,
- newValue: string,
-): string {
- const lines = existingEnvBlob.split("\n");
- let keyFound = false;
-
- const updatedLines = lines.map((line) => {
- if (line.startsWith(`${key}=`)) {
- keyFound = true;
- return `${key}=${newValue}`;
- }
- return line;
- });
-
- if (!keyFound) {
- updatedLines.push(`${key}=${newValue}`);
- }
-
- const newBlob = updatedLines
- .filter((line, index) => !(line === "" && index === 0))
- .join("\n")
- .trim();
-
- return newBlob;
-}
-
-export function deleteEnvVariable(
- existingEnvBlob: string,
- key: string,
-): string {
- const lines = existingEnvBlob.split("\n");
- const updatedLines = lines.filter((line) => !line.startsWith(`${key}=`));
- const newBlob = updatedLines
- .filter((line, index) => !(line === "" && index === 0))
- .join("\n")
- .trim();
-
- return newBlob;
-}
-
-/**
- * Get a generic ALL_CAPS environment variable name for a connector property.
- * If schema provides x-env-var-name, use it directly.
- * Otherwise uses DRIVER_NAME_PROPERTY_KEY format.
- *
- * @param driverName - The connector driver name (e.g., "clickhouse", "s3")
- * @param propertyKey - The property key (e.g., "password", "aws_access_key_id")
- * @param schema - Optional schema with x-env-var-name annotations
- * @returns The environment variable name in SCREAMING_SNAKE_CASE
- *
- * @example
- * getGenericEnvVarName("clickhouse", "password") // "CLICKHOUSE_PASSWORD"
- * getGenericEnvVarName("s3", "aws_access_key_id", s3Schema) // "AWS_ACCESS_KEY_ID" (from x-env-var-name)
- */
-export function getGenericEnvVarName(
- driverName: string,
- propertyKey: string,
- schema?: { properties?: Record },
-): string {
- // If schema provides explicit env var name, use it
- const field = schema?.properties?.[propertyKey];
- if (field?.["x-env-var-name"]) {
- return field["x-env-var-name"];
- }
-
- // Convert property key to SCREAMING_SNAKE_CASE
- const propertyKeyUpper = propertyKey
- .replace(/([a-z])([A-Z])/g, "$1_$2")
- .replace(/[._-]+/g, "_")
- .toUpperCase();
-
- // Otherwise, use DriverName_PropertyKey format
- const driverNameUpper = driverName
- .replace(/([a-z])([A-Z])/g, "$1_$2")
- .replace(/[._-]+/g, "_")
- .toUpperCase();
-
- return `${driverNameUpper}_${propertyKeyUpper}`;
-}
-
-/**
- * Check if an environment variable exists in the env blob.
- *
- * @param envBlob - The contents of the .env file as a string
- * @param varName - The environment variable name to check for
- * @returns true if the variable exists, false otherwise
- */
-export function envVarExists(envBlob: string, varName: string): boolean {
- const lines = envBlob.split("\n");
- return lines.some((line) => line.startsWith(`${varName}=`));
-}
-
-/**
- * Find the next available environment variable name by appending _1, _2, etc.
- * Used to avoid conflicts when creating multiple connectors of the same type.
- *
- * @param envBlob - The contents of the .env file as a string
- * @param baseName - The base environment variable name to check
- * @returns The first available name (baseName, baseName_1, baseName_2, etc.)
- *
- * @example
- * // If .env contains "AWS_ACCESS_KEY_ID=xxx"
- * findAvailableEnvVarName(envBlob, "AWS_ACCESS_KEY_ID") // "AWS_ACCESS_KEY_ID_1"
- */
-export function findAvailableEnvVarName(
- envBlob: string,
- baseName: string,
-): string {
- let varName = baseName;
- let counter = 1;
-
- while (envVarExists(envBlob, varName)) {
- varName = `${baseName}_${counter}`;
- counter++;
- }
-
- return varName;
-}
-
-/**
- * Generate an environment variable key for a property.
- * Uses schema-defined x-env-var-name when available, otherwise generates
- * DRIVER_NAME_PROPERTY_KEY format. Handles conflicts by appending _1, _2, etc.
- *
- * @param driverName - The connector driver name (e.g., "clickhouse", "s3")
- * @param key - The property key (e.g., "password", "dsn")
- * @param existingEnvBlob - Optional existing .env content for conflict detection
- * @param schema - Optional schema with x-env-var-name annotations
- * @returns The environment variable name, with suffix if needed to avoid conflicts
- */
-export function makeEnvVarKey(
- driverName: string,
- key: string,
- existingEnvBlob?: string,
- schema?: { properties?: Record },
-): string {
- // Generate generic ALL_CAPS environment variable name
- const baseGenericName = getGenericEnvVarName(driverName, key, schema);
-
- // If no existing env blob is provided, just return the base generic name
- if (!existingEnvBlob) {
- return baseGenericName;
- }
-
- // Check for conflicts and append _# if necessary
- return findAvailableEnvVarName(existingEnvBlob, baseGenericName);
-}
-
export async function updateRillYAMLWithOlapConnector(
client: RuntimeClient,
queryClient: QueryClient,
diff --git a/web-common/src/features/connectors/env-utils.ts b/web-common/src/features/connectors/env-utils.ts
new file mode 100644
index 000000000000..a340e143fd5b
--- /dev/null
+++ b/web-common/src/features/connectors/env-utils.ts
@@ -0,0 +1,41 @@
+/**
+ * Get a generic ALL_CAPS environment variable name for a connector property.
+ * If schema provides x-env-var-name, use it directly.
+ * Otherwise uses DRIVER_NAME_PROPERTY_KEY format.
+ *
+ * @param namespace - Usually the connector driver name (e.g., "clickhouse", "s3")
+ * @param propertyKey - The property key (e.g., "password", "aws_access_key_id")
+ * @param schema - Optional schema with x-env-var-name annotations
+ * @returns The environment variable name in SCREAMING_SNAKE_CASE
+ *
+ * @example
+ * getGenericEnvVarName("clickhouse", "password") // "CLICKHOUSE_PASSWORD"
+ * getGenericEnvVarName("s3", "aws_access_key_id", s3Schema) // "AWS_ACCESS_KEY_ID" (from x-env-var-name)
+ */
+export function getGenericEnvVarName(
+ namespace: string,
+ propertyKey: string,
+ schema: {
+ properties?: Record;
+ } | null = null,
+): string {
+ // If schema provides explicit env var name, use it
+ const field = schema?.properties?.[propertyKey];
+ if (field?.["x-env-var-name"]) {
+ return field["x-env-var-name"];
+ }
+
+ // Convert property key to SCREAMING_SNAKE_CASE
+ const propertyKeyUpper = propertyKey
+ .replace(/([a-z])([A-Z])/g, "$1_$2")
+ .replace(/[._-]+/g, "_")
+ .toUpperCase();
+
+ // Otherwise, use Namespace_PropertyKey format
+ const namespaceUpper = namespace
+ .replace(/([a-z])([A-Z])/g, "$1_$2")
+ .replace(/[._-]+/g, "_")
+ .toUpperCase();
+
+ return `${namespaceUpper}_${propertyKeyUpper}`;
+}
diff --git a/web-common/src/features/env-management/dot-env.spec.ts b/web-common/src/features/env-management/dot-env.spec.ts
new file mode 100644
index 000000000000..53132a48ed67
--- /dev/null
+++ b/web-common/src/features/env-management/dot-env.spec.ts
@@ -0,0 +1,134 @@
+import { describe, expect, it } from "vitest";
+import { parseDotEnv, serializeDotEnv } from "./dot-env";
+
+describe("parseDotEnv", () => {
+ it("parses basic KEY=VALUE pairs", () => {
+ expect(parseDotEnv("FOO=bar\nBAZ=qux")).toEqual({ FOO: "bar", BAZ: "qux" });
+ });
+
+ it("ignores comment-only lines and blank lines", () => {
+ const src = "# leading\n\nFOO=bar\n # indented\nBAZ=qux\n";
+ expect(parseDotEnv(src)).toEqual({ FOO: "bar", BAZ: "qux" });
+ });
+
+ it("strips trailing inline comments from unquoted values", () => {
+ expect(parseDotEnv("FOO=bar # trailing comment")).toEqual({ FOO: "bar" });
+ });
+
+ it("treats unquoted `#` as the start of a comment", () => {
+ // Standard dotenv behavior: unquoted `#` is a comment marker. Callers
+ // that need a literal `#` in the value must quote it.
+ expect(parseDotEnv("URL=https://example.com#section")).toEqual({
+ URL: "https://example.com",
+ });
+ });
+
+ it("preserves `#` inside quoted values", () => {
+ expect(parseDotEnv('URL="https://example.com#section"')).toEqual({
+ URL: "https://example.com#section",
+ });
+ expect(parseDotEnv("URL='https://example.com#section'")).toEqual({
+ URL: "https://example.com#section",
+ });
+ });
+
+ it("preserves `=` in unquoted values (splits on the first `=` only)", () => {
+ expect(parseDotEnv("DSN=postgres://u:p@h/db?opt=1&other=2")).toEqual({
+ DSN: "postgres://u:p@h/db?opt=1&other=2",
+ });
+ });
+
+ it("strips matching surrounding single, double, and backtick quotes", () => {
+ expect(parseDotEnv("A='single'\nB=\"double\"\nC=`back`")).toEqual({
+ A: "single",
+ B: "double",
+ C: "back",
+ });
+ });
+
+ it("expands \\n and \\r escapes inside double quotes", () => {
+ expect(parseDotEnv('JSON="{\\n \\"k\\": 1\\n}"').JSON).toBe(
+ '{\n \\"k\\": 1\n}',
+ );
+ });
+
+ it("does not expand escapes inside single quotes", () => {
+ expect(parseDotEnv("RAW='line1\\nline2'")).toEqual({
+ RAW: "line1\\nline2",
+ });
+ });
+
+ it("supports the `export` prefix", () => {
+ expect(parseDotEnv("export FOO=bar")).toEqual({ FOO: "bar" });
+ });
+
+ it("returns empty values for KEY= (no value after equals)", () => {
+ expect(parseDotEnv("EMPTY=\nFOO=bar")).toEqual({ EMPTY: "", FOO: "bar" });
+ });
+
+ it("normalizes CRLF line endings", () => {
+ expect(parseDotEnv("FOO=bar\r\nBAZ=qux")).toEqual({
+ FOO: "bar",
+ BAZ: "qux",
+ });
+ });
+
+ it("trims whitespace around bare values", () => {
+ expect(parseDotEnv("FOO= bar ")).toEqual({ FOO: "bar" });
+ });
+
+ it("returns an empty object for empty input", () => {
+ expect(parseDotEnv("")).toEqual({});
+ });
+});
+
+describe("serializeDotEnv", () => {
+ it("writes plain KEY=VALUE for simple values", () => {
+ expect(serializeDotEnv({ FOO: "bar", BAZ: "qux" })).toBe(
+ "FOO=bar\nBAZ=qux",
+ );
+ });
+
+ it("writes KEY= for empty values", () => {
+ expect(serializeDotEnv({ EMPTY: "", FOO: "bar" })).toBe("EMPTY=\nFOO=bar");
+ });
+
+ it("quotes values containing `#`", () => {
+ expect(serializeDotEnv({ URL: "https://example.com#section" })).toBe(
+ "URL='https://example.com#section'",
+ );
+ });
+
+ it("quotes values containing whitespace", () => {
+ expect(serializeDotEnv({ NOTE: "two words" })).toBe("NOTE='two words'");
+ });
+
+ it("uses double quotes when the value contains a single quote", () => {
+ expect(serializeDotEnv({ MSG: "it's fine" })).toBe(`MSG="it's fine"`);
+ });
+
+ it("escapes newlines using double-quoted \\n", () => {
+ expect(serializeDotEnv({ MULTI: "line1\nline2" })).toBe(
+ `MULTI="line1\\nline2"`,
+ );
+ });
+});
+
+describe("round-trip", () => {
+ it.each([
+ { FOO: "bar" },
+ { URL: "https://example.com#section" },
+ { PASSWORD: "p@ss#w0rd!" },
+ { DSN: "postgres://u:p@h/db?opt=1&other=2" },
+ { NOTE: "two words" },
+ { MSG: "it's fine" },
+ { MULTI: "line1\nline2" },
+ { EMPTY: "" },
+ { FOO: "bar", URL: "https://example.com#section", MSG: "it's fine" },
+ ] as Record[])(
+ "serialize → parse preserves %j",
+ (entries) => {
+ expect(parseDotEnv(serializeDotEnv(entries))).toEqual(entries);
+ },
+ );
+});
diff --git a/web-common/src/features/env-management/dot-env.ts b/web-common/src/features/env-management/dot-env.ts
new file mode 100644
index 000000000000..af3a3d278b7f
--- /dev/null
+++ b/web-common/src/features/env-management/dot-env.ts
@@ -0,0 +1,46 @@
+// Parser borrowed from dotenv:
+// https://github.com/motdotla/dotenv/blob/master/lib/main.js
+const LINE_RE =
+ /^\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?$/gm;
+
+export function parseDotEnv(src: string): Record {
+ const obj: Record = {};
+ const normalized = src.replace(/\r\n?/g, "\n");
+
+ LINE_RE.lastIndex = 0;
+ let match: RegExpExecArray | null;
+ while ((match = LINE_RE.exec(normalized)) != null) {
+ const key = match[1];
+ let value = (match[2] ?? "").trim();
+ const quote = value[0];
+ value = value.replace(/^(['"`])([\s\S]*)\1$/, "$2");
+ if (quote === '"') {
+ value = value.replace(/\\n/g, "\n").replace(/\\r/g, "\r");
+ }
+ obj[key] = value;
+ }
+ return obj;
+}
+
+// Characters that change a bare value's meaning: whitespace (would be trimmed),
+// `#` (starts a comment), or any quote char (would be read as the opening quote).
+const NEEDS_QUOTING_RE = /[\s#"'`]/;
+
+export function serializeDotEnv(entries: Record): string {
+ return Object.entries(entries)
+ .map(([k, v]) => {
+ if (v === "") return `${k}=`;
+ if (!NEEDS_QUOTING_RE.test(v)) return `${k}=${v}`;
+ // Prefer single quotes when possible: dotenv treats single-quoted
+ // content literally, so we don't have to escape anything inside.
+ // Newlines can't appear inside single quotes since the value must
+ // stay on one line; fall through to double quotes in that case.
+ if (!v.includes("'") && !/[\n\r]/.test(v)) return `${k}='${v}'`;
+ // Double-quoted: the parser only unescapes \n and \r, so those are
+ // the only sequences we escape. Backslashes are passed through
+ // verbatim because the parser would not unescape them either.
+ const escaped = v.replace(/\n/g, "\\n").replace(/\r/g, "\\r");
+ return `${k}="${escaped}"`;
+ })
+ .join("\n");
+}
diff --git a/web-common/src/features/env-management/env-edit-session-variable.ts b/web-common/src/features/env-management/env-edit-session-variable.ts
new file mode 100644
index 000000000000..3a04b2f95746
--- /dev/null
+++ b/web-common/src/features/env-management/env-edit-session-variable.ts
@@ -0,0 +1,7 @@
+export class EnvEditSessionVariable {
+ constructor(
+ public readonly key: string,
+ public value: string,
+ public readonly mappedEnvVarName: string,
+ ) {}
+}
diff --git a/web-common/src/features/env-management/env-edit-session.ts b/web-common/src/features/env-management/env-edit-session.ts
new file mode 100644
index 000000000000..b455810fbad3
--- /dev/null
+++ b/web-common/src/features/env-management/env-edit-session.ts
@@ -0,0 +1,89 @@
+import { EnvEditSessionVariable } from "@rilldata/web-common/features/env-management/env-edit-session-variable.ts";
+import type { EnvStore } from "@rilldata/web-common/features/env-management/env-store.ts";
+import { getName } from "@rilldata/web-common/features/entity-management/name-utils.ts";
+import type { JSONSchemaObject } from "@rilldata/web-common/features/templates/schemas/types.ts";
+import { getGenericEnvVarName } from "@rilldata/web-common/features/connectors/env-utils.ts";
+
+export class EnvEditSession {
+ public readonly entries = new Map();
+ public readonly inflightEntries = new Map();
+
+ // Final env var name → value this session has successfully committed.
+ // Read by rollback to distinguish session-owned vars from externally
+ // changed ones, and by startEdit to decide whether a prior allocation can
+ // still be reused.
+ private committed = new Map();
+
+ private assignedVars = new Set();
+
+ public constructor(
+ public readonly parentStore: EnvStore,
+ private readonly namespace: string = "",
+ private readonly schema: JSONSchemaObject | null = null,
+ ) {
+ this.assignedVars = new Set(this.parentStore.store.keys());
+ }
+
+ public startEdit() {
+ this.inflightEntries.clear();
+ this.entries.forEach((v: EnvEditSessionVariable) => {
+ const writtenValue = this.committed.get(v.mappedEnvVarName);
+ if (writtenValue !== undefined) {
+ // This entry was committed earlier in the session. If parent.store
+ // no longer matches what we wrote, an external party has taken over
+ // the name — drop our claim so the next acquire allocates a fresh
+ // suffixed name instead of overwriting the external value.
+ const current = this.parentStore.store.get(v.mappedEnvVarName);
+ if (current?.value !== writtenValue) {
+ this.committed.delete(v.mappedEnvVarName);
+ return;
+ }
+ }
+ this.inflightEntries.set(v.key, v);
+ });
+ this.entries.clear();
+
+ this.assignedVars = new Set(this.parentStore.store.keys());
+ }
+
+ public acquire(key: string, value: string, envVarName?: string) {
+ if (this.inflightEntries.has(key)) {
+ const entry = this.inflightEntries.get(key)!;
+ entry.value = value;
+ this.inflightEntries.delete(key);
+ this.entries.set(key, entry);
+ this.assignedVars.add(key);
+ return entry;
+ }
+
+ envVarName ??= getGenericEnvVarName(this.namespace, key, this.schema);
+ const mappedEnvVarName = this.acquireVarName(envVarName);
+ const entry = new EnvEditSessionVariable(key, value, mappedEnvVarName);
+
+ this.entries.set(key, entry);
+ this.assignedVars.add(key);
+ return entry;
+ }
+
+ public async commit() {
+ await this.parentStore.flush(this);
+ // Record successful writes so rollback can compare against them.
+ this.entries.forEach((entry) => {
+ this.committed.set(entry.mappedEnvVarName, entry.value);
+ });
+ this.inflightEntries.forEach((entry) => {
+ this.committed.delete(entry.mappedEnvVarName);
+ });
+ }
+
+ public async rollback() {
+ await this.parentStore.rollbackOwned(this.committed);
+ this.committed.clear();
+ }
+
+ private acquireVarName(varName: string) {
+ const assignedVarName = getName(varName, [...this.assignedVars]);
+ this.assignedVars.add(assignedVarName);
+ return assignedVarName;
+ }
+}
diff --git a/web-common/src/features/env-management/env-file-store.spec.ts b/web-common/src/features/env-management/env-file-store.spec.ts
new file mode 100644
index 000000000000..3ba121663dcc
--- /dev/null
+++ b/web-common/src/features/env-management/env-file-store.spec.ts
@@ -0,0 +1,197 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { EnvStore } from "@rilldata/web-common/features/env-management/env-store.ts";
+
+// fileArtifacts.getFileArtifact("/.env") drives the getter. Each test sets
+// `mockEnvContent`, then triggers `envStore.pull()` to exercise the parser.
+let mockEnvContent: string | undefined = undefined;
+const fetchContent = vi.fn(async () => mockEnvContent);
+
+vi.mock(
+ "@rilldata/web-common/features/entity-management/file-artifacts",
+ () => ({
+ fileArtifacts: {
+ getFileArtifact: vi.fn(() => ({ fetchContent })),
+ },
+ }),
+);
+
+vi.mock(
+ "@rilldata/web-common/features/entity-management/edit-environment.ts",
+ () => ({
+ isCloudRuntimeEditEnvironment: vi.fn(() => false),
+ }),
+);
+
+vi.mock("@rilldata/web-common/runtime-client", () => ({
+ runtimeServicePutFile: vi.fn(async () => ({})),
+ runtimeServicePushEnv: vi.fn(async () => ({})),
+}));
+
+// createEnvFileStore calls setContext, which only works inside a Svelte
+// component lifecycle. Replace it with a simple in-memory slot so we can
+// retrieve the store via getEnvFileStore the same way callers do.
+let capturedStore: EnvStore | null = null;
+vi.mock("svelte", () => ({
+ setContext: vi.fn((_key: string, value: EnvStore) => {
+ capturedStore = value;
+ }),
+ getContext: vi.fn(() => capturedStore),
+}));
+
+import {
+ runtimeServicePushEnv,
+ runtimeServicePutFile,
+} from "@rilldata/web-common/runtime-client";
+import { isCloudRuntimeEditEnvironment } from "@rilldata/web-common/features/entity-management/edit-environment.ts";
+import { eventBus } from "@rilldata/web-common/lib/event-bus/event-bus.ts";
+import { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
+import { createEnvFileStore, getEnvFileStore } from "./env-file-store";
+
+const runtimeClient = { instanceId: "inst-1" } as never;
+
+async function setupStore(envContent: string | undefined) {
+ mockEnvContent = envContent;
+ const unsubscribe = createEnvFileStore(runtimeClient);
+ const store = getEnvFileStore();
+ // createEnvFileStore fires `void envStore.pull()`; await an explicit pull
+ // so assertions don't race with that floating promise.
+ await store.pull();
+ return { store, unsubscribe };
+}
+
+describe("env-file-store", () => {
+ beforeEach(() => {
+ capturedStore = null;
+ mockEnvContent = undefined;
+ vi.mocked(isCloudRuntimeEditEnvironment).mockReturnValue(false);
+ });
+
+ afterEach(() => {
+ vi.clearAllMocks();
+ });
+
+ describe("pull / getter", () => {
+ it("parses basic KEY=VALUE pairs", async () => {
+ const { store, unsubscribe } = await setupStore("FOO=bar\nBAZ=qux");
+ expect(store.store.get("FOO")?.value).toBe("bar");
+ expect(store.store.get("BAZ")?.value).toBe("qux");
+ unsubscribe();
+ });
+
+ it("strips empty and whitespace-only lines", async () => {
+ const { store, unsubscribe } = await setupStore(
+ "\nFOO=bar\n \n\nBAZ=qux\n",
+ );
+ expect([...store.store.keys()]).toEqual(["FOO", "BAZ"]);
+ unsubscribe();
+ });
+
+ it("strips comment-only lines", async () => {
+ const { store, unsubscribe } = await setupStore(
+ "# leading comment\nFOO=bar\n # indented comment\nBAZ=qux",
+ );
+ expect([...store.store.keys()]).toEqual(["FOO", "BAZ"]);
+ unsubscribe();
+ });
+
+ it("retains '#' inside quoted values", async () => {
+ const { store, unsubscribe } = await setupStore(
+ `URL="https://example.com#section"\nPASSWORD='foo#bar'`,
+ );
+ expect(store.store.get("URL")?.value).toBe("https://example.com#section");
+ expect(store.store.get("PASSWORD")?.value).toBe("foo#bar");
+ unsubscribe();
+ });
+
+ it("preserves '=' inside values", async () => {
+ const { store, unsubscribe } = await setupStore(
+ "DSN=postgres://u:p@h/db?opt=1&other=2",
+ );
+ expect(store.store.get("DSN")?.value).toBe(
+ "postgres://u:p@h/db?opt=1&other=2",
+ );
+ unsubscribe();
+ });
+
+ it("returns an empty store when the .env file is missing", async () => {
+ const { store, unsubscribe } = await setupStore(undefined);
+ expect(store.store.size).toBe(0);
+ unsubscribe();
+ });
+
+ it("returns an empty store when the .env file is empty", async () => {
+ const { store, unsubscribe } = await setupStore("");
+ expect(store.store.size).toBe(0);
+ unsubscribe();
+ });
+ });
+
+ describe("flush / setter", () => {
+ it("writes key=value lines via runtimeServicePutFile", async () => {
+ const { store, unsubscribe } = await setupStore("");
+ const session = new EnvEditSession(store, "clickhouse");
+ session.acquire("password", "secret", "CLICKHOUSE_PASSWORD");
+ await session.commit();
+
+ expect(runtimeServicePutFile).toHaveBeenCalledWith(runtimeClient, {
+ path: "/.env",
+ blob: "CLICKHOUSE_PASSWORD=secret",
+ });
+ unsubscribe();
+ });
+
+ it("calls runtimeServicePushEnv on cloud", async () => {
+ vi.mocked(isCloudRuntimeEditEnvironment).mockReturnValue(true);
+ const { store, unsubscribe } = await setupStore("");
+ const session = new EnvEditSession(store, "clickhouse");
+ session.acquire("password", "secret", "CLICKHOUSE_PASSWORD");
+ await session.commit();
+
+ expect(runtimeServicePushEnv).toHaveBeenCalledWith(runtimeClient, {});
+ unsubscribe();
+ });
+
+ it("skips runtimeServicePushEnv when not on cloud", async () => {
+ const { store, unsubscribe } = await setupStore("");
+ const session = new EnvEditSession(store, "clickhouse");
+ session.acquire("password", "secret", "CLICKHOUSE_PASSWORD");
+ await session.commit();
+
+ expect(runtimeServicePushEnv).not.toHaveBeenCalled();
+ unsubscribe();
+ });
+ });
+
+ describe("event-bus subscription", () => {
+ it("re-pulls when an env-file-updated event fires", async () => {
+ mockEnvContent = "FOO=initial";
+ const unsubscribe = createEnvFileStore(runtimeClient);
+ const store = getEnvFileStore();
+ await store.pull();
+ expect(store.store.get("FOO")?.value).toBe("initial");
+
+ mockEnvContent = "FOO=updated";
+ eventBus.emit("env-file-updated", "/.env");
+ // The handler calls `void envStore.pull()`; wait for the floating
+ // promise to settle before asserting.
+ await new Promise((r) => setTimeout(r, 0));
+
+ expect(store.store.get("FOO")?.value).toBe("updated");
+ unsubscribe();
+ });
+
+ it("stops re-pulling after the returned unsubscribe is called", async () => {
+ mockEnvContent = "FOO=initial";
+ const unsubscribe = createEnvFileStore(runtimeClient);
+ const store = getEnvFileStore();
+ await store.pull();
+ unsubscribe();
+
+ mockEnvContent = "FOO=updated";
+ eventBus.emit("env-file-updated", "/.env");
+ await new Promise((r) => setTimeout(r, 0));
+
+ expect(store.store.get("FOO")?.value).toBe("initial");
+ });
+ });
+});
diff --git a/web-common/src/features/env-management/env-file-store.ts b/web-common/src/features/env-management/env-file-store.ts
new file mode 100644
index 000000000000..26891d244d2e
--- /dev/null
+++ b/web-common/src/features/env-management/env-file-store.ts
@@ -0,0 +1,45 @@
+import { RuntimeClient } from "@rilldata/web-common/runtime-client/v2";
+import { EnvStore } from "@rilldata/web-common/features/env-management/env-store.ts";
+import {
+ runtimeServicePushEnv,
+ runtimeServicePutFile,
+} from "@rilldata/web-common/runtime-client";
+import { getContext, setContext } from "svelte";
+import { isCloudRuntimeEditEnvironment } from "@rilldata/web-common/features/entity-management/edit-environment.ts";
+import { fileArtifacts } from "@rilldata/web-common/features/entity-management/file-artifacts.ts";
+import { eventBus } from "@rilldata/web-common/lib/event-bus/event-bus.ts";
+import {
+ parseDotEnv,
+ serializeDotEnv,
+} from "@rilldata/web-common/features/env-management/dot-env.ts";
+
+const EnvFileStoreKey = "rill:app:env-file-store";
+
+export function createEnvFileStore(runtimeClient: RuntimeClient) {
+ const envArtifact = fileArtifacts.getFileArtifact("/.env");
+ const envStore = new EnvStore(
+ async () => {
+ const envBlob = await envArtifact.fetchContent();
+ return envBlob ? parseDotEnv(envBlob) : {};
+ },
+ async (entries) => {
+ await runtimeServicePutFile(runtimeClient, {
+ path: "/.env",
+ blob: serializeDotEnv(entries),
+ });
+ if (isCloudRuntimeEditEnvironment()) {
+ // Only push env on cloud for now. We will revisit this for rill-dev.
+ await runtimeServicePushEnv(runtimeClient, {});
+ }
+ },
+ );
+ setContext(EnvFileStoreKey, envStore);
+ void envStore.pull();
+ return eventBus.on("env-file-updated", () => {
+ void envStore.pull();
+ });
+}
+
+export function getEnvFileStore() {
+ return getContext(EnvFileStoreKey);
+}
diff --git a/web-common/src/features/env-management/env-store.ts b/web-common/src/features/env-management/env-store.ts
new file mode 100644
index 000000000000..9265531a1c32
--- /dev/null
+++ b/web-common/src/features/env-management/env-store.ts
@@ -0,0 +1,71 @@
+import type { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
+import { EnvVariable } from "@rilldata/web-common/features/env-management/env-variable.ts";
+
+export class EnvStore {
+ public store = new Map();
+
+ public constructor(
+ private readonly getter: () => Promise>,
+ private readonly setter: (entries: Record) => Promise,
+ ) {}
+
+ public async pull() {
+ const newEntries = await this.getter();
+ const newStore = new Map();
+ for (const key in newEntries) {
+ newStore.set(key, new EnvVariable(key, newEntries[key]));
+ }
+ this.store = newStore;
+ }
+
+ public async flush(editSession: EnvEditSession) {
+ if (
+ editSession.entries.size === 0 &&
+ editSession.inflightEntries.size === 0
+ ) {
+ return;
+ }
+
+ const newStore = new Map(this.store.entries());
+
+ editSession.entries.forEach((entry) => {
+ newStore.set(
+ entry.mappedEnvVarName,
+ new EnvVariable(entry.mappedEnvVarName, entry.value),
+ );
+ });
+ editSession.inflightEntries.forEach((entry) => {
+ newStore.delete(entry.mappedEnvVarName);
+ });
+
+ await this.setter(
+ Object.fromEntries([...newStore.values()].map((v) => [v.key, v.value])),
+ );
+
+ this.store = newStore;
+ }
+
+ // Remove only the vars whose current value still matches what the session
+ // wrote. Anything an external party has changed (or removed) since the
+ // commit is left alone — we only revert our own writes.
+ public async rollbackOwned(committed: Map) {
+ if (committed.size === 0) return;
+
+ const newStore = new Map(this.store.entries());
+ let changed = false;
+ for (const [name, writtenValue] of committed) {
+ const current = newStore.get(name);
+ if (current?.value === writtenValue) {
+ newStore.delete(name);
+ changed = true;
+ }
+ }
+ if (!changed) return;
+
+ await this.setter(
+ Object.fromEntries([...newStore.values()].map((v) => [v.key, v.value])),
+ );
+
+ this.store = newStore;
+ }
+}
diff --git a/web-common/src/features/env-management/env-variable.ts b/web-common/src/features/env-management/env-variable.ts
new file mode 100644
index 000000000000..6c715fa86a3a
--- /dev/null
+++ b/web-common/src/features/env-management/env-variable.ts
@@ -0,0 +1,6 @@
+export class EnvVariable {
+ constructor(
+ public readonly key: string,
+ public value: string,
+ ) {}
+}
diff --git a/web-common/src/features/env-management/test/test-env-store.ts b/web-common/src/features/env-management/test/test-env-store.ts
new file mode 100644
index 000000000000..f0c418d66671
--- /dev/null
+++ b/web-common/src/features/env-management/test/test-env-store.ts
@@ -0,0 +1,50 @@
+import { EnvEditSessionVariable } from "@rilldata/web-common/features/env-management/env-edit-session-variable.ts";
+import { EnvStore } from "@rilldata/web-common/features/env-management/env-store.ts";
+import { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
+import type { JSONSchemaObject } from "@rilldata/web-common/features/templates/schemas/types.ts";
+
+export async function makeTestEnvStore(
+ initValues: Record = {},
+) {
+ // set and get will be on this map that can be read and written to by tests.
+ const testEnvs = initValues;
+ const envStore = new EnvStore(
+ () => Promise.resolve(initValues),
+ async (entries) => {
+ Object.keys(testEnvs).forEach((key) => delete testEnvs[key]);
+ Object.assign(testEnvs, entries);
+ },
+ );
+ await envStore.pull();
+
+ return {
+ testEnvs,
+ envStore,
+ };
+}
+
+export async function makeTestEnvEditSession(
+ connectorName: string | undefined,
+ schema: JSONSchemaObject | undefined,
+ initEditValues: Record = {},
+ initStoreValues: Record = {},
+) {
+ const { testEnvs, envStore } = await makeTestEnvStore(initStoreValues);
+ const envEditSession = new EnvEditSession(
+ envStore,
+ connectorName ?? "",
+ schema,
+ );
+ Object.entries(initEditValues).forEach(([key, value]) => {
+ envEditSession.acquire(key, value);
+ });
+ return { testEnvs, envStore, envEditSession };
+}
+
+export function envMappedVarsAndValuesToObject(
+ vars: Map,
+) {
+ return Object.fromEntries(
+ vars.entries().map(([_, e]) => [e.mappedEnvVarName, e.value]),
+ );
+}
diff --git a/web-common/src/features/sources/sourceUtils.spec.ts b/web-common/src/features/sources/sourceUtils.spec.ts
index 65fbd2508c3d..d839c65381bc 100644
--- a/web-common/src/features/sources/sourceUtils.spec.ts
+++ b/web-common/src/features/sources/sourceUtils.spec.ts
@@ -5,9 +5,10 @@ import {
inferSourceName,
buildDuckDbQuery,
maybeRewriteToDuckDb,
- compileSourceYAML,
+ generateSourceYAML,
prepareSourceFormData,
} from "./sourceUtils";
+import { makeTestEnvEditSession } from "@rilldata/web-common/features/env-management/test/test-env-store.ts";
const gcsTests = [
{
@@ -328,12 +329,20 @@ describe("maybeRewriteToDuckDb", () => {
});
});
-describe("compileSourceYAML", () => {
- it("should produce basic model YAML with SQL", () => {
+describe("generateSourceYAML", () => {
+ it("should produce basic model YAML with SQL", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileSourceYAML(connector, {
- sql: "SELECT * FROM events",
- });
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
+ connector,
+ {
+ sql: "SELECT * FROM events",
+ },
+ envEditSession,
+ );
expect(result).toContain("# Model YAML");
expect(result).toContain("type: model");
expect(result).toContain("materialize: true");
@@ -342,124 +351,208 @@ describe("compileSourceYAML", () => {
expect(result).toContain(" SELECT * FROM events");
});
- it("should replace secret properties with env var placeholders", () => {
+ it("should replace secret properties with env var placeholders", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileSourceYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
connector,
{ password: "super_secret", sql: "SELECT 1" },
+ envEditSession,
{ secretKeys: ["password"] },
);
expect(result).toContain("{{ .env.CLICKHOUSE_PASSWORD }}");
expect(result).not.toContain("super_secret");
});
- it("should quote string properties", () => {
+ it("should quote string properties", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileSourceYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
connector,
{ host: "ch.example.com", sql: "SELECT 1" },
+ envEditSession,
{ stringKeys: ["host"] },
);
expect(result).toContain('host: "ch.example.com"');
});
- it("should not quote non-string properties", () => {
+ it("should not quote non-string properties", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileSourceYAML(connector, {
- port: 9000,
- sql: "SELECT 1",
- });
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
+ connector,
+ {
+ port: 9000,
+ sql: "SELECT 1",
+ },
+ envEditSession,
+ );
expect(result).toContain("port: 9000");
expect(result).not.toContain('port: "9000"');
});
- it("should filter out empty string values", () => {
+ it("should filter out empty string values", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileSourceYAML(connector, {
- database: "",
- sql: "SELECT 1",
- });
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
+ connector,
+ {
+ database: "",
+ sql: "SELECT 1",
+ },
+ envEditSession,
+ );
expect(result).not.toContain("database:");
});
- it("should filter out undefined values", () => {
+ it("should filter out undefined values", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileSourceYAML(connector, {
- database: undefined,
- sql: "SELECT 1",
- });
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
+ connector,
+ {
+ database: undefined,
+ sql: "SELECT 1",
+ },
+ envEditSession,
+ );
expect(result).not.toContain("database:");
});
- it("should always exclude the name field", () => {
+ it("should always exclude the name field", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileSourceYAML(connector, {
- name: "my_source",
- sql: "SELECT 1",
- });
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
+ connector,
+ {
+ name: "my_source",
+ sql: "SELECT 1",
+ },
+ envEditSession,
+ );
expect(result).not.toContain("name: my_source");
});
- it("should include dev section for warehouse connectors", () => {
+ it("should include dev section for warehouse connectors", async () => {
const connector: V1ConnectorDriver = {
name: "clickhouse",
implementsWarehouse: true,
};
- const result = compileSourceYAML(connector, {
- sql: "SELECT * FROM events;",
- });
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
+ connector,
+ {
+ sql: "SELECT * FROM events;",
+ },
+ envEditSession,
+ );
expect(result).toContain("dev:");
expect(result).toContain("limit 10000");
// Dev SQL should strip trailing semicolons
expect(result).toContain("SELECT * FROM events limit 10000");
});
- it("should skip dev section for redshift", () => {
+ it("should skip dev section for redshift", async () => {
const connector: V1ConnectorDriver = {
name: "redshift",
implementsWarehouse: true,
};
- const result = compileSourceYAML(connector, {
- sql: "SELECT * FROM events",
- });
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
+ connector,
+ {
+ sql: "SELECT * FROM events",
+ },
+ envEditSession,
+ );
expect(result).not.toContain("dev:");
});
- it("should skip dev section for non-warehouse connectors", () => {
+ it("should skip dev section for non-warehouse connectors", async () => {
const connector: V1ConnectorDriver = {
name: "duckdb",
implementsWarehouse: false,
};
- const result = compileSourceYAML(connector, {
- sql: "SELECT * FROM events",
- });
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
+ connector,
+ {
+ sql: "SELECT * FROM events",
+ },
+ envEditSession,
+ );
expect(result).not.toContain("dev:");
});
- it("should skip dev section when no SQL", () => {
+ it("should skip dev section when no SQL", async () => {
const connector: V1ConnectorDriver = {
name: "clickhouse",
implementsWarehouse: true,
};
- const result = compileSourceYAML(connector, { host: "ch.example.com" });
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
+ connector,
+ { host: "ch.example.com" },
+ envEditSession,
+ );
expect(result).not.toContain("dev:");
});
- it("should use connectorInstanceName as connector value", () => {
+ it("should use connectorInstanceName as connector value", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileSourceYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
connector,
{ sql: "SELECT 1" },
+ envEditSession,
{ connectorInstanceName: "clickhouse_prod" },
);
expect(result).toContain("connector: clickhouse_prod");
});
- it("should use originalDriverName in header comment", () => {
+ it("should use originalDriverName in header comment", async () => {
const connector: V1ConnectorDriver = { name: "duckdb" };
- const result = compileSourceYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ );
+ const result = generateSourceYAML(
connector,
{ sql: "SELECT 1" },
+ envEditSession,
{ originalDriverName: "s3" },
);
expect(result).toContain(
@@ -467,14 +560,20 @@ describe("compileSourceYAML", () => {
);
});
- it("should handle env var conflict resolution with existingEnvBlob", () => {
+ it("should handle env var conflict resolution with existingEnvBlob", async () => {
const connector: V1ConnectorDriver = { name: "clickhouse" };
- const result = compileSourceYAML(
+ const { envEditSession } = await makeTestEnvEditSession(
+ connector.name,
+ undefined,
+ {},
+ { CLICKHOUSE_PASSWORD: "old_value" },
+ );
+ const result = generateSourceYAML(
connector,
{ password: "secret", sql: "SELECT 1" },
+ envEditSession,
{
secretKeys: ["password"],
- existingEnvBlob: "CLICKHOUSE_PASSWORD=old_value",
},
);
expect(result).toContain("CLICKHOUSE_PASSWORD_1");
diff --git a/web-common/src/features/sources/sourceUtils.ts b/web-common/src/features/sources/sourceUtils.ts
index c473dc37fe69..1e5efa62b901 100644
--- a/web-common/src/features/sources/sourceUtils.ts
+++ b/web-common/src/features/sources/sourceUtils.ts
@@ -3,7 +3,6 @@ import type {
V1ConnectorDriver,
V1Source,
} from "@rilldata/web-common/runtime-client";
-import { makeEnvVarKey } from "../connectors/code-utils";
import { sanitizeEntityName } from "../entity-management/name-utils";
import { getConnectorSchema } from "./modal/connector-schemas";
import {
@@ -11,6 +10,7 @@ import {
getSchemaSecretKeys,
getSchemaStringKeys,
} from "../templates/schema-utils";
+import type { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
// Helper text that we put at the top of every Model YAML file
function sourceModelFileTop(driverName: string) {
@@ -21,15 +21,15 @@ type: model
materialize: true`;
}
-export function compileSourceYAML(
+export function generateSourceYAML(
connector: V1ConnectorDriver,
formValues: Record,
+ envEditSession: EnvEditSession,
opts?: {
secretKeys?: string[];
stringKeys?: string[];
connectorInstanceName?: string;
originalDriverName?: string;
- existingEnvBlob?: string;
outputConnector?: string;
},
) {
@@ -39,6 +39,7 @@ export function compileSourceYAML(
const secretPropertyKeys =
opts?.secretKeys ??
(schema ? getSchemaSecretKeys(schema, { step: "source" }) : []);
+ envEditSession.startEdit();
// Get the string property keys
const stringPropertyKeys =
@@ -70,13 +71,9 @@ export function compileSourceYAML(
const isSecretProperty = secretPropertyKeys.includes(key);
if (isSecretProperty) {
+ const entry = envEditSession.acquire(key, String(value));
// For source files, we include secret properties
- return `${key}: "{{ .env.${makeEnvVarKey(
- connector.name as string,
- key,
- opts?.existingEnvBlob,
- schema ?? undefined,
- )} }}"`; // uses standard Go template syntax
+ return `${key}: "{{ .env.${entry.mappedEnvVarName} }}"`; // uses standard Go template syntax
}
if (key === "sql") {
diff --git a/web-common/src/features/templates/schemas/ducklake-utils.spec.ts b/web-common/src/features/templates/schemas/ducklake-utils.spec.ts
index 7c893d80db0f..34f6508e1251 100644
--- a/web-common/src/features/templates/schemas/ducklake-utils.spec.ts
+++ b/web-common/src/features/templates/schemas/ducklake-utils.spec.ts
@@ -1,90 +1,132 @@
import { describe, it, expect } from "vitest";
import {
applyDuckLakeFormTransform,
- buildDuckLakeSecretRefs,
composeDuckLakeAttach,
extractDuckLakeAttachSecrets,
shouldExtractDuckLakeAttachSecrets,
validateDuckLakeAttach,
} from "./ducklake-utils";
import { ducklakeSchema } from "./ducklake";
+import {
+ envMappedVarsAndValuesToObject,
+ makeTestEnvEditSession,
+} from "@rilldata/web-common/features/env-management/test/test-env-store.ts";
describe("composeDuckLakeAttach", () => {
- it("returns empty string when catalog identifier is missing", () => {
- expect(composeDuckLakeAttach({})).toBe("");
+ it("returns empty string when catalog identifier is missing", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
+ expect(composeDuckLakeAttach({}, envEditSession)).toBe("");
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: " ",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: " ",
+ },
+ envEditSession,
+ ),
).toBe("");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("defaults to duckdb catalog when catalog_type is unset", () => {
+ it("defaults to duckdb catalog when catalog_type is unset", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({ catalog_duckdb_path: "catalog.ducklake" }),
+ composeDuckLakeAttach(
+ { catalog_duckdb_path: "catalog.ducklake" },
+ envEditSession,
+ ),
).toBe("'ducklake:catalog.ducklake'");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("builds a minimal clause from a duckdb catalog path", () => {
+ it("builds a minimal clause from a duckdb catalog path", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "duckdb_database.ducklake",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "duckdb_database.ducklake",
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:duckdb_database.ducklake'");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("builds a sqlite catalog clause with the sqlite: prefix", () => {
+ it("builds a sqlite catalog clause with the sqlite: prefix", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "sqlite",
- catalog_sqlite_path: "catalog.sqlite",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "sqlite",
+ catalog_sqlite_path: "catalog.sqlite",
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:sqlite:catalog.sqlite'");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("builds a postgres catalog clause from individual fields", () => {
- expect(
- composeDuckLakeAttach({
- catalog_type: "postgres",
- catalog_postgres_dbname: "mydb",
- catalog_postgres_host: "localhost",
- catalog_postgres_port: "5432",
- catalog_postgres_user: "postgres",
- catalog_postgres_password: "secret",
- }),
- ).toBe(
- "'ducklake:postgres:dbname=mydb host=localhost port=5432 user=postgres password=secret'",
+ it("omits missing postgres params when composing the connection string", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
);
- });
-
- it("omits missing postgres params when composing the connection string", () => {
expect(
- composeDuckLakeAttach({
- catalog_type: "postgres",
- catalog_postgres_dbname: "mydb",
- catalog_postgres_host: "localhost",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "postgres",
+ catalog_postgres_dbname: "mydb",
+ catalog_postgres_host: "localhost",
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:postgres:dbname=mydb host=localhost'");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("builds a mysql catalog clause using the database= key", () => {
+ it("builds a mysql catalog clause using the database= key", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "mysql",
- catalog_mysql_database: "mydb",
- catalog_mysql_host: "localhost",
- catalog_mysql_port: "3306",
- catalog_mysql_user: "root",
- catalog_mysql_password: "secret",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "mysql",
+ catalog_mysql_database: "mydb",
+ catalog_mysql_host: "localhost",
+ catalog_mysql_port: "3306",
+ catalog_mysql_user: "root",
+ catalog_mysql_password: "secret",
+ },
+ envEditSession,
+ ),
).toBe(
- "'ducklake:mysql:database=mydb host=localhost port=3306 user=root password=secret'",
+ "'ducklake:mysql:database=mydb host=localhost port=3306 user=root password={{ .env.DUCKLAKE_CATALOG_MYSQL_PASSWORD }}'",
);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({
+ DUCKLAKE_CATALOG_MYSQL_PASSWORD: "secret",
+ });
});
- it("substitutes postgres password with an env template reference when secretRefs is provided", () => {
+ it("substitutes postgres password with an env template reference", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
composeDuckLakeAttach(
{
@@ -94,18 +136,21 @@ describe("composeDuckLakeAttach", () => {
catalog_postgres_user: "postgres",
catalog_postgres_password: "secret",
},
- {
- secretRefs: {
- catalog_postgres_password: "DUCKLAKE_CATALOG_POSTGRES_PASSWORD",
- },
- },
+ envEditSession,
),
).toBe(
"'ducklake:postgres:dbname=mydb host=localhost user=postgres password={{ .env.DUCKLAKE_CATALOG_POSTGRES_PASSWORD }}'",
);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({
+ DUCKLAKE_CATALOG_POSTGRES_PASSWORD: "secret",
+ });
});
- it("substitutes mysql password with an env template reference when secretRefs is provided", () => {
+ it("substitutes mysql password with an env template reference", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
composeDuckLakeAttach(
{
@@ -115,18 +160,21 @@ describe("composeDuckLakeAttach", () => {
catalog_mysql_user: "root",
catalog_mysql_password: "secret",
},
- {
- secretRefs: {
- catalog_mysql_password: "DUCKLAKE_CATALOG_MYSQL_PASSWORD",
- },
- },
+ envEditSession,
),
).toBe(
"'ducklake:mysql:database=mydb host=localhost user=root password={{ .env.DUCKLAKE_CATALOG_MYSQL_PASSWORD }}'",
);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({
+ DUCKLAKE_CATALOG_MYSQL_PASSWORD: "secret",
+ });
});
- it("omits the password pair when secretRefs is provided but the password is empty", () => {
+ it("omits the password pair when secretRefs is provided but the password is empty", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
composeDuckLakeAttach(
{
@@ -135,142 +183,214 @@ describe("composeDuckLakeAttach", () => {
catalog_postgres_host: "localhost",
catalog_postgres_password: "",
},
- {
- secretRefs: {
- catalog_postgres_password: "DUCKLAKE_CATALOG_POSTGRES_PASSWORD",
- },
- },
+ envEditSession,
),
).toBe("'ducklake:postgres:dbname=mydb host=localhost'");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("includes an alias when provided", () => {
+ it("includes an alias when provided", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "duckdb_database.ducklake",
- alias: "my_ducklake",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "duckdb_database.ducklake",
+ alias: "my_ducklake",
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:duckdb_database.ducklake' AS my_ducklake");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("appends DATA_PATH based on data_path_type", () => {
+ it("appends DATA_PATH based on data_path_type", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "duckdb_database.ducklake",
- data_path_type: "local",
- data_path_local: "other_data_path/",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "duckdb_database.ducklake",
+ data_path_type: "local",
+ data_path_local: "other_data_path/",
+ },
+ envEditSession,
+ ),
).toBe(
"'ducklake:duckdb_database.ducklake' (DATA_PATH 'other_data_path/')",
);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("picks the data path for the active storage type only", () => {
+ it("picks the data path for the active storage type only", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
// A stale s3 value should not leak into DATA_PATH when local is active.
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "c.ducklake",
- data_path_type: "local",
- data_path_local: "local/",
- data_path_s3: "s3://stale/",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "c.ducklake",
+ data_path_type: "local",
+ data_path_local: "local/",
+ data_path_s3: "s3://stale/",
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:c.ducklake' (DATA_PATH 'local/')");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "c.ducklake",
- data_path_type: "s3",
- data_path_local: "local/",
- data_path_s3: "s3://bucket/",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "c.ducklake",
+ data_path_type: "s3",
+ data_path_local: "local/",
+ data_path_s3: "s3://bucket/",
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:c.ducklake' (DATA_PATH 's3://bucket/')");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("emits boolean options regardless of value", () => {
+ it("emits boolean options regardless of value", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "c.ducklake",
- override_data_path: true,
- create_if_not_exists: true,
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "c.ducklake",
+ override_data_path: true,
+ create_if_not_exists: true,
+ },
+ envEditSession,
+ ),
).toBe(
"'ducklake:c.ducklake' (OVERRIDE_DATA_PATH true, CREATE_IF_NOT_EXISTS true)",
);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("emits false and true boolean options", () => {
+ it("emits false and true boolean options", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "c.ducklake",
- override_data_path: false,
- encrypted: true,
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "c.ducklake",
+ override_data_path: false,
+ encrypted: true,
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:c.ducklake' (OVERRIDE_DATA_PATH false, ENCRYPTED true)");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("emits METADATA_PARAMETERS without wrapping quotes", () => {
+ it("emits METADATA_PARAMETERS without wrapping quotes", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "c.ducklake",
- metadata_parameters: "{foo: 'bar'}",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "c.ducklake",
+ metadata_parameters: "{foo: 'bar'}",
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:c.ducklake' (METADATA_PARAMETERS {foo: 'bar'})");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("escapes single quotes inside data path values", () => {
+ it("escapes single quotes inside data path values", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "c.ducklake",
- data_path_type: "local",
- data_path_local: "path/with'quote",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "c.ducklake",
+ data_path_type: "local",
+ data_path_local: "path/with'quote",
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:c.ducklake' (DATA_PATH 'path/with''quote')");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("composes the example from the DuckLake docs", () => {
+ it("composes the example from the DuckLake docs", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "duckdb_database.ducklake",
- data_path_type: "local",
- data_path_local: "other_data_path/",
- override_data_path: true,
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "duckdb_database.ducklake",
+ data_path_type: "local",
+ data_path_local: "other_data_path/",
+ override_data_path: true,
+ },
+ envEditSession,
+ ),
).toBe(
"'ducklake:duckdb_database.ducklake' (DATA_PATH 'other_data_path/', OVERRIDE_DATA_PATH true)",
);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("emits every advanced parameter when set", () => {
+ it("emits every advanced parameter when set", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "c.ducklake",
- alias: "my_ducklake",
- data_path_type: "s3",
- data_path_s3: "s3://bucket/",
- mode: true,
- override_data_path: false,
- create_if_not_exists: false,
- data_inlining_row_limit: 100,
- encrypted: true,
-
- meta_parameter_name: "foo",
- metadata_catalog: "meta_cat",
- metadata_parameters: "{a: 1}",
- metadata_schema: "my_schema",
- automatic_migration: true,
- snapshot_time: "2024-01-01",
- snapshot_version: "v1",
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "c.ducklake",
+ alias: "my_ducklake",
+ data_path_type: "s3",
+ data_path_s3: "s3://bucket/",
+ mode: true,
+ override_data_path: false,
+ create_if_not_exists: false,
+ data_inlining_row_limit: 100,
+ encrypted: true,
+
+ meta_parameter_name: "foo",
+ metadata_catalog: "meta_cat",
+ metadata_parameters: "{a: 1}",
+ metadata_schema: "my_schema",
+ automatic_migration: true,
+ snapshot_time: "2024-01-01",
+ snapshot_version: "v1",
+ },
+ envEditSession,
+ ),
).toBe(
"'ducklake:c.ducklake' AS my_ducklake (" +
"DATA_PATH 's3://bucket/', " +
@@ -288,58 +408,95 @@ describe("composeDuckLakeAttach", () => {
"AUTOMATIC_MIGRATION true" +
")",
);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("emits READ_ONLY from the mode toggle in both states", () => {
+ it("emits READ_ONLY from the mode toggle in both states", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "c.ducklake",
- mode: true,
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "c.ducklake",
+ mode: true,
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:c.ducklake' (READ_ONLY true)");
expect(
- composeDuckLakeAttach({
- catalog_type: "duckdb",
- catalog_duckdb_path: "c.ducklake",
- mode: false,
- }),
+ composeDuckLakeAttach(
+ {
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "c.ducklake",
+ mode: false,
+ },
+ envEditSession,
+ ),
).toBe("'ducklake:c.ducklake' (READ_ONLY false)");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
});
describe("applyDuckLakeFormTransform", () => {
- it("returns values unchanged for non-DuckLake schemas", () => {
+ it("returns values unchanged for non-DuckLake schemas", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
const values = { attach: "foo" };
- expect(applyDuckLakeFormTransform(null, values)).toBe(values);
+ expect(applyDuckLakeFormTransform(null, values, envEditSession)).toBe(
+ values,
+ );
expect(
applyDuckLakeFormTransform(
{ type: "object", title: "DuckLake", properties: {} },
values,
+ envEditSession,
),
).toBe(values);
});
- it("leaves attach alone when in SQL mode", () => {
+ it("leaves attach alone when in SQL mode", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
const values = { connection_mode: "sql", attach: "user input" };
- expect(applyDuckLakeFormTransform(ducklakeSchema, values)).toBe(values);
+ expect(
+ applyDuckLakeFormTransform(ducklakeSchema, values, envEditSession),
+ ).toBe(values);
});
- it("synthesises attach from params when in parameters mode", () => {
- const result = applyDuckLakeFormTransform(ducklakeSchema, {
- connection_mode: "parameters",
- catalog_type: "duckdb",
- catalog_duckdb_path: "duckdb_database.ducklake",
- data_path_type: "local",
- data_path_local: "other_data_path/",
- });
+ it("synthesises attach from params when in parameters mode", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
+ const result = applyDuckLakeFormTransform(
+ ducklakeSchema,
+ {
+ connection_mode: "parameters",
+ catalog_type: "duckdb",
+ catalog_duckdb_path: "duckdb_database.ducklake",
+ data_path_type: "local",
+ data_path_local: "other_data_path/",
+ },
+ envEditSession,
+ );
expect(result.attach).toBe(
"'ducklake:duckdb_database.ducklake' (DATA_PATH 'other_data_path/')",
);
});
- it("threads secretRefs through to composeDuckLakeAttach", () => {
+ it("threads secretRefs through to composeDuckLakeAttach", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
const result = applyDuckLakeFormTransform(
ducklakeSchema,
{
@@ -350,11 +507,7 @@ describe("applyDuckLakeFormTransform", () => {
catalog_postgres_user: "postgres",
catalog_postgres_password: "secret",
},
- {
- secretRefs: {
- catalog_postgres_password: "DUCKLAKE_CATALOG_POSTGRES_PASSWORD",
- },
- },
+ envEditSession,
);
expect(result.attach).toBe(
"'ducklake:postgres:dbname=mydb host=localhost user=postgres password={{ .env.DUCKLAKE_CATALOG_POSTGRES_PASSWORD }}'",
@@ -362,128 +515,139 @@ describe("applyDuckLakeFormTransform", () => {
});
});
-describe("buildDuckLakeSecretRefs", () => {
- it("returns an empty object for non-DuckLake schemas", () => {
- expect(buildDuckLakeSecretRefs(null, "postgres", "")).toEqual({});
- expect(
- buildDuckLakeSecretRefs(
- { type: "object", title: "Other", properties: {} },
- "duckdb",
- "",
- ),
- ).toEqual({});
- });
-
- it("resolves env var names for the DuckLake password fields", () => {
- const refs = buildDuckLakeSecretRefs(ducklakeSchema, "duckdb", "");
- expect(refs).toEqual({
- catalog_postgres_password: "DUCKLAKE_CATALOG_POSTGRES_PASSWORD",
- catalog_mysql_password: "DUCKLAKE_CATALOG_MYSQL_PASSWORD",
- });
- });
-
- it("suffixes env var names to avoid conflicts in an existing .env blob", () => {
- const envBlob =
- "DUCKLAKE_CATALOG_POSTGRES_PASSWORD=existing\n" +
- "DUCKLAKE_CATALOG_MYSQL_PASSWORD=existing";
- const refs = buildDuckLakeSecretRefs(ducklakeSchema, "duckdb", envBlob);
- expect(refs).toEqual({
- catalog_postgres_password: "DUCKLAKE_CATALOG_POSTGRES_PASSWORD_1",
- catalog_mysql_password: "DUCKLAKE_CATALOG_MYSQL_PASSWORD_1",
- });
- });
-});
-
describe("extractDuckLakeAttachSecrets", () => {
- it("returns empty extraction for empty input", () => {
- expect(extractDuckLakeAttachSecrets("", "")).toEqual({
- rewrittenAttach: "",
- extractedSecrets: {},
- });
+ it("returns empty extraction for empty input", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
+ expect(extractDuckLakeAttachSecrets("", envEditSession)).toEqual("");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("leaves non-credential catalog URIs alone", () => {
+ it("leaves non-credential catalog URIs alone", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
const attach =
"'ducklake:catalog.ducklake' AS my_ducklake (DATA_PATH 'files/')";
- const result = extractDuckLakeAttachSecrets(attach, "");
- expect(result.rewrittenAttach).toBe(attach);
- expect(result.extractedSecrets).toEqual({});
+ const rewrittenAttach = extractDuckLakeAttachSecrets(
+ attach,
+ envEditSession,
+ );
+ expect(rewrittenAttach).toBe(attach);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
const sqliteAttach = "'ducklake:sqlite:catalog.sqlite' AS x";
- const sqliteResult = extractDuckLakeAttachSecrets(sqliteAttach, "");
- expect(sqliteResult.rewrittenAttach).toBe(sqliteAttach);
- expect(sqliteResult.extractedSecrets).toEqual({});
+ const rewrittenSqliteAttach = extractDuckLakeAttachSecrets(
+ sqliteAttach,
+ envEditSession,
+ );
+ expect(rewrittenSqliteAttach).toBe(sqliteAttach);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("extracts a postgres catalog body into DUCKLAKE_POSTGRES", () => {
+ it("extracts a postgres catalog body into DUCKLAKE_POSTGRES", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
const attach =
"'ducklake:postgres:dbname=mydb host=localhost password=secret' AS my_ducklake (DATA_PATH 'files/')";
- const result = extractDuckLakeAttachSecrets(attach, "");
- expect(result.rewrittenAttach).toBe(
+ const rewrittenAttach = extractDuckLakeAttachSecrets(
+ attach,
+ envEditSession,
+ );
+ expect(rewrittenAttach).toBe(
"'ducklake:postgres:{{ .env.DUCKLAKE_POSTGRES }}' AS my_ducklake (DATA_PATH 'files/')",
);
- expect(result.extractedSecrets).toEqual({
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({
DUCKLAKE_POSTGRES: "dbname=mydb host=localhost password=secret",
});
});
- it("extracts mysql and motherduck catalog bodies", () => {
- const mysql = extractDuckLakeAttachSecrets(
+ it("extracts mysql and motherduck catalog bodies", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
+
+ envEditSession.startEdit();
+ const mysqlAttach = extractDuckLakeAttachSecrets(
"'ducklake:mysql:database=x host=y password=z'",
- "",
+ envEditSession,
);
- expect(mysql.extractedSecrets).toEqual({
+ expect(mysqlAttach).toBe("'ducklake:mysql:{{ .env.DUCKLAKE_MYSQL }}'");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({
DUCKLAKE_MYSQL: "database=x host=y password=z",
});
- expect(mysql.rewrittenAttach).toBe(
- "'ducklake:mysql:{{ .env.DUCKLAKE_MYSQL }}'",
- );
- const md = extractDuckLakeAttachSecrets(
+ envEditSession.startEdit();
+ const mdAttach = extractDuckLakeAttachSecrets(
"'ducklake:md:my_db?motherduck_token=abc'",
- "",
+ envEditSession,
);
- expect(md.extractedSecrets).toEqual({
+ expect(mdAttach).toEqual("'ducklake:md:{{ .env.DUCKLAKE_MOTHERDUCK }}'");
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({
DUCKLAKE_MOTHERDUCK: "my_db?motherduck_token=abc",
});
- expect(md.rewrittenAttach).toBe(
- "'ducklake:md:{{ .env.DUCKLAKE_MOTHERDUCK }}'",
- );
});
- it("suffixes when the base env var is already defined", () => {
- const envBlob = "DUCKLAKE_POSTGRES=existing";
- const result = extractDuckLakeAttachSecrets(
+ it("suffixes when the base env var is already defined", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ {},
+ {
+ DUCKLAKE_POSTGRES: "existing",
+ },
+ );
+ const rewrittenAttach = extractDuckLakeAttachSecrets(
"'ducklake:postgres:dbname=mydb' AS x",
- envBlob,
+ envEditSession,
);
- expect(result.extractedSecrets).toEqual({
- DUCKLAKE_POSTGRES_1: "dbname=mydb",
- });
- expect(result.rewrittenAttach).toBe(
+ expect(rewrittenAttach).toBe(
"'ducklake:postgres:{{ .env.DUCKLAKE_POSTGRES_1 }}' AS x",
);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({
+ DUCKLAKE_POSTGRES_1: "dbname=mydb",
+ });
});
- it("is idempotent when the body is already a single env template", () => {
+ it("is idempotent when the body is already a single env template", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
const attach =
"'ducklake:postgres:{{ .env.DUCKLAKE_POSTGRES }}' AS my_ducklake";
- const result = extractDuckLakeAttachSecrets(attach, "");
- expect(result.rewrittenAttach).toBe(attach);
- expect(result.extractedSecrets).toEqual({});
+ const rewrittenAttach = extractDuckLakeAttachSecrets(
+ attach,
+ envEditSession,
+ );
+ expect(rewrittenAttach).toBe(attach);
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({});
});
- it("allocates distinct env vars when the same driver appears twice", () => {
+ it("allocates distinct env vars when the same driver appears twice", async () => {
+ const { envEditSession } = await makeTestEnvEditSession(
+ "ducklake",
+ undefined,
+ );
const attach =
"'ducklake:postgres:dbname=a password=1' vs 'ducklake:postgres:dbname=b password=2'";
- const result = extractDuckLakeAttachSecrets(attach, "");
- expect(result.extractedSecrets).toEqual({
+ const rewrittenAttach = extractDuckLakeAttachSecrets(
+ attach,
+ envEditSession,
+ );
+ expect(rewrittenAttach).toBe(
+ "'ducklake:postgres:{{ .env.DUCKLAKE_POSTGRES }}' vs 'ducklake:postgres:{{ .env.DUCKLAKE_POSTGRES_1 }}'",
+ );
+ expect(envMappedVarsAndValuesToObject(envEditSession.entries)).toEqual({
DUCKLAKE_POSTGRES: "dbname=a password=1",
DUCKLAKE_POSTGRES_1: "dbname=b password=2",
});
- expect(result.rewrittenAttach).toBe(
- "'ducklake:postgres:{{ .env.DUCKLAKE_POSTGRES }}' vs 'ducklake:postgres:{{ .env.DUCKLAKE_POSTGRES_1 }}'",
- );
});
});
diff --git a/web-common/src/features/templates/schemas/ducklake-utils.ts b/web-common/src/features/templates/schemas/ducklake-utils.ts
index 8e2794825711..cf5fd51f1b0b 100644
--- a/web-common/src/features/templates/schemas/ducklake-utils.ts
+++ b/web-common/src/features/templates/schemas/ducklake-utils.ts
@@ -1,21 +1,7 @@
-import {
- findAvailableEnvVarName,
- makeEnvVarKey,
- replaceOrAddEnvVariable,
-} from "@rilldata/web-common/features/connectors/code-utils";
import { ducklakeSchema } from "./ducklake";
import type { MultiStepFormSchema } from "./types";
-
-/**
- * Maps DuckLake password field keys (e.g. `catalog_postgres_password`) to the
- * resolved `.env` variable name used in the generated YAML. When present, the
- * composer emits `password={{ .env. }}` instead of the raw value.
- */
-export type DuckLakeSecretRefs = Record;
-
-export interface ComposeDuckLakeAttachOptions {
- secretRefs?: DuckLakeSecretRefs;
-}
+import type { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
+import { getName } from "@rilldata/web-common/features/entity-management/name-utils.ts";
/**
* Compose a DuckDB `ATTACH` clause string (without the leading `ATTACH`
@@ -26,9 +12,9 @@ export interface ComposeDuckLakeAttachOptions {
*/
export function composeDuckLakeAttach(
values: Record,
- opts?: ComposeDuckLakeAttachOptions,
+ envEditSession: EnvEditSession,
): string {
- const identifier = composeCatalogIdentifier(values, opts?.secretRefs);
+ const identifier = composeCatalogIdentifier(values, envEditSession);
if (!identifier) return "";
const alias = stringValue(values.alias);
@@ -92,7 +78,7 @@ export function composeDuckLakeAttach(
*/
function composeCatalogIdentifier(
values: Record,
- secretRefs?: DuckLakeSecretRefs,
+ envEditSession: EnvEditSession,
): string {
const type = stringValue(values.catalog_type) || "duckdb";
@@ -114,7 +100,13 @@ function composeCatalogIdentifier(
["user", values.catalog_postgres_user],
["password", values.catalog_postgres_password],
],
- { password: fieldSecretRef(secretRefs, "catalog_postgres_password") },
+ {
+ password: fieldSecretRef(
+ envEditSession,
+ "catalog_postgres_password",
+ values.catalog_postgres_password,
+ ),
+ },
);
return kv ? `postgres:${kv}` : "";
}
@@ -128,7 +120,13 @@ function composeCatalogIdentifier(
["user", values.catalog_mysql_user],
["password", values.catalog_mysql_password],
],
- { password: fieldSecretRef(secretRefs, "catalog_mysql_password") },
+ {
+ password: fieldSecretRef(
+ envEditSession,
+ "catalog_mysql_password",
+ values.catalog_mysql_password,
+ ),
+ },
);
return kv ? `mysql:${kv}` : "";
}
@@ -139,12 +137,13 @@ function composeCatalogIdentifier(
}
function fieldSecretRef(
- secretRefs: DuckLakeSecretRefs | undefined,
+ envEditSession: EnvEditSession,
fieldKey: string,
+ value: unknown,
): string | undefined {
- const envVarName = secretRefs?.[fieldKey];
- if (!envVarName) return undefined;
- return `{{ .env.${envVarName} }}`;
+ if (!value) return undefined;
+ const entry = envEditSession.acquire(fieldKey, String(value));
+ return `{{ .env.${entry.mappedEnvVarName} }}`;
}
/**
@@ -191,11 +190,11 @@ function keyValuePairs(
export function applyDuckLakeFormTransform(
schema: MultiStepFormSchema | null | undefined,
values: Record,
- opts?: ComposeDuckLakeAttachOptions,
+ envEditSession: EnvEditSession,
): Record {
if (schema !== ducklakeSchema) return values;
if (values.connection_mode === "parameters") {
- const attach = composeDuckLakeAttach(values, opts);
+ const attach = composeDuckLakeAttach(values, envEditSession);
return { ...values, attach };
}
// SQL mode: keep the user-typed value visible in the textarea, but strip
@@ -234,25 +233,6 @@ export const DUCKLAKE_SECRET_FIELD_KEYS = [
"catalog_mysql_password",
] as const;
-/**
- * Resolve the `.env` variable names for DuckLake password fields, matching
- * the name `makeEnvVarKey` will use when writing secrets and compiling YAML.
- * Returns an empty object for non-DuckLake schemas so callers can pass the
- * result through unconditionally.
- */
-export function buildDuckLakeSecretRefs(
- schema: MultiStepFormSchema | null | undefined,
- driverName: string,
- existingEnvBlob: string,
-): DuckLakeSecretRefs {
- if (schema !== ducklakeSchema) return {};
- const refs: DuckLakeSecretRefs = {};
- for (const key of DUCKLAKE_SECRET_FIELD_KEYS) {
- refs[key] = makeEnvVarKey(driverName, key, existingEnvBlob, schema);
- }
- return refs;
-}
-
function stringValue(v: unknown): string {
if (typeof v !== "string") return "";
return v.trim();
@@ -286,13 +266,6 @@ const DUCKLAKE_CATALOG_URI_PATTERN = /'ducklake:(postgres|mysql|md):([^']*)'/g;
const ENV_TEMPLATE_ONLY_PATTERN = /^{{\s*\.env\.[^{}\s]+\s*}}$/;
-export interface DuckLakeAttachExtraction {
- /** The attach string with raw catalog bodies replaced by `{{ .env.X }}` refs. */
- rewrittenAttach: string;
- /** Map of allocated env var name to the raw catalog body to persist in `.env`. */
- extractedSecrets: Record;
-}
-
/**
* Extract credential-bearing catalog URIs from a raw DuckLake ATTACH string.
*
@@ -308,14 +281,11 @@ export interface DuckLakeAttachExtraction {
*/
export function extractDuckLakeAttachSecrets(
attach: string,
- existingEnvBlob: string,
-): DuckLakeAttachExtraction {
- if (!attach) return { rewrittenAttach: attach, extractedSecrets: {} };
- const extractedSecrets: Record = {};
- // Track allocations across multiple matches in a single attach so two
- // postgres catalogs in one string don't collide on the same env var name.
- let reservedBlob = existingEnvBlob;
+ envEditSession: EnvEditSession,
+): string {
+ if (!attach) return attach;
+ const varNames: string[] = [];
const rewrittenAttach = attach.replace(
DUCKLAKE_CATALOG_URI_PATTERN,
(_match, driver: string, body: string) => {
@@ -324,14 +294,17 @@ export function extractDuckLakeAttachSecrets(
if (ENV_TEMPLATE_ONLY_PATTERN.test(trimmed)) {
return `'ducklake:${driver}:${body}'`;
}
+
const base = DUCKLAKE_CATALOG_ENV_VAR_BASE[driver];
- const envVarName = findAvailableEnvVarName(reservedBlob, base);
- extractedSecrets[envVarName] = trimmed;
- reservedBlob = replaceOrAddEnvVariable(reservedBlob, envVarName, trimmed);
- return `'ducklake:${driver}:{{ .env.${envVarName} }}'`;
+ const envName = getName(base, varNames);
+ varNames.push(envName);
+
+ const entry = envEditSession.acquire(envName, trimmed, envName);
+ return `'ducklake:${driver}:{{ .env.${entry.mappedEnvVarName} }}'`;
},
);
- return { rewrittenAttach, extractedSecrets };
+
+ return rewrittenAttach;
}
/**
@@ -347,19 +320,6 @@ export function shouldExtractDuckLakeAttachSecrets(
return values.connection_mode !== "parameters";
}
-export interface DuckLakePipelineResult {
- /**
- * Form values after Parameters-tab composition and SQL-tab catalog secret
- * extraction. Use these as the source of truth for YAML emission.
- */
- transformedValues: Record;
- /**
- * Catalog secrets newly extracted from a raw `attach` string. Submit paths
- * append these to the `.env` blob; preview paths discard them.
- */
- extractedSecrets: Record;
-}
-
/**
* Run the full DuckLake form-value pipeline: compose Parameters fields into
* `attach`, route password fields through env-var refs, and extract catalog
@@ -372,40 +332,36 @@ export interface DuckLakePipelineResult {
export function applyDuckLakeFormPipeline(
schema: MultiStepFormSchema | null | undefined,
formValues: Record,
- opts: { connectorName: string; existingEnvBlob: string },
-): DuckLakePipelineResult {
+ opts: {
+ connectorName: string;
+ envEditSession: EnvEditSession;
+ },
+): Record {
if (schema !== ducklakeSchema) {
- return { transformedValues: formValues, extractedSecrets: {} };
+ return formValues;
}
- const secretRefs = buildDuckLakeSecretRefs(
+ let transformedValues = applyDuckLakeFormTransform(
schema,
- opts.connectorName,
- opts.existingEnvBlob,
+ formValues,
+ opts.envEditSession,
);
- let transformedValues = applyDuckLakeFormTransform(schema, formValues, {
- secretRefs,
- });
- let extractedSecrets: Record = {};
if (shouldExtractDuckLakeAttachSecrets(schema, transformedValues)) {
const rawAttach = transformedValues.attach;
if (typeof rawAttach === "string") {
- const result = extractDuckLakeAttachSecrets(
+ const rewrittenAttach = extractDuckLakeAttachSecrets(
rawAttach,
- opts.existingEnvBlob,
+ opts.envEditSession,
);
- if (Object.keys(result.extractedSecrets).length > 0) {
- transformedValues = {
- ...transformedValues,
- attach: result.rewrittenAttach,
- };
- extractedSecrets = result.extractedSecrets;
- }
+ transformedValues = {
+ ...transformedValues,
+ attach: rewrittenAttach,
+ };
}
}
- return { transformedValues, extractedSecrets };
+ return transformedValues;
}
const DUCKLAKE_KNOWN_CATALOG_SCHEMES = new Set([
diff --git a/web-common/src/lib/event-bus/event-bus.ts b/web-common/src/lib/event-bus/event-bus.ts
index bd207140bce2..28858c952d95 100644
--- a/web-common/src/lib/event-bus/event-bus.ts
+++ b/web-common/src/lib/event-bus/event-bus.ts
@@ -17,6 +17,7 @@ export interface Events {
"page-content-resized": PageContentResized;
"start-chat": string;
"rill-yaml-updated": void;
+ "env-file-updated": string;
"remote-changes-detected": void;
}
diff --git a/web-common/src/runtime-client/invalidation/file-invalidators.ts b/web-common/src/runtime-client/invalidation/file-invalidators.ts
index 9294ea875f8b..4e72e5ff21b9 100644
--- a/web-common/src/runtime-client/invalidation/file-invalidators.ts
+++ b/web-common/src/runtime-client/invalidation/file-invalidators.ts
@@ -60,6 +60,8 @@ export async function handleFileEvent(
});
await invalidate("app:init");
eventBus.emit("rill-yaml-updated");
+ } else if (event.path === "/.env") {
+ eventBus.emit("env-file-updated", event.path);
}
state.seenFiles.add(event.path);
break;
diff --git a/web-common/src/runtime-client/v2/RuntimeProvider.svelte b/web-common/src/runtime-client/v2/RuntimeProvider.svelte
index 1b9d1006f9fc..599d67850bc7 100644
--- a/web-common/src/runtime-client/v2/RuntimeProvider.svelte
+++ b/web-common/src/runtime-client/v2/RuntimeProvider.svelte
@@ -9,6 +9,7 @@
RUNTIME_CONTEXT_KEY,
} from "./context";
import type { AuthContext } from "./runtime-client";
+ import { createEnvFileStore } from "@rilldata/web-common/features/env-management/env-file-store.ts";
const queryClient = useQueryClient();
@@ -23,6 +24,8 @@
setContext(RUNTIME_CONTEXT_KEY, client);
featureFlags.setRuntimeClient(client);
+ const envStoreUnsub = createEnvFileStore(client);
+
// Handle JWT-only changes (15-min refresh, View As with same host)
$: {
const authContextChanged = client.updateJwt(jwt, authContext);
@@ -34,6 +37,7 @@
featureFlags.clearRuntimeClient();
evictRuntimeClient(client);
client.dispose();
+ envStoreUnsub();
});