diff --git a/runtime/templates/definitions/clickhouse-models/azure-clickhouse.json b/runtime/templates/definitions/clickhouse-models/azure-clickhouse.json index 011a6cffa9b..7379d5c146b 100644 --- a/runtime/templates/definitions/clickhouse-models/azure-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/azure-clickhouse.json @@ -1,7 +1,7 @@ { "name": "azure-clickhouse", "display_name": "Azure Blob Storage", - "description": "Read Azure Blob Storage files into ClickHouse using table functions", + "description": "Read Azure Blob Storage files into ClickHouse via a named collection", "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/azure", "driver": "azure", "olap": "clickhouse", @@ -12,12 +12,13 @@ "microsoft", "object-storage", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "objectStore", "properties": { "auth_method": { "type": "string", @@ -50,7 +51,8 @@ "azure_storage_key" ], "public": [] - } + }, + "x-step": "connector" }, "azure_storage_connection_string": { "type": "string", @@ -59,6 +61,7 @@ "x-placeholder": "DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=core.windows.net", "x-secret": true, "x-env-var": "AZURE_STORAGE_CONNECTION_STRING", + "x-step": "connector", "x-visible-if": { "auth_method": "connection_string" } @@ -68,6 +71,7 @@ "title": "Storage account", "description": "Azure storage account name", "x-placeholder": "mystorageaccount", + "x-step": "connector", "x-visible-if": { "auth_method": "account_key" } @@ -79,6 +83,7 @@ "x-placeholder": "Enter storage key", "x-secret": true, "x-env-var": "AZURE_STORAGE_KEY", + "x-step": "connector", "x-visible-if": { "auth_method": "account_key" } @@ -91,14 +96,16 @@ "errorMessage": { "pattern": "Must be an Azure URI (e.g. azure://container/path or https://account.blob.core.windows.net/container/path)" }, - "x-placeholder": "azure://container/path" + "x-placeholder": "azure://container/path", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_model" + "x-placeholder": "my_model", + "x-step": "source" } }, "required": [ @@ -138,10 +145,15 @@ ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "[[ if .config_props -]]\n# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: azure\n[[ renderProps .config_props ]]\n[[ end -]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if eq .auth_method \"connection_string\" ]]\n SELECT * FROM azureBlobStorage(\n '[[ propVal .props \"azure_storage_connection_string\" ]]',\n '[[ azureContainer .path ]]',\n '[[ azureBlobPath .path ]]'\n )[[ else if eq .auth_method \"account_key\" ]]\n SELECT * FROM azureBlobStorage(\n '[[ azureEndpoint .path .azure_storage_account ]]',\n '[[ azureContainer .path ]]',\n '[[ azureBlobPath .path ]]',\n '[[ propVal .props \"azure_storage_account\" ]]',\n '[[ propVal .props \"azure_storage_key\" ]]'\n )[[ else ]]\n SELECT * FROM azureBlobStorage(\n '[[ azureEndpoint .path \"\" ]]',\n '[[ azureContainer .path ]]',\n '[[ azureBlobPath .path ]]'\n )[[ end ]]\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if ne .auth_method \"public\" ]]\n SELECT * FROM azureBlobStorage(rill_[[ .connector_name ]], container='[[ azureContainer .path ]]', blob_path='[[ azureBlobPath .path ]]')[[ else ]]\n SELECT * FROM azureBlobStorage('[[ azureEndpoint .path \"\" ]]', '[[ azureContainer .path ]]', '[[ azureBlobPath .path ]]')[[ end ]]\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/delta-clickhouse.json b/runtime/templates/definitions/clickhouse-models/delta-clickhouse.json index 9032cb4c942..f2039aa5eca 100644 --- a/runtime/templates/definitions/clickhouse-models/delta-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/delta-clickhouse.json @@ -1,7 +1,7 @@ { "name": "delta-clickhouse", "display_name": "Delta Lake", - "description": "Query Delta Lake tables in ClickHouse using deltaLake table function", + "description": "Query Delta Lake tables in ClickHouse via a named collection", "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/delta", "driver": "delta", "olap": "clickhouse", @@ -11,12 +11,13 @@ "delta", "table-format", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "objectStore", "properties": { "aws_access_key_id": { "type": "string", @@ -24,7 +25,8 @@ "description": "Access key for the S3 bucket containing your Delta table", "x-placeholder": "Enter AWS access key ID", "x-secret": true, - "x-env-var": "AWS_ACCESS_KEY_ID" + "x-env-var": "AWS_ACCESS_KEY_ID", + "x-step": "connector" }, "aws_secret_access_key": { "type": "string", @@ -32,7 +34,8 @@ "description": "Secret key for the S3 bucket containing your Delta table", "x-placeholder": "Enter AWS secret access key", "x-secret": true, - "x-env-var": "AWS_SECRET_ACCESS_KEY" + "x-env-var": "AWS_SECRET_ACCESS_KEY", + "x-step": "connector" }, "path": { "type": "string", @@ -42,14 +45,16 @@ "errorMessage": { "pattern": "Must be an S3 URI (e.g. s3://bucket/delta_table)" }, - "x-placeholder": "s3://bucket/delta_table" + "x-placeholder": "s3://bucket/delta_table", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_delta_table" + "x-placeholder": "my_delta_table", + "x-step": "source" } }, "required": [ @@ -60,10 +65,15 @@ ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: delta\n[[ renderProps .config_props ]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM deltaLake(\n '[[ s3ToHTTPS .path ]]',\n '[[ propVal .props \"aws_access_key_id\" ]]',\n '[[ propVal .props \"aws_secret_access_key\" ]]'\n )\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM deltaLake(rill_[[ .connector_name ]], url='[[ s3ToHTTPS .path ]]')\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/gcs-clickhouse.json b/runtime/templates/definitions/clickhouse-models/gcs-clickhouse.json index 444d0b5de74..7ba952daea7 100644 --- a/runtime/templates/definitions/clickhouse-models/gcs-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/gcs-clickhouse.json @@ -1,7 +1,7 @@ { "name": "gcs-clickhouse", "display_name": "Google Cloud Storage", - "description": "Read GCS files into ClickHouse using table functions (HMAC keys)", + "description": "Read GCS files into ClickHouse via a named collection (HMAC keys)", "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/gcs", "driver": "gcs", "olap": "clickhouse", @@ -12,12 +12,13 @@ "google", "object-storage", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "objectStore", "properties": { "auth_method": { "type": "string", @@ -44,7 +45,8 @@ "secret" ], "public": [] - } + }, + "x-step": "connector" }, "key_id": { "type": "string", @@ -53,6 +55,7 @@ "x-placeholder": "Enter HMAC access key", "x-secret": true, "x-env-var": "GCP_ACCESS_KEY_ID", + "x-step": "connector", "x-visible-if": { "auth_method": "hmac_key" } @@ -64,6 +67,7 @@ "x-placeholder": "Enter HMAC secret", "x-secret": true, "x-env-var": "GCP_SECRET_ACCESS_KEY", + "x-step": "connector", "x-visible-if": { "auth_method": "hmac_key" } @@ -76,14 +80,16 @@ "errorMessage": { "pattern": "Must be a GCS URI (e.g. gs://bucket/path or https://storage.googleapis.com/bucket/path)" }, - "x-placeholder": "gs://bucket/path" + "x-placeholder": "gs://bucket/path", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_model" + "x-placeholder": "my_model", + "x-step": "source" } }, "required": [ @@ -109,10 +115,15 @@ ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "[[ if .config_props -]]\n# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: gcs\n[[ renderProps .config_props ]]\n[[ end -]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if eq .auth_method \"hmac_key\" ]]\n SELECT * FROM gcs(\n '[[ gcsToHTTPS .path ]]',\n '[[ propVal .props \"key_id\" ]]',\n '[[ propVal .props \"secret\" ]]'\n )[[ else ]]\n SELECT * FROM gcs('[[ gcsToHTTPS .path ]]')[[ end ]]\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if ne .auth_method \"public\" ]]\n SELECT * FROM gcs(rill_[[ .connector_name ]], url='[[ gcsToHTTPS .path ]]')[[ else ]]\n SELECT * FROM gcs('[[ gcsToHTTPS .path ]]')[[ end ]]\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/https-clickhouse.json b/runtime/templates/definitions/clickhouse-models/https-clickhouse.json index 694832e169a..15774f87305 100644 --- a/runtime/templates/definitions/clickhouse-models/https-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/https-clickhouse.json @@ -12,13 +12,14 @@ "http", "url", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "fileStore", "properties": { "headers": { @@ -26,7 +27,8 @@ "description": "HTTP headers to include in the request", "x-display": "key-value", "x-placeholder": "Header name", - "x-hint": "e.g. Authorization: Bearer " + "x-hint": "e.g. Authorization: Bearer ", + "x-step": "connector" }, "path": { @@ -37,7 +39,8 @@ "errorMessage": { "pattern": "Must be a valid HTTP(S) URL" }, - "x-placeholder": "https://example.com/data.csv" + "x-placeholder": "https://example.com/data.csv", + "x-step": "source" }, "name": { @@ -45,7 +48,8 @@ "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_model" + "x-placeholder": "my_model", + "x-step": "source" } }, @@ -53,10 +57,15 @@ }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "[[ if .config_props -]]\n# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: https\n[[ renderProps .config_props ]]\n[[ end -]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\n\ntype: model\nmaterialize: true\n\nconnector: clickhouse\n\nsql: |\n SELECT * FROM url('[[ .path ]]'[[ clickhouseURLSuffix .path .props ]])\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if .config_props ]]\n SELECT * FROM url(rill_[[ .connector_name ]], url='[[ .path ]]')[[ else ]]\n SELECT * FROM url('[[ .path ]]')[[ end ]]\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/hudi-clickhouse.json b/runtime/templates/definitions/clickhouse-models/hudi-clickhouse.json index 9662ba663bc..3394cbac802 100644 --- a/runtime/templates/definitions/clickhouse-models/hudi-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/hudi-clickhouse.json @@ -1,7 +1,7 @@ { "name": "hudi-clickhouse", "display_name": "Hudi", - "description": "Query Apache Hudi tables in ClickHouse using hudi table function", + "description": "Query Apache Hudi tables in ClickHouse via a named collection", "docs_url": "https://clickhouse.com/docs/en/sql-reference/table-functions/hudi", "driver": "hudi", "olap": "clickhouse", @@ -11,12 +11,13 @@ "hudi", "table-format", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "objectStore", "properties": { "aws_access_key_id": { "type": "string", @@ -24,7 +25,8 @@ "description": "Access key for the S3 bucket containing your Hudi table", "x-placeholder": "Enter AWS access key ID", "x-secret": true, - "x-env-var": "AWS_ACCESS_KEY_ID" + "x-env-var": "AWS_ACCESS_KEY_ID", + "x-step": "connector" }, "aws_secret_access_key": { "type": "string", @@ -32,7 +34,8 @@ "description": "Secret key for the S3 bucket containing your Hudi table", "x-placeholder": "Enter AWS secret access key", "x-secret": true, - "x-env-var": "AWS_SECRET_ACCESS_KEY" + "x-env-var": "AWS_SECRET_ACCESS_KEY", + "x-step": "connector" }, "path": { "type": "string", @@ -42,14 +45,16 @@ "errorMessage": { "pattern": "Must be an S3 URI (e.g. s3://bucket/hudi_table)" }, - "x-placeholder": "s3://bucket/hudi_table" + "x-placeholder": "s3://bucket/hudi_table", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_hudi_table" + "x-placeholder": "my_hudi_table", + "x-step": "source" } }, "required": [ @@ -60,10 +65,15 @@ ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: hudi\n[[ renderProps .config_props ]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM hudi(\n '[[ s3ToHTTPS .path ]]',\n '[[ propVal .props \"aws_access_key_id\" ]]',\n '[[ propVal .props \"aws_secret_access_key\" ]]'\n )\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM hudi(rill_[[ .connector_name ]], url='[[ s3ToHTTPS .path ]]')\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/iceberg-clickhouse.json b/runtime/templates/definitions/clickhouse-models/iceberg-clickhouse.json index 6dc5010c623..a99f90223f8 100644 --- a/runtime/templates/definitions/clickhouse-models/iceberg-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/iceberg-clickhouse.json @@ -1,7 +1,7 @@ { "name": "iceberg-clickhouse", "display_name": "Iceberg", - "description": "Query Apache Iceberg tables in ClickHouse using icebergS3 table function", + "description": "Query Apache Iceberg tables in ClickHouse via a named collection", "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/iceberg", "driver": "iceberg", "olap": "clickhouse", @@ -11,12 +11,13 @@ "iceberg", "table-format", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "objectStore", "properties": { "aws_access_key_id": { "type": "string", @@ -24,7 +25,8 @@ "description": "Access key for the S3 bucket containing your Iceberg table", "x-placeholder": "Enter AWS access key ID", "x-secret": true, - "x-env-var": "AWS_ACCESS_KEY_ID" + "x-env-var": "AWS_ACCESS_KEY_ID", + "x-step": "connector" }, "aws_secret_access_key": { "type": "string", @@ -32,7 +34,8 @@ "description": "Secret key for the S3 bucket containing your Iceberg table", "x-placeholder": "Enter AWS secret access key", "x-secret": true, - "x-env-var": "AWS_SECRET_ACCESS_KEY" + "x-env-var": "AWS_SECRET_ACCESS_KEY", + "x-step": "connector" }, "path": { "type": "string", @@ -42,14 +45,16 @@ "errorMessage": { "pattern": "Must be an S3 URI (e.g. s3://bucket/warehouse/my_table)" }, - "x-placeholder": "s3://bucket/warehouse/my_table" + "x-placeholder": "s3://bucket/warehouse/my_table", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_iceberg_table" + "x-placeholder": "my_iceberg_table", + "x-step": "source" } }, "required": [ @@ -60,10 +65,15 @@ ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: iceberg\n[[ renderProps .config_props ]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM icebergS3(\n '[[ s3ToHTTPS .path ]]',\n '[[ propVal .props \"aws_access_key_id\" ]]',\n '[[ propVal .props \"aws_secret_access_key\" ]]'\n )\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM icebergS3(rill_[[ .connector_name ]], url='[[ s3ToHTTPS .path ]]')\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/kafka-clickhouse.json b/runtime/templates/definitions/clickhouse-models/kafka-clickhouse.json index c3a36ae7741..9f0ee4b741a 100644 --- a/runtime/templates/definitions/clickhouse-models/kafka-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/kafka-clickhouse.json @@ -1,7 +1,7 @@ { "name": "kafka-clickhouse", "display_name": "Kafka", - "description": "Read Kafka topics into ClickHouse using Kafka table engine", + "description": "Read Kafka topics into ClickHouse via a named collection", "docs_url": "https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka", "driver": "kafka", "olap": "clickhouse", @@ -11,7 +11,8 @@ "kafka", "streaming", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", @@ -22,19 +23,22 @@ "type": "string", "title": "Broker List", "description": "Comma-separated list of Kafka brokers", - "x-placeholder": "broker1:9092,broker2:9092" + "x-placeholder": "broker1:9092,broker2:9092", + "x-step": "connector" }, "topic": { "type": "string", "title": "Topic", "description": "Kafka topic to consume from", - "x-placeholder": "my_topic" + "x-placeholder": "my_topic", + "x-step": "source" }, "group_name": { "type": "string", "title": "Consumer Group", "description": "Kafka consumer group name", - "x-placeholder": "rill_consumer_group" + "x-placeholder": "rill_consumer_group", + "x-step": "source" }, "format": { "type": "string", @@ -48,14 +52,16 @@ "Parquet" ], "default": "JSONEachRow", - "x-display": "select" + "x-display": "select", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_model" + "x-placeholder": "my_model", + "x-step": "source" } }, "required": [ @@ -67,10 +73,15 @@ ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: kafka\n[[ renderProps .config_props ]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM kafka(\n '[[ .broker_list ]]',\n '[[ .topic ]]',\n '[[ .group_name ]]',\n '[[ .format ]]'\n )\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM kafka(rill_[[ .connector_name ]], kafka_topic_list='[[ .topic ]]', kafka_group_name='[[ .group_name ]]', kafka_format='[[ .format ]]')\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/mongodb-clickhouse.json b/runtime/templates/definitions/clickhouse-models/mongodb-clickhouse.json index 0f405faa94e..920ea8641ec 100644 --- a/runtime/templates/definitions/clickhouse-models/mongodb-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/mongodb-clickhouse.json @@ -1,7 +1,7 @@ { "name": "mongodb-clickhouse", "display_name": "MongoDB", - "description": "Query MongoDB collections in ClickHouse using mongodb table function", + "description": "Query MongoDB collections in ClickHouse via a named collection", "docs_url": "https://clickhouse.com/docs/en/sql-reference/table-functions/mongodb", "driver": "mongodb", "olap": "clickhouse", @@ -11,18 +11,20 @@ "mongodb", "database", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "sqlStore", "properties": { "host": { "type": "string", "title": "Host", "description": "MongoDB server hostname or IP", - "x-placeholder": "localhost" + "x-placeholder": "localhost", + "x-step": "connector" }, "port": { "type": "string", @@ -33,25 +35,22 @@ "pattern": "Port must be a number" }, "default": "27017", - "x-placeholder": "27017" + "x-placeholder": "27017", + "x-step": "connector" }, "database": { "type": "string", "title": "Database", "description": "MongoDB database name", - "x-placeholder": "my_database" - }, - "collection": { - "type": "string", - "title": "Collection", - "description": "MongoDB collection name", - "x-placeholder": "my_collection" + "x-placeholder": "my_database", + "x-step": "connector" }, "user": { "type": "string", "title": "Username", "description": "MongoDB user", - "x-placeholder": "mongo_user" + "x-placeholder": "mongo_user", + "x-step": "connector" }, "password": { "type": "string", @@ -59,29 +58,43 @@ "description": "MongoDB password", "x-placeholder": "your_password", "x-secret": true, - "x-env-var": "MONGODB_PASSWORD" + "x-env-var": "MONGODB_PASSWORD", + "x-step": "connector" + }, + "collection": { + "type": "string", + "title": "Collection", + "description": "MongoDB collection name", + "x-placeholder": "my_collection", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_model" + "x-placeholder": "my_model", + "x-step": "source" } }, "required": [ "host", "database", - "collection", "user", + "collection", "name" ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: mongodb\n[[ renderProps .config_props ]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM mongodb(\n '[[ .host ]]:[[ default (propVal .props \"port\") \"27017\" ]]',\n '[[ .database ]]',\n '[[ .collection ]]',\n '[[ propVal .props \"user\" ]]',\n '[[ propVal .props \"password\" ]]'\n )\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM mongodb(rill_[[ .connector_name ]], collection='[[ .collection ]]')\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/mysql-clickhouse.json b/runtime/templates/definitions/clickhouse-models/mysql-clickhouse.json index f7438e52141..14e6eb29170 100644 --- a/runtime/templates/definitions/clickhouse-models/mysql-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/mysql-clickhouse.json @@ -1,7 +1,7 @@ { "name": "mysql-clickhouse", "display_name": "MySQL", - "description": "Read MySQL tables into ClickHouse using table functions", + "description": "Read MySQL tables into ClickHouse via a named collection", "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/mysql", "driver": "mysql", "olap": "clickhouse", @@ -11,18 +11,20 @@ "mysql", "database", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "sqlStore", "properties": { "host": { "type": "string", "title": "Host", "description": "MySQL server hostname or IP", - "x-placeholder": "localhost" + "x-placeholder": "localhost", + "x-step": "connector" }, "port": { "type": "string", @@ -33,25 +35,22 @@ "pattern": "Port must be a number" }, "default": "3306", - "x-placeholder": "3306" + "x-placeholder": "3306", + "x-step": "connector" }, "database": { "type": "string", "title": "Database", "description": "Database name", - "x-placeholder": "my_database" - }, - "table": { - "type": "string", - "title": "Table", - "description": "Table name to query", - "x-placeholder": "my_table" + "x-placeholder": "my_database", + "x-step": "connector" }, "user": { "type": "string", "title": "Username", "description": "MySQL user", - "x-placeholder": "mysql" + "x-placeholder": "mysql", + "x-step": "connector" }, "password": { "type": "string", @@ -59,29 +58,43 @@ "description": "MySQL password", "x-placeholder": "your_password", "x-secret": true, - "x-env-var": "MYSQL_PASSWORD" + "x-env-var": "MYSQL_PASSWORD", + "x-step": "connector" + }, + "table": { + "type": "string", + "title": "Table", + "description": "Table name to query", + "x-placeholder": "my_table", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_model" + "x-placeholder": "my_model", + "x-step": "source" } }, "required": [ "host", "database", - "table", "user", + "table", "name" ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: mysql\n[[ renderProps .config_props ]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM mysql(\n '[[ .host ]]:[[ default (propVal .props \"port\") \"3306\" ]]',\n '[[ .database ]]',\n '[[ .table ]]',\n '[[ propVal .props \"user\" ]]',\n '[[ propVal .props \"password\" ]]'\n )\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM mysql(rill_[[ .connector_name ]], table='[[ .table ]]')\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/postgres-clickhouse.json b/runtime/templates/definitions/clickhouse-models/postgres-clickhouse.json index 97eac39a41f..deab913f45a 100644 --- a/runtime/templates/definitions/clickhouse-models/postgres-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/postgres-clickhouse.json @@ -1,7 +1,7 @@ { "name": "postgres-clickhouse", "display_name": "PostgreSQL", - "description": "Read PostgreSQL tables into ClickHouse using table functions", + "description": "Read PostgreSQL tables into ClickHouse via a named collection", "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/postgres", "driver": "postgres", "olap": "clickhouse", @@ -12,18 +12,20 @@ "postgresql", "database", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "sqlStore", "properties": { "host": { "type": "string", "title": "Host", "description": "Postgres server hostname or IP", - "x-placeholder": "localhost" + "x-placeholder": "localhost", + "x-step": "connector" }, "port": { "type": "string", @@ -34,25 +36,22 @@ "pattern": "Port must be a number" }, "default": "5432", - "x-placeholder": "5432" + "x-placeholder": "5432", + "x-step": "connector" }, "dbname": { "type": "string", "title": "Database", "description": "Database name", - "x-placeholder": "postgres" - }, - "table": { - "type": "string", - "title": "Table", - "description": "Table name to query", - "x-placeholder": "my_table" + "x-placeholder": "postgres", + "x-step": "connector" }, "user": { "type": "string", "title": "Username", "description": "Postgres user", - "x-placeholder": "postgres" + "x-placeholder": "postgres", + "x-step": "connector" }, "password": { "type": "string", @@ -60,29 +59,43 @@ "description": "Postgres password", "x-placeholder": "your_password", "x-secret": true, - "x-env-var": "POSTGRES_PASSWORD" + "x-env-var": "POSTGRES_PASSWORD", + "x-step": "connector" + }, + "table": { + "type": "string", + "title": "Table", + "description": "Table name to query", + "x-placeholder": "my_table", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_model" + "x-placeholder": "my_model", + "x-step": "source" } }, "required": [ "host", "dbname", - "table", "user", + "table", "name" ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: postgres\n[[ renderProps .config_props ]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM postgresql(\n '[[ .host ]]:[[ default (propVal .props \"port\") \"5432\" ]]',\n '[[ .dbname ]]',\n '[[ .table ]]',\n '[[ propVal .props \"user\" ]]',\n '[[ propVal .props \"password\" ]]'\n )\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM postgresql(rill_[[ .connector_name ]], table='[[ .table ]]')\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/s3-clickhouse.json b/runtime/templates/definitions/clickhouse-models/s3-clickhouse.json index 5f92243b79b..eacd008694d 100644 --- a/runtime/templates/definitions/clickhouse-models/s3-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/s3-clickhouse.json @@ -1,7 +1,7 @@ { "name": "s3-clickhouse", "display_name": "Amazon S3", - "description": "Read S3 files into ClickHouse using table functions", + "description": "Read S3 files into ClickHouse via a named collection", "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/s3", "driver": "s3", "olap": "clickhouse", @@ -12,12 +12,13 @@ "aws", "object-storage", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "objectStore", "properties": { "auth_method": { "type": "string", @@ -44,7 +45,8 @@ "aws_secret_access_key" ], "public": [] - } + }, + "x-step": "connector" }, "aws_access_key_id": { "type": "string", @@ -53,6 +55,7 @@ "x-placeholder": "Enter AWS access key ID", "x-secret": true, "x-env-var": "AWS_ACCESS_KEY_ID", + "x-step": "connector", "x-visible-if": { "auth_method": "access_key" } @@ -64,6 +67,7 @@ "x-placeholder": "Enter AWS secret access key", "x-secret": true, "x-env-var": "AWS_SECRET_ACCESS_KEY", + "x-step": "connector", "x-visible-if": { "auth_method": "access_key" } @@ -76,14 +80,16 @@ "errorMessage": { "pattern": "Must be an S3 URI (e.g. s3://bucket/path or https://bucket.s3.amazonaws.com/path)" }, - "x-placeholder": "s3://bucket/path" + "x-placeholder": "s3://bucket/path", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_model" + "x-placeholder": "my_model", + "x-step": "source" } }, "required": [ @@ -109,10 +115,15 @@ ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "[[ if .config_props -]]\n# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: s3\n[[ renderProps .config_props ]]\n[[ end -]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if eq .auth_method \"access_key\" ]]\n SELECT * FROM s3(\n '[[ s3ToHTTPS .path ]]',\n '[[ propVal .props \"aws_access_key_id\" ]]',\n '[[ propVal .props \"aws_secret_access_key\" ]]'\n )[[ else ]]\n SELECT * FROM s3('[[ s3ToHTTPS .path ]]')[[ end ]]\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if ne .auth_method \"public\" ]]\n SELECT * FROM s3(rill_[[ .connector_name ]], url='[[ s3ToHTTPS .path ]]')[[ else ]]\n SELECT * FROM s3('[[ s3ToHTTPS .path ]]')[[ end ]]\n" } ] } diff --git a/runtime/templates/definitions/clickhouse-models/supabase-clickhouse.json b/runtime/templates/definitions/clickhouse-models/supabase-clickhouse.json index 9ede0b9a27d..af74ab6f9f7 100644 --- a/runtime/templates/definitions/clickhouse-models/supabase-clickhouse.json +++ b/runtime/templates/definitions/clickhouse-models/supabase-clickhouse.json @@ -1,7 +1,7 @@ { "name": "supabase-clickhouse", "display_name": "Supabase", - "description": "Read Supabase tables into ClickHouse using table functions", + "description": "Read Supabase tables into ClickHouse via a named collection", "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/postgres", "driver": "supabase", "olap": "clickhouse", @@ -12,18 +12,20 @@ "postgres", "database", "clickhouse", - "source" + "source", + "connector" ], "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "x-category": "sourceOnly", + "x-category": "sqlStore", "properties": { "host": { "type": "string", "title": "Host", "description": "Supabase database host", - "x-placeholder": "aws-0-[region].pooler.supabase.com" + "x-placeholder": "aws-0-[region].pooler.supabase.com", + "x-step": "connector" }, "port": { "type": "string", @@ -34,25 +36,22 @@ "pattern": "Port must be a number" }, "default": "5432", - "x-placeholder": "5432" + "x-placeholder": "5432", + "x-step": "connector" }, "dbname": { "type": "string", "title": "Database", "description": "Database name", - "x-placeholder": "postgres" - }, - "table": { - "type": "string", - "title": "Table", - "description": "Table name to query", - "x-placeholder": "my_table" + "x-placeholder": "postgres", + "x-step": "connector" }, "user": { "type": "string", "title": "Username", "description": "Supabase database user", - "x-placeholder": "postgres.[ref]" + "x-placeholder": "postgres.[ref]", + "x-step": "connector" }, "password": { "type": "string", @@ -60,29 +59,43 @@ "description": "Supabase database password", "x-placeholder": "your_password", "x-secret": true, - "x-env-var": "SUPABASE_PASSWORD" + "x-env-var": "SUPABASE_PASSWORD", + "x-step": "connector" + }, + "table": { + "type": "string", + "title": "Table", + "description": "Table name to query", + "x-placeholder": "my_table", + "x-step": "source" }, "name": { "type": "string", "title": "Model name", "description": "Name for the source model", "pattern": "^[a-zA-Z0-9_]+$", - "x-placeholder": "my_model" + "x-placeholder": "my_model", + "x-step": "source" } }, "required": [ "host", "dbname", - "table", "user", + "table", "name" ] }, "files": [ + { + "name": "connector", + "path_template": "connectors/[[ .connector_name ]].yaml", + "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: supabase\n[[ renderProps .config_props ]]\n" + }, { "name": "model", "path_template": "models/[[ .model_name ]].yaml", - "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM postgresql(\n '[[ .host ]]:[[ default (propVal .props \"port\") \"5432\" ]]',\n '[[ .dbname ]]',\n '[[ .table ]]',\n '[[ propVal .props \"user\" ]]',\n '[[ propVal .props \"password\" ]]'\n )\n" + "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n SELECT * FROM postgresql(rill_[[ .connector_name ]], table='[[ .table ]]')\n" } ] } diff --git a/runtime/templates/definitions/duckdb-models/azure-duckdb.json b/runtime/templates/definitions/duckdb-models/azure-duckdb.json index 49040dd0a36..26a232fd13b 100644 --- a/runtime/templates/definitions/duckdb-models/azure-duckdb.json +++ b/runtime/templates/definitions/duckdb-models/azure-duckdb.json @@ -185,7 +185,7 @@ { "name": "connector", "path_template": "connectors/[[ .connector_name ]].yaml", - "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: azure\n[[ renderProps .config_props ]]\n" + "code_template": "[[ if .config_props -]]\n# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: azure\n[[ renderProps .config_props ]]\n[[ end -]]\n" }, { "name": "model", diff --git a/runtime/templates/definitions/duckdb-models/gcs-duckdb.json b/runtime/templates/definitions/duckdb-models/gcs-duckdb.json index 7a47a9d0dd8..af08f910b97 100644 --- a/runtime/templates/definitions/duckdb-models/gcs-duckdb.json +++ b/runtime/templates/definitions/duckdb-models/gcs-duckdb.json @@ -153,7 +153,7 @@ { "name": "connector", "path_template": "connectors/[[ .connector_name ]].yaml", - "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: gcs\n[[ renderProps .config_props ]]\n" + "code_template": "[[ if .config_props -]]\n# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: gcs\n[[ renderProps .config_props ]]\n[[ end -]]\n" }, { "name": "model", diff --git a/runtime/templates/definitions/duckdb-models/s3-duckdb.json b/runtime/templates/definitions/duckdb-models/s3-duckdb.json index 3b6827fac7a..89c2557a309 100644 --- a/runtime/templates/definitions/duckdb-models/s3-duckdb.json +++ b/runtime/templates/definitions/duckdb-models/s3-duckdb.json @@ -153,7 +153,7 @@ { "name": "connector", "path_template": "connectors/[[ .connector_name ]].yaml", - "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: s3\n[[ renderProps .config_props ]]\n" + "code_template": "[[ if .config_props -]]\n# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: s3\n[[ renderProps .config_props ]]\n[[ end -]]\n" }, { "name": "model", diff --git a/runtime/templates/render.go b/runtime/templates/render.go index dfc15ad06ca..93659296a0e 100644 --- a/runtime/templates/render.go +++ b/runtime/templates/render.go @@ -62,6 +62,15 @@ func Render(input *RenderInput) (*RenderOutput, error) { return nil, fmt.Errorf("rendering code template for %q: %w", f.Name, err) } + // Skip files that render to whitespace-only output. This lets templates + // emit nothing (e.g. via [[ if .config_props -]]...[[ end -]]) when an + // optional output isn't applicable to the current form values — such as + // the "Public" auth path on object-store connectors that don't need a + // connector YAML at all. + if strings.TrimSpace(blob) == "" { + continue + } + files = append(files, RenderedFile{ Path: strings.TrimSpace(path), Blob: blob, diff --git a/runtime/templates/render_test.go b/runtime/templates/render_test.go index dbb54faa00a..20e286e9449 100644 --- a/runtime/templates/render_test.go +++ b/runtime/templates/render_test.go @@ -171,10 +171,14 @@ func TestRenderS3ClickHouseModel(t *testing.T) { tmpl, ok := registry.Get("s3-clickhouse") require.True(t, ok) + // Access-key auth: render both files. The connector YAML carries the + // credentials (the backend turns it into a CH named collection), and + // the model SQL references that collection by name. result, err := Render(&RenderInput{ - Template: tmpl, - Output: "model", + Template: tmpl, + ConnectorName: "my_s3", Properties: map[string]any{ + "auth_method": "access_key", "aws_access_key_id": "AKIAIOSFODNN7EXAMPLE", "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", "path": "s3://my-bucket/data/events.parquet", @@ -183,24 +187,54 @@ func TestRenderS3ClickHouseModel(t *testing.T) { ExistingEnv: make(map[string]bool), }) require.NoError(t, err) - require.Len(t, result.Files, 1) + require.Len(t, result.Files, 2) - blob := result.Files[0].Blob - require.Contains(t, blob, "type: model") - require.Contains(t, blob, "connector: clickhouse") - require.Contains(t, blob, "materialize: true") - // SQL should show the s3() function with env var refs - require.Contains(t, blob, "FROM s3(") - require.Contains(t, blob, "https://my-bucket.s3.amazonaws.com/data/events.parquet") - require.Contains(t, blob, "{{ .env.AWS_ACCESS_KEY_ID }}") - require.Contains(t, blob, "{{ .env.AWS_SECRET_ACCESS_KEY }}") - // Raw secrets should NOT appear in the blob - require.NotContains(t, blob, "AKIAIOSFODNN7EXAMPLE") + connectorBlob := result.Files[0].Blob + require.Equal(t, "connectors/my_s3.yaml", result.Files[0].Path) + require.Contains(t, connectorBlob, "type: connector") + require.Contains(t, connectorBlob, "driver: s3") + require.Contains(t, connectorBlob, "{{ .env.AWS_ACCESS_KEY_ID }}") + require.Contains(t, connectorBlob, "{{ .env.AWS_SECRET_ACCESS_KEY }}") + require.NotContains(t, connectorBlob, "AKIAIOSFODNN7EXAMPLE") + + modelBlob := result.Files[1].Blob + require.Equal(t, "models/s3_events.yaml", result.Files[1].Path) + require.Contains(t, modelBlob, "type: model") + require.Contains(t, modelBlob, "connector: clickhouse") + require.Contains(t, modelBlob, "materialize: true") + require.Contains(t, modelBlob, "FROM s3(rill_my_s3, url='https://my-bucket.s3.amazonaws.com/data/events.parquet')") - // Env vars should be extracted require.Equal(t, "AKIAIOSFODNN7EXAMPLE", result.EnvVars["AWS_ACCESS_KEY_ID"]) +} + +func TestRenderS3ClickHouseModelPublic(t *testing.T) { + registry, err := NewRegistry() + require.NoError(t, err) + + tmpl, ok := registry.Get("s3-clickhouse") + require.True(t, ok) + + // Public auth: no creds, no named collection. The connector YAML renders + // to whitespace and is skipped; only the model file is emitted, with an + // inline URL on the s3() function. + result, err := Render(&RenderInput{ + Template: tmpl, + ConnectorName: "my_s3", + Properties: map[string]any{ + "auth_method": "public", + "path": "s3://my-bucket/public/data.parquet", + "name": "public_events", + }, + ExistingEnv: make(map[string]bool), + }) + require.NoError(t, err) + require.Len(t, result.Files, 1) - require.Equal(t, "models/s3_events.yaml", result.Files[0].Path) + modelBlob := result.Files[0].Blob + require.Equal(t, "models/public_events.yaml", result.Files[0].Path) + require.Contains(t, modelBlob, "FROM s3('https://my-bucket.s3.amazonaws.com/public/data.parquet')") + require.NotContains(t, modelBlob, "rill_my_s3") + require.Empty(t, result.EnvVars) } func TestRenderMySQLClickHouseModel(t *testing.T) { @@ -210,9 +244,12 @@ func TestRenderMySQLClickHouseModel(t *testing.T) { tmpl, ok := registry.Get("mysql-clickhouse") require.True(t, ok) + // Renders both files: the connector YAML carries the credentials (the + // backend turns it into a CH named collection), and the model SQL refers + // to that collection by name with just the table override. result, err := Render(&RenderInput{ - Template: tmpl, - Output: "model", + Template: tmpl, + ConnectorName: "my_mysql", Properties: map[string]any{ "host": "db.example.com", "port": "3306", @@ -225,17 +262,23 @@ func TestRenderMySQLClickHouseModel(t *testing.T) { ExistingEnv: make(map[string]bool), }) require.NoError(t, err) - require.Len(t, result.Files, 1) + require.Len(t, result.Files, 2) - blob := result.Files[0].Blob - require.Contains(t, blob, "connector: clickhouse") - require.Contains(t, blob, "FROM mysql(") - require.Contains(t, blob, "db.example.com:3306") - require.Contains(t, blob, "mydb") - require.Contains(t, blob, "events") - require.Contains(t, blob, "myuser") - require.Contains(t, blob, "{{ .env.MYSQL_PASSWORD }}") - require.NotContains(t, blob, "secret123") + connectorBlob := result.Files[0].Blob + require.Equal(t, "connectors/my_mysql.yaml", result.Files[0].Path) + require.Contains(t, connectorBlob, "type: connector") + require.Contains(t, connectorBlob, "driver: mysql") + require.Contains(t, connectorBlob, `host: "db.example.com"`) + require.Contains(t, connectorBlob, `database: "mydb"`) + require.Contains(t, connectorBlob, `user: "myuser"`) + require.Contains(t, connectorBlob, `password: "{{ .env.MYSQL_PASSWORD }}"`) + require.NotContains(t, connectorBlob, "secret123") + + modelBlob := result.Files[1].Blob + require.Equal(t, "models/mysql_events.yaml", result.Files[1].Path) + require.Contains(t, modelBlob, "FROM mysql(rill_my_mysql, table='events')") + + require.Equal(t, "secret123", result.EnvVars["MYSQL_PASSWORD"]) } func TestRenderHTTPSClickHouseWithHeaders(t *testing.T) { @@ -245,9 +288,11 @@ func TestRenderHTTPSClickHouseWithHeaders(t *testing.T) { tmpl, ok := registry.Get("https-clickhouse") require.True(t, ok) + // With headers: render both files. The connector YAML carries the headers + // (the backend turns them into a CH named collection); the model SQL + // references the collection by name. result, err := Render(&RenderInput{ Template: tmpl, - Output: "model", Properties: map[string]any{ // Frontend key-value editor sends [{key, value}, ...] format "headers": []any{ @@ -261,22 +306,21 @@ func TestRenderHTTPSClickHouseWithHeaders(t *testing.T) { ExistingEnv: make(map[string]bool), }) require.NoError(t, err) - require.Len(t, result.Files, 1) + require.Len(t, result.Files, 2) - blob := result.Files[0].Blob - require.Contains(t, blob, "type: model") - require.Contains(t, blob, "connector: clickhouse") - require.Contains(t, blob, "url(") - require.Contains(t, blob, "https://example.com/data.csv") - require.Contains(t, blob, "CSVWithNames") - require.Contains(t, blob, "headers(") - require.Contains(t, blob, "'Authorization'=") - require.Contains(t, blob, "'X-API-Key'=") - // Raw secrets should NOT appear in the blob - require.NotContains(t, blob, "my-secret-token") - require.NotContains(t, blob, "key123") + connectorBlob := result.Files[0].Blob + require.Equal(t, "connectors/my_https.yaml", result.Files[0].Path) + require.Contains(t, connectorBlob, "type: connector") + require.Contains(t, connectorBlob, "driver: https") + require.Contains(t, connectorBlob, "Authorization") + require.Contains(t, connectorBlob, "X-API-Key") + require.NotContains(t, connectorBlob, "my-secret-token") + require.NotContains(t, connectorBlob, "key123") + + modelBlob := result.Files[1].Blob + require.Equal(t, "models/api_data.yaml", result.Files[1].Path) + require.Contains(t, modelBlob, "url(rill_my_https, url='https://example.com/data.csv')") - // Env vars should be extracted for sensitive headers require.Contains(t, result.EnvVars, "connector.https.authorization") require.Contains(t, result.EnvVars, "connector.https.x_api_key") } @@ -288,9 +332,10 @@ func TestRenderHTTPSClickHouseNoHeaders(t *testing.T) { tmpl, ok := registry.Get("https-clickhouse") require.True(t, ok) + // No headers: connector YAML renders to whitespace and is skipped; only + // the model file is emitted with an inline URL on the url() function. result, err := Render(&RenderInput{ Template: tmpl, - Output: "model", Properties: map[string]any{ "path": "https://example.com/data.csv", "name": "simple_data", @@ -301,10 +346,10 @@ func TestRenderHTTPSClickHouseNoHeaders(t *testing.T) { require.NoError(t, err) require.Len(t, result.Files, 1) - blob := result.Files[0].Blob - require.Contains(t, blob, "url('https://example.com/data.csv')") - // No headers() clause when no headers provided - require.NotContains(t, blob, "headers(") + modelBlob := result.Files[0].Blob + require.Equal(t, "models/simple_data.yaml", result.Files[0].Path) + require.Contains(t, modelBlob, "url('https://example.com/data.csv')") + require.NotContains(t, modelBlob, "rill_my_https") } func TestRenderEnvVarConflict(t *testing.T) { diff --git a/web-common/src/features/add-data/manager/selectors.ts b/web-common/src/features/add-data/manager/selectors.ts index 036c462ef6f..014d6a8d6ba 100644 --- a/web-common/src/features/add-data/manager/selectors.ts +++ b/web-common/src/features/add-data/manager/selectors.ts @@ -1,33 +1,119 @@ -import { RuntimeClient } from "@rilldata/web-common/runtime-client/v2"; +import type { RuntimeClient } from "@rilldata/web-common/runtime-client/v2"; +import { + createRuntimeServiceGetInstance, + createRuntimeServiceListTemplates, + getRuntimeServiceListTemplatesQueryOptions, +} from "@rilldata/web-common/runtime-client/v2/gen/runtime-service"; import { useIsModelingSupportedForDefaultOlapDriverOLAP as useIsModelingSupportedForDefaultOlapDriver } from "@rilldata/web-common/features/connectors/selectors.ts"; +import { connectorKeywordMapping } from "@rilldata/web-common/features/connectors/connector-metadata.ts"; +import { createQuery } from "@tanstack/svelte-query"; import { derived } from "svelte/store"; -import { connectors } from "@rilldata/web-common/features/sources/modal/connector-schemas.ts"; +import { + registerTemplateSchema, + type ConnectorInfo, +} from "@rilldata/web-common/features/sources/modal/connector-schemas.ts"; +import { setOlapCache } from "@rilldata/web-common/features/sources/modal/generate-template.ts"; import type { AddDataConfig } from "@rilldata/web-common/features/add-data/manager/steps/types.ts"; +import type { + ConnectorCategory, + MultiStepFormSchema, +} from "@rilldata/web-common/features/templates/schemas/types"; +import type { Template as V1Template } from "@rilldata/web-common/proto/gen/rill/runtime/v1/api_pb"; + +/** + * Register schemas from `ListTemplates` responses so getConnectorSchema and + * connectorInfoMap resolve drivers that only exist as templates (e.g. kafka, + * hudi when ClickHouse is the OLAP). + */ +function registerTemplatesIfNeeded(templates: V1Template[]) { + for (const t of templates) { + const driver = t.driver ?? t.name ?? ""; + const templateName = t.name ?? ""; + if (!driver || !t.jsonSchema) continue; + registerTemplateSchema( + driver, + templateName, + t.jsonSchema as unknown as MultiStepFormSchema, + t.displayName ?? t.name ?? driver, + ); + } +} + +function templateToConnectorInfo( + t: V1Template, + fallbackCategory: ConnectorCategory, +): ConnectorInfo { + const driver = t.driver ?? t.name ?? ""; + const schema = t.jsonSchema as Record | undefined; + const category = + (schema?.["x-category"] as ConnectorCategory | undefined) ?? + fallbackCategory; + return { + name: driver, + displayName: t.displayName ?? t.name ?? driver, + category, + keywords: connectorKeywordMapping[driver] ?? [], + }; +} export function getSupportedConnectorInfos( runtimeClient: RuntimeClient, config: AddDataConfig, ) { - const isModelingSupportedForDefaultOlapDriver = - useIsModelingSupportedForDefaultOlapDriver(runtimeClient); + const instanceQuery = createRuntimeServiceGetInstance(runtimeClient, { + sensitive: true, + }); - return derived( - isModelingSupportedForDefaultOlapDriver, - (isModellingSupportedResp) => { - return connectors - .filter( - (c) => - (config.importOnly ? true : c.name !== "duckdb") && - c.category !== "ai" && - (isModellingSupportedResp.data || c.category === "olap"), - ) - .sort((a, b) => { - if (a.name === "https" || a.name === "local_file") return 1; - if (b.name === "https" || b.name === "local_file") return -1; - return a.displayName.localeCompare(b.displayName); - }); - }, - ); + // Source templates re-fetch whenever the instance OLAP changes. + const sourceQueryOptions = derived([instanceQuery], ([$instance]) => { + const olap = $instance.data?.instance?.olapConnector || ""; + if (olap) setOlapCache(runtimeClient.instanceId, olap); + return getRuntimeServiceListTemplatesQueryOptions( + runtimeClient, + { tags: ["source", olap] }, + { query: { enabled: !!olap } }, + ); + }); + const sourceTemplates = createQuery(sourceQueryOptions); + const olapTemplates = createRuntimeServiceListTemplates(runtimeClient, { + tags: ["olap"], + }); + + return derived([sourceTemplates, olapTemplates], ([$sources, $olap]) => { + const sourceList = ($sources.data?.templates ?? []) as V1Template[]; + const olapList = ($olap.data?.templates ?? []) as V1Template[]; + registerTemplatesIfNeeded([...sourceList, ...olapList]); + + const sources = sourceList.map((t) => + templateToConnectorInfo(t, "sourceOnly" as ConnectorCategory), + ); + const olaps = olapList.map((t) => + templateToConnectorInfo(t, "olap" as ConnectorCategory), + ); + + // Deduplicate by driver name; source entries take priority over OLAP + // entries when a connector appears in both lists (e.g. ClickHouse). + const seen = new Set(); + const merged: ConnectorInfo[] = []; + for (const c of [...sources, ...olaps]) { + if (!seen.has(c.name)) { + seen.add(c.name); + merged.push(c); + } + } + + return merged + .filter( + (c) => + (config.importOnly ? true : c.name !== "duckdb") && + c.category !== "ai", + ) + .sort((a, b) => { + if (a.name === "https" || a.name === "local_file") return 1; + if (b.name === "https" || b.name === "local_file") return -1; + return a.displayName.localeCompare(b.displayName); + }); + }); } const TopConnectors = ["clickhouse", "gcs", "s3", "snowflake"]; diff --git a/web-common/src/features/sources/modal/AddDataForm.svelte b/web-common/src/features/sources/modal/AddDataForm.svelte index 08e244948f3..ae4cbf3750d 100644 --- a/web-common/src/features/sources/modal/AddDataForm.svelte +++ b/web-common/src/features/sources/modal/AddDataForm.svelte @@ -18,7 +18,7 @@ import { AddDataFormManager } from "./AddDataFormManager"; import { createConnectorForm } from "./FormValidation"; import AddDataFormSection from "./AddDataFormSection.svelte"; - import { onMount } from "svelte"; + import { onDestroy } from "svelte"; import { get } from "svelte/store"; import { getConnectorSchema, @@ -29,7 +29,6 @@ getSchemaButtonLabels, isVisibleForValues, } from "../../templates/schema-utils"; - import { runtimeServiceGetFile } from "@rilldata/web-common/runtime-client"; import { ICONS } from "./icons"; export let connector: V1ConnectorDriver; @@ -111,22 +110,6 @@ const connectorSchema = getConnectorSchema(schemaName); - // Capture .env blob ONCE on mount for consistent conflict detection in YAML preview. - // This prevents the preview from updating when Test and Connect writes to .env. - // Use null to indicate "not yet loaded" vs "" for "loaded but empty" - let existingEnvBlob: string | null = null; - onMount(async () => { - try { - const envFile = await runtimeServiceGetFile(runtimeClient, { - path: ".env", - }); - existingEnvBlob = envFile.blob ?? ""; - } catch { - // .env doesn't exist yet - existingEnvBlob = ""; - } - }); - // Clear errors when connection type changes $: { const currentDeploymentType = $form.deployment_type as string | undefined; @@ -218,7 +201,9 @@ client: runtimeClient, queryClient, values: $form, - existingEnvBlob: existingEnvBlob ?? undefined, + // Submit re-fetches .env when undefined; the preview no longer needs + // a captured blob since GenerateFile resolves env-var conflicts server-side. + existingEnvBlob: undefined, }); if (result.ok) { // Use quiet close — saveConnector already navigated via goto(). @@ -232,13 +217,35 @@ saving = false; } - // Re-compute preview when existingEnvBlob is loaded (changes from null to string) - $: yamlPreview = formManager.computeYamlPreview({ - stepState, - isMultiStepConnector: isStepFlowConnector, - isConnectorForm, - formValues: $form, - existingEnvBlob: existingEnvBlob ?? "", + // Async, debounced YAML preview. Each form change kicks off a 150 ms timer; + // the latest server response replaces the visible preview. We keep the last + // valid blob on error so the YAML pane doesn't blank out while the user types. + let yamlPreview = ""; + let previewTimer: ReturnType | undefined; + let previewSeq = 0; + $: void $form, stepState, schedulePreview(); + + function schedulePreview() { + if (previewTimer) clearTimeout(previewTimer); + previewTimer = setTimeout(async () => { + const seq = ++previewSeq; + try { + const blob = await formManager.computeYamlPreview({ + client: runtimeClient, + stepState, + isMultiStepConnector: isStepFlowConnector, + isConnectorForm, + formValues: $form, + }); + if (seq === previewSeq) yamlPreview = blob; + } catch { + // Keep last-valid preview on error so the pane doesn't flicker. + } + }, 150); + } + + onDestroy(() => { + if (previewTimer) clearTimeout(previewTimer); }); // Show Save button for connector forms on the connector step (not for public auth which skips connection test). // Intentionally not disabled when fields are empty: Save persists whatever the user has entered so far, @@ -369,10 +376,12 @@ >
{#if paramsError} - +
+ +
{/if} = { + gcs: { + type: "object", + "x-category": "objectStore", + properties: { + key: { type: "string", "x-step": "connector" }, + path: { type: "string", "x-step": "source" }, + }, + } as unknown as MultiStepFormSchema, + snowflake: { + type: "object", + "x-category": "warehouse", + properties: { + account: { type: "string", "x-step": "connector" }, + sql: { type: "string", "x-step": "explorer" }, + }, + } as unknown as MultiStepFormSchema, +}; + describe("AddDataFormManager", () => { + beforeAll(() => { + populateSchemaCache(testSchemas); + }); beforeEach(() => { resetConnectorStep(); }); diff --git a/web-common/src/features/sources/modal/AddDataFormManager.ts b/web-common/src/features/sources/modal/AddDataFormManager.ts index e724961bc57..756ad57e503 100644 --- a/web-common/src/features/sources/modal/AddDataFormManager.ts +++ b/web-common/src/features/sources/modal/AddDataFormManager.ts @@ -24,17 +24,14 @@ import { type ConnectorStepState, } from "./connectorStepStore"; import { get } from "svelte/store"; -import { compileConnectorYAML } from "../../connectors/code-utils"; -import { compileSourceYAML, prepareSourceFormData } from "../sourceUtils"; import type { ActionResult } from "@sveltejs/kit"; import type { QueryClient } from "@tanstack/query-core"; import { filterSchemaValuesForSubmit, findRadioEnumKey, getSchemaFieldMetaList, - getSchemaSecretKeys, - getSchemaStringKeys, } from "../../templates/schema-utils"; +import { generateTemplate } from "./generate-template"; import type { ButtonLabels } from "../../templates/schemas/types"; import { processFileContent } from "../../templates/file-encoding"; @@ -485,124 +482,96 @@ export class AddDataFormManager { } /** - * Compute YAML preview for the current form state. - * Schema conditionals handle connector-specific requirements. + * Compute YAML preview for the current form state via the GenerateFile RPC. + * The runtime handles secret extraction, env-var name resolution, and YAML + * formatting from the active template's declarative definition. */ - computeYamlPreview(ctx: { + async computeYamlPreview(ctx: { + client: RuntimeClient; stepState: ConnectorStepState | undefined; isMultiStepConnector: boolean; isConnectorForm: boolean; formValues: Record; - existingEnvBlob?: string; - }): string { + }): Promise { const connector = this.connector; const { + client, stepState, isMultiStepConnector, isConnectorForm, formValues, - existingEnvBlob, } = ctx; const schema = getConnectorSchema(this.schemaName); - const schemaConnectorFields = schema - ? getSchemaFieldMetaList(schema, { step: "connector" }) - : null; - const schemaConnectorSecretKeys = schema - ? getSchemaSecretKeys(schema, { step: "connector" }) - : undefined; - const schemaConnectorStringKeys = schema - ? getSchemaStringKeys(schema, { step: "connector" }) - : undefined; - - const connectorPropertiesForPreview = schemaConnectorFields ?? []; - - const getConnectorYamlPreview = (values: Record) => { + const isOnConnectorStep = !stepState || stepState.step === "connector"; + const isOnSourceOrExplorerStep = + stepState?.step === "source" || stepState?.step === "explorer"; + + if (isMultiStepConnector && isOnConnectorStep) { const filteredValues = schema - ? filterSchemaValuesForSubmit(schema, values, { step: "connector" }) - : values; - return compileConnectorYAML(connector, filteredValues, { - fieldFilter: (property) => { - if ("internal" in property && property.internal) return false; - return !("noPrompt" in property && property.noPrompt); - }, - orderedProperties: connectorPropertiesForPreview, - secretKeys: schemaConnectorSecretKeys, - stringKeys: schemaConnectorStringKeys, - schema: schema ?? undefined, - existingEnvBlob, + ? filterSchemaValuesForSubmit(schema, formValues, { + step: "connector", + }) + : formValues; + const response = await generateTemplate(client, { + resourceType: "connector", + driver: connector.name as string, + properties: filteredValues, }); - }; + return response.blob ?? ""; + } + + if (isMultiStepConnector && isOnSourceOrExplorerStep) { + const combinedValues = { + ...(stepState?.connectorConfig || {}), + ...formValues, + } as Record; - const getSourceYamlPreview = (values: Record) => { - // For multi-step connectors in step 2, filter out connector properties - let filteredValues = values; - if ( - (isMultiStepConnector && stepState?.step === "source") || - stepState?.step === "explorer" - ) { + let sourceValues = combinedValues; + if (schema) { const connectorPropertyKeys = new Set( - schema - ? getSchemaFieldMetaList(schema, { step: "connector" }).map( - (field) => field.key, - ) - : [], + getSchemaFieldMetaList(schema, { step: "connector" }).map( + (field) => field.key, + ), ); - filteredValues = Object.fromEntries( - Object.entries(values).filter( + sourceValues = Object.fromEntries( + Object.entries(combinedValues).filter( ([key]) => !connectorPropertyKeys.has(key), ), ); } - const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData( - connector, - filteredValues, - { - connectorInstanceName: stepState?.connectorInstanceName || undefined, - }, - ); - const isExplorerStep = stepState?.step === "explorer"; - const isRewrittenToDuckDb = rewrittenConnector.name === "duckdb"; - const rewrittenSchema = getConnectorSchema(rewrittenConnector.name ?? ""); - const sourceStep = isExplorerStep ? "explorer" : "source"; - const rewrittenSecretKeys = rewrittenSchema - ? getSchemaSecretKeys(rewrittenSchema, { step: sourceStep }) - : undefined; - const rewrittenStringKeys = rewrittenSchema - ? getSchemaStringKeys(rewrittenSchema, { step: sourceStep }) - : undefined; - if (isRewrittenToDuckDb || isExplorerStep) { - // When rewritten to DuckDB, don't use the original connectorInstanceName. - // The original connector is referenced via create_secrets_from_connectors. - const yamlConnectorInstanceName = isRewrittenToDuckDb - ? undefined - : stepState?.connectorInstanceName || undefined; - return compileSourceYAML(rewrittenConnector, rewrittenFormValues, { - secretKeys: rewrittenSecretKeys, - stringKeys: rewrittenStringKeys, - originalDriverName: connector.name || undefined, - connectorInstanceName: yamlConnectorInstanceName, - }); - } - return getConnectorYamlPreview(rewrittenFormValues); - }; + const response = await generateTemplate(client, { + resourceType: "model", + driver: connector.name as string, + properties: sourceValues, + connectorName: + stepState?.connectorInstanceName || (connector.name as string), + }); + return response.blob ?? ""; + } - // Multi-step connectors (S3, GCS, Azure) - if (isMultiStepConnector) { - if (stepState?.step === "connector") { - return getConnectorYamlPreview(formValues); - } else { - const combinedValues = { - ...(stepState?.connectorConfig || {}), - ...formValues, - } as Record; - return getSourceYamlPreview(combinedValues); - } + if (isConnectorForm) { + const filteredValues = schema + ? filterSchemaValuesForSubmit(schema, formValues, { + step: "connector", + }) + : formValues; + const response = await generateTemplate(client, { + resourceType: "connector", + driver: connector.name as string, + properties: filteredValues, + }); + return response.blob ?? ""; } - if (isConnectorForm) return getConnectorYamlPreview(formValues); - return getSourceYamlPreview(formValues); + // Single-step source form + const response = await generateTemplate(client, { + resourceType: "model", + driver: connector.name as string, + properties: formValues, + }); + return response.blob ?? ""; } /** diff --git a/web-common/src/features/sources/modal/AddDataModal.svelte b/web-common/src/features/sources/modal/AddDataModal.svelte index cdef51edba4..de6c33cbaab 100644 --- a/web-common/src/features/sources/modal/AddDataModal.svelte +++ b/web-common/src/features/sources/modal/AddDataModal.svelte @@ -3,6 +3,8 @@ import { getScreenNameFromPage } from "@rilldata/web-common/features/file-explorer/telemetry"; import { cn } from "@rilldata/web-common/lib/shadcn"; import type { V1ConnectorDriver } from "@rilldata/web-common/runtime-client"; + import { createQuery } from "@tanstack/svelte-query"; + import { derived } from "svelte/store"; import { onMount } from "svelte"; import { behaviourEvent } from "../../../metrics/initMetrics"; import { @@ -11,7 +13,15 @@ } from "../../../metrics/service/BehaviourEventTypes"; import { MetricsEventSpace } from "../../../metrics/service/MetricsTypes"; import { useRuntimeClient } from "../../../runtime-client/v2"; - import { connectorIconMapping } from "../../connectors/connector-metadata.ts"; + import { + createRuntimeServiceGetInstance, + createRuntimeServiceListTemplates, + getRuntimeServiceListTemplatesQueryOptions, + } from "../../../runtime-client/v2/gen/runtime-service"; + import { + connectorIconMapping, + connectorKeywordMapping, + } from "../../connectors/connector-metadata.ts"; import { useIsModelingSupportedForDefaultOlapDriverOLAP as useIsModelingSupportedForDefaultOlapDriver } from "../../connectors/selectors"; import { duplicateSourceName } from "../sources-store"; import AddDataForm from "./AddDataForm.svelte"; @@ -19,17 +29,44 @@ import LocalSourceUpload from "./LocalSourceUpload.svelte"; import RequestConnectorForm from "./RequestConnectorForm.svelte"; import { - connectors, getConnectorSchema, getFormWidth, isMultiStepConnector as isMultiStepConnectorSchema, + registerTemplateSchema, toConnectorDriver as toConnectorDriverFromSchema, type ConnectorInfo, } from "./connector-schemas"; + import { setOlapCache } from "./generate-template"; + import type { ConnectorCategory } from "../../templates/schemas/types"; import { ICONS } from "./icons"; import { resetConnectorStep } from "./connectorStepStore"; import LoadingSpinner from "@rilldata/web-common/components/icons/LoadingSpinner.svelte"; + const runtimeClient = useRuntimeClient(); + + // Drive the connector picker from `ListTemplates` so source vs OLAP membership + // and OLAP-compatibility live in the template tags rather than frontend constants. + const instanceQuery = createRuntimeServiceGetInstance(runtimeClient, { + sensitive: true, + }); + + // Source templates re-fetch whenever the instance OLAP changes. + const sourceTemplatesQueryOptions = derived( + [instanceQuery], + ([$instance]) => { + const olap = $instance.data?.instance?.olapConnector || ""; + return getRuntimeServiceListTemplatesQueryOptions( + runtimeClient, + { tags: ["source", olap] }, + { query: { enabled: !!olap } }, + ); + }, + ); + const sourceTemplatesQuery = createQuery(sourceTemplatesQueryOptions); + const olapTemplatesQuery = createRuntimeServiceListTemplates(runtimeClient, { + tags: ["olap"], + }); + let step = 0; let selectedConnector: null | V1ConnectorDriver = null; let selectedSchemaName: string | null = null; @@ -38,11 +75,54 @@ let requestConnector = false; let isSubmittingForm = false; - // Filter connectors by category from JSON schemas - $: sourceConnectors = connectors.filter( - (c) => c.category !== "olap" && c.category !== "ai", + // Cache OLAP for generate-template once the instance resolves. + $: { + const olap = $instanceQuery.data?.instance?.olapConnector; + if (olap) setOlapCache(runtimeClient.instanceId, olap); + } + + // Map ListTemplates responses → ConnectorInfo, registering each schema in the cache + // so the form renderer (which still reads from `multiStepFormSchemas`) can resolve it. + function templatesToConnectors( + templates: + | { + name?: string; + driver?: string; + displayName?: string; + jsonSchema?: unknown; + }[] + | undefined, + fallbackCategory: ConnectorCategory, + ): ConnectorInfo[] { + if (!templates) return []; + return templates.map((t) => { + const driver = t.driver || t.name || ""; + const displayName = t.displayName || driver; + const schema = (t.jsonSchema ?? null) as Record | null; + if (schema && typeof schema === "object" && t.name) { + registerTemplateSchema(driver, t.name, schema as never, displayName); + } + const category = + (schema?.["x-category"] as ConnectorCategory | undefined) ?? + fallbackCategory; + return { + name: driver, + displayName, + category, + keywords: connectorKeywordMapping[driver] ?? [], + }; + }); + } + + $: sourceConnectors = templatesToConnectors( + $sourceTemplatesQuery.data?.templates, + "sourceOnly" as ConnectorCategory, + ); + $: olapConnectors = templatesToConnectors( + $olapTemplatesQuery.data?.templates, + "olap" as ConnectorCategory, ); - $: olapConnectors = connectors.filter((c) => c.category === "olap"); + $: connectors = [...sourceConnectors, ...olapConnectors]; // Get the form width class for the selected connector $: selectedSchema = selectedSchemaName @@ -184,8 +264,6 @@ resetModal(); } - const runtimeClient = useRuntimeClient(); - $: isModelingSupportedForDefaultOlapDriver = useIsModelingSupportedForDefaultOlapDriver(runtimeClient); $: isModelingSupported = $isModelingSupportedForDefaultOlapDriver.data; diff --git a/web-common/src/features/sources/modal/FormValidation.test.ts b/web-common/src/features/sources/modal/FormValidation.test.ts index 722052755c4..0dd72436db8 100644 --- a/web-common/src/features/sources/modal/FormValidation.test.ts +++ b/web-common/src/features/sources/modal/FormValidation.test.ts @@ -1,8 +1,19 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeAll } from "vitest"; import { getValidationSchemaForConnector } from "./FormValidation"; +import { populateSchemaCache } from "./connector-schemas"; +import type { MultiStepFormSchema } from "../../templates/schemas/types"; +// Import the runtime template's JSON schema directly so this spec catches +// drift between frontend validation and the backend's source-of-truth schema. +import s3DuckdbTemplate from "../../../../../runtime/templates/definitions/duckdb-models/s3-duckdb.json"; describe("getValidationSchemaForConnector (multi-step auth)", () => { + beforeAll(() => { + populateSchemaCache({ + s3: s3DuckdbTemplate.json_schema as unknown as MultiStepFormSchema, + }); + }); + it("enforces required fields for access key auth", async () => { const schema = getValidationSchemaForConnector("s3", "connector"); diff --git a/web-common/src/features/sources/modal/add-source-visibility.spec.ts b/web-common/src/features/sources/modal/add-source-visibility.spec.ts index 2de9e06f51e..1e3949ecf40 100644 --- a/web-common/src/features/sources/modal/add-source-visibility.spec.ts +++ b/web-common/src/features/sources/modal/add-source-visibility.spec.ts @@ -1,9 +1,33 @@ -import { describe, it, expect, beforeEach } from "vitest"; +import { describe, it, expect, beforeAll, beforeEach } from "vitest"; import { addSourceModal } from "./add-source-visibility"; import { resetConnectorStep, connectorStepStore } from "./connectorStepStore"; +import { populateSchemaCache } from "./connector-schemas"; +import type { MultiStepFormSchema } from "../../templates/schemas/types"; import { get } from "svelte/store"; +const testSchemas: Record = { + gcs: { + type: "object", + "x-category": "objectStore", + properties: { + key: { type: "string", "x-step": "connector" }, + path: { type: "string", "x-step": "source" }, + }, + } as unknown as MultiStepFormSchema, + snowflake: { + type: "object", + "x-category": "warehouse", + properties: { + account: { type: "string", "x-step": "connector" }, + sql: { type: "string", "x-step": "explorer" }, + }, + } as unknown as MultiStepFormSchema, +}; + describe("addSourceModal", () => { + beforeAll(() => { + populateSchemaCache(testSchemas); + }); beforeEach(() => { resetConnectorStep(); }); diff --git a/web-common/src/features/sources/modal/connector-schemas.spec.ts b/web-common/src/features/sources/modal/connector-schemas.spec.ts index 36e3995f074..da3ee1a4227 100644 --- a/web-common/src/features/sources/modal/connector-schemas.spec.ts +++ b/web-common/src/features/sources/modal/connector-schemas.spec.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, beforeAll } from "vitest"; import { getSchemaNameFromDriver, getConnectorSchema, @@ -9,9 +9,129 @@ import { shouldShowSkipLink, toConnectorDriver, multiStepFormSchemas, + populateSchemaCache, } from "./connector-schemas"; +import type { MultiStepFormSchema } from "../../templates/schemas/types"; + +// Test fixtures. The static schema imports were removed in PR 3 — at runtime +// schemas come from the `ListTemplates` RPC, so these specs seed the cache with +// just enough shape for the helper functions under test. Each fixture mirrors +// the real schema's category/step/auth setup; details that aren't exercised +// here (placeholders, hints, validation) are omitted. +const testSchemas: Record = { + s3: { + type: "object", + title: "Amazon S3", + "x-category": "objectStore", + properties: { + access_key: { type: "string", "x-step": "connector" }, + path: { type: "string", "x-step": "source" }, + }, + } as unknown as MultiStepFormSchema, + gcs: { + type: "object", + title: "Google Cloud Storage", + "x-category": "objectStore", + properties: { + key: { type: "string", "x-step": "connector" }, + path: { type: "string", "x-step": "source" }, + }, + } as unknown as MultiStepFormSchema, + azure: { + type: "object", + title: "Azure Blob Storage", + "x-category": "objectStore", + properties: { + account: { type: "string", "x-step": "connector" }, + path: { type: "string", "x-step": "source" }, + }, + } as unknown as MultiStepFormSchema, + postgres: { + type: "object", + title: "Postgres", + "x-category": "sqlStore", + properties: { + host: { type: "string", "x-step": "connector" }, + sql: { type: "string", "x-step": "explorer" }, + }, + } as unknown as MultiStepFormSchema, + mysql: { + type: "object", + title: "MySQL", + "x-category": "sqlStore", + properties: { + host: { type: "string", "x-step": "connector" }, + sql: { type: "string", "x-step": "explorer" }, + }, + } as unknown as MultiStepFormSchema, + snowflake: { + type: "object", + title: "Snowflake", + "x-category": "warehouse", + "x-form-height": "tall", + properties: { + account: { type: "string", "x-step": "connector" }, + sql: { type: "string", "x-step": "explorer" }, + }, + } as unknown as MultiStepFormSchema, + bigquery: { + type: "object", + title: "BigQuery", + "x-category": "warehouse", + properties: { + project_id: { type: "string", "x-step": "connector" }, + sql: { type: "string", "x-step": "explorer" }, + }, + } as unknown as MultiStepFormSchema, + salesforce: { + type: "object", + title: "Salesforce", + "x-category": "warehouse", + properties: { + username: { type: "string", "x-step": "connector" }, + }, + } as unknown as MultiStepFormSchema, + sqlite: { + type: "object", + title: "SQLite", + "x-category": "sqlStore", + properties: { + path: { type: "string", "x-step": "connector" }, + }, + } as unknown as MultiStepFormSchema, + clickhouse: { + type: "object", + title: "ClickHouse", + "x-category": "olap", + properties: { + host: { type: "string", "x-step": "connector" }, + }, + } as unknown as MultiStepFormSchema, + duckdb: { + type: "object", + title: "DuckDB", + "x-category": "olap", + properties: { + path: { type: "string", "x-step": "connector" }, + }, + } as unknown as MultiStepFormSchema, + // x-driver override: schema name differs from the backend driver name + motherduck: { + type: "object", + title: "MotherDuck", + "x-category": "olap", + "x-driver": "duckdb", + properties: { + token: { type: "string", "x-step": "connector" }, + }, + } as unknown as MultiStepFormSchema, +}; describe("connector-schemas", () => { + beforeAll(() => { + populateSchemaCache(testSchemas); + }); + describe("getSchemaNameFromDriver", () => { it("returns driver name when it directly matches a schema name", () => { expect(getSchemaNameFromDriver("postgres")).toBe("postgres"); @@ -21,19 +141,10 @@ describe("connector-schemas", () => { }); it("returns schema name for drivers with x-driver override (when no direct match)", () => { - // Find schemas with x-driver overrides that don't directly match another schema name - for (const [schemaName, schema] of Object.entries(multiStepFormSchemas)) { - const xDriver = schema?.["x-driver"]; - // Only test if x-driver is set and doesn't match an existing schema name - // (because direct schema name matches take precedence) - if ( - xDriver && - xDriver !== schemaName && - !(xDriver in multiStepFormSchemas) - ) { - expect(getSchemaNameFromDriver(xDriver)).toBe(schemaName); - } - } + // motherduck has x-driver: "duckdb"; reverse lookup of "duckdb" should + // return "duckdb" because that's a direct schema name match (which + // takes precedence over x-driver overrides). + expect(getSchemaNameFromDriver("duckdb")).toBe("duckdb"); }); it("returns the driver name as fallback for unknown drivers", () => { @@ -78,10 +189,7 @@ describe("connector-schemas", () => { }); it("returns x-driver value when specified in schema", () => { - for (const [schemaName, schema] of Object.entries(multiStepFormSchemas)) { - const expected = schema?.["x-driver"] ?? schemaName; - expect(getBackendConnectorName(schemaName)).toBe(expected); - } + expect(getBackendConnectorName("motherduck")).toBe("duckdb"); }); it("returns schema name for unknown connectors", () => { @@ -91,25 +199,14 @@ describe("connector-schemas", () => { describe("isMultiStepConnector", () => { it("returns true for object store connectors", () => { - const s3Schema = getConnectorSchema("s3"); - const gcsSchema = getConnectorSchema("gcs"); - const azureSchema = getConnectorSchema("azure"); - - expect(isMultiStepConnector(s3Schema)).toBe(true); - expect(isMultiStepConnector(gcsSchema)).toBe(true); - expect(isMultiStepConnector(azureSchema)).toBe(true); + expect(isMultiStepConnector(getConnectorSchema("s3"))).toBe(true); + expect(isMultiStepConnector(getConnectorSchema("gcs"))).toBe(true); + expect(isMultiStepConnector(getConnectorSchema("azure"))).toBe(true); }); it("returns false for non-object store connectors", () => { - const postgresSchema = getConnectorSchema("postgres"); - const mysqlSchema = getConnectorSchema("mysql"); - - expect(isMultiStepConnector(postgresSchema)).toBe(false); - expect(isMultiStepConnector(mysqlSchema)).toBe(false); - }); - - it("returns false for AI connectors", () => { - expect(isMultiStepConnector(getConnectorSchema("claude"))).toBe(false); + expect(isMultiStepConnector(getConnectorSchema("postgres"))).toBe(false); + expect(isMultiStepConnector(getConnectorSchema("mysql"))).toBe(false); }); it("returns false for null schema", () => { @@ -118,26 +215,13 @@ describe("connector-schemas", () => { }); describe("hasExplorerStep", () => { - it("returns true for SQL store and warehouse connectors", () => { - const snowflakeSchema = getConnectorSchema("snowflake"); - const postgresSchema = getConnectorSchema("postgres"); - - // Check based on category - if (snowflakeSchema?.["x-category"] === "warehouse") { - expect(hasExplorerStep(snowflakeSchema)).toBe(true); - } - if (postgresSchema?.["x-category"] === "sqlStore") { - expect(hasExplorerStep(postgresSchema)).toBe(true); - } + it("returns true for warehouse and SQL store connectors with explorer step", () => { + expect(hasExplorerStep(getConnectorSchema("snowflake"))).toBe(true); + expect(hasExplorerStep(getConnectorSchema("postgres"))).toBe(true); }); it("returns false for object store connectors", () => { - const s3Schema = getConnectorSchema("s3"); - expect(hasExplorerStep(s3Schema)).toBe(false); - }); - - it("returns false for AI connectors", () => { - expect(hasExplorerStep(getConnectorSchema("claude"))).toBe(false); + expect(hasExplorerStep(getConnectorSchema("s3"))).toBe(false); }); it("returns false for null schema", () => { @@ -146,28 +230,22 @@ describe("connector-schemas", () => { }); describe("getFormHeight", () => { - it("returns tall height for schemas with x-form-height: tall", () => { - const FORM_HEIGHT_TALL = "max-h-[40rem] min-h-[40rem]"; + const FORM_HEIGHT_TALL = "max-h-[40rem] min-h-[40rem]"; + const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; - for (const [, schema] of Object.entries(multiStepFormSchemas)) { - if (schema?.["x-form-height"] === "tall") { - expect(getFormHeight(schema)).toBe(FORM_HEIGHT_TALL); - } - } + it("returns tall height for schemas with x-form-height: tall", () => { + expect(getFormHeight(getConnectorSchema("snowflake"))).toBe( + FORM_HEIGHT_TALL, + ); }); it("returns default height for schemas without x-form-height", () => { - const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; - - for (const [, schema] of Object.entries(multiStepFormSchemas)) { - if (!schema?.["x-form-height"]) { - expect(getFormHeight(schema)).toBe(FORM_HEIGHT_DEFAULT); - } - } + expect(getFormHeight(getConnectorSchema("postgres"))).toBe( + FORM_HEIGHT_DEFAULT, + ); }); it("returns default height for null schema", () => { - const FORM_HEIGHT_DEFAULT = "max-h-[34.5rem] min-h-[34.5rem]"; expect(getFormHeight(null)).toBe(FORM_HEIGHT_DEFAULT); }); }); @@ -215,18 +293,6 @@ describe("connector-schemas", () => { expect(toConnectorDriver("nonexistent")).toBeNull(); }); - it("sets implementsAi for AI connectors", () => { - const claude = toConnectorDriver("claude"); - expect(claude).not.toBeNull(); - expect(claude!.name).toBe("claude"); - expect(claude!.displayName).toBe("Claude"); - expect(claude!.implementsAi).toBe(true); - expect(claude!.implementsOlap).toBe(false); - expect(claude!.implementsWarehouse).toBe(false); - expect(claude!.implementsObjectStore).toBe(false); - expect(claude!.implementsSqlStore).toBe(false); - }); - it("sets implementsWarehouse for warehouse connectors", () => { const bq = toConnectorDriver("bigquery"); expect(bq).not.toBeNull(); diff --git a/web-common/src/features/sources/modal/connector-schemas.ts b/web-common/src/features/sources/modal/connector-schemas.ts index d459aaf4af9..55bf18e5861 100644 --- a/web-common/src/features/sources/modal/connector-schemas.ts +++ b/web-common/src/features/sources/modal/connector-schemas.ts @@ -4,59 +4,21 @@ import type { MultiStepFormSchema, } from "../../templates/schemas/types"; import type { ConnectorStep } from "./connectorStepStore"; -import { athenaSchema } from "../../templates/schemas/athena"; -import { azureSchema } from "../../templates/schemas/azure"; -import { bigquerySchema } from "../../templates/schemas/bigquery"; import { claudeSchema } from "../../templates/schemas/claude"; -import { clickhouseSchema } from "../../templates/schemas/clickhouse"; -import { gcsSchema } from "../../templates/schemas/gcs"; import { geminiSchema } from "../../templates/schemas/gemini"; -import { mysqlSchema } from "../../templates/schemas/mysql"; import { openaiSchema } from "../../templates/schemas/openai"; -import { postgresSchema } from "../../templates/schemas/postgres"; -import { redshiftSchema } from "../../templates/schemas/redshift"; -import { salesforceSchema } from "../../templates/schemas/salesforce"; -import { snowflakeSchema } from "../../templates/schemas/snowflake"; -import { sqliteSchema } from "../../templates/schemas/sqlite"; -import { localFileSchema } from "../../templates/schemas/local_file"; -import { duckdbSchema } from "../../templates/schemas/duckdb"; import { ducklakeSchema } from "../../templates/schemas/ducklake"; -import { deltaSchema } from "../../templates/schemas/delta"; -import { httpsSchema } from "../../templates/schemas/https"; -import { icebergSchema } from "../../templates/schemas/iceberg"; -import { motherduckSchema } from "../../templates/schemas/motherduck"; -import { druidSchema } from "../../templates/schemas/druid"; -import { pinotSchema } from "../../templates/schemas/pinot"; -import { s3Schema } from "../../templates/schemas/s3"; -import { starrocksSchema } from "../../templates/schemas/starrocks"; -import { supabaseSchema } from "../../templates/schemas/supabase"; -import { SOURCES, OLAP_ENGINES, AI_CONNECTORS } from "./constants"; import { connectorKeywordMapping } from "@rilldata/web-common/features/connectors/connector-metadata.ts"; +/** + * Connector schemas registered for synchronous lookup. Source-connector schemas + * are populated dynamically via `registerTemplateSchema` when the `ListTemplates` + * RPC resolves; AI connectors and DuckLake (which has client-side composer logic + * in `ducklake-utils.ts`) stay as static imports because their flows are not + * driven by the runtime templates registry. + */ export const multiStepFormSchemas: Record = { - athena: athenaSchema, - bigquery: bigquerySchema, - clickhouse: clickhouseSchema, - mysql: mysqlSchema, - postgres: postgresSchema, - redshift: redshiftSchema, - salesforce: salesforceSchema, - snowflake: snowflakeSchema, - sqlite: sqliteSchema, - motherduck: motherduckSchema, - duckdb: duckdbSchema, ducklake: ducklakeSchema, - druid: druidSchema, - pinot: pinotSchema, - starrocks: starrocksSchema, - supabase: supabaseSchema, - local_file: localFileSchema, - https: httpsSchema, - s3: s3Schema, - gcs: gcsSchema, - iceberg: icebergSchema, - azure: azureSchema, - delta: deltaSchema, claude: claudeSchema, openai: openaiSchema, gemini: geminiSchema, @@ -73,29 +35,22 @@ export interface ConnectorInfo { } /** - * All connectors enumerated from JSON schemas, sorted by display order. - */ -export const connectors: ConnectorInfo[] = [ - ...SOURCES, - ...OLAP_ENGINES, - ...AI_CONNECTORS, -] - .filter((name) => multiStepFormSchemas[name]?.["x-category"]) - .map((name) => { - const schema = multiStepFormSchemas[name]; - return { - name, - displayName: schema.title ?? name, - category: schema["x-category"] as ConnectorCategory, - keywords: connectorKeywordMapping[name] ?? [], - }; - }); -/** - * Map of connector names to ConnectorInfo objects. - * We need connector info by name in a lot of places, so we have a map to optimize lookups. + * Map of connector names to ConnectorInfo objects, populated dynamically by + * `registerTemplateSchema` when `ListTemplates` resolves. We need connector + * info by name in a lot of places, so we have a map to optimize lookups. */ export const connectorInfoMap = new Map( - connectors.map((connector) => [connector.name, connector]), + Object.entries(multiStepFormSchemas) + .filter(([, schema]) => schema?.["x-category"]) + .map(([name, schema]) => [ + name, + { + name, + displayName: schema.title ?? name, + category: schema["x-category"] as ConnectorCategory, + keywords: connectorKeywordMapping[name] ?? [], + }, + ]), ); export function getConnectorSchema( @@ -105,6 +60,48 @@ export function getConnectorSchema( return schema?.properties ? schema : null; } +/** + * Maps driver names to their full template names (e.g. "kafka" → "kafka-clickhouse"). + * Populated when templates are fetched from the ListTemplates RPC so that + * AddDataFormManager can route to the right template for the active OLAP. + */ +export const templateNameMap = new Map(); + +/** + * Register a template schema dynamically. Called when templates are fetched + * from the ListTemplates RPC so that connectors not in the static schema map + * (e.g. kafka, hudi, mongodb when ClickHouse is the OLAP) work in the form + * flow. Also updates connectorInfoMap so getConnectorDriverForSchema resolves. + */ +export function registerTemplateSchema( + driverName: string, + templateName: string, + schema: MultiStepFormSchema, + displayName: string, +) { + multiStepFormSchemas[driverName] = schema; + templateNameMap.set(driverName, templateName); + const category = (schema["x-category"] ?? "sourceOnly") as ConnectorCategory; + connectorInfoMap.set(driverName, { + name: driverName, + displayName, + category, + keywords: connectorKeywordMapping[driverName] ?? [], + }); +} + +/** + * Test seam: replace the schema cache with a fixture map. Used by specs that + * need a deterministic set of schemas without invoking the runtime. + */ +export function populateSchemaCache( + schemas: Record, +) { + for (const [driverName, schema] of Object.entries(schemas)) { + multiStepFormSchemas[driverName] = schema; + } +} + /** * Get the backend driver name for a given schema name. * Returns x-driver if specified, otherwise returns the schema name. diff --git a/web-common/src/features/sources/modal/generate-template.spec.ts b/web-common/src/features/sources/modal/generate-template.spec.ts new file mode 100644 index 00000000000..6a2b46d8ad3 --- /dev/null +++ b/web-common/src/features/sources/modal/generate-template.spec.ts @@ -0,0 +1,175 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { mergeEnvVars } from "./generate-template"; +import type { RuntimeClient } from "../../../runtime-client/v2"; + +const mockGetFile = vi.fn(); +vi.mock("../../../runtime-client/v2/gen/runtime-service", () => ({ + getRuntimeServiceGetFileQueryKey: vi.fn( + (instanceId: string, params: { path: string }) => [ + "runtimeServiceGetFile", + instanceId, + params, + ], + ), + runtimeServiceGetFile: (...args: unknown[]) => mockGetFile(...args), + runtimeServiceGenerateFile: vi.fn(), +})); + +vi.mock("../../connectors/code-utils", async () => { + return { + replaceOrAddEnvVariable: ( + existingEnvBlob: string, + key: string, + newValue: string, + ): string => { + const lines = existingEnvBlob.split("\n"); + let keyFound = false; + + const updatedLines = lines.map((line) => { + if (line.startsWith(`${key}=`)) { + keyFound = true; + return `${key}=${newValue}`; + } + return line; + }); + + if (!keyFound) { + updatedLines.push(`${key}=${newValue}`); + } + + return updatedLines + .filter((line, index) => !(line === "" && index === 0)) + .join("\n") + .trim(); + }, + }; +}); + +const mockClient = { + instanceId: "test-instance", +} as unknown as RuntimeClient; + +describe("mergeEnvVars", () => { + let queryClient: { + invalidateQueries: ReturnType; + fetchQuery: ReturnType; + }; + + beforeEach(() => { + vi.clearAllMocks(); + queryClient = { + invalidateQueries: vi.fn().mockResolvedValue(undefined), + fetchQuery: vi.fn(), + }; + }); + + it("merges env vars into existing .env content", async () => { + queryClient.fetchQuery.mockResolvedValue({ + blob: "EXISTING_VAR=existing_value", + }); + + const result = await mergeEnvVars(mockClient, queryClient as never, { + CLICKHOUSE_PASSWORD: "secret123", + CLICKHOUSE_HOST: "ch.example.com", + }); + + expect(result.originalBlob).toBe("EXISTING_VAR=existing_value"); + expect(result.newBlob).toContain("EXISTING_VAR=existing_value"); + expect(result.newBlob).toContain("CLICKHOUSE_PASSWORD=secret123"); + expect(result.newBlob).toContain("CLICKHOUSE_HOST=ch.example.com"); + }); + + it("handles empty .env file", async () => { + queryClient.fetchQuery.mockResolvedValue({ blob: "" }); + + const result = await mergeEnvVars(mockClient, queryClient as never, { + S3_ACCESS_KEY: "AKID123", + }); + + expect(result.originalBlob).toBe(""); + expect(result.newBlob).toContain("S3_ACCESS_KEY=AKID123"); + }); + + it("handles .env file not found", async () => { + queryClient.fetchQuery.mockRejectedValue( + new Error("open .env: no such file or directory"), + ); + + const result = await mergeEnvVars(mockClient, queryClient as never, { + NEW_VAR: "new_value", + }); + + expect(result.originalBlob).toBe(""); + expect(result.newBlob).toContain("NEW_VAR=new_value"); + }); + + it("updates existing env var values", async () => { + queryClient.fetchQuery.mockResolvedValue({ + blob: "CLICKHOUSE_PASSWORD=old_secret", + }); + + const result = await mergeEnvVars(mockClient, queryClient as never, { + CLICKHOUSE_PASSWORD: "new_secret", + }); + + expect(result.newBlob).toContain("CLICKHOUSE_PASSWORD=new_secret"); + expect(result.newBlob).not.toContain("old_secret"); + }); + + it("handles empty envVars map", async () => { + queryClient.fetchQuery.mockResolvedValue({ + blob: "EXISTING=value", + }); + + const result = await mergeEnvVars(mockClient, queryClient as never, {}); + + expect(result.originalBlob).toBe("EXISTING=value"); + expect(result.newBlob).toBe("EXISTING=value"); + }); + + it("skips entries with empty keys or values", async () => { + queryClient.fetchQuery.mockResolvedValue({ blob: "" }); + + const result = await mergeEnvVars(mockClient, queryClient as never, { + "": "no_key", + VALID_KEY: "", + REAL_KEY: "real_value", + }); + + expect(result.newBlob).toContain("REAL_KEY=real_value"); + expect(result.newBlob).not.toContain("no_key"); + expect(result.newBlob).not.toContain("VALID_KEY"); + }); + + it("invalidates query cache before fetching", async () => { + queryClient.fetchQuery.mockResolvedValue({ blob: "" }); + + await mergeEnvVars(mockClient, queryClient as never, { KEY: "value" }); + + expect(queryClient.invalidateQueries).toHaveBeenCalledBefore( + queryClient.fetchQuery, + ); + }); + + it("re-throws non-file-not-found errors", async () => { + const error = new Error("network error"); + queryClient.fetchQuery.mockRejectedValue(error); + + await expect( + mergeEnvVars(mockClient, queryClient as never, { KEY: "value" }), + ).rejects.toThrow("network error"); + }); + + it("handles suffixed env var names from backend", async () => { + queryClient.fetchQuery.mockResolvedValue({ + blob: "CLICKHOUSE_PASSWORD=first_secret", + }); + + const result = await mergeEnvVars(mockClient, queryClient as never, { + CLICKHOUSE_PASSWORD_1: "second_secret", + }); + + expect(result.newBlob).toContain("CLICKHOUSE_PASSWORD=first_secret"); + expect(result.newBlob).toContain("CLICKHOUSE_PASSWORD_1=second_secret"); + }); +}); diff --git a/web-common/src/features/sources/modal/generate-template.ts b/web-common/src/features/sources/modal/generate-template.ts new file mode 100644 index 00000000000..b8515692483 --- /dev/null +++ b/web-common/src/features/sources/modal/generate-template.ts @@ -0,0 +1,135 @@ +import type { QueryClient } from "@tanstack/query-core"; +import type { RuntimeClient } from "../../../runtime-client/v2"; +import { + getRuntimeServiceGetFileQueryKey, + runtimeServiceGenerateFile, + runtimeServiceGetFile, +} from "../../../runtime-client/v2/gen/runtime-service"; +import { replaceOrAddEnvVariable } from "../../connectors/code-utils"; +import { OLAP_ENGINES } from "./constants"; + +const OLAP_SET = new Set(OLAP_ENGINES); + +// OLAP per instance, populated by AddDataModal when the instance OLAP is known. +// Avoids a redundant GetInstance round-trip on first generateTemplate invocation. +const olapCache = new Map(); + +/** Set the cached OLAP value for an instance. */ +export function setOlapCache(instanceId: string, olap: string) { + olapCache.set(instanceId, olap); +} + +/** Test seam: clear the OLAP cache between tests. */ +export function _clearOlapCache() { + olapCache.clear(); +} + +/** + * Resolve the template name from (driver, olap). + * OLAP engine drivers have standalone templates (e.g. "clickhouse"). + * Source drivers use combined templates (e.g. "s3-duckdb", "postgres-clickhouse"), + * regardless of whether we're rendering the connector or model output. + */ +function resolveTemplateName(driver: string, olap: string): string { + if (OLAP_SET.has(driver)) return driver; + return `${driver}-${olap}`; +} + +/** + * Call the GenerateFile RPC to produce YAML and env-var names from + * structured form data. The backend handles env-var naming, conflict + * suffixes, and YAML formatting via declarative templates. + * + * Always uses preview mode so the server renders without writing files; + * the caller is responsible for persisting the YAML and `.env`. + */ +export async function generateTemplate( + client: RuntimeClient, + opts: { + resourceType: string; + driver: string; + properties: Record; + connectorName?: string; + }, +): Promise<{ blob: string; envVars: Record }> { + // Resolve OLAP from cache (populated when the modal mounts). + // Falls back to "duckdb" if the cache is empty (shouldn't happen in practice). + const olap = OLAP_SET.has(opts.driver) + ? opts.driver + : (olapCache.get(client.instanceId) ?? "duckdb"); + + const templateName = resolveTemplateName(opts.driver, olap); + + const response = await runtimeServiceGenerateFile(client, { + templateName, + output: opts.resourceType, + properties: opts.properties, + connectorName: opts.connectorName, + preview: true, + }); + + return { + blob: response.files?.[0]?.blob ?? "", + envVars: response.envVars ?? {}, + }; +} + +/** + * Merge env vars returned by GenerateFile into the existing `.env` file. + * The backend has already resolved names and conflict suffixes, so this + * is a straight key=value merge. + * + * Returns the updated blob and the original blob (for rollback). + */ +export async function mergeEnvVars( + client: RuntimeClient, + queryClient: QueryClient, + envVars: Record, +): Promise<{ newBlob: string; originalBlob: string }> { + await queryClient.invalidateQueries({ + queryKey: getRuntimeServiceGetFileQueryKey(client.instanceId, { + path: ".env", + }), + }); + + let blob: string; + let originalBlob: string; + try { + const file = await queryClient.fetchQuery({ + queryKey: getRuntimeServiceGetFileQueryKey(client.instanceId, { + path: ".env", + }), + queryFn: () => runtimeServiceGetFile(client, { path: ".env" }), + }); + blob = file.blob || ""; + originalBlob = blob; + } catch (error) { + const msg = + ( + error as { + message?: string; + response?: { data?: { message?: string } }; + } + )?.message ?? + ( + error as { + message?: string; + response?: { data?: { message?: string } }; + } + )?.response?.data?.message ?? + ""; + if (msg.includes("no such file")) { + blob = ""; + originalBlob = ""; + } else { + throw error; + } + } + + for (const [key, value] of Object.entries(envVars)) { + if (!key || !value) continue; + blob = replaceOrAddEnvVariable(blob, key, value); + } + + return { newBlob: blob, originalBlob }; +}