feat: rewire add-data modal to use template RPCs (template-connectors v2, PR 3)

runtime/templates/definitions/clickhouse-models/azure-clickhouse.json

@@ -1,7 +1,7 @@
 {
   "name": "azure-clickhouse",
   "display_name": "Azure Blob Storage",
-  "description": "Read Azure Blob Storage files into ClickHouse using table functions",
+  "description": "Read Azure Blob Storage files into ClickHouse via a named collection",
   "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/azure",
   "driver": "azure",
   "olap": "clickhouse",
@@ -12,12 +12,13 @@
     "microsoft",
     "object-storage",
     "clickhouse",
-    "source"
+    "source",
+    "connector"
   ],
   "json_schema": {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "type": "object",
-    "x-category": "sourceOnly",
+    "x-category": "objectStore",
     "properties": {
       "auth_method": {
         "type": "string",
@@ -50,7 +51,8 @@
             "azure_storage_key"
           ],
           "public": []
-        }
+        },
+        "x-step": "connector"
       },
       "azure_storage_connection_string": {
         "type": "string",
@@ -59,6 +61,7 @@
         "x-placeholder": "DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=core.windows.net",
         "x-secret": true,
         "x-env-var": "AZURE_STORAGE_CONNECTION_STRING",
+        "x-step": "connector",
         "x-visible-if": {
           "auth_method": "connection_string"
         }
@@ -68,6 +71,7 @@
         "title": "Storage account",
         "description": "Azure storage account name",
         "x-placeholder": "mystorageaccount",
+        "x-step": "connector",
         "x-visible-if": {
           "auth_method": "account_key"
         }
@@ -79,6 +83,7 @@
         "x-placeholder": "Enter storage key",
         "x-secret": true,
         "x-env-var": "AZURE_STORAGE_KEY",
+        "x-step": "connector",
         "x-visible-if": {
           "auth_method": "account_key"
         }
@@ -91,14 +96,16 @@
         "errorMessage": {
           "pattern": "Must be an Azure URI (e.g. azure://container/path or https://account.blob.core.windows.net/container/path)"
         },
-        "x-placeholder": "azure://container/path"
+        "x-placeholder": "azure://container/path",
+        "x-step": "source"
       },
       "name": {
         "type": "string",
         "title": "Model name",
         "description": "Name for the source model",
         "pattern": "^[a-zA-Z0-9_]+$",
-        "x-placeholder": "my_model"
+        "x-placeholder": "my_model",
+        "x-step": "source"
       }
     },
     "required": [
@@ -138,10 +145,15 @@
     ]
   },
   "files": [
+    {
+      "name": "connector",
+      "path_template": "connectors/[[ .connector_name ]].yaml",
+      "code_template": "[[ if .config_props -]]\n# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: azure\n[[ renderProps .config_props ]]\n[[ end -]]\n"
+    },
     {
       "name": "model",
       "path_template": "models/[[ .model_name ]].yaml",
-      "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if eq .auth_method \"connection_string\" ]]\n  SELECT * FROM azureBlobStorage(\n      '[[ propVal .props \"azure_storage_connection_string\" ]]',\n      '[[ azureContainer .path ]]',\n      '[[ azureBlobPath .path ]]'\n  )[[ else if eq .auth_method \"account_key\" ]]\n  SELECT * FROM azureBlobStorage(\n      '[[ azureEndpoint .path .azure_storage_account ]]',\n      '[[ azureContainer .path ]]',\n      '[[ azureBlobPath .path ]]',\n      '[[ propVal .props \"azure_storage_account\" ]]',\n      '[[ propVal .props \"azure_storage_key\" ]]'\n  )[[ else ]]\n  SELECT * FROM azureBlobStorage(\n      '[[ azureEndpoint .path \"\" ]]',\n      '[[ azureContainer .path ]]',\n      '[[ azureBlobPath .path ]]'\n  )[[ end ]]\n"
+      "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if ne .auth_method \"public\" ]]\n  SELECT * FROM azureBlobStorage(rill_[[ .connector_name ]], container='[[ azureContainer .path ]]', blob_path='[[ azureBlobPath .path ]]')[[ else ]]\n  SELECT * FROM azureBlobStorage('[[ azureEndpoint .path \"\" ]]', '[[ azureContainer .path ]]', '[[ azureBlobPath .path ]]')[[ end ]]\n"
     }
   ]
 }

runtime/templates/definitions/clickhouse-models/delta-clickhouse.json

@@ -1,7 +1,7 @@
 {
   "name": "delta-clickhouse",
   "display_name": "Delta Lake",
-  "description": "Query Delta Lake tables in ClickHouse using deltaLake table function",
+  "description": "Query Delta Lake tables in ClickHouse via a named collection",
   "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/delta",
   "driver": "delta",
   "olap": "clickhouse",
@@ -11,28 +11,31 @@
     "delta",
     "table-format",
     "clickhouse",
-    "source"
+    "source",
+    "connector"
   ],
   "json_schema": {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "type": "object",
-    "x-category": "sourceOnly",
+    "x-category": "objectStore",
     "properties": {
       "aws_access_key_id": {
         "type": "string",
         "title": "AWS Access Key ID",
         "description": "Access key for the S3 bucket containing your Delta table",
         "x-placeholder": "Enter AWS access key ID",
         "x-secret": true,
-        "x-env-var": "AWS_ACCESS_KEY_ID"
+        "x-env-var": "AWS_ACCESS_KEY_ID",
+        "x-step": "connector"
       },
       "aws_secret_access_key": {
         "type": "string",
         "title": "AWS Secret Access Key",
         "description": "Secret key for the S3 bucket containing your Delta table",
         "x-placeholder": "Enter AWS secret access key",
         "x-secret": true,
-        "x-env-var": "AWS_SECRET_ACCESS_KEY"
+        "x-env-var": "AWS_SECRET_ACCESS_KEY",
+        "x-step": "connector"
       },
       "path": {
         "type": "string",
@@ -42,14 +45,16 @@
         "errorMessage": {
           "pattern": "Must be an S3 URI (e.g. s3://bucket/delta_table)"
         },
-        "x-placeholder": "s3://bucket/delta_table"
+        "x-placeholder": "s3://bucket/delta_table",
+        "x-step": "source"
       },
       "name": {
         "type": "string",
         "title": "Model name",
         "description": "Name for the source model",
         "pattern": "^[a-zA-Z0-9_]+$",
-        "x-placeholder": "my_delta_table"
+        "x-placeholder": "my_delta_table",
+        "x-step": "source"
       }
     },
     "required": [
@@ -60,10 +65,15 @@
     ]
   },
   "files": [
+    {
+      "name": "connector",
+      "path_template": "connectors/[[ .connector_name ]].yaml",
+      "code_template": "# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: delta\n[[ renderProps .config_props ]]\n"
+    },
     {
       "name": "model",
       "path_template": "models/[[ .model_name ]].yaml",
-      "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n  SELECT * FROM deltaLake(\n      '[[ s3ToHTTPS .path ]]',\n      '[[ propVal .props \"aws_access_key_id\" ]]',\n      '[[ propVal .props \"aws_secret_access_key\" ]]'\n  )\n"
+      "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |\n  SELECT * FROM deltaLake(rill_[[ .connector_name ]], url='[[ s3ToHTTPS .path ]]')\n"
     }
   ]
 }

runtime/templates/definitions/clickhouse-models/gcs-clickhouse.json

@@ -1,7 +1,7 @@
 {
   "name": "gcs-clickhouse",
   "display_name": "Google Cloud Storage",
-  "description": "Read GCS files into ClickHouse using table functions (HMAC keys)",
+  "description": "Read GCS files into ClickHouse via a named collection (HMAC keys)",
   "docs_url": "https://docs.rilldata.com/developers/build/connectors/data-source/gcs",
   "driver": "gcs",
   "olap": "clickhouse",
@@ -12,12 +12,13 @@
     "google",
     "object-storage",
     "clickhouse",
-    "source"
+    "source",
+    "connector"
   ],
   "json_schema": {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "type": "object",
-    "x-category": "sourceOnly",
+    "x-category": "objectStore",
     "properties": {
       "auth_method": {
         "type": "string",
@@ -44,7 +45,8 @@
             "secret"
           ],
           "public": []
-        }
+        },
+        "x-step": "connector"
       },
       "key_id": {
         "type": "string",
@@ -53,6 +55,7 @@
         "x-placeholder": "Enter HMAC access key",
         "x-secret": true,
         "x-env-var": "GCP_ACCESS_KEY_ID",
+        "x-step": "connector",
         "x-visible-if": {
           "auth_method": "hmac_key"
         }
@@ -64,6 +67,7 @@
         "x-placeholder": "Enter HMAC secret",
         "x-secret": true,
         "x-env-var": "GCP_SECRET_ACCESS_KEY",
+        "x-step": "connector",
         "x-visible-if": {
           "auth_method": "hmac_key"
         }
@@ -76,14 +80,16 @@
         "errorMessage": {
           "pattern": "Must be a GCS URI (e.g. gs://bucket/path or https://storage.googleapis.com/bucket/path)"
         },
-        "x-placeholder": "gs://bucket/path"
+        "x-placeholder": "gs://bucket/path",
+        "x-step": "source"
       },
       "name": {
         "type": "string",
         "title": "Model name",
         "description": "Name for the source model",
         "pattern": "^[a-zA-Z0-9_]+$",
-        "x-placeholder": "my_model"
+        "x-placeholder": "my_model",
+        "x-step": "source"
       }
     },
     "required": [
@@ -109,10 +115,15 @@
     ]
   },
   "files": [
+    {
+      "name": "connector",
+      "path_template": "connectors/[[ .connector_name ]].yaml",
+      "code_template": "[[ if .config_props -]]\n# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: gcs\n[[ renderProps .config_props ]]\n[[ end -]]\n"
+    },
     {
       "name": "model",
       "path_template": "models/[[ .model_name ]].yaml",
-      "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if eq .auth_method \"hmac_key\" ]]\n  SELECT * FROM gcs(\n      '[[ gcsToHTTPS .path ]]',\n      '[[ propVal .props \"key_id\" ]]',\n      '[[ propVal .props \"secret\" ]]'\n  )[[ else ]]\n  SELECT * FROM gcs('[[ gcsToHTTPS .path ]]')[[ end ]]\n"
+      "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if ne .auth_method \"public\" ]]\n  SELECT * FROM gcs(rill_[[ .connector_name ]], url='[[ gcsToHTTPS .path ]]')[[ else ]]\n  SELECT * FROM gcs('[[ gcsToHTTPS .path ]]')[[ end ]]\n"
     }
   ]
 }

runtime/templates/definitions/clickhouse-models/https-clickhouse.json

@@ -12,21 +12,23 @@
     "http",
     "url",
     "clickhouse",
-    "source"
+    "source",
+    "connector"
   ],
 
   "json_schema": {
     "$schema": "http://json-schema.org/draft-07/schema#",
     "type": "object",
-    "x-category": "sourceOnly",
+    "x-category": "fileStore",
 
     "properties": {
       "headers": {
         "title": "Headers",
         "description": "HTTP headers to include in the request",
         "x-display": "key-value",
         "x-placeholder": "Header name",
-        "x-hint": "e.g. Authorization: Bearer <token>"
+        "x-hint": "e.g. Authorization: Bearer <token>",
+        "x-step": "connector"
       },
 
       "path": {
@@ -37,26 +39,33 @@
         "errorMessage": {
           "pattern": "Must be a valid HTTP(S) URL"
         },
-        "x-placeholder": "https://example.com/data.csv"
+        "x-placeholder": "https://example.com/data.csv",
+        "x-step": "source"
       },
 
       "name": {
         "type": "string",
         "title": "Model name",
         "description": "Name for the source model",
         "pattern": "^[a-zA-Z0-9_]+$",
-        "x-placeholder": "my_model"
+        "x-placeholder": "my_model",
+        "x-step": "source"
       }
     },
 
     "required": ["path", "name"]
   },
 
   "files": [
+    {
+      "name": "connector",
+      "path_template": "connectors/[[ .connector_name ]].yaml",
+      "code_template": "[[ if .config_props -]]\n# Connector YAML\n# Reference documentation: [[ .docs_url ]]\ntype: connector\ndriver: https\n[[ renderProps .config_props ]]\n[[ end -]]\n"
+    },
     {
       "name": "model",
       "path_template": "models/[[ .model_name ]].yaml",
-      "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\n\ntype: model\nmaterialize: true\n\nconnector: clickhouse\n\nsql: |\n  SELECT * FROM url('[[ .path ]]'[[ clickhouseURLSuffix .path .props ]])\n"
+      "code_template": "# Model YAML\n# Reference documentation: https://docs.rilldata.com/reference/project-files/models\ntype: model\nmaterialize: true\nconnector: clickhouse\nsql: |[[ if .config_props ]]\n  SELECT * FROM url(rill_[[ .connector_name ]], url='[[ .path ]]')[[ else ]]\n  SELECT * FROM url('[[ .path ]]')[[ end ]]\n"
     }
   ]
 }