Add support for UC secrets

acceptance/bundle/invariant/configs/secrets.yml.tmpl

@@ -0,0 +1,20 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+variables:
+  secret_value:
+    description: "Test secret value"
+    default: "initial-secret-value"
+
+resources:
+  secrets:
+    secret1:
+      catalog_name: main
+      schema_name: default
+      name: test_secret_$UNIQUE_NAME
+      value: ${var.secret_value}
+      comment: "Test secret"
+      grants:
+      - principal: account users
+        privileges:
+          - READ_SECRET

acceptance/bundle/invariant/migrate/test.toml

@@ -34,3 +34,6 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml
 # migrate deploys via Terraform first, and the TF provider rejects an uppercase
 # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct.
 EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"]
+
+# Secrets are not supported in terraform mode, so the terraform deploy that seeds the migration fails for them.
+EnvMatrixExclude.no_secret = ["INPUT_CONFIG=secrets.yml.tmpl"]

acceptance/bundle/invariant/test.toml

@@ -63,6 +63,7 @@ EnvMatrix.INPUT_CONFIG = [
     "schema_uppercase_name.yml.tmpl",
     "secret_scope.yml.tmpl",
     "secret_scope_default_backend_type.yml.tmpl",
+    "secrets.yml.tmpl",
     "sql_warehouse.yml.tmpl",
     "synced_database_table.yml.tmpl",
     "vector_search_endpoint.yml.tmpl",

acceptance/bundle/refschema/out.fields.txt

@@ -3344,6 +3344,32 @@ resources.secret_scopes.*.permissions.acls[*]	workspace.AclItem	ALL
 resources.secret_scopes.*.permissions.acls[*].permission	workspace.AclPermission	ALL
 resources.secret_scopes.*.permissions.acls[*].principal	string	ALL
 resources.secret_scopes.*.permissions.scope_name	string	ALL
+resources.secrets.*.catalog_name	string	ALL
+resources.secrets.*.comment	string	ALL
+resources.secrets.*.create_time	*time.Time	REMOTE	STATE
+resources.secrets.*.created_by	string	REMOTE	STATE
+resources.secrets.*.effective_owner	string	REMOTE	STATE
+resources.secrets.*.effective_value	string	REMOTE	STATE
+resources.secrets.*.expire_time	*time.Time	ALL
+resources.secrets.*.full_name	string	REMOTE	STATE
+resources.secrets.*.id	string	INPUT
+resources.secrets.*.lifecycle	resources.Lifecycle	INPUT
+resources.secrets.*.lifecycle.prevent_destroy	bool	INPUT
+resources.secrets.*.metastore_id	string	REMOTE	STATE
+resources.secrets.*.modified_status	string	INPUT
+resources.secrets.*.name	string	ALL
+resources.secrets.*.owner	string	ALL
+resources.secrets.*.schema_name	string	ALL
+resources.secrets.*.update_time	*time.Time	REMOTE	STATE
+resources.secrets.*.updated_by	string	REMOTE	STATE
+resources.secrets.*.url	string	INPUT
+resources.secrets.*.value	string	ALL
+resources.secrets.*.grants.full_name	string	ALL
+resources.secrets.*.grants.securable_type	string	ALL
+resources.secrets.*.grants[*]	catalog.PrivilegeAssignment	ALL
+resources.secrets.*.grants[*].principal	string	ALL
+resources.secrets.*.grants[*].privileges	[]catalog.Privilege	ALL
+resources.secrets.*.grants[*].privileges[*]	catalog.Privilege	ALL
 resources.sql_warehouses.*.auto_stop_mins	int	ALL
 resources.sql_warehouses.*.channel	*sql.Channel	ALL
 resources.sql_warehouses.*.channel.dbsql_version	string	ALL

acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl

@@ -0,0 +1,16 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+variables:
+  secret_value:
+    description: "Test secret value"
+    default: "initial-secret-value"
+
+resources:
+  secrets:
+    secret1:
+      catalog_name: main
+      schema_name: default
+      name: test_secret
+      value: ${var.secret_value}
+      comment: "Test secret for acceptance testing"

acceptance/bundle/resources/secrets/basic/output.txt

@@ -0,0 +1,102 @@
+
+>>> [CLI] bundle validate
+Name: test-bundle-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Validation OK!
+
+>>> [CLI] bundle summary
+Name: test-bundle-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+Resources:
+  Secrets:
+    secret1:
+      Name: main.default.test_secret
+      URL:  (not deployed)
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //unity-catalog
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/secrets",
+  "body": {
+    "catalog_name": "main",
+    "comment": "Test secret for acceptance testing",
+    "name": "test_secret",
+    "schema_name": "default",
+    "value": "initial-secret-value"
+  }
+}
+secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret for acceptance testing'
+
+=== Update comment and re-deploy
+>>> update_file.py databricks.yml Test secret for acceptance testing Updated comment
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //unity-catalog
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret",
+  "q": {
+    "update_mask": "*"
+  },
+  "body": {
+    "catalog_name": "main",
+    "comment": "Updated comment",
+    "name": "test_secret",
+    "schema_name": "default",
+    "value": "initial-secret-value"
+  }
+}
+secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Updated comment'
+
+=== Restore comment and re-deploy
+>>> update_file.py databricks.yml Updated comment Test secret for acceptance testing
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //unity-catalog
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret",
+  "q": {
+    "update_mask": "*"
+  },
+  "body": {
+    "catalog_name": "main",
+    "comment": "Test secret for acceptance testing",
+    "name": "test_secret",
+    "schema_name": "default",
+    "value": "initial-secret-value"
+  }
+}
+secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret for acceptance testing'
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.secrets.secret1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/secrets/basic/script

@@ -0,0 +1,28 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm out.requests.txt
+}
+trap cleanup EXIT
+
+trace $CLI bundle validate
+trace $CLI bundle summary
+trace $CLI bundle deploy
+
+trace print_requests.py //unity-catalog
+read_state.py secrets secret1 catalog_name schema_name name comment
+
+title "Update comment and re-deploy"
+trace update_file.py databricks.yml "Test secret for acceptance testing" "Updated comment"
+trace $CLI bundle deploy
+trace print_requests.py //unity-catalog
+read_state.py secrets secret1 catalog_name schema_name name comment
+
+title "Restore comment and re-deploy"
+trace update_file.py databricks.yml "Updated comment" "Test secret for acceptance testing"
+trace $CLI bundle deploy
+trace print_requests.py //unity-catalog
+read_state.py secrets secret1 catalog_name schema_name name comment
+
+rm -f out.requests.txt

acceptance/bundle/resources/secrets/basic/test.toml

@@ -0,0 +1,8 @@
+Local = true
+Cloud = true
+RecordRequests = true
+
+Ignore = [".databricks"]
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/resources/secrets/direct-only/databricks.yml

@@ -0,0 +1,15 @@
+bundle:
+  name: test-bundle
+
+variables:
+  secret_value:
+    description: "Test secret value"
+    default: "test-value"
+
+resources:
+  secrets:
+    secret1:
+      catalog_name: main
+      schema_name: default
+      name: test_secret
+      value: ${var.secret_value}

acceptance/bundle/resources/secrets/direct-only/output.txt

@@ -0,0 +1,11 @@
+
+=== Deploy should fail in terraform mode
+>>> [CLI] bundle deploy
+Error: Secret resources are only supported with direct deployment mode
+  in databricks.yml:11:5
+
+Secret resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use secret resources.
+Learn more at https://docs.databricks.com/dev-tools/bundles/direct
+
+
+Exit code: 1

acceptance/bundle/resources/secrets/direct-only/script

@@ -0,0 +1,4 @@
+title "Deploy should fail in terraform mode"
+trace $CLI bundle deploy 2>&1 | contains.py \
+  "Secret resources are only supported with direct deployment mode" \
+  "DATABRICKS_BUNDLE_ENGINE"

acceptance/bundle/resources/secrets/direct-only/test.toml

@@ -0,0 +1,8 @@
+Local = true
+Cloud = false
+RecordRequests = false
+
+Ignore = [".databricks"]
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["terraform"]

acceptance/bundle/resources/secrets/update-value/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: test-bundle
+
+variables:
+  secret_value:
+    description: "Test secret value"
+    default: "initial-secret-value"
+
+resources:
+  secrets:
+    secret1:
+      catalog_name: main
+      schema_name: default
+      name: test_secret
+      value: ${var.secret_value}
+      comment: "Test secret"

acceptance/bundle/resources/secrets/update-value/output.txt

@@ -0,0 +1,68 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //unity-catalog
+{
+  "method": "POST",
+  "path": "/api/2.1/unity-catalog/secrets",
+  "body": {
+    "catalog_name": "main",
+    "comment": "Test secret",
+    "name": "test_secret",
+    "schema_name": "default",
+    "value": "initial-secret-value"
+  }
+}
+secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='initial-secret-value'
+
+=== Update secret value by changing variable default
+>>> update_file.py databricks.yml initial-secret-value updated-secret-value
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //unity-catalog
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret",
+  "q": {
+    "update_mask": "*"
+  },
+  "body": {
+    "catalog_name": "main",
+    "comment": "Test secret",
+    "name": "test_secret",
+    "schema_name": "default",
+    "value": "updated-secret-value"
+  }
+}
+secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='updated-secret-value'
+
+=== Verify state does not contain actual secret value
+>>> print_state.py
+{
+ "state_version": 2,
+ "cli_version": "[CLI_VERSION]",
+ "lineage": "[UUID]",
+ "serial": 2,
+ "state": {
+  "resources.secrets.secret1": {
+   "__id__": "main.default.test_secret",
+   "state": {
+    "catalog_name": "main",
+    "comment": "Test secret",
+    "name": "test_secret",
+    "schema_name": "default",
+    "value": "updated-secret-value"
+contains error: 'updated-secret-value' was not expected: '"value": "updated-secret-value"'
+   }
+  }
+ }
+}

acceptance/bundle/resources/secrets/update-value/script

@@ -0,0 +1,15 @@
+trace $CLI bundle deploy
+
+trace print_requests.py //unity-catalog
+read_state.py secrets secret1 catalog_name schema_name name comment value
+
+title "Update secret value by changing variable default"
+trace update_file.py databricks.yml "initial-secret-value" "updated-secret-value"
+trace $CLI bundle deploy
+trace print_requests.py //unity-catalog
+read_state.py secrets secret1 catalog_name schema_name name comment value
+
+title "Verify state does not contain actual secret value"
+{ trace print_state.py | contains.py "!initial-secret-value" "!updated-secret-value"; } || true
+
+rm -f out.requests.txt