From cf222861c6ec9a7f0e1281383253ba1f8aa0bdb3 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 17 Jun 2026 11:57:29 +0200 Subject: [PATCH 1/9] Added support for UC secrets --- .../secrets/basic/databricks.yml.tmpl | 16 ++ .../resources/secrets/basic/out.test.toml | 3 + .../bundle/resources/secrets/basic/output.txt | 102 +++++++++ .../bundle/resources/secrets/basic/script | 28 +++ .../bundle/resources/secrets/basic/test.toml | 8 + .../secrets/direct-only/databricks.yml | 15 ++ .../secrets/direct-only/out.test.toml | 3 + .../resources/secrets/direct-only/output.txt | 11 + .../resources/secrets/direct-only/script | 4 + .../resources/secrets/direct-only/test.toml | 8 + .../secrets/update-value/databricks.yml | 16 ++ .../secrets/update-value/out.test.toml | 3 + .../resources/secrets/update-value/output.txt | 68 ++++++ .../resources/secrets/update-value/script | 15 ++ .../resources/secrets/update-value/test.toml | 8 + .../validate-no-plain-text/databricks.yml | 20 ++ .../validate-no-plain-text/out.test.toml | 3 + .../secrets/validate-no-plain-text/output.txt | 14 ++ .../secrets/validate-no-plain-text/script | 4 + .../secrets/validate-no-plain-text/test.toml | 8 + .../validate_secret_value_is_variable.go | 57 +++++ bundle/config/resources.go | 3 + bundle/config/resources/secret.go | 102 +++++++++ bundle/direct/dresources/all.go | 2 + bundle/direct/dresources/resources.yml | 39 ++++ bundle/direct/dresources/secret.go | 98 +++++++++ bundle/phases/initialize.go | 1 + libs/testserver/fake_workspace.go | 1 + libs/testserver/handlers.go | 24 +++ libs/testserver/uc_secrets.go | 198 ++++++++++++++++++ 30 files changed, 882 insertions(+) create mode 100644 acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/secrets/basic/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/basic/output.txt create mode 100755 acceptance/bundle/resources/secrets/basic/script create mode 100644 acceptance/bundle/resources/secrets/basic/test.toml create mode 100644 acceptance/bundle/resources/secrets/direct-only/databricks.yml create mode 100644 acceptance/bundle/resources/secrets/direct-only/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/direct-only/output.txt create mode 100755 acceptance/bundle/resources/secrets/direct-only/script create mode 100644 acceptance/bundle/resources/secrets/direct-only/test.toml create mode 100644 acceptance/bundle/resources/secrets/update-value/databricks.yml create mode 100644 acceptance/bundle/resources/secrets/update-value/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/update-value/output.txt create mode 100755 acceptance/bundle/resources/secrets/update-value/script create mode 100644 acceptance/bundle/resources/secrets/update-value/test.toml create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt create mode 100755 acceptance/bundle/resources/secrets/validate-no-plain-text/script create mode 100644 acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml create mode 100644 bundle/config/mutator/validate_secret_value_is_variable.go create mode 100644 bundle/config/resources/secret.go create mode 100644 bundle/direct/dresources/secret.go create mode 100644 libs/testserver/uc_secrets.go diff --git a/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl b/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl new file mode 100644 index 00000000000..518dbfbb9a5 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/databricks.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: "Test secret value" + default: "initial-secret-value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret + value: ${var.secret_value} + comment: "Test secret for acceptance testing" diff --git a/acceptance/bundle/resources/secrets/basic/out.test.toml b/acceptance/bundle/resources/secrets/basic/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/basic/output.txt b/acceptance/bundle/resources/secrets/basic/output.txt new file mode 100644 index 00000000000..584a62b4258 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/output.txt @@ -0,0 +1,102 @@ + +>>> [CLI] bundle validate +Name: test-bundle-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: test-bundle-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default +Resources: + Secrets: + secret1: + Name: main.default.test_secret + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "comment": "Test secret for acceptance testing", + "name": "test_secret", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret for acceptance testing' + +=== Update comment and re-deploy +>>> update_file.py databricks.yml Test secret for acceptance testing Updated comment + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "main", + "comment": "Updated comment", + "name": "test_secret", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Updated comment' + +=== Restore comment and re-deploy +>>> update_file.py databricks.yml Updated comment Test secret for acceptance testing + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "main", + "comment": "Test secret for acceptance testing", + "name": "test_secret", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret for acceptance testing' + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.secrets.secret1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/secrets/basic/script b/acceptance/bundle/resources/secrets/basic/script new file mode 100755 index 00000000000..f959b83eb59 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate +trace $CLI bundle summary +trace $CLI bundle deploy + +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment + +title "Update comment and re-deploy" +trace update_file.py databricks.yml "Test secret for acceptance testing" "Updated comment" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment + +title "Restore comment and re-deploy" +trace update_file.py databricks.yml "Updated comment" "Test secret for acceptance testing" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment + +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/secrets/basic/test.toml b/acceptance/bundle/resources/secrets/basic/test.toml new file mode 100644 index 00000000000..6b3ba16e605 --- /dev/null +++ b/acceptance/bundle/resources/secrets/basic/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/direct-only/databricks.yml b/acceptance/bundle/resources/secrets/direct-only/databricks.yml new file mode 100644 index 00000000000..ae64aaa6ede --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +variables: + secret_value: + description: "Test secret value" + default: "test-value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret + value: ${var.secret_value} diff --git a/acceptance/bundle/resources/secrets/direct-only/out.test.toml b/acceptance/bundle/resources/secrets/direct-only/out.test.toml new file mode 100644 index 00000000000..65156e0457c --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/secrets/direct-only/output.txt b/acceptance/bundle/resources/secrets/direct-only/output.txt new file mode 100644 index 00000000000..7d6d276152b --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/output.txt @@ -0,0 +1,11 @@ + +=== Deploy should fail in terraform mode +>>> [CLI] bundle deploy +Error: Secret resources are only supported with direct deployment mode + in databricks.yml:11:5 + +Secret resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use secret resources. +Learn more at https://docs.databricks.com/dev-tools/bundles/direct + + +Exit code: 1 diff --git a/acceptance/bundle/resources/secrets/direct-only/script b/acceptance/bundle/resources/secrets/direct-only/script new file mode 100755 index 00000000000..4e7454f402e --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/script @@ -0,0 +1,4 @@ +title "Deploy should fail in terraform mode" +trace $CLI bundle deploy 2>&1 | contains.py \ + "Secret resources are only supported with direct deployment mode" \ + "DATABRICKS_BUNDLE_ENGINE" diff --git a/acceptance/bundle/resources/secrets/direct-only/test.toml b/acceptance/bundle/resources/secrets/direct-only/test.toml new file mode 100644 index 00000000000..38ca093e146 --- /dev/null +++ b/acceptance/bundle/resources/secrets/direct-only/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false +RecordRequests = false + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/secrets/update-value/databricks.yml b/acceptance/bundle/resources/secrets/update-value/databricks.yml new file mode 100644 index 00000000000..89208023eea --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: test-bundle + +variables: + secret_value: + description: "Test secret value" + default: "initial-secret-value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret + value: ${var.secret_value} + comment: "Test secret" diff --git a/acceptance/bundle/resources/secrets/update-value/out.test.toml b/acceptance/bundle/resources/secrets/update-value/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt new file mode 100644 index 00000000000..6c92ba765a0 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -0,0 +1,68 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/secrets", + "body": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "initial-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='initial-secret-value' + +=== Update secret value by changing variable default +>>> update_file.py databricks.yml initial-secret-value updated-secret-value + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //unity-catalog +{ + "method": "PATCH", + "path": "/api/2.1/unity-catalog/secrets/main.default.test_secret", + "q": { + "update_mask": "*" + }, + "body": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "updated-secret-value" + } +} +secrets secret1 catalog_name='main' schema_name='default' name='test_secret' comment='Test secret' value='updated-secret-value' + +=== Verify state does not contain actual secret value +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "state": { + "resources.secrets.secret1": { + "__id__": "main.default.test_secret", + "state": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "updated-secret-value" +contains error: 'updated-secret-value' was not expected: '"value": "updated-secret-value"' + } + } + } +} diff --git a/acceptance/bundle/resources/secrets/update-value/script b/acceptance/bundle/resources/secrets/update-value/script new file mode 100755 index 00000000000..6c0355d3ba1 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/script @@ -0,0 +1,15 @@ +trace $CLI bundle deploy + +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment value + +title "Update secret value by changing variable default" +trace update_file.py databricks.yml "initial-secret-value" "updated-secret-value" +trace $CLI bundle deploy +trace print_requests.py //unity-catalog +read_state.py secrets secret1 catalog_name schema_name name comment value + +title "Verify state does not contain actual secret value" +{ trace print_state.py | contains.py "!initial-secret-value" "!updated-secret-value"; } || true + +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/secrets/update-value/test.toml b/acceptance/bundle/resources/secrets/update-value/test.toml new file mode 100644 index 00000000000..4402e189994 --- /dev/null +++ b/acceptance/bundle/resources/secrets/update-value/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true +RecordRequests = true + +Ignore = [".databricks", ".gitignore"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml b/acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml new file mode 100644 index 00000000000..d183373d3b0 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/databricks.yml @@ -0,0 +1,20 @@ +bundle: + name: test-bundle + +resources: + catalogs: + test_catalog: + name: ${workspace.resource_prefix}_catalog + + schemas: + test_schema: + name: ${workspace.resource_prefix}_schema + catalog_name: ${resources.catalogs.test_catalog.name} + + secrets: + secret1: + catalog_name: ${resources.catalogs.test_catalog.name} + schema_name: ${resources.schemas.test_schema.name} + name: test_secret + value: "plain-text-secret-not-allowed" + comment: "This should fail validation" diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt b/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt new file mode 100644 index 00000000000..0f3c88f9905 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/output.txt @@ -0,0 +1,14 @@ + +=== Deploy should fail due to plain text secret value +>>> [CLI] bundle deploy +Error: Secret value must be a variable reference + at resources.secrets.secret1.value + in databricks.yml:19:14 + +The secret value for "secret1" must be a variable reference (e.g., ${var.my_secret}). +Plain text secret values are not allowed to prevent leaking secrets in configuration files. +Use bundle variables to pass secret values at deployment time. + +contains error: 'plain text secret values are not allowed' not found in the output. + +Exit code: 1 diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/script b/acceptance/bundle/resources/secrets/validate-no-plain-text/script new file mode 100755 index 00000000000..8e8d002a480 --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/script @@ -0,0 +1,4 @@ +title "Deploy should fail due to plain text secret value" +trace $CLI bundle deploy 2>&1 | contains.py \ + "Secret value must be a variable reference" \ + "plain text secret values are not allowed" diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml new file mode 100644 index 00000000000..9a9e1194d6c --- /dev/null +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false +RecordRequests = false + +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/validate_secret_value_is_variable.go b/bundle/config/mutator/validate_secret_value_is_variable.go new file mode 100644 index 00000000000..49a4031ceaf --- /dev/null +++ b/bundle/config/mutator/validate_secret_value_is_variable.go @@ -0,0 +1,57 @@ +package mutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" +) + +type validateSecretValueIsVariable struct{} + +func ValidateSecretValueIsVariable() bundle.Mutator { + return &validateSecretValueIsVariable{} +} + +func (v *validateSecretValueIsVariable) Name() string { + return "ValidateSecretValueIsVariable" +} + +func (v *validateSecretValueIsVariable) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + // Iterate over all secrets in the bundle + for key := range b.Config.Resources.Secrets { + p := dyn.NewPath(dyn.Key("resources"), dyn.Key("secrets"), dyn.Key(key), dyn.Key("value")) + val, err := dyn.GetByPath(b.Config.Value(), p) + if dyn.IsNoSuchKeyError(err) { + continue + } + if err != nil { + return diag.FromErr(err) + } + + valueStr, ok := val.AsString() + if !ok { + continue + } + + // Value must be a variable reference to prevent leaking secrets in config files + if !dynvar.ContainsVariableReference(valueStr) { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "Secret value must be a variable reference", + Detail: fmt.Sprintf(`The secret value for "%s" must be a variable reference (e.g., ${var.my_secret}). +Plain text secret values are not allowed to prevent leaking secrets in configuration files. +Use bundle variables to pass secret values at deployment time.`, key), + Locations: val.Locations(), + Paths: []dyn.Path{p}, + }) + } + } + + return diags +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 3dc7dc295d3..da6e9c5314a 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -40,6 +40,7 @@ type Resources struct { PostgresSyncedTables map[string]*resources.PostgresSyncedTable `json:"postgres_synced_tables,omitempty"` VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` VectorSearchIndexes map[string]*resources.VectorSearchIndex `json:"vector_search_indexes,omitempty"` + Secrets map[string]*resources.Secret `json:"secrets,omitempty"` } type ConfigResource interface { @@ -121,6 +122,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_synced_tables"], r.PostgresSyncedTables), collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), collectResourceMap(descriptions["vector_search_indexes"], r.VectorSearchIndexes), + collectResourceMap(descriptions["secrets"], r.Secrets), } } @@ -180,5 +182,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_synced_tables": (&resources.PostgresSyncedTable{}).ResourceDescription(), "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), "vector_search_indexes": (&resources.VectorSearchIndex{}).ResourceDescription(), + "secrets": (&resources.Secret{}).ResourceDescription(), } } diff --git a/bundle/config/resources/secret.go b/bundle/config/resources/secret.go new file mode 100644 index 00000000000..cc182a5cbec --- /dev/null +++ b/bundle/config/resources/secret.go @@ -0,0 +1,102 @@ +package resources + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +type Secret struct { + BaseResource + + // The name of the catalog where the schema and the secret reside. + CatalogName string `json:"catalog_name"` + + // The name of the schema where the secret resides. + SchemaName string `json:"schema_name"` + + // The name of the secret, relative to its parent schema. + Name string `json:"name"` + + // The secret value to store. This field must be a variable reference (e.g., ${var.my_secret}) + // to prevent leaking secrets in configuration files. Plain text values are not allowed. + // The maximum size is 60 KiB (pre-encryption). + Value string `json:"value"` + + // User-provided free-form text description of the secret. + Comment string `json:"comment,omitempty"` + + // The owner of the secret. Defaults to the creating principal on creation. + // Can be updated to transfer ownership of the secret to another principal. + Owner string `json:"owner,omitempty"` + + // User-provided expiration time of the secret. This field indicates when + // the secret should no longer be used and may be displayed as a warning in + // the UI. It is purely informational and does not trigger any automatic + // actions or affect the secret's lifecycle. + ExpireTime *time.Time `json:"expire_time,omitempty"` + + // List of grants to apply on this secret. + Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` +} + +func (s *Secret) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s Secret) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s Secret) Exists(ctx context.Context, w *databricks.WorkspaceClient, fullName string) (bool, error) { + log.Tracef(ctx, "Checking if secret with fullName=%s exists", fullName) + + _, err := w.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ + FullName: fullName, + }) + if err != nil { + log.Debugf(ctx, "secret with full name %s does not exist: %v", fullName, err) + + if apierr.IsMissing(err) { + return false, nil + } + + return false, err + } + return true, nil +} + +func (*Secret) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "secret", + PluralName: "secrets", + SingularTitle: "Secret", + PluralTitle: "Secrets", + } +} + +func (s *Secret) InitializeURL(baseURL url.URL) { + if s.ID == "" { + return + } + s.URL = workspaceurls.ResourceURL(baseURL, "secrets", s.ID) +} + +func (s *Secret) GetURL() string { + return s.URL +} + +func (s *Secret) GetName() string { + if s.ID != "" { + return s.ID + } + return fmt.Sprintf("%s.%s.%s", s.CatalogName, s.SchemaName, s.Name) +} diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 6cc1eb55437..cae0f017f52 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -35,6 +35,7 @@ var SupportedResources = map[string]any{ "quality_monitors": (*ResourceQualityMonitor)(nil), "vector_search_endpoints": (*ResourceVectorSearchEndpoint)(nil), "vector_search_indexes": (*ResourceVectorSearchIndex)(nil), + "secrets": (*ResourceSecret)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), @@ -60,6 +61,7 @@ var SupportedResources = map[string]any{ "volumes.grants": (*ResourceGrants)(nil), "registered_models.grants": (*ResourceGrants)(nil), "vector_search_indexes.grants": (*ResourceGrants)(nil), + "secrets.grants": (*ResourceGrants)(nil), } func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 9c12da9f39b..8ad71a19511 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -651,3 +651,42 @@ resources: backend_defaults: # The Vector Search API assigns index_subtype when the config omits it - field: index_subtype + + secrets: + recreate_on_changes: + # Secrets are three-level namespace objects (catalog.schema.secret) + # and cannot be renamed or moved between schemas + - field: catalog_name + reason: immutable + - field: schema_name + reason: immutable + - field: name + reason: immutable + ignore_remote_changes: + # These fields are computed by the backend and not settable by the user + - field: browse_only + reason: output_only + - field: create_time + reason: output_only + - field: created_by + reason: output_only + - field: effective_owner + reason: output_only + - field: effective_value + reason: output_only + - field: external_secret_id + reason: output_only + - field: full_name + reason: output_only + - field: metastore_id + reason: output_only + - field: update_time + reason: output_only + - field: updated_by + reason: output_only + # The actual secret value is never stored in state for security reasons. + # RemapState always returns empty string for value to prevent persisting + # secrets in the deployment state file. Remote changes to the value are + # ignored since we can't read the plaintext value back from the API. + - field: value + reason: security - value not persisted in state diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go new file mode 100644 index 00000000000..1c303f9429f --- /dev/null +++ b/bundle/direct/dresources/secret.go @@ -0,0 +1,98 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +// Terraform provider implementation: +// https://github.com/databricks/terraform-provider-databricks/blob/main/catalog/resource_secret.go + +type ResourceSecret struct { + client *databricks.WorkspaceClient +} + +func (*ResourceSecret) New(client *databricks.WorkspaceClient) *ResourceSecret { + return &ResourceSecret{client: client} +} + +func (*ResourceSecret) PrepareState(input *resources.Secret) *catalog.Secret { + var expireTime *sdktime.Time + if input.ExpireTime != nil { + expireTime = sdktime.New(*input.ExpireTime) + } + return &catalog.Secret{ + CatalogName: input.CatalogName, + SchemaName: input.SchemaName, + Name: input.Name, + Value: input.Value, + Comment: input.Comment, + Owner: input.Owner, + ExpireTime: expireTime, + } +} + +func (*ResourceSecret) RemapState(remote *catalog.Secret) *catalog.Secret { + // Do not store the actual secret value in state for security reasons. + // The API does not return the plaintext value in GetSecret responses (only metadata), + // and we should not persist secrets in the deployment state file. + // We store an empty value and rely on update_time changes to detect drift. + return &catalog.Secret{ + CatalogName: remote.CatalogName, + SchemaName: remote.SchemaName, + Name: remote.Name, + Value: "", // Never store actual secret value in state + Comment: remote.Comment, + Owner: remote.Owner, + ExpireTime: remote.ExpireTime, + ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), + } +} + +// DoRead fetches the secret by full name. +func (r *ResourceSecret) DoRead(ctx context.Context, id string) (*catalog.Secret, error) { + return r.client.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ + FullName: id, + }) +} + +// DoCreate creates a new UC secret. +func (r *ResourceSecret) DoCreate(ctx context.Context, state *catalog.Secret) (string, *catalog.Secret, error) { + response, err := r.client.SecretsUc.CreateSecret(ctx, catalog.CreateSecretRequest{ + Secret: *state, + }) + if err != nil || response == nil { + return "", nil, err + } + return response.FullName, response, nil +} + +// DoUpdate updates the secret in place and returns remote state. +func (r *ResourceSecret) DoUpdate(ctx context.Context, id string, state *catalog.Secret, _ *PlanEntry) (*catalog.Secret, error) { + // The UpdateSecret API accepts a field mask specifying which fields to update. + // Supported fields: value, comment, owner, expire_time + response, err := r.client.SecretsUc.UpdateSecret(ctx, catalog.UpdateSecretRequest{ + FullName: id, + Secret: *state, + UpdateMask: fieldmask.FieldMask{ + Paths: []string{"*"}, + }, + }) + if err != nil { + return nil, err + } + return response, nil +} + +// DoDelete deletes the secret. +func (r *ResourceSecret) DoDelete(ctx context.Context, id string, _ *catalog.Secret) error { + return r.client.SecretsUc.DeleteSecret(ctx, catalog.DeleteSecretRequest{ + FullName: id, + }) +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 1b86e37dcee..3d74d6bf8cd 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -33,6 +33,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { validate.NoInterpolationInBundleName(), validate.ValidateEngine(), validate.Scripts(), + mutator.ValidateSecretValueIsVariable(), // Updates (dynamic): sync.{paths,include,exclude} (makes them relative to bundle root rather than to definition file) // Rewrites sync paths to be relative to the bundle root instead of the file they were defined in. diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 6c3766ccaeb..3e9875ca175 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -158,6 +158,7 @@ type FakeWorkspace struct { SecretScopes map[string]workspace.SecretScope Secrets map[string]map[string]string // scope -> key -> value Acls map[string][]workspace.AclItem + UCSecrets map[string]catalog.Secret // full_name -> secret (Unity Catalog secrets) // Generic permissions storage: key is "{object_type}:{object_id}" Permissions map[string]iam.ObjectPermissions diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 3b0a154a6ee..d0a7ad2def0 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -671,6 +671,30 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.SecretsAclsDelete(req) }) + // Unity Catalog base endpoint (used for UC availability check): + server.Handle("GET", "/api/2.1/unity-catalog", func(req Request) any { + return map[string]any{ + "metastore_id": "test-metastore-id", + } + }) + + // Unity Catalog Secrets: + server.Handle("POST", "/api/2.1/unity-catalog/secrets", func(req Request) any { + return req.Workspace.SecretsUcCreateSecret(req) + }) + + server.Handle("GET", "/api/2.1/unity-catalog/secrets/{full_name}", func(req Request) any { + return req.Workspace.SecretsUcGetSecret(req) + }) + + server.Handle("PATCH", "/api/2.1/unity-catalog/secrets/{full_name}", func(req Request) any { + return req.Workspace.SecretsUcUpdateSecret(req) + }) + + server.Handle("DELETE", "/api/2.1/unity-catalog/secrets/{full_name}", func(req Request) any { + return req.Workspace.SecretsUcDeleteSecret(req) + }) + // Groups: server.Handle("POST", "/api/2.0/preview/scim/v2/Groups", func(req Request) any { return req.Workspace.GroupsCreate(req) diff --git a/libs/testserver/uc_secrets.go b/libs/testserver/uc_secrets.go new file mode 100644 index 00000000000..98537afed35 --- /dev/null +++ b/libs/testserver/uc_secrets.go @@ -0,0 +1,198 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + sdktime "github.com/databricks/databricks-sdk-go/common/types/time" + "github.com/databricks/databricks-sdk-go/service/catalog" +) + +// SecretsUcCreateSecret handles POST /api/2.1/unity-catalog/secrets +func (s *FakeWorkspace) SecretsUcCreateSecret(req Request) Response { + defer s.LockUnlock()() + + // The API accepts flat fields, not wrapped in a "secret" object + var inputSecret catalog.Secret + if err := json.Unmarshal(req.Body, &inputSecret); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{"message": fmt.Sprintf("Failed to parse request: %s", err)}, + } + } + + if s.UCSecrets == nil { + s.UCSecrets = make(map[string]catalog.Secret) + } + + fullName := fmt.Sprintf("%s.%s.%s", + inputSecret.CatalogName, + inputSecret.SchemaName, + inputSecret.Name) + + // Check if secret already exists + if _, exists := s.UCSecrets[fullName]; exists { + return Response{ + StatusCode: 409, + Body: map[string]any{ + "error_code": "RESOURCE_ALREADY_EXISTS", + "message": fmt.Sprintf("Secret %s already exists", fullName), + }, + } + } + + now := sdktime.New(time.Now()) + secret := catalog.Secret{ + CatalogName: inputSecret.CatalogName, + SchemaName: inputSecret.SchemaName, + Name: inputSecret.Name, + FullName: fullName, + Value: inputSecret.Value, + Comment: inputSecret.Comment, + Owner: inputSecret.Owner, + ExpireTime: inputSecret.ExpireTime, + CreateTime: now, + UpdateTime: now, + CreatedBy: "test-user@databricks.com", + UpdatedBy: "test-user@databricks.com", + EffectiveOwner: inputSecret.Owner, + MetastoreId: "test-metastore-id", + } + + if secret.Owner == "" { + secret.Owner = "test-user@databricks.com" + secret.EffectiveOwner = "test-user@databricks.com" + } + + s.UCSecrets[fullName] = secret + + return Response{ + Body: secret, + } +} + +// SecretsUcGetSecret handles GET /api/2.1/unity-catalog/secrets/{full_name} +func (s *FakeWorkspace) SecretsUcGetSecret(req Request) Response { + defer s.LockUnlock()() + + // Extract full_name from path parameter + fullName := req.Vars["full_name"] + if fullName == "" { + // Fallback: extract from path + parts := strings.Split(req.URL.Path, "/") + if len(parts) >= 6 { + fullName = parts[5] + } + } + + secret, exists := s.UCSecrets[fullName] + if !exists { + return Response{ + StatusCode: 404, + Body: map[string]any{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Secret %s not found", fullName), + }, + } + } + + // Return secret without the actual value (only metadata) + // The real API doesn't return the value unless specifically requested + returnSecret := secret + returnSecret.EffectiveValue = "" // Don't expose value in GET + returnSecret.Value = "" + + return Response{ + Body: returnSecret, + } +} + +// SecretsUcUpdateSecret handles PATCH /api/2.1/unity-catalog/secrets/{full_name} +func (s *FakeWorkspace) SecretsUcUpdateSecret(req Request) Response { + defer s.LockUnlock()() + + // Extract full_name from path parameter + fullName := req.Vars["full_name"] + if fullName == "" { + // Fallback: extract from path + parts := strings.Split(req.URL.Path, "/") + if len(parts) >= 6 { + fullName = parts[5] + } + } + + // The API accepts flat fields + var updateSecret catalog.Secret + if err := json.Unmarshal(req.Body, &updateSecret); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{"message": fmt.Sprintf("Failed to parse request: %s", err)}, + } + } + + secret, exists := s.UCSecrets[fullName] + if !exists { + return Response{ + StatusCode: 404, + Body: map[string]any{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Secret %s not found", fullName), + }, + } + } + + // Update fields based on update mask + if updateSecret.Value != "" { + secret.Value = updateSecret.Value + } + if updateSecret.Comment != "" { + secret.Comment = updateSecret.Comment + } + if updateSecret.Owner != "" { + secret.Owner = updateSecret.Owner + secret.EffectiveOwner = updateSecret.Owner + } + if updateSecret.ExpireTime != nil { + secret.ExpireTime = updateSecret.ExpireTime + } + + secret.UpdateTime = sdktime.New(time.Now()) + secret.UpdatedBy = "test-user@databricks.com" + + s.UCSecrets[fullName] = secret + + return Response{ + Body: secret, + } +} + +// SecretsUcDeleteSecret handles DELETE /api/2.1/unity-catalog/secrets/{full_name} +func (s *FakeWorkspace) SecretsUcDeleteSecret(req Request) Response { + defer s.LockUnlock()() + + // Extract full_name from path parameter + fullName := req.Vars["full_name"] + if fullName == "" { + // Fallback: extract from path + parts := strings.Split(req.URL.Path, "/") + if len(parts) >= 6 { + fullName = parts[5] + } + } + + if _, exists := s.UCSecrets[fullName]; !exists { + return Response{ + StatusCode: 404, + Body: map[string]any{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": fmt.Sprintf("Secret %s not found", fullName), + }, + } + } + + delete(s.UCSecrets, fullName) + + return Response{} +} From 668279511681b70ad165925296d1adf264f91129 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 10:47:32 +0200 Subject: [PATCH 2/9] fixes --- .../apply_bundle_permissions_test.go | 1 + .../resourcemutator/apply_target_mode_test.go | 8 +++ .../config/mutator/resourcemutator/run_as.go | 10 ++++ .../mutator/resourcemutator/run_as_test.go | 1 + bundle/config/resources/secret.go | 4 +- bundle/config/resources_test.go | 8 +++ bundle/deploy/terraform/lifecycle_test.go | 1 + bundle/direct/dresources/all_test.go | 19 +++++++ bundle/direct/dresources/grants.go | 1 + bundle/direct/dresources/secret.go | 55 +++++++++++++------ bundle/internal/schema/annotations.yml | 33 +++++++++++ bundle/statemgmt/state_load_test.go | 35 ++++++++++++ libs/workspaceurls/urls.go | 2 + 13 files changed, 160 insertions(+), 18 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 3fa97c41a7f..196917f17b7 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -31,6 +31,7 @@ var unsupportedResources = []string{ "postgres_catalogs", "postgres_synced_tables", "vector_search_indexes", + "secrets", } func TestApplyBundlePermissions(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 440221001a7..ca79928db2f 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -173,6 +173,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Name: "secretScope1", }, }, + Secrets: map[string]*resources.Secret{ + "secret1": { + CatalogName: "main", + SchemaName: "default", + Name: "secret1", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "sql_warehouse1": { CreateWarehouseRequest: sql.CreateWarehouseRequest{ @@ -473,6 +480,7 @@ func TestAppropriateResourcesAreRenamed(t *testing.T) { notUserNamed := []string{ "Apps", "SecretScopes", + "Secrets", "DatabaseInstances", "DatabaseCatalogs", "SyncedDatabaseTables", diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index 4f5e3ce9036..9f13b31eabe 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -126,6 +126,16 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { )) } + // UC secrets do not support run_as in the API. + if len(b.Config.Resources.Secrets) > 0 { + diags = diags.Extend(reportRunAsNotSupported( + "secrets", + b.Config.GetLocation("resources.secrets"), + b.Config.Workspace.CurrentUser.UserName, + identity, + )) + } + return diags } diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index af1470848d7..88311780b94 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -55,6 +55,7 @@ func allResourceTypes(t *testing.T) []string { "registered_models", "schemas", "secret_scopes", + "secrets", "sql_warehouses", "synced_database_tables", "vector_search_endpoints", diff --git a/bundle/config/resources/secret.go b/bundle/config/resources/secret.go index cc182a5cbec..15832f51aaa 100644 --- a/bundle/config/resources/secret.go +++ b/bundle/config/resources/secret.go @@ -52,11 +52,11 @@ func (s *Secret) UnmarshalJSON(b []byte) error { return marshal.Unmarshal(b, s) } -func (s Secret) MarshalJSON() ([]byte, error) { +func (s *Secret) MarshalJSON() ([]byte, error) { return marshal.Marshal(s) } -func (s Secret) Exists(ctx context.Context, w *databricks.WorkspaceClient, fullName string) (bool, error) { +func (s *Secret) Exists(ctx context.Context, w *databricks.WorkspaceClient, fullName string) (bool, error) { log.Tracef(ctx, "Checking if secret with fullName=%s exists", fullName) _, err := w.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index d56a24ced46..85a410a661c 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -226,6 +226,13 @@ func TestResourcesBindSupport(t *testing.T) { Name: "0", }, }, + Secrets: map[string]*resources.Secret{ + "my_secret": { + CatalogName: "main", + SchemaName: "default", + Name: "my_secret", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "my_sql_warehouse": { CreateWarehouseRequest: sql.CreateWarehouseRequest{}, @@ -343,6 +350,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetSyncedTable(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVectorSearchIndexesAPI().EXPECT().GetIndexByIndexName(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockSecretsUcAPI().EXPECT().GetSecret(mock.Anything, mock.Anything).Return(&catalog.Secret{FullName: "0"}, nil) allResources := supportedResources.AllResources() for _, group := range allResources { diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index b60bff612c7..4a3c5280891 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -18,6 +18,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { "catalogs", "external_locations", "genie_spaces", + "secrets", "vector_search_endpoints", "vector_search_indexes", } diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 30adb4640cc..a0d85548a2a 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -121,6 +121,14 @@ var testConfig map[string]any = map[string]any{ }, }, + "secrets": &resources.Secret{ + CatalogName: "main", + SchemaName: "default", + Name: "my_secret", + Value: "my_secret_value", + Comment: "Test secret", + }, + "secret_scopes": &resources.SecretScope{ Name: "my_secret_scope", BackendType: workspace.ScopeBackendTypeAzureKeyvault, @@ -665,6 +673,17 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "secrets.grants": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + return &GrantsState{ + SecurableType: "secret", + FullName: "main.default.my_secret", + EmbeddedSlice: []catalog.PrivilegeAssignment{{ + Privileges: []catalog.Privilege{catalog.PrivilegeSelect}, + Principal: "user@example.com", + }}, + }, nil + }, + "secret_scopes.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { err := client.Secrets.CreateScope(ctx, workspace.CreateScope{ Scope: "permissions_test_scope", diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 596346f1614..8f10c17f917 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -18,6 +18,7 @@ var grantResourceToSecurableType = map[string]string{ "external_locations": "external_location", "volumes": "volume", "registered_models": "function", + "secrets": "secret", "vector_search_indexes": "table", } diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 1c303f9429f..78720a2f67c 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -28,13 +28,24 @@ func (*ResourceSecret) PrepareState(input *resources.Secret) *catalog.Secret { expireTime = sdktime.New(*input.ExpireTime) } return &catalog.Secret{ - CatalogName: input.CatalogName, - SchemaName: input.SchemaName, - Name: input.Name, - Value: input.Value, - Comment: input.Comment, - Owner: input.Owner, - ExpireTime: expireTime, + CatalogName: input.CatalogName, + SchemaName: input.SchemaName, + Name: input.Name, + Value: input.Value, + Comment: input.Comment, + Owner: input.Owner, + ExpireTime: expireTime, + BrowseOnly: false, + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: "", + ExternalSecretId: "", + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", + ForceSendFields: nil, } } @@ -44,21 +55,33 @@ func (*ResourceSecret) RemapState(remote *catalog.Secret) *catalog.Secret { // and we should not persist secrets in the deployment state file. // We store an empty value and rely on update_time changes to detect drift. return &catalog.Secret{ - CatalogName: remote.CatalogName, - SchemaName: remote.SchemaName, - Name: remote.Name, - Value: "", // Never store actual secret value in state - Comment: remote.Comment, - Owner: remote.Owner, - ExpireTime: remote.ExpireTime, - ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), + CatalogName: remote.CatalogName, + SchemaName: remote.SchemaName, + Name: remote.Name, + Value: "", // Never store actual secret value in state + Comment: remote.Comment, + Owner: remote.Owner, + ExpireTime: remote.ExpireTime, + BrowseOnly: false, + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: "", + ExternalSecretId: "", + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", + ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), } } // DoRead fetches the secret by full name. func (r *ResourceSecret) DoRead(ctx context.Context, id string) (*catalog.Secret, error) { return r.client.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ - FullName: id, + FullName: id, + IncludeBrowse: false, + ForceSendFields: nil, }) } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 69d7c9d025d..69da6acc1b0 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -253,6 +253,11 @@ github.com/databricks/cli/bundle/config.Resources: The secret scope definitions for the bundle, where each key is the name of the secret scope. "markdown_description": |- The secret scope definitions for the bundle, where each key is the name of the secret scope. See [\_](/dev-tools/bundles/resources.md#secret_scopes). + "secrets": + "description": |- + The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. + "markdown_description": |- + The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. See [\_](/dev-tools/bundles/resources.md#secrets). "sql_warehouses": "description": |- The SQL warehouse definitions for the bundle, where each key is the name of the warehouse. @@ -1012,6 +1017,34 @@ github.com/databricks/cli/bundle/config/resources.PostgresSyncedTable: "timeseries_key": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.Secret: + "catalog_name": + "description": |- + The name of the catalog where the schema and the secret reside. + "comment": + "description": |- + User-provided free-form text description of the secret. + "expire_time": + "description": |- + User-provided expiration time of the secret. Purely informational; does not trigger automatic actions. + "grants": + "description": |- + The grants to apply on this secret. + "lifecycle": + "description": |- + Lifecycle is a struct that contains the lifecycle settings for a resource. It controls the behavior of the resource when it is deployed or destroyed. + "name": + "description": |- + The name of the secret, relative to its parent schema. + "owner": + "description": |- + The owner of the secret. Defaults to the creating principal on creation. + "schema_name": + "description": |- + The name of the schema where the secret resides. + "value": + "description": |- + The secret value to store. Must be a variable reference (e.g. ${var.my_secret}) to prevent plain-text secrets in configuration files. github.com/databricks/cli/bundle/config/resources.SecretScope: "backend_type": "description": |- diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 672cd9855b2..2c6611a2ff6 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -54,6 +54,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.postgres_synced_tables.test_postgres_synced_table": {ID: "synced_tables/main.public.test_synced_table"}, "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, + "resources.secrets.test_secret": {ID: "main.default.test_secret"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -134,6 +135,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "vs-index-1", config.Resources.VectorSearchIndexes["test_vector_search_index"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchIndexes["test_vector_search_index"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Secrets["test_secret"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -251,6 +255,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { Name: "test_secret_scope", }, }, + Secrets: map[string]*resources.Secret{ + "test_secret": { + CatalogName: "main", + SchemaName: "default", + Name: "test_secret", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "test_sql_warehouse": { CreateWarehouseRequest: sql.CreateWarehouseRequest{ @@ -394,6 +405,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.SecretScopes["test_secret_scope"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.SecretScopes["test_secret_scope"].ModifiedStatus) + assert.Empty(t, config.Resources.Secrets["test_secret"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Secrets["test_secret"].ModifiedStatus) + assert.Empty(t, config.Resources.SqlWarehouses["test_sql_warehouse"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.SqlWarehouses["test_sql_warehouse"].ModifiedStatus) @@ -617,6 +631,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { Name: "test_secret_scope_new", }, }, + Secrets: map[string]*resources.Secret{ + "test_secret": { + CatalogName: "main", + SchemaName: "default", + Name: "test_secret", + }, + "test_secret_new": { + CatalogName: "main", + SchemaName: "default", + Name: "test_secret_new", + }, + }, SqlWarehouses: map[string]*resources.SqlWarehouse{ "test_sql_warehouse": { CreateWarehouseRequest: sql.CreateWarehouseRequest{ @@ -817,6 +843,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.vector_search_endpoints.test_vector_search_endpoint_old": {ID: "vs-endpoint-old"}, "resources.vector_search_indexes.test_vector_search_index": {ID: "vs-index-1"}, "resources.vector_search_indexes.test_vector_search_index_old": {ID: "vs-index-old"}, + "resources.secrets.test_secret": {ID: "main.default.test_secret"}, + "resources.secrets.test_secret_old": {ID: "main.default.test_secret_old"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -991,6 +1019,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.VectorSearchIndexes["test_vector_search_index_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchIndexes["test_vector_search_index_new"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret", config.Resources.Secrets["test_secret"].ID) + assert.Empty(t, config.Resources.Secrets["test_secret"].ModifiedStatus) + assert.Equal(t, "main.default.test_secret_old", config.Resources.Secrets["test_secret_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Secrets["test_secret_old"].ModifiedStatus) + assert.Empty(t, config.Resources.Secrets["test_secret_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Secrets["test_secret_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 46930c92239..53e308c532a 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -28,6 +28,7 @@ var resourceURLPatterns = map[string]string{ "queries": "sql/editor/%s", "registered_models": "explore/data/models/%s", "schemas": "explore/data/%s", + "secrets": "explore/data/%s", "synced_database_tables": "explore/data/%s", "vector_search_endpoints": "compute/vector-search/%s", "vector_search_indexes": "explore/data/%s", @@ -53,6 +54,7 @@ var dotSeparatedResources = map[string]bool{ "quality_monitors": true, "registered_models": true, "schemas": true, + "secrets": true, "vector_search_indexes": true, "volumes": true, } From 4a52fd3c2d90bc5d3326333ea116318ae2c19622 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 11:24:12 +0200 Subject: [PATCH 3/9] fixed test + lint --- .../bundle/invariant/configs/secrets.yml.tmpl | 16 +++++ acceptance/bundle/invariant/test.toml | 1 + bundle/direct/dresources/resources.yml | 4 -- bundle/direct/dresources/secret.go | 58 +++++++++---------- bundle/internal/schema/annotations.yml | 33 +++++++++++ 5 files changed, 76 insertions(+), 36 deletions(-) create mode 100644 acceptance/bundle/invariant/configs/secrets.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/secrets.yml.tmpl b/acceptance/bundle/invariant/configs/secrets.yml.tmpl new file mode 100644 index 00000000000..96aa784e16f --- /dev/null +++ b/acceptance/bundle/invariant/configs/secrets.yml.tmpl @@ -0,0 +1,16 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +variables: + secret_value: + description: "Test secret value" + default: "initial-secret-value" + +resources: + secrets: + secret1: + catalog_name: main + schema_name: default + name: test_secret_$UNIQUE_NAME + value: ${var.secret_value} + comment: "Test secret" diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index ec4233c8c59..706292504e4 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -63,6 +63,7 @@ EnvMatrix.INPUT_CONFIG = [ "schema_uppercase_name.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", + "secrets.yml.tmpl", "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 10b63f2f9cf..06c27b3272f 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -786,8 +786,6 @@ resources: reason: immutable ignore_remote_changes: # These fields are computed by the backend and not settable by the user - - field: browse_only - reason: output_only - field: create_time reason: output_only - field: created_by @@ -796,8 +794,6 @@ resources: reason: output_only - field: effective_value reason: output_only - - field: external_secret_id - reason: output_only - field: full_name reason: output_only - field: metastore_id diff --git a/bundle/direct/dresources/secret.go b/bundle/direct/dresources/secret.go index 78720a2f67c..4de72f1efb2 100644 --- a/bundle/direct/dresources/secret.go +++ b/bundle/direct/dresources/secret.go @@ -35,17 +35,15 @@ func (*ResourceSecret) PrepareState(input *resources.Secret) *catalog.Secret { Comment: input.Comment, Owner: input.Owner, ExpireTime: expireTime, - BrowseOnly: false, - CreateTime: nil, - CreatedBy: "", - EffectiveOwner: "", - EffectiveValue: "", - ExternalSecretId: "", - FullName: "", - MetastoreId: "", - UpdateTime: nil, - UpdatedBy: "", - ForceSendFields: nil, + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: "", + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", + ForceSendFields: nil, } } @@ -55,33 +53,29 @@ func (*ResourceSecret) RemapState(remote *catalog.Secret) *catalog.Secret { // and we should not persist secrets in the deployment state file. // We store an empty value and rely on update_time changes to detect drift. return &catalog.Secret{ - CatalogName: remote.CatalogName, - SchemaName: remote.SchemaName, - Name: remote.Name, - Value: "", // Never store actual secret value in state - Comment: remote.Comment, - Owner: remote.Owner, - ExpireTime: remote.ExpireTime, - BrowseOnly: false, - CreateTime: nil, - CreatedBy: "", - EffectiveOwner: "", - EffectiveValue: "", - ExternalSecretId: "", - FullName: "", - MetastoreId: "", - UpdateTime: nil, - UpdatedBy: "", - ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), + CatalogName: remote.CatalogName, + SchemaName: remote.SchemaName, + Name: remote.Name, + Value: "", // Never store actual secret value in state + Comment: remote.Comment, + Owner: remote.Owner, + ExpireTime: remote.ExpireTime, + CreateTime: nil, + CreatedBy: "", + EffectiveOwner: "", + EffectiveValue: "", + FullName: "", + MetastoreId: "", + UpdateTime: nil, + UpdatedBy: "", + ForceSendFields: utils.FilterFields[catalog.Secret](remote.ForceSendFields), } } // DoRead fetches the secret by full name. func (r *ResourceSecret) DoRead(ctx context.Context, id string) (*catalog.Secret, error) { return r.client.SecretsUc.GetSecret(ctx, catalog.GetSecretRequest{ - FullName: id, - IncludeBrowse: false, - ForceSendFields: nil, + FullName: id, }) } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 7fafe145330..112f219c3d1 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -1797,6 +1797,39 @@ resources: "user_name": "description": |- The name of the user granted the permission level. This field translates to a `principal` field in secret scope ACL. + "secrets": + "description": |- + The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. + "markdown_description": |- + The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. See [\_](/dev-tools/bundles/resources.md#secrets). + "$fields": + "catalog_name": + "description": |- + The name of the catalog where the schema and the secret reside. + "comment": + "description": |- + User-provided free-form text description of the secret. + "expire_time": + "description": |- + User-provided expiration time of the secret. Purely informational; does not trigger automatic actions. + "grants": + "description": |- + The grants to apply on this secret. + "lifecycle": + "description": |- + Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed. + "name": + "description": |- + The name of the secret, relative to its parent schema. + "owner": + "description": |- + The owner of the secret. Defaults to the creating principal on creation. + "schema_name": + "description": |- + The name of the schema where the secret resides. + "value": + "description": |- + The secret value to store. Must be a variable reference (e.g. ${var.my_secret}) to prevent plain-text secrets in configuration files. "sql_warehouses": "description": |- The SQL warehouse definitions for the bundle, where each key is the name of the warehouse. From e8c51ad2660733d2bad1b7920f1ca75cae188d37 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 11:50:21 +0200 Subject: [PATCH 4/9] fixed tests --- cmd/experimental/workspace_open_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 9589b48a126..cc2255d580a 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -127,6 +127,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "queries", "registered_models", "schemas", + "secrets", "synced_database_tables", "vector_search_endpoints", "vector_search_indexes", @@ -146,7 +147,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") From 2ed69603d507eb4f1dfd4e8ea58eb96f60651dcd Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 12:03:09 +0200 Subject: [PATCH 5/9] updated generation --- acceptance/bundle/refschema/out.fields.txt | 26 ++++++ .../direct/dresources/apitypes.generated.yml | 2 + .../direct/dresources/resources.generated.yml | 33 +++++++ .../validation/generated/enum_fields.go | 2 + .../validation/generated/required_fields.go | 2 + bundle/schema/jsonschema.json | 87 +++++++++++++++++++ 6 files changed, 152 insertions(+) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 7f93d586b83..616aec700d3 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -3344,6 +3344,32 @@ resources.secret_scopes.*.permissions.acls[*] workspace.AclItem ALL resources.secret_scopes.*.permissions.acls[*].permission workspace.AclPermission ALL resources.secret_scopes.*.permissions.acls[*].principal string ALL resources.secret_scopes.*.permissions.scope_name string ALL +resources.secrets.*.catalog_name string ALL +resources.secrets.*.comment string ALL +resources.secrets.*.create_time *time.Time REMOTE STATE +resources.secrets.*.created_by string REMOTE STATE +resources.secrets.*.effective_owner string REMOTE STATE +resources.secrets.*.effective_value string REMOTE STATE +resources.secrets.*.expire_time *time.Time ALL +resources.secrets.*.full_name string REMOTE STATE +resources.secrets.*.id string INPUT +resources.secrets.*.lifecycle resources.Lifecycle INPUT +resources.secrets.*.lifecycle.prevent_destroy bool INPUT +resources.secrets.*.metastore_id string REMOTE STATE +resources.secrets.*.modified_status string INPUT +resources.secrets.*.name string ALL +resources.secrets.*.owner string ALL +resources.secrets.*.schema_name string ALL +resources.secrets.*.update_time *time.Time REMOTE STATE +resources.secrets.*.updated_by string REMOTE STATE +resources.secrets.*.url string INPUT +resources.secrets.*.value string ALL +resources.secrets.*.grants.full_name string ALL +resources.secrets.*.grants.securable_type string ALL +resources.secrets.*.grants[*] catalog.PrivilegeAssignment ALL +resources.secrets.*.grants[*].principal string ALL +resources.secrets.*.grants[*].privileges []catalog.Privilege ALL +resources.secrets.*.grants[*].privileges[*] catalog.Privilege ALL resources.sql_warehouses.*.auto_stop_mins int ALL resources.sql_warehouses.*.channel *sql.Channel ALL resources.sql_warehouses.*.channel.dbsql_version string ALL diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 8fc2eec34a0..90f4a7b3815 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -52,6 +52,8 @@ schemas: catalog.CreateSchema secret_scopes: workspace.CreateScope +secrets: catalog.Secret + sql_warehouses: sql.EditWarehouseRequest synced_database_tables: database.SyncedDatabaseTable diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index e4154180970..c5e662626f8 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -338,6 +338,39 @@ resources: # secret_scopes: no api field behaviors + secrets: + + recreate_on_changes: + - field: catalog_name + reason: spec:immutable + - field: name + reason: spec:immutable + - field: schema_name + reason: spec:immutable + + ignore_remote_changes: + - field: owner + reason: spec:input_only + - field: value + reason: spec:input_only + + - field: create_time + reason: spec:output_only + - field: created_by + reason: spec:output_only + - field: effective_owner + reason: spec:output_only + - field: effective_value + reason: spec:output_only + - field: full_name + reason: spec:output_only + - field: metastore_id + reason: spec:output_only + - field: update_time + reason: spec:output_only + - field: updated_by + reason: spec:output_only + # sql_warehouses: no api field behaviors synced_database_tables: diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index f223ab7ef52..ec2e6077993 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -217,6 +217,8 @@ var EnumFields = map[string][]string{ "resources.secret_scopes.*.backend_type": {"AZURE_KEYVAULT", "DATABRICKS"}, + "resources.secrets.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_METADATA", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, + "resources.sql_warehouses.*.channel.name": {"CHANNEL_NAME_CURRENT", "CHANNEL_NAME_CUSTOM", "CHANNEL_NAME_PREVIEW", "CHANNEL_NAME_PREVIOUS"}, "resources.sql_warehouses.*.permissions[*].level": {"CAN_MANAGE", "CAN_MONITOR", "CAN_USE", "CAN_VIEW", "IS_OWNER"}, "resources.sql_warehouses.*.spot_instance_policy": {"COST_OPTIMIZED", "POLICY_UNSPECIFIED", "RELIABILITY_OPTIMIZED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index 1eb3d7443c0..3ced7963c1a 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -259,6 +259,8 @@ var RequiredFields = map[string][]string{ "resources.secret_scopes.*.keyvault_metadata": {"dns_name", "resource_id"}, "resources.secret_scopes.*.permissions[*]": {"level"}, + "resources.secrets.*": {"catalog_name", "schema_name", "name", "value"}, + "resources.sql_warehouses.*.permissions[*]": {"level"}, "resources.synced_database_tables.*": {"name"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 503ddbf2ec4..841635e30ed 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2203,6 +2203,62 @@ } ] }, + "resources.Secret": { + "oneOf": [ + { + "type": "object", + "properties": { + "catalog_name": { + "description": "The name of the catalog where the schema and the secret reside.", + "$ref": "#/$defs/string" + }, + "comment": { + "description": "User-provided free-form text description of the secret.", + "$ref": "#/$defs/string" + }, + "expire_time": { + "description": "User-provided expiration time of the secret. Purely informational; does not trigger automatic actions.", + "$ref": "#/$defs/time.Time" + }, + "grants": { + "description": "The grants to apply on this secret.", + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment" + }, + "lifecycle": { + "description": "Settings that control the deployment lifecycle of the resource, such as preventing it from being destroyed.", + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "name": { + "description": "The name of the secret, relative to its parent schema.", + "$ref": "#/$defs/string" + }, + "owner": { + "description": "The owner of the secret. Defaults to the creating principal on creation.", + "$ref": "#/$defs/string" + }, + "schema_name": { + "description": "The name of the schema where the secret resides.", + "$ref": "#/$defs/string" + }, + "value": { + "description": "The secret value to store. Must be a variable reference (e.g. ${var.my_secret}) to prevent plain-text secrets in configuration files.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "catalog_name", + "schema_name", + "name", + "value" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScope": { "oneOf": [ { @@ -3152,6 +3208,11 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SecretScope", "markdownDescription": "The secret scope definitions for the bundle, where each key is the name of the secret scope. See [secret_scopes](https://docs.databricks.com/dev-tools/bundles/resources.html#secret_scopes)." }, + "secrets": { + "description": "The Unity Catalog secret definitions for the bundle, where each key is the name of the secret.", + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Secret", + "markdownDescription": "The Unity Catalog secret definitions for the bundle, where each key is the name of the secret. See [secrets](https://docs.databricks.com/dev-tools/bundles/resources.html#secrets)." + }, "sql_warehouses": { "description": "The SQL warehouse definitions for the bundle, where each key is the name of the warehouse.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SqlWarehouse", @@ -13977,6 +14038,20 @@ } ] }, + "resources.Secret": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Secret" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.SecretScope": { "oneOf": [ { @@ -15037,6 +15112,18 @@ }, "string": { "type": "string" + }, + "time.Time": { + "oneOf": [ + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.\\p{L}+([-_]*[\\p{L}\\p{N}]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] } }, "type": "object", From e1b0b77fed13ed558efc77b0047f6ab442493f10 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 12:49:56 +0200 Subject: [PATCH 6/9] fixed resources.yml --- bundle/direct/dresources/resources.yml | 35 -------------------------- 1 file changed, 35 deletions(-) diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 06c27b3272f..8baa03536a2 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -773,38 +773,3 @@ resources: backend_defaults: # The Vector Search API assigns index_subtype when the config omits it - field: index_subtype - - secrets: - recreate_on_changes: - # Secrets are three-level namespace objects (catalog.schema.secret) - # and cannot be renamed or moved between schemas - - field: catalog_name - reason: immutable - - field: schema_name - reason: immutable - - field: name - reason: immutable - ignore_remote_changes: - # These fields are computed by the backend and not settable by the user - - field: create_time - reason: output_only - - field: created_by - reason: output_only - - field: effective_owner - reason: output_only - - field: effective_value - reason: output_only - - field: full_name - reason: output_only - - field: metastore_id - reason: output_only - - field: update_time - reason: output_only - - field: updated_by - reason: output_only - # The actual secret value is never stored in state for security reasons. - # RemapState always returns empty string for value to prevent persisting - # secrets in the deployment state file. Remote changes to the value are - # ignored since we can't read the plaintext value back from the API. - - field: value - reason: security - value not persisted in state From 44965b1fb2e23ca6b34f5a64f50ee91f3a4afaf8 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 13:20:20 +0200 Subject: [PATCH 7/9] fixed acc output --- acceptance/bundle/invariant/migrate/test.toml | 3 +++ acceptance/bundle/resources/secrets/update-value/output.txt | 2 +- acceptance/experimental/open/output.txt | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index af25cbb62dd..a8c26c46109 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -34,3 +34,6 @@ EnvMatrixExclude.no_pydabs_1000_tasks = ["INPUT_CONFIG=job_pydabs_1000_tasks.yml # migrate deploys via Terraform first, and the TF provider rejects an uppercase # volume schema_name ("inconsistent final plan"). Covered by no_drift on direct. EnvMatrixExclude.no_volume_uppercase = ["INPUT_CONFIG=volume_uppercase_name.yml.tmpl"] + +# Secrets are not supported in terraform mode, so the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_secret = ["INPUT_CONFIG=secrets.yml.tmpl"] diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt index 6c92ba765a0..03aadc3e5bc 100644 --- a/acceptance/bundle/resources/secrets/update-value/output.txt +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -49,7 +49,7 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com >>> print_state.py { "state_version": 2, - "cli_version": "[DEV_VERSION]", + "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, "state": { diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index f634d630fc8..d60765969d3 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, secrets, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses === test auto-completion handler >>> [CLI] __complete experimental open , @@ -33,6 +33,7 @@ quality_monitors queries registered_models schemas +secrets synced_database_tables vector_search_endpoints vector_search_indexes From 1f56144ceb1efee7c550b9c33d10611c11cd7840 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 13:43:34 +0200 Subject: [PATCH 8/9] add schema.grants to invariant --- acceptance/bundle/invariant/configs/secrets.yml.tmpl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/invariant/configs/secrets.yml.tmpl b/acceptance/bundle/invariant/configs/secrets.yml.tmpl index 96aa784e16f..b402d8a8cdc 100644 --- a/acceptance/bundle/invariant/configs/secrets.yml.tmpl +++ b/acceptance/bundle/invariant/configs/secrets.yml.tmpl @@ -14,3 +14,7 @@ resources: name: test_secret_$UNIQUE_NAME value: ${var.secret_value} comment: "Test secret" + grants: + - principal: account users + privileges: + - READ_SECRET From 3821f05865e375219be07ffe269be957f8d13fff Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 9 Jul 2026 14:35:57 +0200 Subject: [PATCH 9/9] updated out.test.toml --- acceptance/bundle/invariant/continue_293/out.test.toml | 1 + acceptance/bundle/invariant/no_drift/out.test.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 407bac1bf19..795910c9ab7 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -37,6 +37,7 @@ EnvMatrix.INPUT_CONFIG = [ "schema_uppercase_name.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", + "secrets.yml.tmpl", "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index e61d97cc42c..267f9e56cec 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -45,6 +45,7 @@ EnvMatrix.INPUT_CONFIG = [ "schema_uppercase_name.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", + "secrets.yml.tmpl", "sql_warehouse.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl",