Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
bundle:
name: test-bundle-$UNIQUE_NAME

resources:
jobs:
test_job:
max_concurrent_runs: 1
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook
new_cluster:
spark_version: $DEFAULT_SPARK_VERSION
node_type_id: $NODE_TYPE_ID
num_workers: 1

targets:
default:
mode: development

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@

=== Missing tf state is reported as skipped, exit 0
{
"status": "skipped",
"error": "state snapshot not available: resources state snapshot not found remotely at resources-config-sync-snapshot.json: state snapshot not found",
"files": null,
"changes": {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
#!/bin/bash

envsubst < databricks.yml.tmpl > databricks.yml

# No deploy, so the terraform state snapshot does not exist: in JSON mode the
# sync is reported as skipped with exit 0, not a hard failure.
title "Missing tf state is reported as skipped, exit 0"
echo
errcode $CLI bundle config-remote-sync -o json
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
Cloud = true

RecordRequests = false
Ignore = [".databricks", "dummy.whl", "databricks.yml"]

Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"

# "Missing tf state" is terraform-specific: the direct engine has no separate
# state snapshot to be absent, so ErrStateSnapshotNotFound (and the skipped
# outcome it drives) only arises on terraform.
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
bundle:
name: test-bundle-$UNIQUE_NAME

resources:
jobs:
test_job:
max_concurrent_runs: 1
tasks:
- task_key: main
notebook_task:
notebook_path: /Users/{{workspace_user_name}}/notebook
new_cluster:
spark_version: $DEFAULT_SPARK_VERSION
node_type_id: $NODE_TYPE_ID
num_workers: 1

targets:
default:
mode: development

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files...
Created jobs.test_job
Files: 6 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

=== A selector that matches nothing is reported as skipped, exit 0
{
"status": "skipped",
"error": "no deployed jobs resource with id no-such-id-999; deployed jobs ids in state: [TEST_JOB_ID]",
"files": null,
"changes": {}
}

=== A malformed selector is reported as failed, exit 0
{
"status": "failed",
"error": "invalid --select-ids value \"no-such-id-999\", expected \u003ctype\u003e:\u003cid\u003e (e.g. jobs:[NUMID])",
"files": null,
"changes": {}
}

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.jobs.test_job

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default

Destroy: 1 deleted
23 changes: 23 additions & 0 deletions acceptance/bundle/config-remote-sync/json_status_selectors/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/bin/bash

envsubst < databricks.yml.tmpl > databricks.yml

cleanup() {
trace $CLI bundle destroy --auto-approve
}
trap cleanup EXIT

touch dummy.whl
$CLI bundle deploy
job_id="$(read_id.py test_job)"

# In JSON mode the command never fails the process: the outcome is carried in
# "status" so the workspace caller can tell a benign no-op apart from an error.

title "A selector that matches nothing is reported as skipped, exit 0"
echo
errcode $CLI bundle config-remote-sync --select-ids "jobs:no-such-id-999" -o json

title "A malformed selector is reported as failed, exit 0"
echo
errcode $CLI bundle config-remote-sync --select-ids "no-such-id-999" -o json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
Cloud = true

RecordRequests = false
Ignore = [".databricks", "dummy.whl", "databricks.yml"]

Env.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"

EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"]
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ Files: 6 uploaded, 0 deleted
Resources: 1 created, 0 changed, 0 deleted, 0 unchanged

=== JSON output format{
"status": "success",
"files": [
{
"path": "[TEST_TMP_DIR]/databricks.yml",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ No changes detected.

=== JSON output
{
"status": "success",
"files": null,
"changes": {}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
bundle:
name: config-remote-sync-skipped-state

resources:
jobs:
foo:
name: test job
tasks:
- task_key: main
notebook_task:
notebook_path: /Workspace/Users/tester@databricks.com/notebook

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@

>>> errcode [CLI] bundle config-remote-sync -o json
{
"status": "skipped",
"error": "state snapshot not available: resources state snapshot not found remotely at resources-config-sync-snapshot.json: state snapshot not found",
"files": null,
"changes": {}
}

>>> cat out.requests.txt
{
"engine": "terraform",
"state_source": "local",
"states_available_count": 0,
"error_message": "state snapshot not available: resources state snapshot not found remotely at resources-config-sync-snapshot.json: state snapshot not found",
"error_category": "STATE_NOT_FOUND"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Running config-remote-sync -o json without a prior deploy: the state snapshot
# does not exist. In JSON mode the command exits 0 and reports the outcome as
# skipped, but telemetry still records the STATE_NOT_FOUND error payload.
trace errcode $CLI bundle config-remote-sync -o json

trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_config_remote_sync_event | select(. != null)'

rm out.requests.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# config-remote-sync is not supported on Windows.
GOOS.windows = false

# STATE_NOT_FOUND is a terraform-only path: the terraform engine pulls a config
# snapshot that doesn't exist without a prior deploy, while the direct engine
# does not. Pin the engine so the test exercises that specific outcome.
EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"]
67 changes: 67 additions & 0 deletions bundle/configsync/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,16 @@ package configsync

import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"

"github.com/databricks/cli/bundle"
"github.com/databricks/cli/libs/telemetry"
"github.com/databricks/cli/libs/telemetry/protos"
)

// FileChange represents a change to a bundle configuration file
Expand All @@ -15,12 +21,73 @@ type FileChange struct {
ModifiedContent string `json:"modifiedContent"`
}

// SyncStatus is the outcome the JSON caller (DABs in the Workspace) branches on.
type SyncStatus string

const (
StatusSuccess SyncStatus = "success"
// StatusSkipped: nothing to sync against (state missing, or no selector
// matched a deployed resource). Not an error for the caller.
StatusSkipped SyncStatus = "skipped"
StatusFailed SyncStatus = "failed"
)

// DiffOutput represents the complete output of the config-remote-sync command
type DiffOutput struct {
Status SyncStatus `json:"status"`
Error string `json:"error,omitempty"`
Files []FileChange `json:"files"`
Changes Changes `json:"changes"`
}

// WriteResult renders the result and records the error payload on stats for
// telemetry. In JSON mode the process never fails: the outcome is carried in
// DiffOutput.Status so the caller branches on it, not the exit code. Text mode
// returns the error so the CLI still fails loudly for humans.
func WriteResult(out io.Writer, jsonOutput bool, stats *Stats, files []FileChange, changes Changes, err error) error {
status := StatusSuccess
if err != nil {
if stats.ErrorCategory == "" {
stats.ErrorCategory = protos.BundleConfigRemoteSyncErrorCategoryBundleLoadFailed
}
stats.ErrorMessage = telemetry.ScrubErrorMessage(err.Error())
// Missing state or a selector that matched nothing: nothing to sync, not a failure.
if errors.Is(err, ErrStateSnapshotNotFound) || errors.Is(err, ErrNoMatchingSelector) {
status = StatusSkipped
} else {
status = StatusFailed
}
}

if !jsonOutput {
if err != nil {
return err
}
_, _ = io.WriteString(out, FormatTextOutput(changes))
_, _ = out.Write([]byte{'\n'})
return nil
}

output := DiffOutput{Status: status, Files: files, Changes: changes}
if output.Changes == nil {
// Serialize as {} rather than null so a consumer can iterate it unconditionally.
output.Changes = Changes{}
}
if err != nil {
output.Error = err.Error()
}
result, marshalErr := json.MarshalIndent(output, "", " ")
if marshalErr != nil {
// Cannot produce JSON: fail rather than emit a partial contract.
stats.ErrorCategory = protos.BundleConfigRemoteSyncErrorCategoryOutputFailed
stats.ErrorMessage = telemetry.ScrubErrorMessage(marshalErr.Error())
return fmt.Errorf("failed to marshal output: %w", marshalErr)
}
_, _ = out.Write(result)
_, _ = out.Write([]byte{'\n'})
return nil
}

// SaveFiles writes all file changes to disk.
func SaveFiles(ctx context.Context, b *bundle.Bundle, files []FileChange) error {
for _, file := range files {
Expand Down
18 changes: 17 additions & 1 deletion bundle/configsync/select.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package configsync

import (
"context"
"errors"
"fmt"
"slices"
"strings"
Expand All @@ -10,6 +11,19 @@ import (
"github.com/databricks/cli/libs/log"
)

// ErrNoMatchingSelector marks an all-stale --select-ids batch (no selector
// matched a deployed resource), distinct from a malformed selector: the former
// is drift the caller reports as skipped, the latter is a failure.
var ErrNoMatchingSelector = errors.New("no matching selector")

// noMatchingSelectorError keeps the descriptive message while matching
// ErrNoMatchingSelector via errors.Is, so the sentinel text never leaks into it.
type noMatchingSelectorError struct{ msg string }

func (e *noMatchingSelectorError) Error() string { return e.msg }

func (e *noMatchingSelectorError) Unwrap() error { return ErrNoMatchingSelector }

// IndexDeployedResources indexes the deployed resources by "<type>:<id>",
// mapping to the plan key ("resources.<type>.<name>"). Indexing by the <type>
// component means a selector can only ever match a resource of that exact type,
Expand Down Expand Up @@ -82,7 +96,9 @@ func ResolveResourceSelectors(ctx context.Context, state *dstate.DeploymentState
// nothing, so the caller does not report a spurious success.
if len(keys) == 0 {
resourceType, id, _ := strings.Cut(missing[0], ":")
return nil, fmt.Errorf("no deployed %s resource with id %s; %s", resourceType, id, describeStateIDs(byTypeID, resourceType))
return nil, &noMatchingSelectorError{
msg: fmt.Sprintf("no deployed %s resource with id %s; %s", resourceType, id, describeStateIDs(byTypeID, resourceType)),
}
}

// Some selectors matched: skip the stale ones so the matched resources still
Expand Down
Loading
Loading