diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go new file mode 100644 index 0000000000..e76f70f625 --- /dev/null +++ b/bundle/configsync/diff.go @@ -0,0 +1,47 @@ +package configsync + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/libs/log" +) + +// DetectChanges compares current remote state with the last deployed state +// and returns a map of resource changes. +func DetectChanges(ctx context.Context, b *bundle.Bundle) (map[string]deployplan.Changes, error) { + changes := make(map[string]deployplan.Changes) + + deployBundle := &direct.DeploymentBundle{} + // TODO: for Terraform engine we should read the state file, converted to direct state format, it should be created during deployment + _, statePath := b.StateFilenameDirect(ctx) + + plan, err := deployBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, statePath) + if err != nil { + return nil, fmt.Errorf("failed to calculate plan: %w", err) + } + + for resourceKey, entry := range plan.Plan { + resourceChanges := make(deployplan.Changes) + + if entry.Changes != nil { + for path, changeDesc := range entry.Changes { + // TODO: distinguish action Skip between actual server-side defaults and remote-side changes + if changeDesc.Remote != nil && changeDesc.Action != deployplan.Skip { + resourceChanges[path] = changeDesc + } + } + } + + if len(resourceChanges) != 0 { + changes[resourceKey] = resourceChanges + } + + log.Debugf(ctx, "Resource %s has %d changes", resourceKey, len(resourceChanges)) + } + + return changes, nil +} diff --git a/bundle/configsync/dyn.go b/bundle/configsync/dyn.go new file mode 100644 index 0000000000..87809c826f --- /dev/null +++ b/bundle/configsync/dyn.go @@ -0,0 +1,116 @@ +package configsync + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" +) + +// structpathToDynPath converts a structpath string to a dyn.Path +// Example: "tasks[0].timeout_seconds" -> Path{Key("tasks"), Index(0), Key("timeout_seconds")} +// Also supports "tasks[task_key='my_task']" syntax for array element selection by field value +func structpathToDynPath(_ context.Context, pathStr string, baseValue dyn.Value) (dyn.Path, error) { + node, err := structpath.Parse(pathStr) + if err != nil { + return nil, fmt.Errorf("failed to parse path %s: %w", pathStr, err) + } + + nodes := node.AsSlice() + + var dynPath dyn.Path + currentValue := baseValue + + for _, n := range nodes { + if key, ok := n.StringKey(); ok { + dynPath = append(dynPath, dyn.Key(key)) + + if currentValue.IsValid() { + currentValue, _ = dyn.GetByPath(currentValue, dyn.Path{dyn.Key(key)}) + } + continue + } + + if idx, ok := n.Index(); ok { + dynPath = append(dynPath, dyn.Index(idx)) + + if currentValue.IsValid() { + currentValue, _ = dyn.GetByPath(currentValue, dyn.Path{dyn.Index(idx)}) + } + continue + } + + // Check for key-value selector: [key='value'] + if key, value, ok := n.KeyValue(); ok { + if !currentValue.IsValid() || currentValue.Kind() != dyn.KindSequence { + return nil, fmt.Errorf("cannot apply [key='value'] selector to non-array value at path %s", dynPath.String()) + } + + seq, _ := currentValue.AsSequence() + foundIndex := -1 + + for i, elem := range seq { + keyValue, err := dyn.GetByPath(elem, dyn.Path{dyn.Key(key)}) + if err != nil { + continue + } + + if keyValue.Kind() == dyn.KindString && keyValue.MustString() == value { + foundIndex = i + break + } + } + + if foundIndex == -1 { + return nil, fmt.Errorf("no array element found with %s='%s' at path %s", key, value, dynPath.String()) + } + + dynPath = append(dynPath, dyn.Index(foundIndex)) + currentValue = seq[foundIndex] + continue + } + + if n.DotStar() || n.BracketStar() { + return nil, errors.New("wildcard patterns are not supported in field paths") + } + } + + return dynPath, nil +} + +// dynPathToJSONPointer converts a dyn.Path to RFC 6902 JSON Pointer format +// Example: [Key("resources"), Key("jobs"), Key("my_job")] -> "/resources/jobs/my_job" +// Example: [Key("tasks"), Index(1), Key("timeout")] -> "/tasks/1/timeout" +func dynPathToJSONPointer(path dyn.Path) string { + if len(path) == 0 { + return "" + } + + var builder strings.Builder + for _, component := range path { + builder.WriteString("/") + + // Handle Key components + if key := component.Key(); key != "" { + // Escape special characters per RFC 6902 + // ~ must be escaped as ~0 + // / must be escaped as ~1 + escaped := strings.ReplaceAll(key, "~", "~0") + escaped = strings.ReplaceAll(escaped, "/", "~1") + builder.WriteString(escaped) + continue + } + + // Handle Index components + if idx := component.Index(); idx >= 0 { + builder.WriteString(strconv.Itoa(idx)) + continue + } + } + + return builder.String() +} diff --git a/bundle/configsync/format.go b/bundle/configsync/format.go new file mode 100644 index 0000000000..e4416e0c78 --- /dev/null +++ b/bundle/configsync/format.go @@ -0,0 +1,30 @@ +package configsync + +import ( + "fmt" + "strings" + + "github.com/databricks/cli/bundle/deployplan" +) + +// FormatTextOutput formats the config changes as human-readable text. Useful for debugging +func FormatTextOutput(changes map[string]deployplan.Changes) string { + var output strings.Builder + + if len(changes) == 0 { + output.WriteString("No changes detected.\n") + return output.String() + } + + output.WriteString(fmt.Sprintf("Detected changes in %d resource(s):\n\n", len(changes))) + + for resourceKey, resourceChanges := range changes { + output.WriteString(fmt.Sprintf("Resource: %s\n", resourceKey)) + + for path, changeDesc := range resourceChanges { + output.WriteString(fmt.Sprintf(" %s: %s\n", path, changeDesc.Action)) + } + } + + return output.String() +} diff --git a/bundle/configsync/output.go b/bundle/configsync/output.go new file mode 100644 index 0000000000..fad2fd7636 --- /dev/null +++ b/bundle/configsync/output.go @@ -0,0 +1,39 @@ +package configsync + +import ( + "context" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deployplan" +) + +// FileChange represents a change to a bundle configuration file +type FileChange struct { + Path string `json:"path"` + OriginalContent string `json:"originalContent"` + ModifiedContent string `json:"modifiedContent"` +} + +// DiffOutput represents the complete output of the config-remote-sync command +type DiffOutput struct { + Files []FileChange `json:"files"` + Changes map[string]deployplan.Changes `json:"changes"` +} + +// SaveFiles writes all file changes to disk. +func SaveFiles(ctx context.Context, b *bundle.Bundle, files []FileChange) error { + for _, file := range files { + err := os.MkdirAll(filepath.Dir(file.Path), 0o755) + if err != nil { + return err + } + + err = os.WriteFile(file.Path, []byte(file.ModifiedContent), 0o644) + if err != nil { + return err + } + } + return nil +} diff --git a/bundle/configsync/output_test.go b/bundle/configsync/output_test.go new file mode 100644 index 0000000000..1b35b807d8 --- /dev/null +++ b/bundle/configsync/output_test.go @@ -0,0 +1,89 @@ +package configsync + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSaveFiles_Success(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + + yamlPath := filepath.Join(tmpDir, "subdir", "databricks.yml") + modifiedContent := `resources: + jobs: + test_job: + name: "Updated Job" + timeout_seconds: 7200 +` + + files := []FileChange{ + { + Path: yamlPath, + OriginalContent: "original content", + ModifiedContent: modifiedContent, + }, + } + + err := SaveFiles(ctx, &bundle.Bundle{}, files) + require.NoError(t, err) + + _, err = os.Stat(yamlPath) + require.NoError(t, err) + + content, err := os.ReadFile(yamlPath) + require.NoError(t, err) + assert.Equal(t, modifiedContent, string(content)) + + _, err = os.Stat(filepath.Dir(yamlPath)) + require.NoError(t, err) +} + +func TestSaveFiles_MultipleFiles(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + + file1Path := filepath.Join(tmpDir, "file1.yml") + file2Path := filepath.Join(tmpDir, "subdir", "file2.yml") + content1 := "content for file 1" + content2 := "content for file 2" + + files := []FileChange{ + { + Path: file1Path, + OriginalContent: "original 1", + ModifiedContent: content1, + }, + { + Path: file2Path, + OriginalContent: "original 2", + ModifiedContent: content2, + }, + } + + err := SaveFiles(ctx, &bundle.Bundle{}, files) + require.NoError(t, err) + + content, err := os.ReadFile(file1Path) + require.NoError(t, err) + assert.Equal(t, content1, string(content)) + + content, err = os.ReadFile(file2Path) + require.NoError(t, err) + assert.Equal(t, content2, string(content)) +} + +func TestSaveFiles_EmptyList(t *testing.T) { + ctx := context.Background() + + err := SaveFiles(ctx, &bundle.Bundle{}, []FileChange{}) + require.NoError(t, err) +} diff --git a/bundle/configsync/patch.go b/bundle/configsync/patch.go new file mode 100644 index 0000000000..6510e5bccc --- /dev/null +++ b/bundle/configsync/patch.go @@ -0,0 +1,206 @@ +package configsync + +import ( + "context" + "fmt" + "os" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" + "github.com/palantir/pkg/yamlpatch/gopkgv3yamlpatcher" + "github.com/palantir/pkg/yamlpatch/yamlpatch" +) + +// applyChanges applies all field changes to a YAML +func applyChanges(ctx context.Context, filePath string, fieldLocations fieldLocations, targetName string) (string, error) { + content, err := os.ReadFile(filePath) + if err != nil { + return "", fmt.Errorf("failed to read file %s: %w", filePath, err) + } + + var operations yamlpatch.Patch + for jsonPointer, changeDesc := range fieldLocations { + yamlValue := changeDesc.Remote + + jsonPointers := []string{jsonPointer} + if targetName != "" { + targetPrefix := "/targets/" + targetName + jsonPointers = append(jsonPointers, targetPrefix+jsonPointer) + } + + var successfulPath string + var opType string + + // Try replace operation first (for existing fields) + for _, jsonPointer := range jsonPointers { + path, err := yamlpatch.ParsePath(jsonPointer) + if err != nil { + continue + } + + testOp := yamlpatch.Operation{ + Type: yamlpatch.OperationReplace, + Path: path, + Value: yamlValue, + } + + patcher := gopkgv3yamlpatcher.New(gopkgv3yamlpatcher.IndentSpaces(2)) + _, err = patcher.Apply(content, yamlpatch.Patch{testOp}) + if err == nil { + successfulPath = jsonPointer + opType = yamlpatch.OperationReplace + break + } + } + + // If replace failed, try add operation (for new fields) + if successfulPath == "" { + for _, jsonPointer := range jsonPointers { + path, err := yamlpatch.ParsePath(jsonPointer) + if err != nil { + continue + } + + testOp := yamlpatch.Operation{ + Type: yamlpatch.OperationAdd, + Path: path, + Value: yamlValue, + } + + patcher := gopkgv3yamlpatcher.New(gopkgv3yamlpatcher.IndentSpaces(2)) + _, err = patcher.Apply(content, yamlpatch.Patch{testOp}) + if err == nil { + successfulPath = jsonPointer + opType = yamlpatch.OperationAdd + break + } + } + } + + if successfulPath == "" { + log.Warnf(ctx, "Failed to find valid path for %s", jsonPointers) + continue + } + + path, err := yamlpatch.ParsePath(successfulPath) + if err != nil { + log.Warnf(ctx, "Failed to parse JSON Pointer %s: %v", successfulPath, err) + continue + } + + op := yamlpatch.Operation{ + Type: opType, + Path: path, + Value: yamlValue, + } + operations = append(operations, op) + } + + patcher := gopkgv3yamlpatcher.New(gopkgv3yamlpatcher.IndentSpaces(2)) + modifiedContent, err := patcher.Apply(content, operations) + if err != nil { + return "", fmt.Errorf("failed to apply patches to %s: %w", filePath, err) + } + + return string(modifiedContent), nil +} + +type fieldLocations map[string]*deployplan.ChangeDesc + +// getFieldLocations builds a map from file paths to lists of field changes +func getFieldLocations(ctx context.Context, b *bundle.Bundle, changes map[string]deployplan.Changes) (map[string]fieldLocations, error) { + configValue := b.Config.Value() + locationsByFile := make(map[string]fieldLocations) + + for resourceKey, resourceChanges := range changes { + for fieldPath, changeDesc := range resourceChanges { + fullPath := resourceKey + "." + fieldPath + path, err := structpathToDynPath(ctx, fullPath, configValue) + if err != nil { + log.Warnf(ctx, "Failed to convert path %s to dyn.Path: %v", fullPath, err) + continue + } + + value, err := dyn.GetByPath(configValue, path) + if err != nil { + log.Debugf(ctx, "Path %s not found in config: %v", path.String(), err) + continue + } + + filePath := value.Location().File + + // If field has no location, find the parent resource's location to then add a new field + if filePath == "" { + filePath = findResourceFileLocation(ctx, b, resourceKey) + if filePath == "" { + continue + } + log.Debugf(ctx, "Field %s has no location, using resource location: %s", fullPath, filePath) + } + + jsonPointer := dynPathToJSONPointer(path) + + if _, ok := locationsByFile[filePath]; !ok { + locationsByFile[filePath] = make(fieldLocations) + } + locationsByFile[filePath][jsonPointer] = changeDesc + } + } + + return locationsByFile, nil +} + +// findResourceFileLocation finds the file where a resource is defined. +// It checks both the root resources and target-specific overrides, +// preferring the target override if it exists. +func findResourceFileLocation(_ context.Context, b *bundle.Bundle, resourceKey string) string { + targetName := b.Config.Bundle.Target + + // Try target override first if we have a target + if targetName != "" { + targetPath := "targets." + targetName + "." + resourceKey + loc := b.Config.GetLocation(targetPath) + if loc.File != "" { + return loc.File + } + } + + // Fall back to root resource location + loc := b.Config.GetLocation(resourceKey) + return loc.File +} + +// ApplyChangesToYAML generates YAML files for the given changes. +func ApplyChangesToYAML(ctx context.Context, b *bundle.Bundle, changes map[string]deployplan.Changes) ([]FileChange, error) { + locationsByFile, err := getFieldLocations(ctx, b, changes) + if err != nil { + return nil, err + } + + var result []FileChange + targetName := b.Config.Bundle.Target + + for filePath, jsonPointers := range locationsByFile { + originalContent, err := os.ReadFile(filePath) + if err != nil { + log.Warnf(ctx, "Failed to read file %s: %v", filePath, err) + continue + } + + modifiedContent, err := applyChanges(ctx, filePath, jsonPointers, targetName) + if err != nil { + log.Warnf(ctx, "Failed to apply changes to file %s: %v", filePath, err) + continue + } + + result = append(result, FileChange{ + Path: filePath, + OriginalContent: string(originalContent), + ModifiedContent: modifiedContent, + }) + } + + return result, nil +} diff --git a/bundle/configsync/patch_test.go b/bundle/configsync/patch_test.go new file mode 100644 index 0000000000..a0a76ef4a2 --- /dev/null +++ b/bundle/configsync/patch_test.go @@ -0,0 +1,617 @@ +package configsync + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestApplyChangesToYAML_SimpleFieldChange(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + timeout_seconds: 3600 + tasks: + - task_key: "main_task" + notebook_task: + notebook_path: "/path/to/notebook" +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 3600, + Remote: 7200, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, yamlPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].OriginalContent, "timeout_seconds: 3600") + assert.Contains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 7200") + assert.NotContains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 3600") +} + +func TestApplyChangesToYAML_NestedFieldChange(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "main_task" + notebook_task: + notebook_path: "/path/to/notebook" + timeout_seconds: 1800 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "tasks[0].timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Contains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 3600") + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + testJob := jobs["test_job"].(map[string]any) + tasks := testJob["tasks"].([]any) + task0 := tasks[0].(map[string]any) + + assert.Equal(t, 3600, task0["timeout_seconds"]) +} + +func TestApplyChangesToYAML_ArrayKeyValueAccess(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "setup_task" + notebook_task: + notebook_path: "/setup" + timeout_seconds: 600 + - task_key: "main_task" + notebook_task: + notebook_path: "/main" + timeout_seconds: 1800 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "tasks[task_key='main_task'].timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + testJob := jobs["test_job"].(map[string]any) + tasks := testJob["tasks"].([]any) + + task0 := tasks[0].(map[string]any) + assert.Equal(t, "setup_task", task0["task_key"]) + assert.Equal(t, 600, task0["timeout_seconds"]) + + task1 := tasks[1].(map[string]any) + assert.Equal(t, "main_task", task1["task_key"]) + assert.Equal(t, 3600, task1["timeout_seconds"]) +} + +func TestApplyChangesToYAML_MultipleResourcesSameFile(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + job1: + name: "Job 1" + timeout_seconds: 3600 + job2: + name: "Job 2" + timeout_seconds: 1800 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.job1": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 3600, + Remote: 7200, + }, + }, + "resources.jobs.job2": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + + require.Len(t, fileChanges, 1) + assert.Equal(t, yamlPath, fileChanges[0].Path) + + assert.Contains(t, fileChanges[0].ModifiedContent, "job1") + assert.Contains(t, fileChanges[0].ModifiedContent, "job2") + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + + job1 := jobs["job1"].(map[string]any) + assert.Equal(t, 7200, job1["timeout_seconds"]) + + job2 := jobs["job2"].(map[string]any) + assert.Equal(t, 3600, job2["timeout_seconds"]) +} + +func TestApplyChangesToYAML_ResourceNotFound(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + existing_job: + name: "Existing Job" +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.nonexistent_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + + assert.Len(t, fileChanges, 0) +} + +func TestApplyChangesToYAML_InvalidFieldPath(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + timeout_seconds: 3600 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "invalid[[[path": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Remote: 7200, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + + if len(fileChanges) > 0 { + assert.Contains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 3600") + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + testJob := jobs["test_job"].(map[string]any) + assert.Equal(t, 3600, testJob["timeout_seconds"]) + } +} + +func TestApplyChangesToYAML_Include(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + mainYAML := `bundle: + name: test-bundle + +include: + - "targets/*.yml" +` + + mainPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(mainPath, []byte(mainYAML), 0o644) + require.NoError(t, err) + + targetsDir := filepath.Join(tmpDir, "targets") + err = os.MkdirAll(targetsDir, 0o755) + require.NoError(t, err) + + devYAML := `resources: + jobs: + dev_job: + name: "Dev Job" + timeout_seconds: 1800 +` + + devPath := filepath.Join(targetsDir, "dev.yml") + err = os.WriteFile(devPath, []byte(devYAML), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.dev_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, devPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].OriginalContent, "timeout_seconds: 1800") + assert.Contains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 3600") + assert.NotContains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 1800") +} + +func TestGenerateYAMLFiles_TargetOverride(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + mainYAML := `bundle: + name: test-bundle +targets: + dev: + resources: + jobs: + dev_job: + name: "Dev Job" + timeout_seconds: 1800 +` + + mainPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(mainPath, []byte(mainYAML), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + diags := bundle.Apply(ctx, b, mutator.SelectTarget("dev")) + require.NoError(t, diags.Error()) + + changes := map[string]deployplan.Changes{ + "resources.jobs.dev_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, mainPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 3600") +} + +func TestApplyChangesToYAML_WithStructValues(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + timeout_seconds: 3600 + email_notifications: + on_success: + - old@example.com +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + type EmailNotifications struct { + OnSuccess []string `json:"on_success,omitempty" yaml:"on_success,omitempty"` + OnFailure []string `json:"on_failure,omitempty" yaml:"on_failure,omitempty"` + } + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "email_notifications": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Remote: &EmailNotifications{ + OnSuccess: []string{"success@example.com"}, + OnFailure: []string{"failure@example.com"}, + }, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, yamlPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].OriginalContent, "on_success:") + assert.Contains(t, fileChanges[0].OriginalContent, "old@example.com") + assert.Contains(t, fileChanges[0].ModifiedContent, "success@example.com") + assert.Contains(t, fileChanges[0].ModifiedContent, "failure@example.com") + + type JobsConfig struct { + Name string `yaml:"name"` + TimeoutSeconds int `yaml:"timeout_seconds"` + EmailNotifications *EmailNotifications `yaml:"email_notifications,omitempty"` + } + + type ResourcesConfig struct { + Jobs map[string]JobsConfig `yaml:"jobs"` + } + + type RootConfig struct { + Resources ResourcesConfig `yaml:"resources"` + } + + var result RootConfig + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + testJob := result.Resources.Jobs["test_job"] + assert.Equal(t, "Test Job", testJob.Name) + assert.Equal(t, 3600, testJob.TimeoutSeconds) + require.NotNil(t, testJob.EmailNotifications) + assert.Equal(t, []string{"success@example.com"}, testJob.EmailNotifications.OnSuccess) + assert.Equal(t, []string{"failure@example.com"}, testJob.EmailNotifications.OnFailure) +} + +func TestApplyChangesToYAML_PreserveComments(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `# test_comment0 +resources: + # test_comment1 + jobs: + test_job: + # test_comment2 + name: "Test Job" + # test_comment3 + timeout_seconds: 3600 + # test_comment4 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Remote: 7200, + }, + "name": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Remote: "New Test Job", + }, + "tags": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Remote: map[string]string{ + "test": "value", + }, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, yamlPath, fileChanges[0].Path) + + assert.Contains(t, fileChanges[0].ModifiedContent, "# test_comment0") + assert.Contains(t, fileChanges[0].ModifiedContent, "# test_comment1") + assert.Contains(t, fileChanges[0].ModifiedContent, "# test_comment2") + assert.Contains(t, fileChanges[0].ModifiedContent, "# test_comment3") + assert.Contains(t, fileChanges[0].ModifiedContent, "# test_comment4") +} + +func TestApplyChangesToYAML_FieldWithoutFileLocation(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + // Create bundle config with a job that doesn't define edit_mode + yamlContent := `bundle: + name: test-bundle +targets: + dev: + resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "main" + notebook_task: + notebook_path: "/notebook" +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + diags := bundle.Apply(ctx, b, mutator.SelectTarget("dev")) + require.NoError(t, diags.Error()) + + // Manually add edit_mode field to the config without a file location + // This simulates a server-side default field that was merged into the config + err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.SetByPath(v, dyn.MustPathFromString("resources.jobs.test_job.edit_mode"), dyn.V("UI_LOCKED")) + }) + require.NoError(t, err) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "edit_mode": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: "UI_LOCKED", + Remote: "EDITABLE", + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, yamlPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].ModifiedContent, "edit_mode: EDITABLE") +} diff --git a/cmd/bundle/debug.go b/cmd/bundle/debug.go index b912e14fe2..f0bd6c83ed 100644 --- a/cmd/bundle/debug.go +++ b/cmd/bundle/debug.go @@ -16,5 +16,6 @@ func newDebugCommand() *cobra.Command { cmd.AddCommand(debug.NewTerraformCommand()) cmd.AddCommand(debug.NewRefSchemaCommand()) cmd.AddCommand(debug.NewStatesCommand()) + cmd.AddCommand(debug.NewConfigRemoteSyncCommand()) return cmd } diff --git a/cmd/bundle/debug/config_remote_sync.go b/cmd/bundle/debug/config_remote_sync.go new file mode 100644 index 0000000000..cb5d6f0aa4 --- /dev/null +++ b/cmd/bundle/debug/config_remote_sync.go @@ -0,0 +1,80 @@ +package debug + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/configsync" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/flags" + "github.com/spf13/cobra" +) + +func NewConfigRemoteSyncCommand() *cobra.Command { + var save bool + + cmd := &cobra.Command{ + Use: "config-remote-sync", + Short: "Sync remote resource changes to bundle configuration (experimental)", + Long: `Compares deployed state with current remote state and generates updated configuration files. + +When --save is specified, writes updated YAML files to disk. +Otherwise, outputs diff without modifying files. + +Examples: + # Show diff without saving + databricks bundle debug config-remote-sync + + # Show diff and save to files + databricks bundle debug config-remote-sync --save`, + Hidden: true, // Used by DABs in the Workspace only + } + + cmd.Flags().BoolVar(&save, "save", false, "Write updated config files to disk") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + b, _, err := utils.ProcessBundleRet(cmd, utils.ProcessOptions{}) + if err != nil { + return err + } + + ctx := cmd.Context() + changes, err := configsync.DetectChanges(ctx, b) + if err != nil { + return fmt.Errorf("failed to detect changes: %w", err) + } + + files, err := configsync.ApplyChangesToYAML(ctx, b, changes) + if err != nil { + return fmt.Errorf("failed to generate YAML files: %w", err) + } + + if save { + if err := configsync.SaveFiles(ctx, b, files); err != nil { + return fmt.Errorf("failed to save files: %w", err) + } + } + + var result []byte + if root.OutputType(cmd) == flags.OutputJSON { + diffOutput := &configsync.DiffOutput{ + Files: files, + Changes: changes, + } + result, err = json.MarshalIndent(diffOutput, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal output: %w", err) + } + } else if root.OutputType(cmd) == flags.OutputText { + result = []byte(configsync.FormatTextOutput(changes)) + } + + out := cmd.OutOrStdout() + _, _ = out.Write(result) + _, _ = out.Write([]byte{'\n'}) + return nil + } + + return cmd +} diff --git a/go.mod b/go.mod index dd1a073f73..c54855bd94 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,8 @@ require ( // Dependencies for experimental MCP commands require github.com/google/jsonschema-go v0.4.2 // MIT +require github.com/palantir/pkg/yamlpatch v1.5.0 + require ( cloud.google.com/go/auth v0.16.5 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect @@ -63,6 +65,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/zclconf/go-cty v1.16.4 // indirect diff --git a/go.sum b/go.sum index 8be5ce6c28..9fea9a2901 100644 --- a/go.sum +++ b/go.sum @@ -113,10 +113,14 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/nwidger/jsoncolor v0.3.2 h1:rVJJlwAWDJShnbTYOQ5RM7yTA20INyKXlJ/fg4JMhHQ= github.com/nwidger/jsoncolor v0.3.2/go.mod h1:Cs34umxLbJvgBMnVNVqhji9BhoT/N/KinHqZptQ7cf4= +github.com/palantir/pkg/yamlpatch v1.5.0 h1:186RUlcHFVf64onUhaI7nUCPzPIaRTQ5HJlKuv0d6NM= +github.com/palantir/pkg/yamlpatch v1.5.0/go.mod h1:45cYAIiv9E0MiZnHjIIT2hGqi6Wah/DL6J1omJf2ny0= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=