diff --git a/acceptance/bin/edit_resource.py b/acceptance/bin/edit_resource.py index e42cb59562..7bca11a070 100755 --- a/acceptance/bin/edit_resource.py +++ b/acceptance/bin/edit_resource.py @@ -32,6 +32,14 @@ def set(self, job_id, value): return run([CLI, "jobs", "reset", job_id, "--json", json.dumps(payload)]) +class pipelines: + def get(self, pipeline_id): + return run_json([CLI, "pipelines", "get", pipeline_id])["spec"] + + def set(self, pipeline_id, value): + return run([CLI, "pipelines", "update", pipeline_id, "--json", json.dumps(value)]) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("type") diff --git a/acceptance/bundle/config-remote-sync/config_edits/databricks.yml b/acceptance/bundle/config-remote-sync/config_edits/databricks.yml new file mode 100644 index 0000000000..40626744d0 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/config_edits/databricks.yml @@ -0,0 +1,42 @@ +bundle: + name: test-bundle + +resources: + jobs: + my_job: + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + new_cluster: + spark_version: 13.3.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + +targets: + default: + resources: + jobs: + my_job: + email_notifications: + on_success: + - success@example.com + parameters: + - name: catalog + default: main + - name: env + default: dev + trigger: + periodic: + interval: 1 + unit: DAYS + tags: + env: dev + version: v1 + max_concurrent_runs: 1 + environments: + - environment_key: default + spec: + environment_version: "3" + dependencies: + - ./*.whl diff --git a/acceptance/bundle/config-remote-sync/config_edits/out.test.toml b/acceptance/bundle/config-remote-sync/config_edits/out.test.toml new file mode 100644 index 0000000000..9620d0ee57 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/config_edits/out.test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true + +[GOOS] + windows = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/config-remote-sync/config_edits/output.txt b/acceptance/bundle/config-remote-sync/config_edits/output.txt new file mode 100644 index 0000000000..202e2b6363 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/config_edits/output.txt @@ -0,0 +1,51 @@ +Uploading dummy.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Edit config locally + +=== Edit job remotely + +=== Detect and save changes +Detected changes in 1 resource(s): + +Resource: resources.jobs.my_job + email_notifications.on_failure[0]: update + max_concurrent_runs: update + tags['env']: update + + +=== Configuration changes + bundle: + name: test-bundle +- + resources: + jobs: + my_job: + spark_version: 13.3.x-scala2.12 + node_type_id: [NODE_TYPE_ID] + num_workers: 1 +- + targets: + default: + resources: + on_success: + - success@example.com + on_failure: +- - config-failure@example.com ++ - remote-failure@example.com + parameters: + - name: catalog + default: main + interval: 1 + unit: DAYS + tags: +- env: config-production +- max_concurrent_runs: 3 ++ env: remote-staging ++ max_concurrent_runs: 5 + timeout_seconds: 3600 + environments: + - environment_key: default diff --git a/acceptance/bundle/config-remote-sync/config_edits/script b/acceptance/bundle/config-remote-sync/config_edits/script new file mode 100755 index 0000000000..420c528e1f --- /dev/null +++ b/acceptance/bundle/config-remote-sync/config_edits/script @@ -0,0 +1,96 @@ +#!/bin/bash + +touch dummy.whl +$CLI bundle deploy +job_id="$(read_id.py my_job)" + +title "Edit config locally" +echo +# Case 1: Add field in config (on_failure) - will also be added remotely with different value +# Case 2: Add field in config (timeout_seconds) - will be removed remotely +# Case 3: Remove field from config (version tag) - will also be removed remotely +# Case 4: Update field in config (max_concurrent_runs) - will also be updated remotely with different value +# Case 5: Update field in config (tags.env) - will also be updated remotely with different value +cat > databricks.yml <<'EOF' +bundle: + name: test-bundle + +resources: + jobs: + my_job: + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + new_cluster: + spark_version: 13.3.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 + +targets: + default: + resources: + jobs: + my_job: + email_notifications: + on_success: + - success@example.com + on_failure: + - config-failure@example.com + parameters: + - name: catalog + default: main + - name: env + default: dev + trigger: + periodic: + interval: 1 + unit: DAYS + tags: + env: config-production + max_concurrent_runs: 3 + timeout_seconds: 3600 + environments: + - environment_key: default + spec: + environment_version: "3" + dependencies: + - ./*.whl +EOF + +title "Edit job remotely" +echo +# Case 1: Add field remotely (on_failure) - also added in config with different value +edit_resource.py jobs $job_id < permissions.json < out.json +cat out.json diff --git a/acceptance/bundle/config-remote-sync/output_json/test.toml b/acceptance/bundle/config-remote-sync/output_json/test.toml new file mode 100644 index 0000000000..1e2683f35a --- /dev/null +++ b/acceptance/bundle/config-remote-sync/output_json/test.toml @@ -0,0 +1,8 @@ +RecordRequests = false +Ignore = [".databricks", "dummy.whl", "out.json"] + +[Env] +DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true" + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/config-remote-sync/output_no_changes/databricks.yml b/acceptance/bundle/config-remote-sync/output_no_changes/databricks.yml new file mode 100644 index 0000000000..970dfcc3a5 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/output_no_changes/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: test-bundle + +resources: + jobs: + test_job: + max_concurrent_runs: 1 + tasks: + - task_key: main + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/notebook + new_cluster: + spark_version: 13.3.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 diff --git a/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml b/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml new file mode 100644 index 0000000000..a84c0304e6 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false + +[GOOS] + windows = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/config-remote-sync/output_no_changes/output.txt b/acceptance/bundle/config-remote-sync/output_no_changes/output.txt new file mode 100644 index 0000000000..e4af37c0aa --- /dev/null +++ b/acceptance/bundle/config-remote-sync/output_no_changes/output.txt @@ -0,0 +1,15 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Check for changes immediately after deploymentNo changes detected. + + +=== Text outputNo changes detected. + + +=== JSON output{ + "files": null, + "changes": {} +} diff --git a/acceptance/bundle/config-remote-sync/output_no_changes/script b/acceptance/bundle/config-remote-sync/output_no_changes/script new file mode 100755 index 0000000000..595473c1ae --- /dev/null +++ b/acceptance/bundle/config-remote-sync/output_no_changes/script @@ -0,0 +1,14 @@ +#!/bin/bash + +touch dummy.whl +$CLI bundle deploy + +title "Check for changes immediately after deployment" +$CLI bundle config-remote-sync + +title "Text output" +$CLI bundle config-remote-sync | contains.py "No changes detected" + +title "JSON output" +$CLI bundle config-remote-sync -o json > out.json +cat out.json diff --git a/acceptance/bundle/config-remote-sync/output_no_changes/test.toml b/acceptance/bundle/config-remote-sync/output_no_changes/test.toml new file mode 100644 index 0000000000..1e2683f35a --- /dev/null +++ b/acceptance/bundle/config-remote-sync/output_no_changes/test.toml @@ -0,0 +1,8 @@ +RecordRequests = false +Ignore = [".databricks", "dummy.whl", "out.json"] + +[Env] +DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true" + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/databricks.yml b/acceptance/bundle/config-remote-sync/pipeline_fields/databricks.yml new file mode 100644 index 0000000000..a5867bccaa --- /dev/null +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: test-bundle + +resources: + pipelines: + my_pipeline: + catalog: main + schema: default + configuration: + key1: value1 + notifications: + - email_recipients: + - success@example.com + alerts: + - on-update-success + permissions: + - level: CAN_VIEW + user_name: viewer@example.com + libraries: + - notebook: + path: /Users/{{workspace_user_name}}/notebook diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml b/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml new file mode 100644 index 0000000000..9620d0ee57 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = true + +[GOOS] + windows = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt new file mode 100644 index 0000000000..ea9c35e38f --- /dev/null +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/output.txt @@ -0,0 +1,39 @@ +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Modify pipeline fields remotely +=== Detect and save changes +Detected changes in 1 resource(s): + +Resource: resources.pipelines.my_pipeline + configuration['key2']: update + notifications[0].alerts: update + notifications[0].email_recipients: update + schema: update + + +=== Configuration changes + bundle: + name: test-bundle +- + resources: + pipelines: + my_pipeline: + catalog: main +- schema: default ++ schema: prod + configuration: + key1: value1 ++ key2: value2 + notifications: + - email_recipients: + - success@example.com ++ - admin@example.com + alerts: + - on-update-success ++ - on-update-failure + permissions: + - level: CAN_VIEW + user_name: viewer@example.com diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/script b/acceptance/bundle/config-remote-sync/pipeline_fields/script new file mode 100755 index 0000000000..b452faee36 --- /dev/null +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/script @@ -0,0 +1,38 @@ +#!/bin/bash + +$CLI bundle deploy +pipeline_id="$(read_id.py my_pipeline)" + + +title "Modify pipeline fields remotely" +edit_resource.py pipelines $pipeline_id < permissions.json < "tasks[1].name" +func resolveSelectors(pathStr string, b *bundle.Bundle) (string, error) { + node, err := structpath.Parse(pathStr) + if err != nil { + return "", fmt.Errorf("failed to parse path %s: %w", pathStr, err) + } + + nodes := node.AsSlice() + var builder strings.Builder + currentValue := b.Config.Value() + + for i, n := range nodes { + if key, ok := n.StringKey(); ok { + if i > 0 { + builder.WriteString(".") + } + builder.WriteString(key) + + if currentValue.IsValid() { + currentValue, _ = dyn.GetByPath(currentValue, dyn.Path{dyn.Key(key)}) + } + continue + } + + if idx, ok := n.Index(); ok { + builder.WriteString("[") + builder.WriteString(strconv.Itoa(idx)) + builder.WriteString("]") + + if currentValue.IsValid() { + currentValue, _ = dyn.GetByPath(currentValue, dyn.Path{dyn.Index(idx)}) + } + continue + } + + // Check for key-value selector: [key='value'] + if key, value, ok := n.KeyValue(); ok { + if !currentValue.IsValid() || currentValue.Kind() != dyn.KindSequence { + return "", fmt.Errorf("cannot apply [%s='%s'] selector to non-array value in path %s", key, value, pathStr) + } + + seq, _ := currentValue.AsSequence() + foundIndex := -1 + + for i, elem := range seq { + keyValue, err := dyn.GetByPath(elem, dyn.Path{dyn.Key(key)}) + if err != nil { + continue + } + + if keyValue.Kind() == dyn.KindString && keyValue.MustString() == value { + foundIndex = i + break + } + } + + if foundIndex == -1 { + return "", fmt.Errorf("no array element found with %s='%s' in path %s", key, value, pathStr) + } + + builder.WriteString("[") + builder.WriteString(strconv.Itoa(foundIndex)) + builder.WriteString("]") + currentValue = seq[foundIndex] + continue + } + + if n.DotStar() || n.BracketStar() { + return "", errors.New("wildcard patterns are not supported in field paths") + } + } + + return builder.String(), nil +} diff --git a/bundle/configsync/dyn_test.go b/bundle/configsync/dyn_test.go new file mode 100644 index 0000000000..d60ea51b30 --- /dev/null +++ b/bundle/configsync/dyn_test.go @@ -0,0 +1,206 @@ +package configsync + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/logdiag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestResolveSelectors_NoSelectors(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" +` + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + result, err := resolveSelectors("resources.jobs.test_job.name", b) + require.NoError(t, err) + assert.Equal(t, "resources.jobs.test_job.name", result) +} + +func TestResolveSelectors_NumericIndices(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + tasks: + - task_key: "task1" + - task_key: "task2" +` + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + result, err := resolveSelectors("resources.jobs.test_job.tasks[0].task_key", b) + require.NoError(t, err) + assert.Equal(t, "resources.jobs.test_job.tasks[0].task_key", result) + + result, err = resolveSelectors("resources.jobs.test_job.tasks[1].task_key", b) + require.NoError(t, err) + assert.Equal(t, "resources.jobs.test_job.tasks[1].task_key", result) +} + +func TestResolveSelectors_KeyValueSelector(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + tasks: + - task_key: "setup" + notebook_task: + notebook_path: "/setup" + - task_key: "main" + notebook_task: + notebook_path: "/main" +` + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + result, err := resolveSelectors("resources.jobs.test_job.tasks[task_key='main'].notebook_task.notebook_path", b) + require.NoError(t, err) + assert.Equal(t, "resources.jobs.test_job.tasks[1].notebook_task.notebook_path", result) + + result, err = resolveSelectors("resources.jobs.test_job.tasks[task_key='setup'].notebook_task.notebook_path", b) + require.NoError(t, err) + assert.Equal(t, "resources.jobs.test_job.tasks[0].notebook_task.notebook_path", result) +} + +func TestResolveSelectors_SelectorNotFound(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + tasks: + - task_key: "setup" + notebook_task: + notebook_path: "/setup" +` + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + _, err = resolveSelectors("resources.jobs.test_job.tasks[task_key='nonexistent'].notebook_task.notebook_path", b) + require.Error(t, err) + assert.Contains(t, err.Error(), "no array element found with task_key='nonexistent'") +} + +func TestResolveSelectors_SelectorOnNonArray(t *testing.T) { + ctx := cmdio.MockDiscard(logdiag.InitContext(context.Background())) + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" +` + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + _, err = resolveSelectors("resources.jobs.test_job[task_key='main'].name", b) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot apply [task_key='main'] selector to non-array value") +} + +func TestResolveSelectors_NestedSelectors(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + tasks: + - task_key: "setup" + libraries: + - pypi: + package: "pandas" + - task_key: "main" + libraries: + - pypi: + package: "numpy" +` + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + result, err := resolveSelectors("resources.jobs.test_job.tasks[task_key='main'].libraries[0].pypi.package", b) + require.NoError(t, err) + assert.Equal(t, "resources.jobs.test_job.tasks[1].libraries[0].pypi.package", result) +} + +func TestResolveSelectors_WildcardNotSupported(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + tasks: + - task_key: "task1" + notebook_task: + notebook_path: "/notebook" +` + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + _, err = resolveSelectors("resources.jobs.test_job.tasks.*.task_key", b) + require.Error(t, err) + assert.Contains(t, err.Error(), "wildcard patterns are not supported") +} diff --git a/bundle/configsync/format.go b/bundle/configsync/format.go new file mode 100644 index 0000000000..d22e621d95 --- /dev/null +++ b/bundle/configsync/format.go @@ -0,0 +1,45 @@ +package configsync + +import ( + "fmt" + "sort" + "strings" + + "github.com/databricks/cli/bundle/deployplan" +) + +// FormatTextOutput formats the config changes as human-readable text. Useful for debugging +func FormatTextOutput(changes map[string]deployplan.Changes) string { + var output strings.Builder + + if len(changes) == 0 { + output.WriteString("No changes detected.\n") + return output.String() + } + + output.WriteString(fmt.Sprintf("Detected changes in %d resource(s):\n\n", len(changes))) + + resourceKeys := make([]string, 0, len(changes)) + for key := range changes { + resourceKeys = append(resourceKeys, key) + } + sort.Strings(resourceKeys) + + for _, resourceKey := range resourceKeys { + resourceChanges := changes[resourceKey] + output.WriteString(fmt.Sprintf("Resource: %s\n", resourceKey)) + + paths := make([]string, 0, len(resourceChanges)) + for path := range resourceChanges { + paths = append(paths, path) + } + sort.Strings(paths) + + for _, path := range paths { + changeDesc := resourceChanges[path] + output.WriteString(fmt.Sprintf(" %s: %s\n", path, changeDesc.Action)) + } + } + + return output.String() +} diff --git a/bundle/configsync/output.go b/bundle/configsync/output.go new file mode 100644 index 0000000000..fad2fd7636 --- /dev/null +++ b/bundle/configsync/output.go @@ -0,0 +1,39 @@ +package configsync + +import ( + "context" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deployplan" +) + +// FileChange represents a change to a bundle configuration file +type FileChange struct { + Path string `json:"path"` + OriginalContent string `json:"originalContent"` + ModifiedContent string `json:"modifiedContent"` +} + +// DiffOutput represents the complete output of the config-remote-sync command +type DiffOutput struct { + Files []FileChange `json:"files"` + Changes map[string]deployplan.Changes `json:"changes"` +} + +// SaveFiles writes all file changes to disk. +func SaveFiles(ctx context.Context, b *bundle.Bundle, files []FileChange) error { + for _, file := range files { + err := os.MkdirAll(filepath.Dir(file.Path), 0o755) + if err != nil { + return err + } + + err = os.WriteFile(file.Path, []byte(file.ModifiedContent), 0o644) + if err != nil { + return err + } + } + return nil +} diff --git a/bundle/configsync/output_test.go b/bundle/configsync/output_test.go new file mode 100644 index 0000000000..1b35b807d8 --- /dev/null +++ b/bundle/configsync/output_test.go @@ -0,0 +1,89 @@ +package configsync + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSaveFiles_Success(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + + yamlPath := filepath.Join(tmpDir, "subdir", "databricks.yml") + modifiedContent := `resources: + jobs: + test_job: + name: "Updated Job" + timeout_seconds: 7200 +` + + files := []FileChange{ + { + Path: yamlPath, + OriginalContent: "original content", + ModifiedContent: modifiedContent, + }, + } + + err := SaveFiles(ctx, &bundle.Bundle{}, files) + require.NoError(t, err) + + _, err = os.Stat(yamlPath) + require.NoError(t, err) + + content, err := os.ReadFile(yamlPath) + require.NoError(t, err) + assert.Equal(t, modifiedContent, string(content)) + + _, err = os.Stat(filepath.Dir(yamlPath)) + require.NoError(t, err) +} + +func TestSaveFiles_MultipleFiles(t *testing.T) { + ctx := context.Background() + + tmpDir := t.TempDir() + + file1Path := filepath.Join(tmpDir, "file1.yml") + file2Path := filepath.Join(tmpDir, "subdir", "file2.yml") + content1 := "content for file 1" + content2 := "content for file 2" + + files := []FileChange{ + { + Path: file1Path, + OriginalContent: "original 1", + ModifiedContent: content1, + }, + { + Path: file2Path, + OriginalContent: "original 2", + ModifiedContent: content2, + }, + } + + err := SaveFiles(ctx, &bundle.Bundle{}, files) + require.NoError(t, err) + + content, err := os.ReadFile(file1Path) + require.NoError(t, err) + assert.Equal(t, content1, string(content)) + + content, err = os.ReadFile(file2Path) + require.NoError(t, err) + assert.Equal(t, content2, string(content)) +} + +func TestSaveFiles_EmptyList(t *testing.T) { + ctx := context.Background() + + err := SaveFiles(ctx, &bundle.Bundle{}, []FileChange{}) + require.NoError(t, err) +} diff --git a/bundle/configsync/patch.go b/bundle/configsync/patch.go new file mode 100644 index 0000000000..a3aeb2fff4 --- /dev/null +++ b/bundle/configsync/patch.go @@ -0,0 +1,251 @@ +package configsync + +import ( + "context" + "encoding/json" + "fmt" + "os" + "reflect" + "sort" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/palantir/pkg/yamlpatch/gopkgv3yamlpatcher" + "github.com/palantir/pkg/yamlpatch/yamlpatch" +) + +type resolvedChanges map[string]*deployplan.ChangeDesc + +// normalizeValue converts values to plain Go types suitable for YAML patching +// by using SDK marshaling which properly handles ForceSendFields and other annotations. +func normalizeValue(v any) (any, error) { + if v == nil { + return nil, nil + } + + switch v.(type) { + case bool, string, int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64: + return v, nil + } + + rv := reflect.ValueOf(v) + rt := rv.Type() + + if rt.Kind() == reflect.Ptr { + rt = rt.Elem() + } + + var data []byte + var err error + + if rt.Kind() == reflect.Struct { + data, err = marshal.Marshal(v) + } else { + data, err = json.Marshal(v) + } + + if err != nil { + return v, fmt.Errorf("failed to marshal value of type %T: %w", v, err) + } + + var normalized any + err = json.Unmarshal(data, &normalized) + if err != nil { + return v, fmt.Errorf("failed to unmarshal value: %w", err) + } + + return normalized, nil +} + +// ApplyChangesToYAML generates YAML files for the given changes. +func ApplyChangesToYAML(ctx context.Context, b *bundle.Bundle, planChanges map[string]deployplan.Changes) ([]FileChange, error) { + changesByFile, err := getResolvedFieldChanges(ctx, b, planChanges) + if err != nil { + return nil, err + } + + var result []FileChange + targetName := b.Config.Bundle.Target + + for filePath, changes := range changesByFile { + originalContent, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", filePath, err) + } + + modifiedContent, err := applyChanges(ctx, filePath, changes, targetName) + if err != nil { + return nil, fmt.Errorf("failed to apply changes to file %s: %w", filePath, err) + } + + if modifiedContent != string(originalContent) { + result = append(result, FileChange{ + Path: filePath, + OriginalContent: string(originalContent), + ModifiedContent: modifiedContent, + }) + } + } + + return result, nil +} + +// applyChanges applies all field changes to a YAML +func applyChanges(ctx context.Context, filePath string, changes resolvedChanges, targetName string) (string, error) { + content, err := os.ReadFile(filePath) + if err != nil { + return "", fmt.Errorf("failed to read file %s: %w", filePath, err) + } + + fieldPaths := make([]string, 0, len(changes)) + for fieldPath := range changes { + fieldPaths = append(fieldPaths, fieldPath) + } + sort.Strings(fieldPaths) + + for _, fieldPath := range fieldPaths { + changeDesc := changes[fieldPath] + jsonPointer := strPathToJSONPointer(fieldPath) + + jsonPointers := []string{jsonPointer} + if targetName != "" { + targetPrefix := "/targets/" + targetName + jsonPointers = append(jsonPointers, targetPrefix+jsonPointer) + } + + hasConfigValue := changeDesc.Old != nil || changeDesc.New != nil + isRemoval := changeDesc.Remote == nil && hasConfigValue + isReplacement := changeDesc.Remote != nil && hasConfigValue + isAddition := changeDesc.Remote != nil && !hasConfigValue + + success := false + var lastErr error + var lastPointer string + + for _, jsonPointer := range jsonPointers { + path, err := yamlpatch.ParsePath(jsonPointer) + if err != nil { + return "", fmt.Errorf("failed to parse JSON Pointer %s: %w", jsonPointer, err) + } + + var testOp yamlpatch.Operation + if isRemoval { + testOp = yamlpatch.Operation{ + Type: yamlpatch.OperationRemove, + Path: path, + } + } else if isReplacement { + normalizedRemote, err := normalizeValue(changeDesc.Remote) + if err != nil { + return "", fmt.Errorf("failed to normalize replacement value for %s: %w", jsonPointer, err) + } + testOp = yamlpatch.Operation{ + Type: yamlpatch.OperationReplace, + Path: path, + Value: normalizedRemote, + } + } else if isAddition { + normalizedRemote, err := normalizeValue(changeDesc.Remote) + if err != nil { + return "", fmt.Errorf("failed to normalize addition value for %s: %w", jsonPointer, err) + } + testOp = yamlpatch.Operation{ + Type: yamlpatch.OperationAdd, + Path: path, + Value: normalizedRemote, + } + } else { + return "", fmt.Errorf("unknown operation type for field %s", fieldPath) + } + + patcher := gopkgv3yamlpatcher.New(gopkgv3yamlpatcher.IndentSpaces(2)) + modifiedContent, err := patcher.Apply(content, yamlpatch.Patch{testOp}) + if err == nil { + content = modifiedContent + log.Debugf(ctx, "Applied %s change to %s", testOp.Type, jsonPointer) + success = true + break + } else { + log.Debugf(ctx, "Failed to apply change to %s: %v", jsonPointer, err) + lastErr = err + lastPointer = jsonPointer + } + } + if !success { + if lastErr != nil { + return "", fmt.Errorf("failed to apply change %s: %w", lastPointer, lastErr) + } + return "", fmt.Errorf("failed to apply change for field %s: no valid target found", fieldPath) + } + } + + return string(content), nil +} + +// getResolvedFieldChanges builds a map from file paths to lists of field changes +func getResolvedFieldChanges(ctx context.Context, b *bundle.Bundle, planChanges map[string]deployplan.Changes) (map[string]resolvedChanges, error) { + resolvedChangesByFile := make(map[string]resolvedChanges) + + for resourceKey, resourceChanges := range planChanges { + for fieldPath, changeDesc := range resourceChanges { + fullPath := resourceKey + "." + fieldPath + + resolvedPath, err := resolveSelectors(fullPath, b) + if err != nil { + return nil, fmt.Errorf("failed to resolve selectors in path %s: %w", fullPath, err) + } + + loc := b.Config.GetLocation(resolvedPath) + filePath := loc.File + + // If field has no location, find the parent resource's location to then add a new field + if filePath == "" { + filePath = findResourceFileLocation(ctx, b, resourceKey) + if filePath == "" { + continue + } + log.Debugf(ctx, "Field %s has no location, using resource location: %s", fullPath, filePath) + } + + if _, ok := resolvedChangesByFile[filePath]; !ok { + resolvedChangesByFile[filePath] = make(resolvedChanges) + } + resolvedChangesByFile[filePath][resolvedPath] = changeDesc + } + } + + return resolvedChangesByFile, nil +} + +// strPathToJSONPointer converts a structpath string to JSON Pointer format. +// Example: "resources.jobs.test[0].name" -> "/resources/jobs/test/0/name" +func strPathToJSONPointer(pathStr string) string { + if pathStr == "" { + return "" + } + res := strings.ReplaceAll(pathStr, ".", "/") + res = strings.ReplaceAll(res, "[", "/") + res = strings.ReplaceAll(res, "]", "") + return "/" + res +} + +// findResourceFileLocation finds the file where a resource is defined. +// It checks both the root resources and target-specific overrides, +// preferring the target override if it exists. +func findResourceFileLocation(_ context.Context, b *bundle.Bundle, resourceKey string) string { + targetName := b.Config.Bundle.Target + + if targetName != "" { + targetPath := "targets." + targetName + "." + resourceKey + loc := b.Config.GetLocation(targetPath) + if loc.File != "" { + return loc.File + } + } + + loc := b.Config.GetLocation(resourceKey) + return loc.File +} diff --git a/bundle/configsync/patch_test.go b/bundle/configsync/patch_test.go new file mode 100644 index 0000000000..5901e190ab --- /dev/null +++ b/bundle/configsync/patch_test.go @@ -0,0 +1,947 @@ +package configsync + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/logdiag" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestApplyChangesToYAML_SimpleFieldChange(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + timeout_seconds: 3600 + tasks: + - task_key: "main_task" + notebook_task: + notebook_path: "/path/to/notebook" +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 3600, + Remote: 7200, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, yamlPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].OriginalContent, "timeout_seconds: 3600") + assert.Contains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 7200") + assert.NotContains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 3600") +} + +func TestApplyChangesToYAML_NestedFieldChange(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "main_task" + notebook_task: + notebook_path: "/path/to/notebook" + timeout_seconds: 1800 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "tasks[0].timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Contains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 3600") + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + testJob := jobs["test_job"].(map[string]any) + tasks := testJob["tasks"].([]any) + task0 := tasks[0].(map[string]any) + + assert.Equal(t, 3600, task0["timeout_seconds"]) +} + +func TestApplyChangesToYAML_ArrayKeyValueAccess(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "setup_task" + notebook_task: + notebook_path: "/setup" + timeout_seconds: 600 + - task_key: "main_task" + notebook_task: + notebook_path: "/main" + timeout_seconds: 1800 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "tasks[task_key='main_task'].timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + testJob := jobs["test_job"].(map[string]any) + tasks := testJob["tasks"].([]any) + + task0 := tasks[0].(map[string]any) + assert.Equal(t, "setup_task", task0["task_key"]) + assert.Equal(t, 600, task0["timeout_seconds"]) + + task1 := tasks[1].(map[string]any) + assert.Equal(t, "main_task", task1["task_key"]) + assert.Equal(t, 3600, task1["timeout_seconds"]) +} + +func TestApplyChangesToYAML_MultipleResourcesSameFile(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + job1: + name: "Job 1" + timeout_seconds: 3600 + job2: + name: "Job 2" + timeout_seconds: 1800 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.job1": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 3600, + Remote: 7200, + }, + }, + "resources.jobs.job2": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + + require.Len(t, fileChanges, 1) + assert.Equal(t, yamlPath, fileChanges[0].Path) + + assert.Contains(t, fileChanges[0].ModifiedContent, "job1") + assert.Contains(t, fileChanges[0].ModifiedContent, "job2") + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + + job1 := jobs["job1"].(map[string]any) + assert.Equal(t, 7200, job1["timeout_seconds"]) + + job2 := jobs["job2"].(map[string]any) + assert.Equal(t, 3600, job2["timeout_seconds"]) +} + +func TestApplyChangesToYAML_ResourceNotFound(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + existing_job: + name: "Existing Job" +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.nonexistent_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + + assert.Len(t, fileChanges, 0) +} + +func TestApplyChangesToYAML_Include(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + mainYAML := `bundle: + name: test-bundle + +include: + - "targets/*.yml" +` + + mainPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(mainPath, []byte(mainYAML), 0o644) + require.NoError(t, err) + + targetsDir := filepath.Join(tmpDir, "targets") + err = os.MkdirAll(targetsDir, 0o755) + require.NoError(t, err) + + devYAML := `resources: + jobs: + dev_job: + name: "Dev Job" + timeout_seconds: 1800 +` + + devPath := filepath.Join(targetsDir, "dev.yml") + err = os.WriteFile(devPath, []byte(devYAML), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.dev_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, devPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].OriginalContent, "timeout_seconds: 1800") + assert.Contains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 3600") + assert.NotContains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 1800") +} + +func TestGenerateYAMLFiles_TargetOverride(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + mainYAML := `bundle: + name: test-bundle +targets: + dev: + resources: + jobs: + dev_job: + name: "Dev Job" + timeout_seconds: 1800 +` + + mainPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(mainPath, []byte(mainYAML), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + diags := bundle.Apply(ctx, b, mutator.SelectTarget("dev")) + require.NoError(t, diags.Error()) + + changes := map[string]deployplan.Changes{ + "resources.jobs.dev_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: 3600, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, mainPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].ModifiedContent, "timeout_seconds: 3600") +} + +func TestApplyChangesToYAML_WithStructValues(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + timeout_seconds: 3600 + email_notifications: + on_success: + - old@example.com +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + type EmailNotifications struct { + OnSuccess []string `json:"on_success,omitempty" yaml:"on_success,omitempty"` + OnFailure []string `json:"on_failure,omitempty" yaml:"on_failure,omitempty"` + } + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "email_notifications": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: &EmailNotifications{ + OnSuccess: []string{"old@example.com"}, + }, + Remote: &EmailNotifications{ + OnSuccess: []string{"success@example.com"}, + OnFailure: []string{"failure@example.com"}, + }, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, yamlPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].OriginalContent, "on_success:") + assert.Contains(t, fileChanges[0].OriginalContent, "old@example.com") + assert.Contains(t, fileChanges[0].ModifiedContent, "success@example.com") + assert.Contains(t, fileChanges[0].ModifiedContent, "failure@example.com") + + type JobsConfig struct { + Name string `yaml:"name"` + TimeoutSeconds int `yaml:"timeout_seconds"` + EmailNotifications *EmailNotifications `yaml:"email_notifications,omitempty"` + } + + type ResourcesConfig struct { + Jobs map[string]JobsConfig `yaml:"jobs"` + } + + type RootConfig struct { + Resources ResourcesConfig `yaml:"resources"` + } + + var result RootConfig + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + testJob := result.Resources.Jobs["test_job"] + assert.Equal(t, "Test Job", testJob.Name) + assert.Equal(t, 3600, testJob.TimeoutSeconds) + require.NotNil(t, testJob.EmailNotifications) + assert.Equal(t, []string{"success@example.com"}, testJob.EmailNotifications.OnSuccess) + assert.Equal(t, []string{"failure@example.com"}, testJob.EmailNotifications.OnFailure) +} + +func TestApplyChangesToYAML_PreserveComments(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `# test_comment0 +resources: + # test_comment1 + jobs: + test_job: + # test_comment2 + name: "Test Job" + # test_comment3 + timeout_seconds: 3600 + # test_comment4 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 3600, + Remote: 7200, + }, + "name": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: "Test Job", + Remote: "New Test Job", + }, + "tags": &deployplan.ChangeDesc{ + Action: deployplan.Create, + Remote: map[string]string{ + "test": "value", + }, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, yamlPath, fileChanges[0].Path) + + assert.Contains(t, fileChanges[0].ModifiedContent, "# test_comment0") + assert.Contains(t, fileChanges[0].ModifiedContent, "# test_comment1") + assert.Contains(t, fileChanges[0].ModifiedContent, "# test_comment2") + assert.Contains(t, fileChanges[0].ModifiedContent, "# test_comment3") + assert.Contains(t, fileChanges[0].ModifiedContent, "# test_comment4") +} + +func TestStrPathToJSONPointer_SimplePaths(t *testing.T) { + pointer := strPathToJSONPointer("resources.jobs.test_job") + assert.Equal(t, "/resources/jobs/test_job", pointer) +} + +func TestStrPathToJSONPointer_WithIndices(t *testing.T) { + pointer := strPathToJSONPointer("tasks[0].name") + assert.Equal(t, "/tasks/0/name", pointer) + + pointer = strPathToJSONPointer("resources.jobs.test[0].tasks[1].timeout") + assert.Equal(t, "/resources/jobs/test/0/tasks/1/timeout", pointer) +} + +func TestStrPathToJSONPointer_EmptyPath(t *testing.T) { + pointer := strPathToJSONPointer("") + assert.Equal(t, "", pointer) +} + +func TestApplyChangesToYAML_RemoveSimpleField(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + timeout_seconds: 3600 + tasks: + - task_key: "main_task" + notebook_task: + notebook_path: "/path/to/notebook" +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 3600, + Remote: nil, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, yamlPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].OriginalContent, "timeout_seconds: 3600") + assert.NotContains(t, fileChanges[0].ModifiedContent, "timeout_seconds") + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + testJob := jobs["test_job"].(map[string]any) + + _, hasTimeout := testJob["timeout_seconds"] + assert.False(t, hasTimeout, "timeout_seconds should be removed") + assert.Equal(t, "Test Job", testJob["name"]) +} + +func TestApplyChangesToYAML_RemoveNestedField(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "main_task" + notebook_task: + notebook_path: "/path/to/notebook" + timeout_seconds: 1800 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "tasks[0].timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: nil, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.NotContains(t, fileChanges[0].ModifiedContent, "timeout_seconds") + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + testJob := jobs["test_job"].(map[string]any) + tasks := testJob["tasks"].([]any) + task0 := tasks[0].(map[string]any) + + _, hasTimeout := task0["timeout_seconds"] + assert.False(t, hasTimeout, "timeout_seconds should be removed from task") + assert.Equal(t, "main_task", task0["task_key"]) +} + +func TestApplyChangesToYAML_RemoveFieldWithKeyValueAccess(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + tasks: + - task_key: "setup_task" + notebook_task: + notebook_path: "/setup" + timeout_seconds: 600 + - task_key: "main_task" + notebook_task: + notebook_path: "/main" + timeout_seconds: 1800 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "tasks[task_key='main_task'].timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: nil, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + testJob := jobs["test_job"].(map[string]any) + tasks := testJob["tasks"].([]any) + + task0 := tasks[0].(map[string]any) + assert.Equal(t, "setup_task", task0["task_key"]) + assert.Equal(t, 600, task0["timeout_seconds"], "setup_task timeout should remain") + + task1 := tasks[1].(map[string]any) + assert.Equal(t, "main_task", task1["task_key"]) + _, hasTimeout := task1["timeout_seconds"] + assert.False(t, hasTimeout, "main_task timeout_seconds should be removed") +} + +func TestApplyChangesToYAML_RemoveStructField(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + test_job: + name: "Test Job" + timeout_seconds: 3600 + email_notifications: + on_success: + - success@example.com + on_failure: + - failure@example.com +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "email_notifications": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: map[string]any{ + "on_success": []string{"success@example.com"}, + "on_failure": []string{"failure@example.com"}, + }, + Remote: nil, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Contains(t, fileChanges[0].OriginalContent, "email_notifications") + assert.NotContains(t, fileChanges[0].ModifiedContent, "email_notifications") + assert.NotContains(t, fileChanges[0].ModifiedContent, "on_success") + assert.NotContains(t, fileChanges[0].ModifiedContent, "on_failure") + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + testJob := jobs["test_job"].(map[string]any) + + _, hasEmailNotifications := testJob["email_notifications"] + assert.False(t, hasEmailNotifications, "email_notifications should be removed") + assert.Equal(t, "Test Job", testJob["name"]) + assert.Equal(t, 3600, testJob["timeout_seconds"]) +} + +func TestApplyChangesToYAML_RemoveFromTargetOverride(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + mainYAML := `bundle: + name: test-bundle +targets: + dev: + resources: + jobs: + dev_job: + name: "Dev Job" + timeout_seconds: 1800 +` + + mainPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(mainPath, []byte(mainYAML), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + diags := bundle.Apply(ctx, b, mutator.SelectTarget("dev")) + require.NoError(t, diags.Error()) + + changes := map[string]deployplan.Changes{ + "resources.jobs.dev_job": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: nil, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, mainPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].OriginalContent, "timeout_seconds: 1800") + assert.NotContains(t, fileChanges[0].ModifiedContent, "timeout_seconds") + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + targets := result["targets"].(map[string]any) + dev := targets["dev"].(map[string]any) + resources := dev["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + devJob := jobs["dev_job"].(map[string]any) + + _, hasTimeout := devJob["timeout_seconds"] + assert.False(t, hasTimeout, "timeout_seconds should be removed from target override") + assert.Equal(t, "Dev Job", devJob["name"]) +} + +func TestApplyChangesToYAML_MultipleRemovalsInSameFile(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + + tmpDir := t.TempDir() + + yamlContent := `resources: + jobs: + job1: + name: "Job 1" + timeout_seconds: 3600 + job2: + name: "Job 2" + timeout_seconds: 1800 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.job1": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 3600, + Remote: nil, + }, + }, + "resources.jobs.job2": { + "timeout_seconds": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Old: 1800, + Remote: nil, + }, + }, + } + + fileChanges, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, fileChanges, 1) + + assert.Equal(t, yamlPath, fileChanges[0].Path) + assert.Contains(t, fileChanges[0].OriginalContent, "timeout_seconds: 3600") + assert.Contains(t, fileChanges[0].OriginalContent, "timeout_seconds: 1800") + assert.NotContains(t, fileChanges[0].ModifiedContent, "timeout_seconds") + + var result map[string]any + err = yaml.Unmarshal([]byte(fileChanges[0].ModifiedContent), &result) + require.NoError(t, err) + + resources := result["resources"].(map[string]any) + jobs := resources["jobs"].(map[string]any) + + job1 := jobs["job1"].(map[string]any) + _, hasTimeout1 := job1["timeout_seconds"] + assert.False(t, hasTimeout1, "job1 timeout_seconds should be removed") + assert.Equal(t, "Job 1", job1["name"]) + + job2 := jobs["job2"].(map[string]any) + _, hasTimeout2 := job2["timeout_seconds"] + assert.False(t, hasTimeout2, "job2 timeout_seconds should be removed") + assert.Equal(t, "Job 2", job2["name"]) +} + +func TestApplyChangesToYAML_WithSDKStructValues(t *testing.T) { + ctx := logdiag.InitContext(context.Background()) + tmpDir := t.TempDir() + + type MockSDKStruct struct { + Name string `json:"name,omitempty"` + Enabled bool `json:"enabled,omitempty"` + ForceSendFields []string `json:"-"` + } + + yamlContent := `resources: + jobs: + test_job: + name: test + timeout_seconds: 0 +` + + yamlPath := filepath.Join(tmpDir, "databricks.yml") + err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) + require.NoError(t, err) + + b, err := bundle.Load(ctx, tmpDir) + require.NoError(t, err) + + mutator.DefaultMutators(ctx, b) + + changes := map[string]deployplan.Changes{ + "resources.jobs.test_job": { + "settings": &deployplan.ChangeDesc{ + Action: deployplan.Update, + Remote: &MockSDKStruct{ + Name: "updated_name", + Enabled: false, + ForceSendFields: []string{"Enabled"}, // Force send even though false + }, + }, + }, + } + + files, err := ApplyChangesToYAML(ctx, b, changes) + require.NoError(t, err) + require.Len(t, files, 1) + + assert.Contains(t, files[0].ModifiedContent, "name: updated_name") + assert.Contains(t, files[0].ModifiedContent, "enabled: false") +} diff --git a/cmd/bundle/bundle.go b/cmd/bundle/bundle.go index f65645ee83..e6ecae4d9a 100644 --- a/cmd/bundle/bundle.go +++ b/cmd/bundle/bundle.go @@ -39,5 +39,6 @@ Online documentation: https://docs.databricks.com/en/dev-tools/bundles/index.htm cmd.AddCommand(deployment.NewDeploymentCommand()) cmd.AddCommand(newOpenCommand()) cmd.AddCommand(newPlanCommand()) + cmd.AddCommand(newConfigRemoteSyncCommand()) return cmd } diff --git a/cmd/bundle/config_remote_sync.go b/cmd/bundle/config_remote_sync.go new file mode 100644 index 0000000000..3d3a8b8264 --- /dev/null +++ b/cmd/bundle/config_remote_sync.go @@ -0,0 +1,83 @@ +package bundle + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/configsync" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/flags" + "github.com/spf13/cobra" +) + +func newConfigRemoteSyncCommand() *cobra.Command { + var save bool + + cmd := &cobra.Command{ + Use: "config-remote-sync", + Short: "Sync remote resource changes to bundle configuration (experimental)", + Long: `Compares deployed state with current remote state and generates updated configuration files. + +When --save is specified, writes updated YAML files to disk. +Otherwise, outputs diff without modifying files. + +Examples: + # Show diff without saving + databricks bundle config-remote-sync + + # Show diff and save to files + databricks bundle config-remote-sync --save`, + Hidden: true, // Used by DABs in the Workspace only + } + + cmd.Flags().BoolVar(&save, "save", false, "Write updated config files to disk") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + b, _, err := utils.ProcessBundleRet(cmd, utils.ProcessOptions{ + Build: true, + AlwaysPull: true, + }) + if err != nil { + return err + } + + ctx := cmd.Context() + changes, err := configsync.DetectChanges(ctx, b) + if err != nil { + return fmt.Errorf("failed to detect changes: %w", err) + } + + files, err := configsync.ApplyChangesToYAML(ctx, b, changes) + if err != nil { + return fmt.Errorf("failed to generate YAML files: %w", err) + } + + if save { + if err := configsync.SaveFiles(ctx, b, files); err != nil { + return fmt.Errorf("failed to save files: %w", err) + } + } + + var result []byte + if root.OutputType(cmd) == flags.OutputJSON { + diffOutput := &configsync.DiffOutput{ + Files: files, + Changes: changes, + } + result, err = json.MarshalIndent(diffOutput, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal output: %w", err) + } + } else if root.OutputType(cmd) == flags.OutputText { + result = []byte(configsync.FormatTextOutput(changes)) + } + + out := cmd.OutOrStdout() + _, _ = out.Write(result) + _, _ = out.Write([]byte{'\n'}) + return nil + } + + return cmd +} diff --git a/go.mod b/go.mod index 32e25e6798..759b9b4895 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/manifoldco/promptui v0.9.0 // BSD-3-Clause github.com/mattn/go-isatty v0.0.20 // MIT github.com/nwidger/jsoncolor v0.3.2 // MIT + github.com/palantir/pkg/yamlpatch v1.5.0 // BSD-3-Clause github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // BSD-2-Clause github.com/quasilyte/go-ruleguard/dsl v0.3.22 // BSD 3-Clause github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // MIT @@ -45,6 +46,8 @@ require ( // Dependencies for experimental MCP commands require github.com/google/jsonschema-go v0.4.2 // MIT +require gopkg.in/yaml.v3 v3.0.1 + require ( cloud.google.com/go/auth v0.16.5 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect @@ -82,6 +85,7 @@ require ( github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/termenv v0.16.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/stretchr/objx v0.5.2 // indirect @@ -98,5 +102,4 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20250922171735-9219d122eba9 // indirect google.golang.org/grpc v1.75.1 // indirect google.golang.org/protobuf v1.36.9 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8c0efd71ef..3d6376294c 100644 --- a/go.sum +++ b/go.sum @@ -169,10 +169,14 @@ github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/nwidger/jsoncolor v0.3.2 h1:rVJJlwAWDJShnbTYOQ5RM7yTA20INyKXlJ/fg4JMhHQ= github.com/nwidger/jsoncolor v0.3.2/go.mod h1:Cs34umxLbJvgBMnVNVqhji9BhoT/N/KinHqZptQ7cf4= +github.com/palantir/pkg/yamlpatch v1.5.0 h1:186RUlcHFVf64onUhaI7nUCPzPIaRTQ5HJlKuv0d6NM= +github.com/palantir/pkg/yamlpatch v1.5.0/go.mod h1:45cYAIiv9E0MiZnHjIIT2hGqi6Wah/DL6J1omJf2ny0= github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=