Config remote sync command

acceptance/bin/edit_resource.py

@@ -32,6 +32,14 @@ def set(self, job_id, value):
         return run([CLI, "jobs", "reset", job_id, "--json", json.dumps(payload)])
 
 
+class pipelines:
+    def get(self, pipeline_id):
+        return run_json([CLI, "pipelines", "get", pipeline_id])["spec"]
+
+    def set(self, pipeline_id, value):
+        return run([CLI, "pipelines", "update", pipeline_id, "--json", json.dumps(value)])
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("type")

acceptance/bundle/config-remote-sync/config_edits/databricks.yml

@@ -0,0 +1,42 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 1
+
+targets:
+  default:
+    resources:
+      jobs:
+        my_job:
+          email_notifications:
+            on_success:
+              - success@example.com
+          parameters:
+            - name: catalog
+              default: main
+            - name: env
+              default: dev
+          trigger:
+            periodic:
+              interval: 1
+              unit: DAYS
+          tags:
+            env: dev
+            version: v1
+          max_concurrent_runs: 1
+          environments:
+            - environment_key: default
+              spec:
+                environment_version: "3"
+                dependencies:
+                  - ./*.whl

acceptance/bundle/config-remote-sync/config_edits/output.txt

@@ -0,0 +1,57 @@
+Uploading dummy.whl...
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Edit config locally
+
+=== Edit job remotely
+
+=== Detect and save changes
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  email_notifications.on_failure[0]: update
+  max_concurrent_runs: update
+  tags['env']: update
+
+
+=== Configuration changes
+--- databricks.yml.backup	[TIMESTAMP]
++++ databricks.yml	[TIMESTAMP]
+@@ -1,6 +1,5 @@
+ bundle:
+   name: test-bundle
+-
+ resources:
+   jobs:
+     my_job:
+@@ -12,7 +11,6 @@
+             spark_version: 13.3.x-scala2.12
+             node_type_id: [NODE_TYPE_ID]
+             num_workers: 1
+-
+ targets:
+   default:
+     resources:
+@@ -22,7 +20,7 @@
+             on_success:
+               - success@example.com
+             on_failure:
+-              - config-failure@example.com
++              - remote-failure@example.com
+           parameters:
+             - name: catalog
+               default: main
+@@ -33,8 +31,8 @@
+               interval: 1
+               unit: DAYS
+           tags:
+-            env: config-production
+-          max_concurrent_runs: 3
++            env: remote-staging
++          max_concurrent_runs: 5
+           timeout_seconds: 3600
+           environments:
+             - environment_key: default

acceptance/bundle/config-remote-sync/config_edits/script

@@ -0,0 +1,96 @@
+#!/bin/bash
+
+touch dummy.whl
+$CLI bundle deploy
+job_id="$(read_id.py my_job)"
+
+title "Edit config locally"
+echo
+# Case 1: Add field in config (on_failure) - will also be added remotely with different value
+# Case 2: Add field in config (timeout_seconds) - will be removed remotely
+# Case 3: Remove field from config (version tag) - will also be removed remotely
+# Case 4: Update field in config (max_concurrent_runs) - will also be updated remotely with different value
+# Case 5: Update field in config (tags.env) - will also be updated remotely with different value
+cat > databricks.yml <<'EOF'
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 1
+
+targets:
+  default:
+    resources:
+      jobs:
+        my_job:
+          email_notifications:
+            on_success:
+              - success@example.com
+            on_failure:
+              - config-failure@example.com
+          parameters:
+            - name: catalog
+              default: main
+            - name: env
+              default: dev
+          trigger:
+            periodic:
+              interval: 1
+              unit: DAYS
+          tags:
+            env: config-production
+          max_concurrent_runs: 3
+          timeout_seconds: 3600
+          environments:
+            - environment_key: default
+              spec:
+                environment_version: "3"
+                dependencies:
+                  - ./*.whl
+EOF
+
+title "Edit job remotely"
+echo
+# Case 1: Add field remotely (on_failure) - also added in config with different value
+edit_resource.py jobs $job_id <<EOF
+r["email_notifications"]["on_failure"] = ["remote-failure@example.com"]
+EOF
+
+# Case 2: Remove field remotely (timeout_seconds) - added in config
+edit_resource.py jobs $job_id <<EOF
+r.pop("timeout_seconds", None)
+EOF
+
+# Case 3: Remove field remotely (version tag) - also removed in config
+edit_resource.py jobs $job_id <<EOF
+r["tags"].pop("version", None)
+EOF
+
+# Case 4: Update field remotely (max_concurrent_runs) - also updated in config with different value
+edit_resource.py jobs $job_id <<EOF
+r["max_concurrent_runs"] = 5
+EOF
+
+# Case 5: Update field remotely (tags.env) - also updated in config with different value
+edit_resource.py jobs $job_id <<EOF
+r["tags"]["env"] = "remote-staging"
+EOF
+
+title "Detect and save changes"
+echo
+cp databricks.yml databricks.yml.backup
+$CLI bundle config-remote-sync --save
+
+title "Configuration changes"
+echo
+diff -u databricks.yml.backup databricks.yml || true
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/config_edits/test.toml

@@ -0,0 +1,10 @@
+RecordRequests = false
+Cloud = true
+
+Ignore = [".databricks", "dummy.whl"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/config-remote-sync/formatting_preserved/databricks.yml

@@ -0,0 +1,25 @@
+# Top-level comment about the bundle
+bundle:
+  name: test-bundle
+
+# Resources section with extra spacing
+resources:
+  jobs:
+    my_job:
+      # Comment about max concurrent runs
+      max_concurrent_runs: 1
+
+      # Task configuration
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 1 # inline comment about workers
+
+      # Tags for categorization
+      tags:
+        env: dev # environment tag
+        team: data-eng

acceptance/bundle/config-remote-sync/formatting_preserved/output.txt

@@ -0,0 +1,40 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Modify max_concurrent_runs from 1 to 5
+=== Detect and save changes
+Detected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  max_concurrent_runs: update
+
+
+=== Configuration changes
+--- databricks.yml.backup	[TIMESTAMP]
++++ databricks.yml	[TIMESTAMP]
+@@ -1,14 +1,12 @@
+ # Top-level comment about the bundle
+ bundle:
+   name: test-bundle
+-
+ # Resources section with extra spacing
+ resources:
+   jobs:
+     my_job:
+       # Comment about max concurrent runs
+-      max_concurrent_runs: 1
+-
++      max_concurrent_runs: 5
+       # Task configuration
+       tasks:
+         - task_key: main
+@@ -18,7 +16,6 @@
+             spark_version: 13.3.x-scala2.12
+             node_type_id: [NODE_TYPE_ID]
+             num_workers: 1 # inline comment about workers
+-
+       # Tags for categorization
+       tags:
+         env: dev # environment tag

acceptance/bundle/config-remote-sync/formatting_preserved/script

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+touch dummy.whl
+$CLI bundle deploy
+job_id="$(read_id.py my_job)"
+
+title "Modify max_concurrent_runs from 1 to 5"
+edit_resource.py jobs $job_id <<EOF
+r["max_concurrent_runs"] = 5
+EOF
+
+title "Detect and save changes"
+echo
+cp databricks.yml databricks.yml.backup
+$CLI bundle config-remote-sync --save
+
+title "Configuration changes"
+echo
+diff -u databricks.yml.backup databricks.yml || true
+rm databricks.yml.backup

acceptance/bundle/config-remote-sync/formatting_preserved/test.toml

@@ -0,0 +1,8 @@
+RecordRequests = false
+Ignore = [".databricks", "dummy.whl"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/config-remote-sync/job_fields/databricks.yml

@@ -0,0 +1,37 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      email_notifications:
+        on_success:
+          - success@example.com
+      parameters:
+        - name: catalog
+          default: main
+        - name: env
+          default: dev
+      trigger:
+        periodic:
+          interval: 1
+          unit: DAYS
+      tags:
+        env: dev
+      permissions:
+        - level: CAN_VIEW
+          user_name: viewer@example.com
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "3"
+            dependencies:
+              - ./*.whl
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 1