diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 00152d550ea..b7982c5a568 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -5,5 +5,6 @@ ### CLI ### Bundles +* engine/direct: Add declarative `bind` blocks under a target to bring existing workspace resources under bundle management at deploy time, with `bind` and `bind_and_update` actions surfaced in `bundle plan` output ([#4630](https://github.com/databricks/cli/pull/4630)). ### Dependency updates diff --git a/acceptance/bundle/deploy/bind/basic/databricks.yml b/acceptance/bundle/deploy/bind/basic/databricks.yml new file mode 100644 index 00000000000..a6b280560ab --- /dev/null +++ b/acceptance/bundle/deploy/bind/basic/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-basic + +resources: + jobs: + foo: + name: test-bind-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "PLACEHOLDER_JOB_ID" diff --git a/acceptance/bundle/deploy/bind/basic/hello.py b/acceptance/bundle/deploy/bind/basic/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/basic/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/basic/out.test.toml b/acceptance/bundle/deploy/bind/basic/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/basic/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/basic/output.txt b/acceptance/bundle/deploy/bind/basic/output.txt new file mode 100644 index 00000000000..74586828ae0 --- /dev/null +++ b/acceptance/bundle/deploy/bind/basic/output.txt @@ -0,0 +1,66 @@ + +>>> [CLI] bundle plan +bind jobs.foo (id: [NEW_JOB_ID]) + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged, 1 to bind + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-basic/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "state": { + "resources.jobs.foo": { + "__id__": "[NEW_JOB_ID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bind-basic/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "default", + "spec": { + "client": "1" + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "test-bind-job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "environment_key": "default", + "spark_python_task": { + "python_file": "/Workspace/Users/[USERNAME]/.bundle/test-bind-basic/default/files/hello.py" + }, + "task_key": "my_task" + } + ] + } + } + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-basic/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/bind/basic/script b/acceptance/bundle/deploy/bind/basic/script new file mode 100644 index 00000000000..29ed7d5e6f9 --- /dev/null +++ b/acceptance/bundle/deploy/bind/basic/script @@ -0,0 +1,37 @@ +# Create a job in the workspace +NEW_JOB_ID=$($CLI jobs create --json '{"name": "test-import-job", "environments": [{"environment_key": "default", "spec": {"client": "1"}}], "tasks": [{"task_key": "my_task", "environment_key": "default", "spark_python_task": {"python_file": "/Workspace/test.py"}}]}' | jq -r .job_id) +add_repl.py $NEW_JOB_ID NEW_JOB_ID + +# Update the databricks.yml with the actual job ID +update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$NEW_JOB_ID" + +# Run plan - should show import action +trace $CLI bundle plan + +# Deploy with auto-approve +trace $CLI bundle deploy --auto-approve + +# Plan again - should show no changes (skip) +trace $CLI bundle plan + +# Verify state file contains the imported ID +trace print_state.py | contains.py "$NEW_JOB_ID" + +# Remove bind block before destroy (destroy blocks on active bind blocks) +python3 << 'PYSCRIPT' +import re +with open('databricks.yml', 'r') as f: + content = f.read() + +# Remove the bind block from the target (everything from "bind:" until the next top-level key or EOF) +content = re.sub(r'\n bind:.*', '', content, flags=re.DOTALL) + +with open('databricks.yml', 'w') as f: + f.write(content) +PYSCRIPT + +# Remove .databricks directory that might cache old config +rm -rf .databricks + +# Cleanup +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/bind/bind-and-update/databricks.yml b/acceptance/bundle/deploy/bind/bind-and-update/databricks.yml new file mode 100644 index 00000000000..3b813772d02 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-and-update/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-update + +resources: + jobs: + foo: + name: updated-job-name + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "PLACEHOLDER_JOB_ID" diff --git a/acceptance/bundle/deploy/bind/bind-and-update/hello.py b/acceptance/bundle/deploy/bind/bind-and-update/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-and-update/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/bind-and-update/out.test.toml b/acceptance/bundle/deploy/bind/bind-and-update/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-and-update/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/bind-and-update/output.txt b/acceptance/bundle/deploy/bind/bind-and-update/output.txt new file mode 100644 index 00000000000..92b414476c2 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-and-update/output.txt @@ -0,0 +1,37 @@ + +>>> [CLI] bundle plan +bind jobs.foo (id: [NEW_JOB_ID]) + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged, 1 to bind + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-update/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] jobs get [NEW_JOB_ID] +updated-job-name + +>>> [CLI] bundle plan +update jobs.foo + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-update/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] jobs get [NEW_JOB_ID] +second-update-name + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-update/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/bind/bind-and-update/script b/acceptance/bundle/deploy/bind/bind-and-update/script new file mode 100644 index 00000000000..080427e7598 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-and-update/script @@ -0,0 +1,44 @@ +# Create a job in the workspace with a different name +NEW_JOB_ID=$($CLI jobs create --json '{"name": "original-job-name", "environments": [{"environment_key": "default", "spec": {"client": "1"}}], "tasks": [{"task_key": "my_task", "environment_key": "default", "spark_python_task": {"python_file": "/Workspace/test.py"}}]}' | jq -r .job_id) +add_repl.py $NEW_JOB_ID NEW_JOB_ID + +# Update the databricks.yml with the actual job ID +update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$NEW_JOB_ID" + +# Run plan - should show bind action (name differs from config) +trace $CLI bundle plan + +# Deploy with auto-approve +trace $CLI bundle deploy --auto-approve + +# Verify the job was updated +trace $CLI jobs get $NEW_JOB_ID | jq -r .settings.name + +# Now update the job name again in the config and deploy again. +# This time the action should be "update", not "bind", since the resource +# is already bound in state. +update_file.py databricks.yml 'updated-job-name' 'second-update-name' +trace $CLI bundle plan +trace $CLI bundle deploy --auto-approve + +# Verify the job was updated with the second name +trace $CLI jobs get $NEW_JOB_ID | jq -r .settings.name + +# Remove bind block before destroy (destroy blocks on active bind blocks) +python3 << 'PYSCRIPT' +import re +with open('databricks.yml', 'r') as f: + content = f.read() + +# Remove the bind block from the target (everything from "bind:" until the next top-level key or EOF) +content = re.sub(r'\n bind:.*', '', content, flags=re.DOTALL) + +with open('databricks.yml', 'w') as f: + f.write(content) +PYSCRIPT + +# Remove .databricks directory that might cache old config +rm -rf .databricks + +# Cleanup +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/bind/bind-permissions/databricks.yml b/acceptance/bundle/deploy/bind/bind-permissions/databricks.yml new file mode 100644 index 00000000000..ff4bc4534d3 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-permissions/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: test-bind-permissions + +resources: + jobs: + foo: + name: test-job + tasks: + - task_key: test + notebook_task: + notebook_path: ./nb.py + permissions: + - group_name: users + level: CAN_MANAGE + +targets: + default: + bind: + jobs.permissions: + foo: + id: "12345" diff --git a/acceptance/bundle/deploy/bind/bind-permissions/nb.py b/acceptance/bundle/deploy/bind/bind-permissions/nb.py new file mode 100644 index 00000000000..38d86b79c70 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-permissions/nb.py @@ -0,0 +1,2 @@ +# Databricks notebook source +print("Hello, World!") diff --git a/acceptance/bundle/deploy/bind/bind-permissions/out.test.toml b/acceptance/bundle/deploy/bind/bind-permissions/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-permissions/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/bind-permissions/output.txt b/acceptance/bundle/deploy/bind/bind-permissions/output.txt new file mode 100644 index 00000000000..3510df058e9 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-permissions/output.txt @@ -0,0 +1,25 @@ + +>>> musterr [CLI] bundle validate +Error: binding jobs.permissions is not allowed + +bind can only be used for resources directly under the resources block, not for child resources like permissions or grants. + +To manage permissions or grants: +1. First bind the parent resource (without .permissions or .grants) +2. Then define permissions or grants in your bundle configuration + +Invalid bind configuration: + bind: + jobs.permissions: + foo: + id: ... + +Instead, remove this bind entry and ensure the parent resource is bound. + +Name: test-bind-permissions +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bind-permissions/default + +Found 1 error diff --git a/acceptance/bundle/deploy/bind/bind-permissions/script b/acceptance/bundle/deploy/bind/bind-permissions/script new file mode 100755 index 00000000000..a34e6208ad9 --- /dev/null +++ b/acceptance/bundle/deploy/bind/bind-permissions/script @@ -0,0 +1,3 @@ +#!/bin/bash + +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/deploy/bind/block-migrate/databricks.yml b/acceptance/bundle/deploy/bind/block-migrate/databricks.yml new file mode 100644 index 00000000000..16cc8b2be6f --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-block-migrate + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "12345" diff --git a/acceptance/bundle/deploy/bind/block-migrate/hello.py b/acceptance/bundle/deploy/bind/block-migrate/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/block-migrate/out.test.toml b/acceptance/bundle/deploy/bind/block-migrate/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/block-migrate/output.txt b/acceptance/bundle/deploy/bind/block-migrate/output.txt new file mode 100644 index 00000000000..6bc78e4ace6 --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/output.txt @@ -0,0 +1,3 @@ + +>>> musterr [CLI] bundle deployment migrate +Error: cannot run 'bundle deployment migrate' when bind blocks are defined in the target configuration; bind blocks are only supported with the direct deployment engine diff --git a/acceptance/bundle/deploy/bind/block-migrate/script b/acceptance/bundle/deploy/bind/block-migrate/script new file mode 100644 index 00000000000..64c538ec319 --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/script @@ -0,0 +1,2 @@ +# Try to run migration with bind blocks - should fail +trace musterr $CLI bundle deployment migrate diff --git a/acceptance/bundle/deploy/bind/block-migrate/test.toml b/acceptance/bundle/deploy/bind/block-migrate/test.toml new file mode 100644 index 00000000000..680c17c1e01 --- /dev/null +++ b/acceptance/bundle/deploy/bind/block-migrate/test.toml @@ -0,0 +1,2 @@ +# Migration test does not need engine matrix +[EnvMatrix] diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks.yml b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks.yml new file mode 100644 index 00000000000..cbb7da54afe --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bind-delete-conflict + +resources: + jobs: + foo: + name: test-bind-delete-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks_conflict.yml b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks_conflict.yml new file mode 100644 index 00000000000..59758817069 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/databricks_conflict.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-delete-conflict + +resources: + jobs: + bar: + name: test-bind-delete-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + bar: + id: "PLACEHOLDER_JOB_ID" diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/hello.py b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/out.test.toml b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/output.txt b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/output.txt new file mode 100644 index 00000000000..57e63fae484 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/output.txt @@ -0,0 +1,22 @@ + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-delete-conflict/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> musterr [CLI] bundle plan +Error: bind block for "resources.jobs.bar" has the same ID "[FOO_ID]" as existing resource "resources.jobs.foo"; remove the bind block or the conflicting resource + at targets.default.bind.jobs.bar + +Error: bind validation failed + + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-delete-conflict/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/script b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/script new file mode 100644 index 00000000000..6547472f40f --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/script @@ -0,0 +1,18 @@ +# Deploy foo to create it in state +trace $CLI bundle deploy --auto-approve + +# Get the job ID from state +JOB_ID=$(read_id.py foo) + +# Switch to a config that renames foo->bar and adds a bind block for bar +# with the same job ID. This creates a conflict: foo is being deleted +# (still in state) while bar is being bound with the same ID. +cp databricks.yml databricks.yml.bak +cp databricks_conflict.yml databricks.yml +update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$JOB_ID" + +trace musterr $CLI bundle plan + +# Cleanup: restore original config and destroy +cp databricks.yml.bak databricks.yml +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/bind/delete-and-bind-conflict/test.toml b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/test.toml new file mode 100644 index 00000000000..a07a7675610 --- /dev/null +++ b/acceptance/bundle/deploy/bind/delete-and-bind-conflict/test.toml @@ -0,0 +1 @@ +Ignore = [".databricks", "databricks.yml.bak", "databricks_conflict.yml"] diff --git a/acceptance/bundle/deploy/bind/destroy-blocked/databricks.yml b/acceptance/bundle/deploy/bind/destroy-blocked/databricks.yml new file mode 100644 index 00000000000..f60645ff51f --- /dev/null +++ b/acceptance/bundle/deploy/bind/destroy-blocked/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-destroy-blocked + +resources: + jobs: + foo: + name: test-bind-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "PLACEHOLDER_JOB_ID" diff --git a/acceptance/bundle/deploy/bind/destroy-blocked/hello.py b/acceptance/bundle/deploy/bind/destroy-blocked/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/destroy-blocked/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/destroy-blocked/out.test.toml b/acceptance/bundle/deploy/bind/destroy-blocked/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/destroy-blocked/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/destroy-blocked/output.txt b/acceptance/bundle/deploy/bind/destroy-blocked/output.txt new file mode 100644 index 00000000000..9f8ca5a5c60 --- /dev/null +++ b/acceptance/bundle/deploy/bind/destroy-blocked/output.txt @@ -0,0 +1,19 @@ + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-destroy-blocked/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> musterr [CLI] bundle destroy --auto-approve +Error: cannot destroy with bind blocks that reference resources in the deployment state: resources.jobs.foo; remove the bind blocks from the target configuration or run 'bundle deployment unbind' before destroying + + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-destroy-blocked/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/bind/destroy-blocked/script b/acceptance/bundle/deploy/bind/destroy-blocked/script new file mode 100644 index 00000000000..508cbc0a043 --- /dev/null +++ b/acceptance/bundle/deploy/bind/destroy-blocked/script @@ -0,0 +1,30 @@ +# Create a job in the workspace +NEW_JOB_ID=$($CLI jobs create --json '{"name": "test-bind-job", "environments": [{"environment_key": "default", "spec": {"client": "1"}}], "tasks": [{"task_key": "my_task", "environment_key": "default", "spark_python_task": {"python_file": "/Workspace/test.py"}}]}' | jq -r .job_id) +add_repl.py $NEW_JOB_ID NEW_JOB_ID + +# Update the databricks.yml with the actual job ID +update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$NEW_JOB_ID" + +# Deploy to bind the resource into state +trace $CLI bundle deploy --auto-approve + +# Try to destroy - should fail because bind blocks reference resources in state +trace musterr $CLI bundle destroy --auto-approve + +# Remove bind block and destroy to clean up +python3 << 'PYSCRIPT' +import re +with open('databricks.yml', 'r') as f: + content = f.read() + +# Remove the bind block from the target (everything from "bind:" until the next top-level key or EOF) +content = re.sub(r'\n bind:.*', '', content, flags=re.DOTALL) + +with open('databricks.yml', 'w') as f: + f.write(content) +PYSCRIPT + +# Remove .databricks directory that might cache old config +rm -rf .databricks + +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks.yml b/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks.yml new file mode 100644 index 00000000000..9ae092f1907 --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bind-duplicate-id + +resources: + jobs: + foo: + name: test-bind-dup-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks_with_bind.yml b/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks_with_bind.yml new file mode 100644 index 00000000000..0b3abd007f2 --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/databricks_with_bind.yml @@ -0,0 +1,34 @@ +bundle: + name: test-bind-duplicate-id + +resources: + jobs: + foo: + name: test-bind-dup-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + bar: + name: test-bind-dup-bar + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + bar: + id: "PLACEHOLDER_JOB_ID" diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/hello.py b/acceptance/bundle/deploy/bind/duplicate-bind-id/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/out.test.toml b/acceptance/bundle/deploy/bind/duplicate-bind-id/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/output.txt b/acceptance/bundle/deploy/bind/duplicate-bind-id/output.txt new file mode 100644 index 00000000000..60a188806ba --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/output.txt @@ -0,0 +1,22 @@ + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bind-duplicate-id/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> musterr [CLI] bundle plan +Error: bind block for "resources.jobs.bar" has the same ID "[FOO_ID]" as existing resource "resources.jobs.foo"; remove the bind block or the conflicting resource + at targets.default.bind.jobs.bar + +Error: bind validation failed + + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bind-duplicate-id/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/script b/acceptance/bundle/deploy/bind/duplicate-bind-id/script new file mode 100644 index 00000000000..57ab78fcf0f --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/script @@ -0,0 +1,18 @@ +# Deploy foo to create it in state +trace $CLI bundle deploy --auto-approve + +# Get foo's job ID from state +JOB_ID=$(read_id.py foo) + +# Switch to a config that keeps foo AND adds bar with a bind block +# pointing to foo's ID. This is a conflict: foo is still managed in +# state and bar tries to bind the same resource ID. +cp databricks.yml databricks.yml.bak +cp databricks_with_bind.yml databricks.yml +update_file.py databricks.yml 'PLACEHOLDER_JOB_ID' "$JOB_ID" + +trace musterr $CLI bundle plan + +# Cleanup +cp databricks.yml.bak databricks.yml +trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/deploy/bind/duplicate-bind-id/test.toml b/acceptance/bundle/deploy/bind/duplicate-bind-id/test.toml new file mode 100644 index 00000000000..f70b5c78fff --- /dev/null +++ b/acceptance/bundle/deploy/bind/duplicate-bind-id/test.toml @@ -0,0 +1 @@ +Ignore = [".databricks", "databricks.yml.bak", "databricks_with_bind.yml"] diff --git a/acceptance/bundle/deploy/bind/invalid-resource-type/databricks.yml b/acceptance/bundle/deploy/bind/invalid-resource-type/databricks.yml new file mode 100644 index 00000000000..84e59cd7f52 --- /dev/null +++ b/acceptance/bundle/deploy/bind/invalid-resource-type/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-invalid-resource-type + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + foobar: + foo: + id: "123" diff --git a/acceptance/bundle/deploy/bind/invalid-resource-type/hello.py b/acceptance/bundle/deploy/bind/invalid-resource-type/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/invalid-resource-type/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/invalid-resource-type/out.test.toml b/acceptance/bundle/deploy/bind/invalid-resource-type/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/invalid-resource-type/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/invalid-resource-type/output.txt b/acceptance/bundle/deploy/bind/invalid-resource-type/output.txt new file mode 100644 index 00000000000..d06b74c8f14 --- /dev/null +++ b/acceptance/bundle/deploy/bind/invalid-resource-type/output.txt @@ -0,0 +1,7 @@ + +>>> musterr [CLI] bundle plan +Error: bind block references undefined resource "resources.foobar.foo"; define it in the resources section or remove the bind block + at targets.default.bind.foobar.foo + +Error: bind validation failed + diff --git a/acceptance/bundle/deploy/bind/invalid-resource-type/script b/acceptance/bundle/deploy/bind/invalid-resource-type/script new file mode 100644 index 00000000000..9d9604578f1 --- /dev/null +++ b/acceptance/bundle/deploy/bind/invalid-resource-type/script @@ -0,0 +1 @@ +trace musterr $CLI bundle plan diff --git a/acceptance/bundle/deploy/bind/orphaned-bind/databricks.yml b/acceptance/bundle/deploy/bind/orphaned-bind/databricks.yml new file mode 100644 index 00000000000..26d0272b267 --- /dev/null +++ b/acceptance/bundle/deploy/bind/orphaned-bind/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-orphaned + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + bar: + id: "12345" diff --git a/acceptance/bundle/deploy/bind/orphaned-bind/hello.py b/acceptance/bundle/deploy/bind/orphaned-bind/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/orphaned-bind/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/orphaned-bind/out.test.toml b/acceptance/bundle/deploy/bind/orphaned-bind/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/orphaned-bind/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/orphaned-bind/output.txt b/acceptance/bundle/deploy/bind/orphaned-bind/output.txt new file mode 100644 index 00000000000..6ae6816b737 --- /dev/null +++ b/acceptance/bundle/deploy/bind/orphaned-bind/output.txt @@ -0,0 +1,7 @@ + +>>> musterr [CLI] bundle plan +Error: bind block references undefined resource "resources.jobs.bar"; define it in the resources section or remove the bind block + at targets.default.bind.jobs.bar + +Error: bind validation failed + diff --git a/acceptance/bundle/deploy/bind/orphaned-bind/script b/acceptance/bundle/deploy/bind/orphaned-bind/script new file mode 100644 index 00000000000..aeae9c12fa1 --- /dev/null +++ b/acceptance/bundle/deploy/bind/orphaned-bind/script @@ -0,0 +1,2 @@ +# Import block references jobs.bar but only jobs.foo exists in resources +trace musterr $CLI bundle plan diff --git a/acceptance/bundle/deploy/bind/recreate-blocked/databricks.yml b/acceptance/bundle/deploy/bind/recreate-blocked/databricks.yml new file mode 100644 index 00000000000..ddad62da6ad --- /dev/null +++ b/acceptance/bundle/deploy/bind/recreate-blocked/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bind-recreate + +resources: + pipelines: + foo: + name: test-pipeline + storage: /new/storage/path + libraries: + - notebook: + path: ./nb.sql + +targets: + default: + bind: + pipelines: + foo: + id: "PLACEHOLDER_PIPELINE_ID" diff --git a/acceptance/bundle/deploy/bind/recreate-blocked/nb.sql b/acceptance/bundle/deploy/bind/recreate-blocked/nb.sql new file mode 100644 index 00000000000..199ff507884 --- /dev/null +++ b/acceptance/bundle/deploy/bind/recreate-blocked/nb.sql @@ -0,0 +1,2 @@ +-- Databricks notebook source +select 1 diff --git a/acceptance/bundle/deploy/bind/recreate-blocked/out.test.toml b/acceptance/bundle/deploy/bind/recreate-blocked/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/recreate-blocked/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/recreate-blocked/output.txt b/acceptance/bundle/deploy/bind/recreate-blocked/output.txt new file mode 100644 index 00000000000..5ddf2b0be75 --- /dev/null +++ b/acceptance/bundle/deploy/bind/recreate-blocked/output.txt @@ -0,0 +1,32 @@ + +>>> musterr [CLI] bundle plan +Error: cannot bind resources.pipelines.foo: cannot recreate resource with bind block + +This would destroy and recreate the existing workspace resource, changing its ID. + +The following fields cannot be modified because they require resource recreation: + - storage (immutable) + +To resolve this issue, you have two options: + +1. Remove the problematic fields from your configuration to make this a bind-only operation: + + resources: + pipelines: + foo: + # Remove or comment out these fields: + # storage: ... + +2. Remove the bind block if you want to allow the resource to be recreated/updated: + + targets: + : + # Remove the bind block: + # bind: + # pipelines: + # foo: + # id: + + +Error: bind planning failed + diff --git a/acceptance/bundle/deploy/bind/recreate-blocked/script b/acceptance/bundle/deploy/bind/recreate-blocked/script new file mode 100644 index 00000000000..f28f6b94ac8 --- /dev/null +++ b/acceptance/bundle/deploy/bind/recreate-blocked/script @@ -0,0 +1,9 @@ +# Create a pipeline with a different storage path +NEW_PIPELINE_ID=$($CLI pipelines create --json '{"name": "test-pipeline", "storage": "/old/storage/path", "allow_duplicate_names": true, "libraries": [{"notebook": {"path": "/Workspace/test"}}]}' | jq -r .pipeline_id) +add_repl.py $NEW_PIPELINE_ID NEW_PIPELINE_ID + +# Update the databricks.yml with the actual pipeline ID +update_file.py databricks.yml 'PLACEHOLDER_PIPELINE_ID' "$NEW_PIPELINE_ID" + +# Run plan - should fail because changing storage requires recreate which is blocked for binds +trace musterr $CLI bundle plan diff --git a/acceptance/bundle/deploy/bind/resource-not-found/databricks.yml b/acceptance/bundle/deploy/bind/resource-not-found/databricks.yml new file mode 100644 index 00000000000..e02258511da --- /dev/null +++ b/acceptance/bundle/deploy/bind/resource-not-found/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-not-found + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "999999999" diff --git a/acceptance/bundle/deploy/bind/resource-not-found/hello.py b/acceptance/bundle/deploy/bind/resource-not-found/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/resource-not-found/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/resource-not-found/out.test.toml b/acceptance/bundle/deploy/bind/resource-not-found/out.test.toml new file mode 100644 index 00000000000..9cfad3fb0d5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/resource-not-found/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/resource-not-found/output.txt b/acceptance/bundle/deploy/bind/resource-not-found/output.txt new file mode 100644 index 00000000000..96ef84cc0a8 --- /dev/null +++ b/acceptance/bundle/deploy/bind/resource-not-found/output.txt @@ -0,0 +1,6 @@ + +>>> musterr [CLI] bundle plan +Error: cannot bind resources.jobs.foo: resource with ID "[NUMID]" does not exist in workspace + +Error: bind planning failed + diff --git a/acceptance/bundle/deploy/bind/resource-not-found/script b/acceptance/bundle/deploy/bind/resource-not-found/script new file mode 100644 index 00000000000..6226cb85916 --- /dev/null +++ b/acceptance/bundle/deploy/bind/resource-not-found/script @@ -0,0 +1,2 @@ +# Try to plan with a non-existent job ID - should fail +trace musterr $CLI bundle plan diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/databricks.yml b/acceptance/bundle/deploy/bind/terraform-with-bind/databricks.yml new file mode 100644 index 00000000000..98450928340 --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/databricks.yml @@ -0,0 +1,23 @@ +bundle: + name: test-bind-terraform + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py + +targets: + default: + bind: + jobs: + foo: + id: "12345" diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/hello.py b/acceptance/bundle/deploy/bind/terraform-with-bind/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/out.test.toml b/acceptance/bundle/deploy/bind/terraform-with-bind/out.test.toml new file mode 100644 index 00000000000..42c0997090a --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/output.txt b/acceptance/bundle/deploy/bind/terraform-with-bind/output.txt new file mode 100644 index 00000000000..62c0d2b6c5f --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/output.txt @@ -0,0 +1,4 @@ + +>>> musterr [CLI] bundle plan +Error: bind blocks in the target configuration are only supported with the direct deployment engine; set DATABRICKS_BUNDLE_ENGINE=direct or remove the bind blocks + diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/script b/acceptance/bundle/deploy/bind/terraform-with-bind/script new file mode 100644 index 00000000000..4b3c7a8ecde --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/script @@ -0,0 +1,2 @@ +# Import blocks should error with terraform engine +trace musterr $CLI bundle plan diff --git a/acceptance/bundle/deploy/bind/terraform-with-bind/test.toml b/acceptance/bundle/deploy/bind/terraform-with-bind/test.toml new file mode 100644 index 00000000000..272dde4b9ce --- /dev/null +++ b/acceptance/bundle/deploy/bind/terraform-with-bind/test.toml @@ -0,0 +1,3 @@ +# Override engine to terraform to test import block rejection +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/bind/test.toml b/acceptance/bundle/deploy/bind/test.toml new file mode 100644 index 00000000000..931833f6cc5 --- /dev/null +++ b/acceptance/bundle/deploy/bind/test.toml @@ -0,0 +1,5 @@ +Cloud = true +Ignore = [".databricks"] + +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/top-level-bind/databricks.yml b/acceptance/bundle/deploy/bind/top-level-bind/databricks.yml new file mode 100644 index 00000000000..67833116972 --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: test-bind-top-level + +bind: + jobs: + foo: + id: "123" + +resources: + jobs: + foo: + name: test-job + environments: + - environment_key: default + spec: + client: "1" + tasks: + - task_key: my_task + environment_key: default + spark_python_task: + python_file: ./hello.py diff --git a/acceptance/bundle/deploy/bind/top-level-bind/hello.py b/acceptance/bundle/deploy/bind/top-level-bind/hello.py new file mode 100644 index 00000000000..11b15b1a458 --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/hello.py @@ -0,0 +1 @@ +print("hello") diff --git a/acceptance/bundle/deploy/bind/top-level-bind/out.test.toml b/acceptance/bundle/deploy/bind/top-level-bind/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/bind/top-level-bind/output.txt b/acceptance/bundle/deploy/bind/top-level-bind/output.txt new file mode 100644 index 00000000000..9425c8feb52 --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/output.txt @@ -0,0 +1,9 @@ + +>>> musterr [CLI] bundle validate +Error: bind blocks are not allowed at the root level + in databricks.yml:4:1 + +bind blocks must be defined within a target. Move the bind configuration under targets..bind + + +Found 1 error diff --git a/acceptance/bundle/deploy/bind/top-level-bind/script b/acceptance/bundle/deploy/bind/top-level-bind/script new file mode 100644 index 00000000000..44dbe24759c --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/script @@ -0,0 +1 @@ +trace musterr $CLI bundle validate diff --git a/acceptance/bundle/deploy/bind/top-level-bind/test.toml b/acceptance/bundle/deploy/bind/top-level-bind/test.toml new file mode 100644 index 00000000000..18b1a88417e --- /dev/null +++ b/acceptance/bundle/deploy/bind/top-level-bind/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/bundle/config/bind.go b/bundle/config/bind.go new file mode 100644 index 00000000000..943f3d9582a --- /dev/null +++ b/bundle/config/bind.go @@ -0,0 +1,86 @@ +package config + +import ( + "fmt" + "slices" + "strings" + + "github.com/databricks/cli/libs/diag" +) + +// BindResource represents a single resource to bind with its workspace ID. +type BindResource struct { + ID string `json:"id"` +} + +// Bind defines existing workspace resources to bring under bundle management at deploy +// time. The outer map key is the resource type (e.g. "jobs", "pipelines"), the inner +// key is the resource name in the bundle config, and the value carries the workspace +// resource ID. Bind blocks are only valid for the direct deployment engine. +type Bind map[string]map[string]BindResource + +// ForEach calls fn for each bind entry in the configuration. Iteration order is +// stable (sorted by resource type, then resource name) so callers that emit user- +// visible diagnostics get deterministic output across runs. +func (b Bind) ForEach(fn func(resourceType, resourceName, bindID string)) { + resourceTypes := make([]string, 0, len(b)) + for resourceType := range b { + resourceTypes = append(resourceTypes, resourceType) + } + slices.Sort(resourceTypes) + for _, resourceType := range resourceTypes { + resources := b[resourceType] + names := make([]string, 0, len(resources)) + for name := range resources { + names = append(names, name) + } + slices.Sort(names) + for _, name := range names { + fn(resourceType, name, resources[name].ID) + } + } +} + +// IsEmpty returns true if no binds are defined. +func (b Bind) IsEmpty() bool { + for _, resources := range b { + if len(resources) > 0 { + return false + } + } + return true +} + +// Validate rejects bind blocks that target child resources (e.g. "jobs.permissions"). +// The direct engine exposes child resources as full keys like "jobs.permissions" in its +// resource registry, but they are not bindable on their own — bind the parent resource +// and let the bundle manage the child entries declaratively. +func (b Bind) Validate() diag.Diagnostics { + var diags diag.Diagnostics + + for resourceType, resources := range b { + if !strings.HasSuffix(resourceType, ".permissions") && !strings.HasSuffix(resourceType, ".grants") { + continue + } + for resourceName := range resources { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("binding %s is not allowed", resourceType), + Detail: fmt.Sprintf( + "bind can only be used for resources directly under the resources block, not for child resources like permissions or grants.\n\n"+ + "To manage permissions or grants:\n"+ + "1. First bind the parent resource (without .permissions or .grants)\n"+ + "2. Then define permissions or grants in your bundle configuration\n\n"+ + "Invalid bind configuration:\n"+ + " bind:\n"+ + " %s:\n"+ + " %s:\n"+ + " id: ...\n\n"+ + "Instead, remove this bind entry and ensure the parent resource is bound.", + resourceType, resourceName), + }) + } + } + + return diags +} diff --git a/bundle/config/bind_test.go b/bundle/config/bind_test.go new file mode 100644 index 00000000000..e5a02a7eceb --- /dev/null +++ b/bundle/config/bind_test.go @@ -0,0 +1,70 @@ +package config_test + +import ( + "testing" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/stretchr/testify/assert" +) + +func TestBindIsEmpty(t *testing.T) { + cases := []struct { + name string + bind config.Bind + want bool + }{ + {"nil", nil, true}, + {"empty outer", config.Bind{}, true}, + {"empty inner", config.Bind{"jobs": {}}, true}, + {"populated", config.Bind{"jobs": {"foo": {ID: "1"}}}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + assert.Equal(t, c.want, c.bind.IsEmpty()) + }) + } +} + +func TestBindForEachIteratesInSortedOrder(t *testing.T) { + bind := config.Bind{ + "pipelines": {"baz": {ID: "3"}}, + "jobs": {"foo": {ID: "1"}, "bar": {ID: "2"}}, + } + type entry struct{ rt, rn, id string } + var got []entry + bind.ForEach(func(rt, rn, id string) { + got = append(got, entry{rt, rn, id}) + }) + // ForEach guarantees stable order so multi-error diagnostics are reproducible. + assert.Equal(t, []entry{ + {"jobs", "bar", "2"}, + {"jobs", "foo", "1"}, + {"pipelines", "baz", "3"}, + }, got) +} + +func TestBindValidate(t *testing.T) { + cases := []struct { + name string + bind config.Bind + wantError bool + }{ + {"top-level resource", config.Bind{"jobs": {"foo": {ID: "1"}}}, false}, + {"permissions child", config.Bind{"jobs.permissions": {"foo": {ID: "1"}}}, true}, + {"grants child", config.Bind{"schemas.grants": {"foo": {ID: "1"}}}, true}, + // Substring match must NOT trigger; only the .permissions / .grants suffix does. + {"name containing permissions", config.Bind{"my_permissions_jobs": {"foo": {ID: "1"}}}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + diags := c.bind.Validate() + if c.wantError { + assert.True(t, diags.HasError()) + assert.Equal(t, diag.Error, diags[0].Severity) + } else { + assert.False(t, diags.HasError()) + } + }) + } +} diff --git a/bundle/config/mutator/validate_bind_resources.go b/bundle/config/mutator/validate_bind_resources.go new file mode 100644 index 00000000000..129b90b3e49 --- /dev/null +++ b/bundle/config/mutator/validate_bind_resources.go @@ -0,0 +1,29 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type validateBindResources struct{} + +// ValidateBindResources validates that bind blocks only contain valid resource types. +// Binding is only allowed for resources directly under the resources block, +// not for child resources like permissions or grants. +func ValidateBindResources() bundle.Mutator { + return &validateBindResources{} +} + +func (m *validateBindResources) Name() string { + return "ValidateBindResources" +} + +func (m *validateBindResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Target == nil { + return nil + } + + return b.Target.Bind.Validate() +} diff --git a/bundle/config/mutator/validate_bind_resources_test.go b/bundle/config/mutator/validate_bind_resources_test.go new file mode 100644 index 00000000000..e8f10ac9303 --- /dev/null +++ b/bundle/config/mutator/validate_bind_resources_test.go @@ -0,0 +1,40 @@ +package mutator + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/assert" +) + +func TestValidateBindResourcesNoTarget(t *testing.T) { + b := &bundle.Bundle{} + diags := bundle.Apply(t.Context(), b, ValidateBindResources()) + assert.NoError(t, diags.Error()) +} + +func TestValidateBindResourcesAcceptsTopLevel(t *testing.T) { + b := &bundle.Bundle{ + Target: &config.Target{ + Bind: config.Bind{ + "jobs": {"foo": {ID: "1"}}, + }, + }, + } + diags := bundle.Apply(t.Context(), b, ValidateBindResources()) + assert.NoError(t, diags.Error()) +} + +func TestValidateBindResourcesRejectsChildResources(t *testing.T) { + b := &bundle.Bundle{ + Target: &config.Target{ + Bind: config.Bind{ + "jobs.permissions": {"foo": {ID: "1"}}, + }, + }, + } + diags := bundle.Apply(t.Context(), b, ValidateBindResources()) + assert.Error(t, diags.Error()) + assert.Contains(t, diags[0].Summary, "binding jobs.permissions is not allowed") +} diff --git a/bundle/config/target.go b/bundle/config/target.go index fae9c940b3f..a58e5191ee1 100644 --- a/bundle/config/target.go +++ b/bundle/config/target.go @@ -69,6 +69,11 @@ type Target struct { Sync *Sync `json:"sync,omitempty"` Permissions []resources.Permission `json:"permissions,omitempty"` + + // Bind specifies existing workspace resources to bind into bundle management. + // Resources listed here will be bound to the bundle at deploy time. + // This field is only valid for the direct deployment engine. + Bind Bind `json:"bind,omitempty"` } const ( diff --git a/bundle/deployplan/action.go b/bundle/deployplan/action.go index e8e14279567..87e144dfa0f 100644 --- a/bundle/deployplan/action.go +++ b/bundle/deployplan/action.go @@ -28,36 +28,45 @@ type ActionType string // If case of several options, action with highest severity wins. // Note, Create/Delete are handled explicitly and never compared. const ( - Undefined ActionType = "" - Skip ActionType = "skip" - Resize ActionType = "resize" - Update ActionType = "update" - UpdateWithID ActionType = "update_id" - Create ActionType = "create" - Recreate ActionType = "recreate" - Delete ActionType = "delete" + Undefined ActionType = "" + Skip ActionType = "skip" + Resize ActionType = "resize" + Update ActionType = "update" + UpdateWithID ActionType = "update_id" + Bind ActionType = "bind" + BindAndUpdate ActionType = "bind_and_update" + Create ActionType = "create" + Recreate ActionType = "recreate" + Delete ActionType = "delete" ) var actionOrder = map[ActionType]int{ - Undefined: 0, - Skip: 1, - Resize: 2, - Update: 3, - UpdateWithID: 4, - Create: 5, - Recreate: 6, - Delete: 7, + Undefined: 0, + Skip: 1, + Resize: 2, + Update: 3, + UpdateWithID: 4, + Bind: 5, + BindAndUpdate: 6, + Create: 7, + Recreate: 8, + Delete: 9, } func (a ActionType) KeepsID() bool { switch a { - case Create, UpdateWithID, Recreate, Delete: + case Create, UpdateWithID, Recreate, Delete, Bind, BindAndUpdate: return false default: return true } } +// IsBind returns true if the action is a bind action. +func (a ActionType) IsBind() bool { + return a == Bind || a == BindAndUpdate +} + // StringShort short version of action string, without suffix func (a ActionType) StringShort() string { items := strings.SplitN(string(a), "_", 2) diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index b35357c7c28..fe39bd7e6cb 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -74,6 +74,7 @@ func LoadPlanFromFile(path string) (*Plan, error) { type PlanEntry struct { ID string `json:"id,omitempty"` + BindID string `json:"bind_id,omitempty"` DependsOn []DependsOnEntry `json:"depends_on,omitempty"` Action ActionType `json:"action,omitempty"` NewState *structvar.StructVarJSON `json:"new_state,omitempty"` diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index e7186f64670..659537904e4 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -47,6 +47,38 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, } } +// DeclarativeBind brings an existing workspace resource under bundle management. +// For Bind, it just persists state with the bind ID. For BindAndUpdate, it also +// applies the configured field changes to the remote resource. +func (d *DeploymentUnit) DeclarativeBind(ctx context.Context, db *dstate.DeploymentState, bindID string, newState any, actionType deployplan.ActionType, planEntry *deployplan.PlanEntry) error { + if actionType == deployplan.BindAndUpdate { + if !d.Adapter.HasDoUpdate() { + return fmt.Errorf("internal error: DoUpdate not implemented for resource %s", d.ResourceKey) + } + + remoteState, err := d.Adapter.DoUpdate(ctx, bindID, newState, planEntry) + if err != nil { + return fmt.Errorf("updating bound resource id=%s: %w", bindID, err) + } + + err = d.SetRemoteState(remoteState) + if err != nil { + return err + } + + log.Infof(ctx, "Bound and updated %s id=%s", d.ResourceKey, bindID) + } else { + log.Infof(ctx, "Bound %s id=%s", d.ResourceKey, bindID) + } + + err := db.SaveState(d.ResourceKey, bindID, newState, d.DependsOn) + if err != nil { + return fmt.Errorf("saving state id=%s: %w", bindID, err) + } + + return nil +} + func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any) error { newID, remoteState, err := d.Adapter.DoCreate(ctx, newState) if err != nil { diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index a7f3ee65fc2..4a8a39365eb 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -119,6 +119,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } err = b.StateDB.SaveState(resourceKey, id, sv.Value, entry.DependsOn) + } else if action.IsBind() { + err = d.DeclarativeBind(ctx, &b.StateDB, entry.BindID, sv.Value, action, entry) } else { // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) diff --git a/bundle/direct/bundle_plan_bind.go b/bundle/direct/bundle_plan_bind.go new file mode 100644 index 00000000000..faf009234eb --- /dev/null +++ b/bundle/direct/bundle_plan_bind.go @@ -0,0 +1,263 @@ +package direct + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/structs/structdiff" +) + +// ApplyBindToPlan layers declarative bind blocks onto a plan that has already been +// produced by CalculatePlan. It validates the bind config and overrides plan +// entries for resources that are being bound for the first time. +// +// For each bind entry: +// - If the resource has no state, override the planned Create with Bind or +// BindAndUpdate by reading the remote resource and diffing against config. +// - If the resource is already in state under the same ID, the bind is a no-op +// (a previously-bound resource deploys normally). +// - If the resource is in state under a different ID, surface an error. +func (b *DeploymentBundle) ApplyBindToPlan(ctx context.Context, configRoot *config.Root, bindConfig config.Bind) error { + if bindConfig.IsEmpty() { + return nil + } + + if b.Plan == nil { + return errors.New("internal error: ApplyBindToPlan called before CalculatePlan") + } + + if !b.validateBindConfig(ctx, configRoot, bindConfig) { + return errors.New("bind validation failed") + } + + bindConfig.ForEach(func(resourceType, resourceName, bindID string) { + resourceKey := "resources." + resourceType + "." + resourceName + entry := b.Plan.Plan[resourceKey] + + dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) + if hasEntry { + if dbentry.ID != bindID { + logdiag.LogError(ctx, fmt.Errorf("%s: resource already bound to ID %q, cannot bind as %q; remove the bind block or unbind the existing resource", resourceKey, dbentry.ID, bindID)) + return + } + // IDs match: leave the plan entry as-is so the resource deploys normally. + return + } + + entry.BindID = bindID + adapter, err := b.getAdapterForKey(resourceKey) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("cannot plan %s: getting adapter: %w", resourceKey, err)) + return + } + + b.handleBindPlan(ctx, resourceKey, entry, adapter) + }) + + if logdiag.HasError(ctx) { + return errors.New("bind planning failed") + } + + return nil +} + +// validateBindConfig logs diagnostics for invalid bind entries. Returns false if any +// errors were logged. +func (b *DeploymentBundle) validateBindConfig(ctx context.Context, configRoot *config.Root, bindConfig config.Bind) bool { + hasError := false + targetName := configRoot.Bundle.Target + + // Bind blocks must reference resources that exist in the merged config. + bindConfig.ForEach(func(resourceType, resourceName, bindID string) { + key := "resources." + resourceType + "." + resourceName + if _, ok := b.Plan.Plan[key]; ok { + return + } + bindPath := dyn.NewPath(dyn.Key("targets"), dyn.Key(targetName), dyn.Key("bind"), dyn.Key(resourceType), dyn.Key(resourceName)) + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("bind block references undefined resource %q; define it in the resources section or remove the bind block", key), + Locations: configRoot.GetLocations(bindPath.String()), + Paths: []dyn.Path{bindPath}, + }) + hasError = true + }) + + // A bind ID must not collide with another resource's ID in state. Such a collision + // would let a bind silently redirect a delete/recreate/update_id at a resource the + // user is also trying to import; reject it instead of choosing one over the other. + stateKeys := make([]string, 0, len(b.StateDB.Data.State)) + for stateKey := range b.StateDB.Data.State { + stateKeys = append(stateKeys, stateKey) + } + slices.Sort(stateKeys) + + bindConfig.ForEach(func(resourceType, resourceName, bindID string) { + bindKey := "resources." + resourceType + "." + resourceName + for _, stateKey := range stateKeys { + stateEntry := b.StateDB.Data.State[stateKey] + if stateKey == bindKey || stateEntry.ID != bindID { + continue + } + bindPath := dyn.NewPath(dyn.Key("targets"), dyn.Key(targetName), dyn.Key("bind"), dyn.Key(resourceType), dyn.Key(resourceName)) + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("bind block for %q has the same ID %q as existing resource %q; remove the bind block or the conflicting resource", bindKey, bindID, stateKey), + Locations: configRoot.GetLocations(bindPath.String()), + Paths: []dyn.Path{bindPath}, + }) + hasError = true + } + }) + + return !hasError +} + +// handleBindPlan reads the remote resource for a resource that is being bound for +// the first time, computes the change set against the local config, and selects +// either Bind (no changes) or BindAndUpdate (config drift can be applied). +// +// Recreate / UpdateWithID actions imply destroying or replacing the workspace +// resource — incompatible with bind — so they surface as actionable errors. +func (b *DeploymentBundle) handleBindPlan(ctx context.Context, resourceKey string, entry *deployplan.PlanEntry, adapter *dresources.Adapter) { + bindID := entry.BindID + errorPrefix := "cannot bind " + resourceKey + + remoteState, err := adapter.DoRead(ctx, bindID) + if err != nil { + if isResourceGone(err) { + logdiag.LogError(ctx, fmt.Errorf("%s: resource with ID %q does not exist in workspace", errorPrefix, bindID)) + } else { + logdiag.LogError(ctx, fmt.Errorf("%s: reading remote resource id=%q: %w", errorPrefix, bindID, err)) + } + return + } + + entry.RemoteState = remoteState + b.RemoteStateCache.Store(resourceKey, remoteState) + + sv, ok := b.StateCache.Load(resourceKey) + if !ok { + logdiag.LogError(ctx, fmt.Errorf("%s: internal error: no state cache entry", errorPrefix)) + return + } + + remoteStateComparable, err := adapter.RemapState(remoteState) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: interpreting remote state: %w", errorPrefix, err)) + return + } + + remoteDiff, err := structdiff.GetStructDiff(remoteStateComparable, sv.Value, adapter.KeyedSlices()) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: diffing remote state: %w", errorPrefix, err)) + return + } + + // No "saved state" exists for a first-time bind; the diff is purely remote vs. config. + entry.Changes, err = prepareChanges(ctx, adapter, nil, remoteDiff, nil, remoteStateComparable) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return + } + + err = addPerFieldActions(ctx, adapter, entry.Changes, remoteState) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) + return + } + + switch maxAction := getMaxAction(entry.Changes); maxAction { + case deployplan.Skip, deployplan.Undefined: + entry.Action = deployplan.Bind + case deployplan.Update, deployplan.Resize: + entry.Action = deployplan.BindAndUpdate + case deployplan.Recreate, deployplan.UpdateWithID: + logdiag.LogError(ctx, buildBindConflictError(errorPrefix, maxAction, entry.Changes, resourceKey)) + default: + logdiag.LogError(ctx, fmt.Errorf("%s: internal error: unexpected action %q during bind planning", errorPrefix, maxAction)) + } +} + +// buildBindConflictError formats a multi-line message that names the fields whose +// changes forced an incompatible action (Recreate / UpdateWithID) and shows the two +// concrete resolutions: drop the offending fields, or drop the bind block. +func buildBindConflictError(errorPrefix string, action deployplan.ActionType, changes deployplan.Changes, resourceKey string) error { + problematicFields := make([]string, 0, len(changes)) + for fieldPath, change := range changes { + if change.Action == action { + problematicFields = append(problematicFields, fieldPath) + } + } + // Stable order so the YAML examples are deterministic across runs. + slices.Sort(problematicFields) + + resourceType, resourceName := splitResourceKey(resourceKey) + + var msg strings.Builder + // buildBindConflictError is only invoked for Recreate / UpdateWithID; other + // action types are guarded by the caller. + switch action { //nolint:exhaustive + case deployplan.Recreate: + msg.WriteString(errorPrefix + ": cannot recreate resource with bind block\n\n") + msg.WriteString("This would destroy and recreate the existing workspace resource, changing its ID.\n\n") + case deployplan.UpdateWithID: + msg.WriteString(errorPrefix + ": cannot update resource ID with bind block\n\n") + msg.WriteString("This would replace the existing workspace resource with a new ID.\n\n") + } + + msg.WriteString("The following fields cannot be modified because they require ") + if action == deployplan.Recreate { + msg.WriteString("resource recreation:\n") + } else { + msg.WriteString("ID changes:\n") + } + for _, field := range problematicFields { + if reason := changes[field].Reason; reason != "" { + fmt.Fprintf(&msg, " - %s (%s)\n", field, reason) + } else { + fmt.Fprintf(&msg, " - %s\n", field) + } + } + msg.WriteString("\n") + + msg.WriteString("To resolve this issue, you have two options:\n\n") + msg.WriteString("1. Remove the problematic fields from your configuration to make this a bind-only operation:\n\n") + msg.WriteString(" resources:\n") + fmt.Fprintf(&msg, " %s:\n", resourceType) + fmt.Fprintf(&msg, " %s:\n", resourceName) + msg.WriteString(" # Remove or comment out these fields:\n") + for _, field := range problematicFields { + fmt.Fprintf(&msg, " # %s: ...\n", field) + } + msg.WriteString("\n") + msg.WriteString("2. Remove the bind block if you want to allow the resource to be recreated/updated:\n\n") + msg.WriteString(" targets:\n") + msg.WriteString(" :\n") + msg.WriteString(" # Remove the bind block:\n") + msg.WriteString(" # bind:\n") + fmt.Fprintf(&msg, " # %s:\n", resourceType) + fmt.Fprintf(&msg, " # %s:\n", resourceName) + msg.WriteString(" # id: \n") + + return errors.New(msg.String()) +} + +// splitResourceKey extracts the resource type and name from a key like +// "resources.jobs.foo" → ("jobs", "foo"). +func splitResourceKey(resourceKey string) (resourceType, resourceName string) { + parts := strings.SplitN(resourceKey, ".", 3) + if len(parts) < 3 { + return "", "" + } + return parts[1], parts[2] +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 2f28ca27596..921c7f585bf 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -23,6 +23,10 @@ github.com/databricks/cli/bundle/config.ArtifactFile: "source": "description": |- Required. The artifact source file. +github.com/databricks/cli/bundle/config.BindResource: + "id": + "description": |- + The ID of the existing workspace resource to bind. github.com/databricks/cli/bundle/config.Bundle: "cluster_id": "description": |- @@ -372,6 +376,9 @@ github.com/databricks/cli/bundle/config.Target: "artifacts": "description": |- The artifacts to include in the target deployment. + "bind": + "description": |- + The existing workspace resources to bind into bundle management for this target. "bundle": "description": |- The bundle attributes when deploying to this target. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 38389b9adb2..844b0af709c 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -256,12 +256,25 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *deployplan.Plan { + // Bind blocks rely on direct-engine planning primitives (DoRead, RemapState), + // so reject them early when the user is on the terraform engine. + if !engine.IsDirect() && b.Target != nil && !b.Target.Bind.IsEmpty() { + logdiag.LogError(ctx, errors.New("bind blocks in the target configuration are only supported with the direct deployment engine; set DATABRICKS_BUNDLE_ENGINE=direct or remove the bind blocks")) + return nil + } + if engine.IsDirect() { plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config) if err != nil { logdiag.LogError(ctx, err) return nil } + if b.Target != nil && !b.Target.Bind.IsEmpty() { + if err := b.DeploymentBundle.ApplyBindToPlan(ctx, &b.Config, b.Target.Bind); err != nil { + logdiag.LogError(ctx, err) + return nil + } + } return plan } diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 4abc6140e45..741ab8351d2 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -3,9 +3,13 @@ package phases import ( "context" "errors" + "fmt" "net/http" + "slices" + "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/deploy/files" @@ -13,6 +17,7 @@ import ( "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" @@ -155,6 +160,26 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e } } +// errIfBoundResourcesInState rejects a destroy when any bind block currently +// matches a resource in state: that resource was imported, not created by the +// bundle, and a blanket destroy would delete a pre-existing workspace resource. +func errIfBoundResourcesInState(stateDB *dstate.DeploymentState, bindConfig config.Bind) error { + var boundInState []string + bindConfig.ForEach(func(resourceType, resourceName, bindID string) { + key := "resources." + resourceType + "." + resourceName + if entry, ok := stateDB.Data.State[key]; ok && entry.ID == bindID { + boundInState = append(boundInState, key) + } + }) + + if len(boundInState) == 0 { + return nil + } + + slices.Sort(boundInState) + return fmt.Errorf("cannot destroy with bind blocks that reference resources in the deployment state: %s; remove the bind blocks from the target configuration or run 'bundle deployment unbind' before destroying", strings.Join(boundInState, ", ")) +} + // The destroy phase deletes artifacts and resources. func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { log.Info(ctx, "Phase: destroy") @@ -199,6 +224,16 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { var plan *deployplan.Plan if engine.IsDirect() { + // Refuse to destroy when bind blocks point at resources that are currently + // in state: those are pre-existing workspace resources the user imported, + // and destroying them would silently delete data the bundle did not create. + if b.Target != nil && !b.Target.Bind.IsEmpty() { + if err := errIfBoundResourcesInState(&b.DeploymentBundle.StateDB, b.Target.Bind); err != nil { + logdiag.LogError(ctx, err) + return + } + } + plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), nil) if err != nil { logdiag.LogError(ctx, err) diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 25bb2f4bd1b..ab26b5c6291 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -185,6 +185,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Validates that when using a shared workspace path, appropriate permissions are configured permissions.ValidateSharedRootPermissions(), + // Reads (typed): b.Config.Targets..Bind (checks for permissions/grants under bind, which are not bindable) + // Reports validation errors when bind blocks reference child resources instead of top-level resources. + mutator.ValidateBindResources(), + // Annotate resources with "deployment" metadata. // // We don't include this step into initializeResources because these mutators set fields that are diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index ee105a6f821..383316a18a6 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -2206,6 +2206,27 @@ "config.ArtifactType": { "type": "string" }, + "config.BindResource": { + "oneOf": [ + { + "type": "object", + "properties": { + "id": { + "description": "The ID of the existing workspace resource to bind.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false, + "required": [ + "id" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "config.Bundle": { "oneOf": [ { @@ -2632,6 +2653,10 @@ "description": "The artifacts to include in the target deployment.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.Artifact" }, + "bind": { + "description": "The existing workspace resources to bind into bundle management for this target.", + "$ref": "#/$defs/map/map/github.com/databricks/cli/bundle/config.BindResource" + }, "bundle": { "description": "The bundle attributes when deploying to this target.", "$ref": "#/$defs/github.com/databricks/cli/bundle/config.Bundle" @@ -11552,6 +11577,20 @@ } ] }, + "config.BindResource": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config.BindResource" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "config.Command": { "oneOf": [ { @@ -11598,6 +11637,30 @@ } } }, + "map": { + "github.com": { + "databricks": { + "cli": { + "bundle": { + "config.BindResource": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config.BindResource" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + } + } + } + } + } + }, "string": { "oneOf": [ { diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 5020d88e73a..400fb248e06 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -166,6 +166,14 @@ to the workspace so that subsequent deploys of this bundle use direct deployment } ctx := cmd.Context() + // `bundle deployment migrate` runs against an existing terraform deployment to + // produce a direct-engine state. Bind is a direct-engine-only feature, so we + // don't have a defined behavior for "migrating" with bind blocks present. Reject + // it and let the user remove the bind blocks, migrate, then add them back. + if b.Target != nil && !b.Target.Bind.IsEmpty() { + return errors.New("cannot run 'bundle deployment migrate' when bind blocks are defined in the target configuration; bind blocks are only supported with the direct deployment engine") + } + if stateDesc.Lineage == "" { cmdio.LogString(ctx, `Error: This command migrates the existing Terraform state file (terraform.tfstate) to a direct deployment state file (resources.json). However, no existing local or remote state was found. diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index e3dd63929ed..ec5854afbc0 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -72,6 +72,7 @@ It is useful for previewing changes before running 'bundle deploy'.`, updateCount := 0 deleteCount := 0 unchangedCount := 0 + bindCount := 0 for _, change := range plan.GetActions() { switch change.ActionType { @@ -85,6 +86,11 @@ It is useful for previewing changes before running 'bundle deploy'.`, // A recreate counts as both a delete and a create deleteCount++ createCount++ + case deployplan.Bind: + bindCount++ + case deployplan.BindAndUpdate: + bindCount++ + updateCount++ case deployplan.Skip, deployplan.Undefined: unchangedCount++ } @@ -95,7 +101,7 @@ It is useful for previewing changes before running 'bundle deploy'.`, switch root.OutputType(cmd) { case flags.OutputText: // Print summary line and actions to stdout - totalChanges := createCount + updateCount + deleteCount + totalChanges := createCount + updateCount + deleteCount + bindCount if totalChanges > 0 { // Print all actions in the order they were processed for _, action := range plan.GetActions() { @@ -103,12 +109,21 @@ It is useful for previewing changes before running 'bundle deploy'.`, continue } key := strings.TrimPrefix(action.ResourceKey, "resources.") - fmt.Fprintf(out, "%s %s\n", action.ActionType.StringShort(), key) + if action.ActionType.IsBind() { + entry := plan.Plan[action.ResourceKey] + fmt.Fprintf(out, "%s %s (id: %s)\n", action.ActionType.StringShort(), key, entry.BindID) + } else { + fmt.Fprintf(out, "%s %s\n", action.ActionType.StringShort(), key) + } } fmt.Fprintln(out) } // Note, this string should not be changed, "bundle deployment migrate" depends on this format: - fmt.Fprintf(out, "Plan: %d to add, %d to change, %d to delete, %d unchanged\n", createCount, updateCount, deleteCount, unchangedCount) + if bindCount > 0 { + fmt.Fprintf(out, "Plan: %d to add, %d to change, %d to delete, %d unchanged, %d to bind\n", createCount, updateCount, deleteCount, unchangedCount, bindCount) + } else { + fmt.Fprintf(out, "Plan: %d to add, %d to change, %d to delete, %d unchanged\n", createCount, updateCount, deleteCount, unchangedCount) + } case flags.OutputJSON: buf, err := json.MarshalIndent(plan, "", " ") if err != nil { diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 79cfee37441..ae496c3a5c4 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -116,6 +116,21 @@ func (n normalizeOptions) normalizeStruct(typ reflect.Type, src dyn.Value, seen } } + // Bind is only valid under a target. Authors sometimes try to put it at + // the root, which would otherwise surface here as a generic "unknown + // field" warning that's easy to miss. Emit a targeted error so the + // misplaced block fails the build with actionable guidance. + if fieldName == "bind" && len(path) == 0 { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "bind blocks are not allowed at the root level", + Detail: "bind blocks must be defined within a target. Move the bind configuration under targets..bind", + Locations: pk.Locations(), + Paths: []dyn.Path{path}, + }) + continue + } + diags = diags.Append(diag.Diagnostic{ Severity: diag.Warning, Summary: "unknown field: " + fieldName, diff --git a/libs/dyn/convert/normalize_test.go b/libs/dyn/convert/normalize_test.go index 7eb50a13b67..371644f1de4 100644 --- a/libs/dyn/convert/normalize_test.go +++ b/libs/dyn/convert/normalize_test.go @@ -90,6 +90,25 @@ func TestNormalizeStructUnknownField(t *testing.T) { }, vout.AsAny()) } +func TestNormalizeStructBindAtRootIsError(t *testing.T) { + // "bind" at the root level is a common mistake — without this special case it + // would silently degrade to an "unknown field" warning. Make sure we surface a + // dedicated error with location info instead. + type Tmp struct { + Foo string `json:"foo"` + } + + var typ Tmp + + m := dyn.NewMapping() + m.SetLoc("bind", []dyn.Location{{File: "databricks.yml", Line: 1, Column: 1}}, dyn.V(map[string]dyn.Value{})) + + _, diags := Normalize(typ, dyn.V(m)) + assert.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "bind blocks are not allowed at the root level", diags[0].Summary) +} + func TestNormalizeStructNil(t *testing.T) { type Tmp struct { Foo string `json:"foo"`