diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 57dc5693dc..132d728640 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -1,11 +1,12 @@ # NEXT CHANGELOG -## Release v0.300.0 +## Release v0.299.2 ### CLI ### Bundles +* Propagate authentication environment (including `DATABRICKS_CONFIG_PROFILE`) to the `experimental.python` subprocess so bundle validate/deploy no longer fails with a multi-profile host ambiguity error when several profiles in `~/.databrickscfg` share the same host. * Fixed `--force-pull` on `bundle summary` and `bundle open` so the flag bypasses the local state cache and reads state from the workspace. ### Dependency updates diff --git a/acceptance/bundle/python/propagates-auth-env/.databrickscfg b/acceptance/bundle/python/propagates-auth-env/.databrickscfg new file mode 100644 index 0000000000..dec8f68358 --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/.databrickscfg @@ -0,0 +1,7 @@ +[my-profile] +host = $DATABRICKS_HOST +token = $DATABRICKS_TOKEN + +[other-profile] +host = $DATABRICKS_HOST +token = other-token diff --git a/acceptance/bundle/python/propagates-auth-env/databricks.yml b/acceptance/bundle/python/propagates-auth-env/databricks.yml new file mode 100644 index 0000000000..ad214e007b --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: my_project + +sync: {paths: []} # don't need to copy files + +python: + mutators: + - "mutators:capture_profile_env" + +workspace: + profile: my-profile + +resources: + jobs: + my_job: + name: "Job" diff --git a/acceptance/bundle/python/propagates-auth-env/mutators.py b/acceptance/bundle/python/propagates-auth-env/mutators.py new file mode 100644 index 0000000000..959d392937 --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/mutators.py @@ -0,0 +1,13 @@ +from databricks.bundles.jobs import Job +from databricks.bundles.core import job_mutator, Bundle +import os + + +@job_mutator +def capture_profile_env(bundle: Bundle, job: Job) -> Job: + # The CLI must propagate DATABRICKS_CONFIG_PROFILE to the python subprocess + # so the Databricks SDK can disambiguate when multiple profiles share a host. + value = os.getenv("DATABRICKS_CONFIG_PROFILE", "") + with open("captured_env.txt", "w") as f: + f.write(value) + return job diff --git a/acceptance/bundle/python/propagates-auth-env/out.test.toml b/acceptance/bundle/python/propagates-auth-env/out.test.toml new file mode 100644 index 0000000000..0969b3f373 --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/propagates-auth-env/output.txt b/acceptance/bundle/python/propagates-auth-env/output.txt new file mode 100644 index 0000000000..7279b3df1a --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/output.txt @@ -0,0 +1,8 @@ + +>>> uv run [UV_ARGS] -q [CLI] bundle summary -o json +{ + "profile": "my-profile" +} + +>>> cat captured_env.txt +my-profile diff --git a/acceptance/bundle/python/propagates-auth-env/script b/acceptance/bundle/python/propagates-auth-env/script new file mode 100644 index 0000000000..73f9542cce --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/script @@ -0,0 +1,18 @@ + +# Two workspace profiles share the same host so picking one is meaningful. +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN +unset DATABRICKS_CONFIG_PROFILE + +trace uv run $UV_ARGS -q $CLI bundle summary -o json | jq '{profile: .workspace.profile}' + +# The python mutator captures DATABRICKS_CONFIG_PROFILE from its subprocess env. +# Without the fix, the CLI does not propagate the bundle's resolved profile, +# so the SDK inside python re-invokes the CLI without a profile and fails on +# multi-profile ambiguity. +trace cat captured_env.txt +echo "" + +rm -fr .databricks __pycache__ captured_env.txt diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 44e19b276a..ed221c00c6 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -104,6 +104,7 @@ type runPythonMutatorOpts struct { bundleRootPath string pythonPath string loadLocations bool + authEnv map[string]string } // getOpts adapts deprecated PyDABs and upcoming Python configuration @@ -217,6 +218,15 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return diag.Errorf("Running Python code is not allowed when DATABRICKS_BUNDLE_RESTRICTED_CODE_EXECUTION is set") } + // Propagate auth env so the Databricks SDK in the Python subprocess uses the + // same credentials as the CLI. In particular this carries DATABRICKS_CONFIG_PROFILE, + // which lets the CLI disambiguate profiles sharing the same host when the SDK + // re-invokes `databricks auth token --host `. + authEnv, err := b.AuthEnv(ctx) + if err != nil { + return diag.FromErr(err) + } + // mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics' var mutateDiags diag.Diagnostics var result applyPythonOutputResult @@ -238,6 +248,7 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno bundleRootPath: b.BundleRootPath, pythonPath: pythonPath, loadLocations: opts.loadLocations, + authEnv: authEnv, }) mutateDiags = diags if diags.HasError() { @@ -364,6 +375,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op process.WithDir(opts.bundleRootPath), process.WithStderrWriter(stderrWriter), process.WithStdoutWriter(stdoutWriter), + process.WithEnvs(opts.authEnv), ) if processErr != nil { logger.Debugf(ctx, "python mutator process failed: %s", processErr)