From 7f73e73ec985b77ba7b5cc6945a708a1af73f98c Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 23 Apr 2026 13:50:41 +0200 Subject: [PATCH 1/2] Propagate auth env to experimental.python subprocess When workspace.profile is set (via --profile, DATABRICKS_CONFIG_PROFILE, or workspace.profile in databricks.yml), bundle commands that run the experimental.python phase need to pass the resolved profile through to the Python subprocess. Otherwise the Databricks SDK inside Python re-invokes the CLI via `databricks auth token --host ` without a profile hint, and the CLI cannot disambiguate profiles sharing the same host in ~/.databrickscfg. Mirrors what the Terraform phase already does via b.AuthEnv() and fixes the remaining path reported on issue #4649. Co-authored-by: Isaac --- NEXT_CHANGELOG.md | 2 + .../config/mutator/python/python_mutator.go | 12 +++++ .../mutator/python/python_mutator_test.go | 49 +++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index c87c2627c96..edbfb1654f3 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,6 +10,8 @@ ### Bundles +* Propagate authentication environment (including `DATABRICKS_CONFIG_PROFILE`) to the `experimental.python` subprocess so bundle validate/deploy no longer fails with a multi-profile host ambiguity error when several profiles in `~/.databrickscfg` share the same host. + ### Dependency updates * Added `github.com/zalando/go-keyring` as a new dependency (dormant until a later release enables experimental secure-storage for OAuth tokens). diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index c20e172c00f..2c1d28649f7 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -104,6 +104,7 @@ type runPythonMutatorOpts struct { bundleRootPath string pythonPath string loadLocations bool + authEnv map[string]string } // getOpts adapts deprecated PyDABs and upcoming Python configuration @@ -217,6 +218,15 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return diag.Errorf("Running Python code is not allowed when DATABRICKS_BUNDLE_RESTRICTED_CODE_EXECUTION is set") } + // Propagate auth env so the Databricks SDK in the Python subprocess uses the + // same credentials as the CLI. In particular this carries DATABRICKS_CONFIG_PROFILE, + // which lets the CLI disambiguate profiles sharing the same host when the SDK + // re-invokes `databricks auth token --host `. + authEnv, err := b.AuthEnv(ctx) + if err != nil { + return diag.FromErr(err) + } + // mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics' var mutateDiags diag.Diagnostics var result applyPythonOutputResult @@ -238,6 +248,7 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno bundleRootPath: b.BundleRootPath, pythonPath: pythonPath, loadLocations: opts.loadLocations, + authEnv: authEnv, }) mutateDiags = diags if diags.HasError() { @@ -364,6 +375,7 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op process.WithDir(opts.bundleRootPath), process.WithStderrWriter(stderrWriter), process.WithStdoutWriter(stdoutWriter), + process.WithEnvs(opts.authEnv), ) if processErr != nil { logger.Debugf(ctx, "python mutator process failed: %s", processErr) diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 5ea8868b170..24fddcbabe5 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -239,6 +239,55 @@ resources: assert.Equal(t, int64(1), b.Metrics.PythonUpdatedResourcesCount) } +func TestPythonMutator_propagatesAuthEnv(t *testing.T) { + withFakeVEnv(t, ".venv") + + // Minimal databrickscfg so that b.AuthEnv() can resolve a profile. + // Use the .invalid TLD (RFC 2606) so SDK host metadata resolution fails + // fast via DNS instead of hanging on a TCP connect. + cfgPath := filepath.Join(t.TempDir(), ".databrickscfg") + err := os.WriteFile(cfgPath, []byte("[my-profile]\nhost = https://bundle-test.invalid\ntoken = dapi-test\n"), 0o600) + require.NoError(t, err) + t.Setenv("DATABRICKS_CONFIG_FILE", cfgPath) + + b := loadYaml("databricks.yml", ` +experimental: + python: + venv_path: .venv + resources: ["resources:load_resources"] +workspace: + profile: my-profile`) + + // Set up process stub directly so we can inspect the subprocess env. + ctx, stub := process.WithStub(t.Context()) + t.Setenv(env.TempDirVariable, t.TempDir()) + cacheDir, err := createCacheDir(ctx) + require.NoError(t, err) + + outputJSON := `{ + "experimental": { + "python": { + "venv_path": ".venv", + "resources": ["resources:load_resources"] + } + }, + "workspace": { + "profile": "my-profile" + } + }` + + stub.WithCallback(func(cmd *exec.Cmd) error { + require.NoError(t, os.WriteFile(filepath.Join(cacheDir, "output.json"), []byte(outputJSON), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(cacheDir, "diagnostics.json"), []byte(""), 0o600)) + return nil + }) + + diags := bundle.Apply(ctx, b, PythonMutator(PythonMutatorPhaseLoadResources)) + assert.NoError(t, diags.Error()) + + assert.Equal(t, "my-profile", stub.LookupEnv("DATABRICKS_CONFIG_PROFILE")) +} + func TestPythonMutator_badOutput(t *testing.T) { withFakeVEnv(t, ".venv") b := loadYaml("databricks.yml", ` From 9f0f66957069f01e3fd46b4b08902c610789298b Mon Sep 17 00:00:00 2001 From: simon Date: Thu, 7 May 2026 13:55:21 +0200 Subject: [PATCH 2/2] Convert auth env propagation test to acceptance test Replace the unit test that stubbed process.Background with an acceptance test under acceptance/bundle/python/propagates-auth-env/ that runs a real python mutator and asserts DATABRICKS_CONFIG_PROFILE is captured from the subprocess env. Co-authored-by: Isaac --- .../python/propagates-auth-env/.databrickscfg | 7 +++ .../python/propagates-auth-env/databricks.yml | 16 ++++++ .../python/propagates-auth-env/mutators.py | 13 +++++ .../python/propagates-auth-env/out.test.toml | 4 ++ .../python/propagates-auth-env/output.txt | 8 +++ .../bundle/python/propagates-auth-env/script | 18 +++++++ .../mutator/python/python_mutator_test.go | 49 ------------------- 7 files changed, 66 insertions(+), 49 deletions(-) create mode 100644 acceptance/bundle/python/propagates-auth-env/.databrickscfg create mode 100644 acceptance/bundle/python/propagates-auth-env/databricks.yml create mode 100644 acceptance/bundle/python/propagates-auth-env/mutators.py create mode 100644 acceptance/bundle/python/propagates-auth-env/out.test.toml create mode 100644 acceptance/bundle/python/propagates-auth-env/output.txt create mode 100644 acceptance/bundle/python/propagates-auth-env/script diff --git a/acceptance/bundle/python/propagates-auth-env/.databrickscfg b/acceptance/bundle/python/propagates-auth-env/.databrickscfg new file mode 100644 index 00000000000..dec8f683589 --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/.databrickscfg @@ -0,0 +1,7 @@ +[my-profile] +host = $DATABRICKS_HOST +token = $DATABRICKS_TOKEN + +[other-profile] +host = $DATABRICKS_HOST +token = other-token diff --git a/acceptance/bundle/python/propagates-auth-env/databricks.yml b/acceptance/bundle/python/propagates-auth-env/databricks.yml new file mode 100644 index 00000000000..ad214e007bf --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: my_project + +sync: {paths: []} # don't need to copy files + +python: + mutators: + - "mutators:capture_profile_env" + +workspace: + profile: my-profile + +resources: + jobs: + my_job: + name: "Job" diff --git a/acceptance/bundle/python/propagates-auth-env/mutators.py b/acceptance/bundle/python/propagates-auth-env/mutators.py new file mode 100644 index 00000000000..959d3929379 --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/mutators.py @@ -0,0 +1,13 @@ +from databricks.bundles.jobs import Job +from databricks.bundles.core import job_mutator, Bundle +import os + + +@job_mutator +def capture_profile_env(bundle: Bundle, job: Job) -> Job: + # The CLI must propagate DATABRICKS_CONFIG_PROFILE to the python subprocess + # so the Databricks SDK can disambiguate when multiple profiles share a host. + value = os.getenv("DATABRICKS_CONFIG_PROFILE", "") + with open("captured_env.txt", "w") as f: + f.write(value) + return job diff --git a/acceptance/bundle/python/propagates-auth-env/out.test.toml b/acceptance/bundle/python/propagates-auth-env/out.test.toml new file mode 100644 index 00000000000..0969b3f3733 --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/propagates-auth-env/output.txt b/acceptance/bundle/python/propagates-auth-env/output.txt new file mode 100644 index 00000000000..7279b3df1a6 --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/output.txt @@ -0,0 +1,8 @@ + +>>> uv run [UV_ARGS] -q [CLI] bundle summary -o json +{ + "profile": "my-profile" +} + +>>> cat captured_env.txt +my-profile diff --git a/acceptance/bundle/python/propagates-auth-env/script b/acceptance/bundle/python/propagates-auth-env/script new file mode 100644 index 00000000000..73f9542cce3 --- /dev/null +++ b/acceptance/bundle/python/propagates-auth-env/script @@ -0,0 +1,18 @@ + +# Two workspace profiles share the same host so picking one is meaningful. +envsubst < .databrickscfg > out && mv out .databrickscfg +export DATABRICKS_CONFIG_FILE=.databrickscfg +unset DATABRICKS_HOST +unset DATABRICKS_TOKEN +unset DATABRICKS_CONFIG_PROFILE + +trace uv run $UV_ARGS -q $CLI bundle summary -o json | jq '{profile: .workspace.profile}' + +# The python mutator captures DATABRICKS_CONFIG_PROFILE from its subprocess env. +# Without the fix, the CLI does not propagate the bundle's resolved profile, +# so the SDK inside python re-invokes the CLI without a profile and fails on +# multi-profile ambiguity. +trace cat captured_env.txt +echo "" + +rm -fr .databricks __pycache__ captured_env.txt diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 28e08c62c3b..cf81da5f78c 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -240,55 +240,6 @@ resources: assert.Equal(t, int64(1), b.Metrics.PythonUpdatedResourcesCount) } -func TestPythonMutator_propagatesAuthEnv(t *testing.T) { - withFakeVEnv(t, ".venv") - - // Minimal databrickscfg so that b.AuthEnv() can resolve a profile. - // Use the .invalid TLD (RFC 2606) so SDK host metadata resolution fails - // fast via DNS instead of hanging on a TCP connect. - cfgPath := filepath.Join(t.TempDir(), ".databrickscfg") - err := os.WriteFile(cfgPath, []byte("[my-profile]\nhost = https://bundle-test.invalid\ntoken = dapi-test\n"), 0o600) - require.NoError(t, err) - t.Setenv("DATABRICKS_CONFIG_FILE", cfgPath) - - b := loadYaml("databricks.yml", ` -experimental: - python: - venv_path: .venv - resources: ["resources:load_resources"] -workspace: - profile: my-profile`) - - // Set up process stub directly so we can inspect the subprocess env. - ctx, stub := process.WithStub(t.Context()) - t.Setenv(env.TempDirVariable, t.TempDir()) - cacheDir, err := createCacheDir(ctx) - require.NoError(t, err) - - outputJSON := `{ - "experimental": { - "python": { - "venv_path": ".venv", - "resources": ["resources:load_resources"] - } - }, - "workspace": { - "profile": "my-profile" - } - }` - - stub.WithCallback(func(cmd *exec.Cmd) error { - require.NoError(t, os.WriteFile(filepath.Join(cacheDir, "output.json"), []byte(outputJSON), 0o600)) - require.NoError(t, os.WriteFile(filepath.Join(cacheDir, "diagnostics.json"), []byte(""), 0o600)) - return nil - }) - - diags := bundle.Apply(ctx, b, PythonMutator(PythonMutatorPhaseLoadResources)) - assert.NoError(t, diags.Error()) - - assert.Equal(t, "my-profile", stub.LookupEnv("DATABRICKS_CONFIG_PROFILE")) -} - func TestPythonMutator_badOutput(t *testing.T) { withFakeVEnv(t, ".venv") b := loadYaml("databricks.yml", `