From 61f5c71ad5ebc2880a04e1d00dca6bf71769cb7d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 21 May 2026 21:02:25 +0200 Subject: [PATCH 1/4] bundle: use patched wheels in pipeline environments Dynamic wheel builds were only substituted in job libraries and environments, leaving pipeline environments pinned to the original wheel name. Add pipeline replacements and regression coverage so artifact dynamic_version behaves consistently across resources. --- .../.gitignore | 0 .../databricks.yml | 0 .../my_test_code/setup.py | 0 .../my_test_code/src/__init__.py | 0 .../my_test_code/src/__main__.py | 0 .../out.test.toml | 0 .../output.txt | 0 .../script | 0 .../databricks.yml | 17 ++++ .../my_test_code/setup.py | 15 ++++ .../my_test_code/src/__init__.py | 2 + .../my_test_code/src/__main__.py | 16 ++++ .../out.test.toml | 3 + .../pipelines_environments_dynamic/output.txt | 38 +++++++++ .../pipelines_environments_dynamic/script | 13 +++ bundle/libraries/switch_to_patched_wheels.go | 21 +++++ .../switch_to_patched_wheels_test.go | 84 +++++++++++++++++++ 17 files changed, 209 insertions(+) rename acceptance/bundle/artifacts/{whl_via_environment_key => job_environments_dynamic}/.gitignore (100%) rename acceptance/bundle/artifacts/{whl_via_environment_key => job_environments_dynamic}/databricks.yml (100%) rename acceptance/bundle/artifacts/{whl_via_environment_key => job_environments_dynamic}/my_test_code/setup.py (100%) rename acceptance/bundle/artifacts/{whl_via_environment_key => job_environments_dynamic}/my_test_code/src/__init__.py (100%) rename acceptance/bundle/artifacts/{whl_via_environment_key => job_environments_dynamic}/my_test_code/src/__main__.py (100%) rename acceptance/bundle/artifacts/{whl_via_environment_key => job_environments_dynamic}/out.test.toml (100%) rename acceptance/bundle/artifacts/{whl_via_environment_key => job_environments_dynamic}/output.txt (100%) rename acceptance/bundle/artifacts/{whl_via_environment_key => job_environments_dynamic}/script (100%) create mode 100644 acceptance/bundle/artifacts/pipelines_environments_dynamic/databricks.yml create mode 100644 acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/setup.py create mode 100644 acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__init__.py create mode 100644 acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__main__.py create mode 100644 acceptance/bundle/artifacts/pipelines_environments_dynamic/out.test.toml create mode 100644 acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt create mode 100644 acceptance/bundle/artifacts/pipelines_environments_dynamic/script create mode 100644 bundle/libraries/switch_to_patched_wheels_test.go diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/.gitignore b/acceptance/bundle/artifacts/job_environments_dynamic/.gitignore similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/.gitignore rename to acceptance/bundle/artifacts/job_environments_dynamic/.gitignore diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml b/acceptance/bundle/artifacts/job_environments_dynamic/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml rename to acceptance/bundle/artifacts/job_environments_dynamic/databricks.yml diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/setup.py b/acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/setup.py similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/setup.py rename to acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/setup.py diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/src/__init__.py similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/src/__init__.py diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/src/__main__.py similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml b/acceptance/bundle/artifacts/job_environments_dynamic/out.test.toml similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml rename to acceptance/bundle/artifacts/job_environments_dynamic/out.test.toml diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/job_environments_dynamic/output.txt similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/output.txt rename to acceptance/bundle/artifacts/job_environments_dynamic/output.txt diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/job_environments_dynamic/script similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/script rename to acceptance/bundle/artifacts/job_environments_dynamic/script diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/databricks.yml b/acceptance/bundle/artifacts/pipelines_environments_dynamic/databricks.yml new file mode 100644 index 00000000000..e8e1d0f8018 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +artifacts: + my_test_code: + type: whl + path: "./my_test_code" + build: python setup.py bdist_wheel + dynamic_version: true + +resources: + pipelines: + test_pipeline: + name: "My Wheel Pipeline" + environment: + dependencies: + - ./my_test_code/dist/*.whl diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/setup.py b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/setup.py new file mode 100644 index 00000000000..0bd871dd349 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import src + +setup( + name="my_test_code", + version=src.__version__, + author=src.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["src"]), + entry_points={"group_1": "run=src.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__init__.py new file mode 100644 index 00000000000..909f1f3220d --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.0.1" +__author__ = "Databricks" diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__main__.py new file mode 100644 index 00000000000..ea918ce2d53 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print("Hello from my func") + print("Got arguments:") + print(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/out.test.toml b/acceptance/bundle/artifacts/pipelines_environments_dynamic/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt b/acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt new file mode 100644 index 00000000000..0c980558ac9 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt @@ -0,0 +1,38 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 2 whl +.databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting the patched wheel in the pipeline environment +>>> print_requests.py --sort --keep //api/2.0/pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + ] + }, + "name": "My Wheel Pipeline" + } +} + +=== Expecting the patched and original wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/script b/acceptance/bundle/artifacts/pipelines_environments_dynamic/script new file mode 100644 index 00000000000..ddddaabee96 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/script @@ -0,0 +1,13 @@ +trace $CLI bundle deploy + +trace find.py --expect 2 whl + +title "Expecting the patched wheel in the pipeline environment" +trace print_requests.py --sort --keep //api/2.0/pipelines | contains.py \ + 'artifacts/.internal/my_test_code-0.0.1+' \ + '!artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl' + +title "Expecting the patched and original wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 0a9d1846041..bfd2c01999f 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -74,6 +74,27 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } + for pipelineName, pipelineRef := range b.Config.Resources.Pipelines { + if pipelineRef == nil { + continue + } + + env := pipelineRef.Environment + if env == nil { + continue + } + + for depInd, dep := range env.Dependencies { + repl := replacements[dep] + if repl != "" { + log.Debugf(ctx, "Updating resources.pipelines.%s.environment.dependencies[%d] from %s to %s", pipelineName, depInd, dep, repl) + env.Dependencies[depInd] = repl + } else { + log.Debugf(ctx, "Not updating resources.pipelines.%s.environment.dependencies[%d] from %s. Available replacements: %v", pipelineName, depInd, dep, slices.Sorted(maps.Keys(replacements))) + } + } + } + return nil } diff --git a/bundle/libraries/switch_to_patched_wheels_test.go b/bundle/libraries/switch_to_patched_wheels_test.go new file mode 100644 index 00000000000..b79578b065e --- /dev/null +++ b/bundle/libraries/switch_to_patched_wheels_test.go @@ -0,0 +1,84 @@ +package libraries + +import ( + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/vfs" + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/require" +) + +func TestSwitchToPatchedWheelsUpdatesPipelineEnvironmentDependencies(t *testing.T) { + tmpDir := t.TempDir() + + source := filepath.Join(tmpDir, "dist", "my_test_code-0.0.1-py3-none-any.whl") + sourceRel := filepath.Join("dist", "my_test_code-0.0.1-py3-none-any.whl") + patched := filepath.Join(tmpDir, ".databricks", "bundle", "default", "patched_wheels", "wheel_my_test_code", "my_test_code-0.0.1+123-py3-none-any.whl") + patchedRel := filepath.Join(".databricks", "bundle", "default", "patched_wheels", "wheel_my_test_code", "my_test_code-0.0.1+123-py3-none-any.whl") + + b := &bundle.Bundle{ + SyncRootPath: tmpDir, + SyncRoot: vfs.MustNew(tmpDir), + Config: config.Root{ + Artifacts: config.Artifacts{ + "wheel": { + Files: []config.ArtifactFile{ + { + Source: source, + Patched: patched, + }, + }, + }, + }, + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "job": { + JobSettings: jobs.JobSettings{ + Environments: []jobs.JobEnvironment{ + { + EnvironmentKey: "env", + Spec: &compute.Environment{ + Dependencies: []string{ + sourceRel, + "simplejson", + }, + }, + }, + }, + }, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Environment: &pipelines.PipelinesEnvironment{ + Dependencies: []string{ + sourceRel, + "simplejson", + }, + }, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, SwitchToPatchedWheels()) + require.Empty(t, diags) + + require.Equal(t, []string{ + patchedRel, + "simplejson", + }, b.Config.Resources.Jobs["job"].Environments[0].Spec.Dependencies) + require.Equal(t, []string{ + patchedRel, + "simplejson", + }, b.Config.Resources.Pipelines["pipeline"].Environment.Dependencies) +} From ce63ed4e103a75baf72223fdff43f2bc944c7e6b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 21 May 2026 21:37:06 +0200 Subject: [PATCH 2/4] clean up --- .../bundle/artifacts/pipelines_environments_dynamic/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/script b/acceptance/bundle/artifacts/pipelines_environments_dynamic/script index ddddaabee96..5f379694431 100644 --- a/acceptance/bundle/artifacts/pipelines_environments_dynamic/script +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 2 whl title "Expecting the patched wheel in the pipeline environment" -trace print_requests.py --sort --keep //api/2.0/pipelines | contains.py \ +trace print_requests.py --keep //api/2.0/pipelines | contains.py \ 'artifacts/.internal/my_test_code-0.0.1+' \ '!artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl' From 97f64ce1f5da41eb65c0556a57fff3c7338627c2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 21 May 2026 21:39:16 +0200 Subject: [PATCH 3/4] clean up --- .../switch_to_patched_wheels_test.go | 84 ------------------- 1 file changed, 84 deletions(-) delete mode 100644 bundle/libraries/switch_to_patched_wheels_test.go diff --git a/bundle/libraries/switch_to_patched_wheels_test.go b/bundle/libraries/switch_to_patched_wheels_test.go deleted file mode 100644 index b79578b065e..00000000000 --- a/bundle/libraries/switch_to_patched_wheels_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package libraries - -import ( - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/vfs" - "github.com/databricks/databricks-sdk-go/service/compute" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/databricks/databricks-sdk-go/service/pipelines" - "github.com/stretchr/testify/require" -) - -func TestSwitchToPatchedWheelsUpdatesPipelineEnvironmentDependencies(t *testing.T) { - tmpDir := t.TempDir() - - source := filepath.Join(tmpDir, "dist", "my_test_code-0.0.1-py3-none-any.whl") - sourceRel := filepath.Join("dist", "my_test_code-0.0.1-py3-none-any.whl") - patched := filepath.Join(tmpDir, ".databricks", "bundle", "default", "patched_wheels", "wheel_my_test_code", "my_test_code-0.0.1+123-py3-none-any.whl") - patchedRel := filepath.Join(".databricks", "bundle", "default", "patched_wheels", "wheel_my_test_code", "my_test_code-0.0.1+123-py3-none-any.whl") - - b := &bundle.Bundle{ - SyncRootPath: tmpDir, - SyncRoot: vfs.MustNew(tmpDir), - Config: config.Root{ - Artifacts: config.Artifacts{ - "wheel": { - Files: []config.ArtifactFile{ - { - Source: source, - Patched: patched, - }, - }, - }, - }, - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "job": { - JobSettings: jobs.JobSettings{ - Environments: []jobs.JobEnvironment{ - { - EnvironmentKey: "env", - Spec: &compute.Environment{ - Dependencies: []string{ - sourceRel, - "simplejson", - }, - }, - }, - }, - }, - }, - }, - Pipelines: map[string]*resources.Pipeline{ - "pipeline": { - CreatePipeline: pipelines.CreatePipeline{ - Environment: &pipelines.PipelinesEnvironment{ - Dependencies: []string{ - sourceRel, - "simplejson", - }, - }, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, SwitchToPatchedWheels()) - require.Empty(t, diags) - - require.Equal(t, []string{ - patchedRel, - "simplejson", - }, b.Config.Resources.Jobs["job"].Environments[0].Spec.Dependencies) - require.Equal(t, []string{ - patchedRel, - "simplejson", - }, b.Config.Resources.Pipelines["pipeline"].Environment.Dependencies) -} From ddadfd9832d50c9406b44daccfd9f341649da07d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 21 May 2026 21:43:02 +0200 Subject: [PATCH 4/4] update --- .../bundle/artifacts/pipelines_environments_dynamic/output.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt b/acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt index 0c980558ac9..e008a71003f 100644 --- a/acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt @@ -12,7 +12,7 @@ Deployment complete! my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting the patched wheel in the pipeline environment ->>> print_requests.py --sort --keep //api/2.0/pipelines +>>> print_requests.py --keep //api/2.0/pipelines { "method": "POST", "path": "/api/2.0/pipelines",