diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/.gitignore b/acceptance/bundle/artifacts/job_environments_dynamic/.gitignore similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/.gitignore rename to acceptance/bundle/artifacts/job_environments_dynamic/.gitignore diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml b/acceptance/bundle/artifacts/job_environments_dynamic/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml rename to acceptance/bundle/artifacts/job_environments_dynamic/databricks.yml diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/setup.py b/acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/setup.py similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/setup.py rename to acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/setup.py diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/src/__init__.py similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/src/__init__.py diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/src/__main__.py similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/job_environments_dynamic/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml b/acceptance/bundle/artifacts/job_environments_dynamic/out.test.toml similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml rename to acceptance/bundle/artifacts/job_environments_dynamic/out.test.toml diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/job_environments_dynamic/output.txt similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/output.txt rename to acceptance/bundle/artifacts/job_environments_dynamic/output.txt diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/job_environments_dynamic/script similarity index 100% rename from acceptance/bundle/artifacts/whl_via_environment_key/script rename to acceptance/bundle/artifacts/job_environments_dynamic/script diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/databricks.yml b/acceptance/bundle/artifacts/pipelines_environments_dynamic/databricks.yml new file mode 100644 index 00000000000..e8e1d0f8018 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +artifacts: + my_test_code: + type: whl + path: "./my_test_code" + build: python setup.py bdist_wheel + dynamic_version: true + +resources: + pipelines: + test_pipeline: + name: "My Wheel Pipeline" + environment: + dependencies: + - ./my_test_code/dist/*.whl diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/setup.py b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/setup.py new file mode 100644 index 00000000000..0bd871dd349 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import src + +setup( + name="my_test_code", + version=src.__version__, + author=src.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my test wheel", + packages=find_packages(include=["src"]), + entry_points={"group_1": "run=src.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__init__.py new file mode 100644 index 00000000000..909f1f3220d --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__init__.py @@ -0,0 +1,2 @@ +__version__ = "0.0.1" +__author__ = "Databricks" diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__main__.py new file mode 100644 index 00000000000..ea918ce2d53 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/my_test_code/src/__main__.py @@ -0,0 +1,16 @@ +""" +The entry point of the Python Wheel +""" + +import sys + + +def main(): + # This method will print the provided arguments + print("Hello from my func") + print("Got arguments:") + print(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/out.test.toml b/acceptance/bundle/artifacts/pipelines_environments_dynamic/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt b/acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt new file mode 100644 index 00000000000..e008a71003f --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/output.txt @@ -0,0 +1,38 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find.py --expect 2 whl +.databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting the patched wheel in the pipeline environment +>>> print_requests.py --keep //api/2.0/pipelines +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edition": "ADVANCED", + "environment": { + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + ] + }, + "name": "My Wheel Pipeline" + } +} + +=== Expecting the patched and original wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/pipelines_environments_dynamic/script b/acceptance/bundle/artifacts/pipelines_environments_dynamic/script new file mode 100644 index 00000000000..5f379694431 --- /dev/null +++ b/acceptance/bundle/artifacts/pipelines_environments_dynamic/script @@ -0,0 +1,13 @@ +trace $CLI bundle deploy + +trace find.py --expect 2 whl + +title "Expecting the patched wheel in the pipeline environment" +trace print_requests.py --keep //api/2.0/pipelines | contains.py \ + 'artifacts/.internal/my_test_code-0.0.1+' \ + '!artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl' + +title "Expecting the patched and original wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 0a9d1846041..bfd2c01999f 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -74,6 +74,27 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } + for pipelineName, pipelineRef := range b.Config.Resources.Pipelines { + if pipelineRef == nil { + continue + } + + env := pipelineRef.Environment + if env == nil { + continue + } + + for depInd, dep := range env.Dependencies { + repl := replacements[dep] + if repl != "" { + log.Debugf(ctx, "Updating resources.pipelines.%s.environment.dependencies[%d] from %s to %s", pipelineName, depInd, dep, repl) + env.Dependencies[depInd] = repl + } else { + log.Debugf(ctx, "Not updating resources.pipelines.%s.environment.dependencies[%d] from %s. Available replacements: %v", pipelineName, depInd, dep, slices.Sorted(maps.Keys(replacements))) + } + } + } + return nil }