From 8a2409106f6bac5642b3e7e0b4231802d3884230 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Tue, 21 Apr 2026 13:13:59 +0200 Subject: [PATCH 01/20] bundle: add genie_spaces resource for direct deploy Adds first-class support for Genie spaces as a bundle resource, complete with CRUD via direct-mode deploy, `bundle generate genie-space` to import an existing space, permissions handling, and acceptance tests. The resource configuration follows the dashboards pattern: a `file_path` field points to a local `.genie.json` file whose contents are inlined into `serialized_space` during deployment. The parent_path defaults to `${workspace.resource_path}` and is normalized to the `/Workspace` prefix, matching the API's expected form. Co-authored-by: Isaac --- .../generate/genie_space/databricks.yml | 2 + .../genie_space/genie_space.json.tmpl | 7 + .../bundle/generate/genie_space/out.test.toml | 5 + .../genie_space/test_genie_space.genie.json | 4 + .../resource/test_genie_space.genie_space.yml | 7 + .../bundle/generate/genie_space/output.txt | 6 + acceptance/bundle/generate/genie_space/script | 8 + .../bundle/generate/genie_space/test.toml | 10 + .../bundle/help/bundle-generate/output.txt | 1 + acceptance/bundle/refschema/out.fields.txt | 18 + .../genie_spaces/simple/databricks.yml.tmpl | 11 + .../genie_spaces/simple/out.plan.json | 26 + .../genie_spaces/simple/out.test.toml | 6 + .../resources/genie_spaces/simple/output.txt | 24 + .../simple/sales_analytics.genie.json | 136 +++++ .../resources/genie_spaces/simple/script | 17 + .../resources/genie_spaces/simple/test.toml | 10 + .../bundle/resources/genie_spaces/test.toml | 3 + .../genie_space_complex/databricks.yml | 51 ++ .../full_featured.genie.json | 160 ++++++ .../genie_space_complex/out.test.toml | 5 + .../validate/genie_space_complex/output.txt | 18 + .../validate/genie_space_complex/script | 19 + .../genie_space_defaults/databricks.yml | 30 ++ .../genie_space_defaults/out.test.toml | 5 + .../validate/genie_space_defaults/output.txt | 14 + .../validate/genie_space_defaults/script | 1 + acceptance/experimental/open/output.txt | 3 +- .../paths/genie_space_paths_visitor.go | 18 + .../apply_bundle_permissions.go | 4 + .../mutator/resourcemutator/apply_presets.go | 8 + .../resourcemutator/apply_target_mode_test.go | 10 + .../configure_genie_space_serialized_space.go | 53 ++ .../resourcemutator/genie_space_fixups.go | 30 ++ .../resourcemutator/resource_mutator.go | 10 + .../mutator/resourcemutator/run_as_test.go | 2 + bundle/config/mutator/translate_paths.go | 1 + .../mutator/translate_paths_genie_spaces.go | 22 + bundle/config/resources.go | 3 + bundle/config/resources/genie_space.go | 94 ++++ bundle/config/resources_test.go | 4 + bundle/deploy/terraform/lifecycle_test.go | 1 + bundle/direct/dresources/all.go | 2 + bundle/direct/dresources/all_test.go | 19 + bundle/direct/dresources/apitypes.yml | 2 + bundle/direct/dresources/genie_space.go | 164 ++++++ bundle/direct/dresources/permissions.go | 1 + bundle/direct/dresources/resources.yml | 13 + bundle/direct/dresources/type_test.go | 3 + bundle/docsgen/output/reference.md | 232 ++++++++ bundle/docsgen/output/resources.md | 112 ++++ bundle/generate/genie_space.go | 22 + bundle/internal/schema/annotations.yml | 31 ++ bundle/schema/jsonschema.json | 58 ++ bundle/schema/jsonschema_for_docs.json | 42 ++ bundle/statemgmt/state_load_test.go | 35 ++ cmd/bundle/generate.go | 1 + cmd/bundle/generate/genie_space.go | 503 ++++++++++++++++++ cmd/experimental/workspace_open_test.go | 7 +- libs/testserver/fake_workspace.go | 2 + libs/testserver/genie_spaces.go | 162 ++++++ libs/testserver/handlers.go | 14 + libs/testserver/permissions.go | 1 + libs/workspaceurls/urls.go | 1 + 64 files changed, 2290 insertions(+), 4 deletions(-) create mode 100644 acceptance/bundle/generate/genie_space/databricks.yml create mode 100644 acceptance/bundle/generate/genie_space/genie_space.json.tmpl create mode 100644 acceptance/bundle/generate/genie_space/out.test.toml create mode 100644 acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json create mode 100644 acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml create mode 100644 acceptance/bundle/generate/genie_space/output.txt create mode 100644 acceptance/bundle/generate/genie_space/script create mode 100644 acceptance/bundle/generate/genie_space/test.toml create mode 100644 acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/genie_spaces/simple/out.plan.json create mode 100644 acceptance/bundle/resources/genie_spaces/simple/out.test.toml create mode 100644 acceptance/bundle/resources/genie_spaces/simple/output.txt create mode 100644 acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json create mode 100644 acceptance/bundle/resources/genie_spaces/simple/script create mode 100644 acceptance/bundle/resources/genie_spaces/simple/test.toml create mode 100644 acceptance/bundle/resources/genie_spaces/test.toml create mode 100644 acceptance/bundle/validate/genie_space_complex/databricks.yml create mode 100644 acceptance/bundle/validate/genie_space_complex/full_featured.genie.json create mode 100644 acceptance/bundle/validate/genie_space_complex/out.test.toml create mode 100644 acceptance/bundle/validate/genie_space_complex/output.txt create mode 100644 acceptance/bundle/validate/genie_space_complex/script create mode 100644 acceptance/bundle/validate/genie_space_defaults/databricks.yml create mode 100644 acceptance/bundle/validate/genie_space_defaults/out.test.toml create mode 100644 acceptance/bundle/validate/genie_space_defaults/output.txt create mode 100644 acceptance/bundle/validate/genie_space_defaults/script create mode 100644 bundle/config/mutator/paths/genie_space_paths_visitor.go create mode 100644 bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go create mode 100644 bundle/config/mutator/resourcemutator/genie_space_fixups.go create mode 100644 bundle/config/mutator/translate_paths_genie_spaces.go create mode 100644 bundle/config/resources/genie_space.go create mode 100644 bundle/direct/dresources/genie_space.go create mode 100644 bundle/generate/genie_space.go create mode 100644 cmd/bundle/generate/genie_space.go create mode 100644 libs/testserver/genie_spaces.go diff --git a/acceptance/bundle/generate/genie_space/databricks.yml b/acceptance/bundle/generate/genie_space/databricks.yml new file mode 100644 index 00000000000..c533b2b6f98 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: genie-space-generate diff --git a/acceptance/bundle/generate/genie_space/genie_space.json.tmpl b/acceptance/bundle/generate/genie_space/genie_space.json.tmpl new file mode 100644 index 00000000000..2056f3061ef --- /dev/null +++ b/acceptance/bundle/generate/genie_space/genie_space.json.tmpl @@ -0,0 +1,7 @@ +{ + "title": "test genie space", + "description": "test description", + "parent_path": "/Workspace/test-$UNIQUE_NAME", + "warehouse_id": "test-warehouse-id", + "serialized_space": "{\"tables\":[],\"questions\":[]}" +} diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml new file mode 100644 index 00000000000..d560f1de043 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json new file mode 100644 index 00000000000..2c12b3c032c --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json @@ -0,0 +1,4 @@ +{ + "questions": [], + "tables": [] +} diff --git a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml new file mode 100644 index 00000000000..14764931f1b --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml @@ -0,0 +1,7 @@ +resources: + genie_spaces: + test_genie_space: + title: "test genie space" + warehouse_id: test-warehouse-id + file_path: ../genie_space/test_genie_space.genie.json + description: test description diff --git a/acceptance/bundle/generate/genie_space/output.txt b/acceptance/bundle/generate/genie_space/output.txt new file mode 100644 index 00000000000..985366ada68 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME] + +>>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --genie-space-dir out/genie_space --resource-dir out/resource +Writing genie space to out/genie_space/test_genie_space.genie.json +Writing configuration to out/resource/test_genie_space.genie_space.yml diff --git a/acceptance/bundle/generate/genie_space/script b/acceptance/bundle/generate/genie_space/script new file mode 100644 index 00000000000..8e2fa32170a --- /dev/null +++ b/acceptance/bundle/generate/genie_space/script @@ -0,0 +1,8 @@ +trace $CLI workspace mkdirs /Workspace/test-$UNIQUE_NAME + +# create a genie space to import +envsubst < genie_space.json.tmpl > genie_space.json +genie_space_id=$($CLI genie create-space --json @genie_space.json | jq -r '.space_id') +rm genie_space.json + +trace $CLI bundle generate genie-space --existing-id $genie_space_id --genie-space-dir out/genie_space --resource-dir out/resource diff --git a/acceptance/bundle/generate/genie_space/test.toml b/acceptance/bundle/generate/genie_space/test.toml new file mode 100644 index 00000000000..e389c33c277 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/test.toml @@ -0,0 +1,10 @@ +[[Repls]] +Old = '\\\\' +New = '/' + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[GENIE_SPACE_ID]" + +[Env] +MSYS_NO_PATHCONV = "1" diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 97e8667ac78..39ce9293539 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -30,6 +30,7 @@ Available Commands: alert Generate configuration for an alert app Generate bundle configuration for a Databricks app dashboard Generate configuration for a dashboard + genie-space Generate configuration for a Genie space job Generate bundle configuration for a job pipeline Generate bundle configuration for a pipeline diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d50035a048f..ad5c9316dc3 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -729,6 +729,24 @@ resources.external_locations.*.grants[*] catalog.PrivilegeAssignment ALL resources.external_locations.*.grants[*].principal string ALL resources.external_locations.*.grants[*].privileges []catalog.Privilege ALL resources.external_locations.*.grants[*].privileges[*] catalog.Privilege ALL +resources.genie_spaces.*.description string ALL +resources.genie_spaces.*.file_path string INPUT +resources.genie_spaces.*.id string INPUT +resources.genie_spaces.*.lifecycle resources.Lifecycle INPUT +resources.genie_spaces.*.lifecycle.prevent_destroy bool INPUT +resources.genie_spaces.*.modified_status string INPUT +resources.genie_spaces.*.parent_path string ALL +resources.genie_spaces.*.serialized_space any ALL +resources.genie_spaces.*.space_id string ALL +resources.genie_spaces.*.title string ALL +resources.genie_spaces.*.url string INPUT +resources.genie_spaces.*.warehouse_id string ALL +resources.genie_spaces.*.permissions.object_id string ALL +resources.genie_spaces.*.permissions[*] dresources.StatePermission ALL +resources.genie_spaces.*.permissions[*].group_name string ALL +resources.genie_spaces.*.permissions[*].level iam.PermissionLevel ALL +resources.genie_spaces.*.permissions[*].service_principal_name string ALL +resources.genie_spaces.*.permissions[*].user_name string ALL resources.jobs.*.budget_policy_id string ALL resources.jobs.*.continuous *jobs.Continuous ALL resources.jobs.*.continuous.pause_status jobs.PauseStatus ALL diff --git a/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl new file mode 100644 index 00000000000..26ed410751a --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: deploy-genie-space-test-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Sales Analytics Genie" + description: "AI assistant for sales data analysis" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + file_path: "sales_analytics.genie.json" diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json new file mode 100644 index 00000000000..19f063bb002 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -0,0 +1,26 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.genie_spaces.sales_analytics": { + "action": "skip", + "remote_state": { + "description": "AI assistant for sales data analysis", + "serialized_space": "{\n \"version\": 1,\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"sq-001\",\n \"question\": [\"What is the total revenue?\"]\n },\n {\n \"id\": \"sq-002\",\n \"question\": [\"Show me the top customers\"]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"identifier\": \"main.sales.orders\",\n \"column_configs\": [\n {\n \"column_name\": \"order_id\",\n \"get_example_values\": true,\n \"build_value_dictionary\": true\n },\n {\n \"column_name\": \"customer_id\",\n \"get_example_values\": true\n },\n {\n \"column_name\": \"amount\",\n \"get_example_values\": false\n }\n ]\n },\n {\n \"identifier\": \"main.sales.customers\"\n }\n ]\n },\n \"instructions\": {\n \"text_instructions\": [\n {\n \"id\": \"ti-001\",\n \"content\": [\n \"This genie space analyzes sales data.\\n\",\n \"Always filter by date when querying orders.\\n\",\n \"Use customer_name instead of customer_id in results.\"\n ]\n }\n ],\n \"example_question_sqls\": [\n {\n \"id\": \"eq-001\",\n \"question\": [\"What are the top customers by revenue?\"],\n \"sql\": [\n \"SELECT\\n\",\n \" c.customer_name,\\n\",\n \" SUM(o.amount) AS total_revenue\\n\",\n \"FROM main.sales.orders o\\n\",\n \"JOIN main.sales.customers c ON o.customer_id = c.id\\n\",\n \"WHERE o.order_date \u003e= :start_date\\n\",\n \"GROUP BY c.customer_name\\n\",\n \"ORDER BY total_revenue DESC\\n\",\n \"LIMIT :limit\"\n ],\n \"parameters\": [\n {\n \"name\": \"start_date\",\n \"type_hint\": \"STRING\",\n \"description\": [\"Start date for the analysis period\"],\n \"default_value\": {\n \"values\": [\"2024-01-01\"]\n }\n },\n {\n \"name\": \"limit\",\n \"type_hint\": \"INTEGER\",\n \"description\": [\"Number of customers to return\"],\n \"default_value\": {\n \"values\": [\"10\"]\n }\n }\n ]\n },\n {\n \"id\": \"eq-002\",\n \"question\": [\"Calculate daily revenue\"],\n \"sql\": [\n \"SELECT\\n\",\n \" order_date,\\n\",\n \" SUM(amount) AS daily_revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY order_date\\n\",\n \"ORDER BY order_date\"\n ]\n }\n ],\n \"sql_snippets\": {\n \"measures\": [\n {\n \"id\": \"m-001\",\n \"alias\": \"total_revenue\",\n \"sql\": [\"SUM(orders.amount)\"],\n \"display_name\": \"Total Revenue\"\n }\n ]\n },\n \"sql_functions\": [\n {\n \"id\": \"sf-001\",\n \"identifier\": \"main.analytics.calculate_churn\"\n }\n ]\n },\n \"benchmarks\": {\n \"questions\": [\n {\n \"id\": \"bq-001\",\n \"question\": [\"What is the monthly revenue trend?\"],\n \"answer\": [\n {\n \"format\": \"SQL\",\n \"content\": [\n \"SELECT\\n\",\n \" DATE_TRUNC('month', order_date) AS month,\\n\",\n \" SUM(amount) AS revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY 1\\n\",\n \"ORDER BY 1\"\n ]\n }\n ]\n }\n ]\n }\n}\n", + "space_id": "[GENIE_SPACE_ID]", + "title": "Sales Analytics Genie", + "warehouse_id": "test-warehouse-id" + }, + "changes": { + "parent_path": { + "action": "skip", + "reason": "input_only", + "old": "/Workspace/Users/[USERNAME]", + "new": "/Workspace/Users/[USERNAME]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml new file mode 100644 index 00000000000..496668716fa --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] + Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/output.txt b/acceptance/bundle/resources/genie_spaces/simple/output.txt new file mode 100644 index 00000000000..f65d9583e18 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-test-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] genie get-space [GENIE_SPACE_ID] +{ + "title": "Sales Analytics Genie", + "description": "AI assistant for sales data analysis", + "warehouse_id": "test-warehouse-id" +} + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.sales_analytics + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-test-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json new file mode 100644 index 00000000000..5d59dff96d5 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json @@ -0,0 +1,136 @@ +{ + "version": 1, + "config": { + "sample_questions": [ + { + "id": "sq-001", + "question": ["What is the total revenue?"] + }, + { + "id": "sq-002", + "question": ["Show me the top customers"] + } + ] + }, + "data_sources": { + "tables": [ + { + "identifier": "main.sales.orders", + "column_configs": [ + { + "column_name": "order_id", + "get_example_values": true, + "build_value_dictionary": true + }, + { + "column_name": "customer_id", + "get_example_values": true + }, + { + "column_name": "amount", + "get_example_values": false + } + ] + }, + { + "identifier": "main.sales.customers" + } + ] + }, + "instructions": { + "text_instructions": [ + { + "id": "ti-001", + "content": [ + "This genie space analyzes sales data.\n", + "Always filter by date when querying orders.\n", + "Use customer_name instead of customer_id in results." + ] + } + ], + "example_question_sqls": [ + { + "id": "eq-001", + "question": ["What are the top customers by revenue?"], + "sql": [ + "SELECT\n", + " c.customer_name,\n", + " SUM(o.amount) AS total_revenue\n", + "FROM main.sales.orders o\n", + "JOIN main.sales.customers c ON o.customer_id = c.id\n", + "WHERE o.order_date >= :start_date\n", + "GROUP BY c.customer_name\n", + "ORDER BY total_revenue DESC\n", + "LIMIT :limit" + ], + "parameters": [ + { + "name": "start_date", + "type_hint": "STRING", + "description": ["Start date for the analysis period"], + "default_value": { + "values": ["2024-01-01"] + } + }, + { + "name": "limit", + "type_hint": "INTEGER", + "description": ["Number of customers to return"], + "default_value": { + "values": ["10"] + } + } + ] + }, + { + "id": "eq-002", + "question": ["Calculate daily revenue"], + "sql": [ + "SELECT\n", + " order_date,\n", + " SUM(amount) AS daily_revenue\n", + "FROM main.sales.orders\n", + "GROUP BY order_date\n", + "ORDER BY order_date" + ] + } + ], + "sql_snippets": { + "measures": [ + { + "id": "m-001", + "alias": "total_revenue", + "sql": ["SUM(orders.amount)"], + "display_name": "Total Revenue" + } + ] + }, + "sql_functions": [ + { + "id": "sf-001", + "identifier": "main.analytics.calculate_churn" + } + ] + }, + "benchmarks": { + "questions": [ + { + "id": "bq-001", + "question": ["What is the monthly revenue trend?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " DATE_TRUNC('month', order_date) AS month,\n", + " SUM(amount) AS revenue\n", + "FROM main.sales.orders\n", + "GROUP BY 1\n", + "ORDER BY 1" + ] + } + ] + } + ] + } +} diff --git a/acceptance/bundle/resources/genie_spaces/simple/script b/acceptance/bundle/resources/genie_spaces/simple/script new file mode 100644 index 00000000000..4db9ef03078 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') + +# Capture the genie space ID as a replacement. +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +trace $CLI genie get-space $GENIE_SPACE_ID | jq '{title, description, warehouse_id}' + +# Verify that there is no drift right after deploy. +trace $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/simple/test.toml b/acceptance/bundle/resources/genie_spaces/simple/test.toml new file mode 100644 index 00000000000..18a07a1b00b --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/test.toml @@ -0,0 +1,10 @@ +Local = true +RecordRequests = false + +# Genie spaces only support direct deployment engine (no Terraform provider yet) +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/test.toml b/acceptance/bundle/resources/genie_spaces/test.toml new file mode 100644 index 00000000000..2569ef7dcb1 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/test.toml @@ -0,0 +1,3 @@ +# Genie spaces are only deployed via direct deployment engine +[Env] +DATABRICKS_BUNDLE_ENGINE = "direct" diff --git a/acceptance/bundle/validate/genie_space_complex/databricks.yml b/acceptance/bundle/validate/genie_space_complex/databricks.yml new file mode 100644 index 00000000000..eae5cdc3f10 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/databricks.yml @@ -0,0 +1,51 @@ +bundle: + name: genie-space-complex + +workspace: + resource_path: /foo/bar + +resources: + genie_spaces: + # Test with all features enabled + full_featured: + warehouse_id: "my-warehouse-1234" + title: "Full Featured Genie Space" + description: "A comprehensive test of all genie space features" + file_path: ./full_featured.genie.json + + # Test with inline serialized_space (YAML syntax) + inline_yaml: + warehouse_id: "my-warehouse-1234" + title: "Inline YAML Genie Space" + serialized_space: + version: 1 + data_sources: + tables: + - identifier: main.schema.table1 + column_configs: + - column_name: id + get_example_values: true + build_value_dictionary: true + - column_name: name + get_example_values: true + instructions: + text_instructions: + - id: inst-001 + content: + - "This is a text instruction.\n" + - "It spans multiple lines." + example_question_sqls: + - id: eq-001 + question: + - "How many records are there?" + sql: + - "SELECT COUNT(*) FROM main.schema.table1" + + # Test with empty but valid structure + minimal_valid: + warehouse_id: "my-warehouse-1234" + title: "Minimal Valid" + serialized_space: + version: 1 + data_sources: + tables: [] diff --git a/acceptance/bundle/validate/genie_space_complex/full_featured.genie.json b/acceptance/bundle/validate/genie_space_complex/full_featured.genie.json new file mode 100644 index 00000000000..9c9221d8328 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/full_featured.genie.json @@ -0,0 +1,160 @@ +{ + "version": 1, + "config": { + "sample_questions": [ + { + "id": "sq-001", + "question": ["What is the total revenue?"] + }, + { + "id": "sq-002", + "question": ["Show me the top customers"] + } + ] + }, + "data_sources": { + "tables": [ + { + "identifier": "main.sales.orders", + "column_configs": [ + { + "column_name": "order_id", + "get_example_values": true, + "build_value_dictionary": true + }, + { + "column_name": "customer_id", + "get_example_values": true + }, + { + "column_name": "amount", + "get_example_values": false + } + ] + }, + { + "identifier": "main.sales.customers" + } + ] + }, + "instructions": { + "text_instructions": [ + { + "id": "ti-001", + "content": [ + "This genie space analyzes sales data.\n", + "Always filter by date when querying orders.\n", + "Use customer_name instead of customer_id in results." + ] + } + ], + "example_question_sqls": [ + { + "id": "eq-001", + "question": ["What are the top customers by revenue?"], + "sql": [ + "SELECT\n", + " c.customer_name,\n", + " SUM(o.amount) AS total_revenue\n", + "FROM main.sales.orders o\n", + "JOIN main.sales.customers c ON o.customer_id = c.id\n", + "WHERE o.order_date >= :start_date\n", + "GROUP BY c.customer_name\n", + "ORDER BY total_revenue DESC\n", + "LIMIT :limit" + ], + "parameters": [ + { + "name": "start_date", + "type_hint": "STRING", + "description": ["Start date for the analysis period"], + "default_value": { + "values": ["2024-01-01"] + } + }, + { + "name": "limit", + "type_hint": "INTEGER", + "description": ["Number of customers to return"], + "default_value": { + "values": ["10"] + } + } + ] + }, + { + "id": "eq-002", + "question": ["Calculate daily revenue"], + "sql": [ + "SELECT\n", + " order_date,\n", + " SUM(amount) AS daily_revenue\n", + "FROM main.sales.orders\n", + "GROUP BY order_date\n", + "ORDER BY order_date" + ] + } + ], + "sql_snippets": { + "measures": [ + { + "id": "m-001", + "alias": "total_revenue", + "sql": ["SUM(orders.amount)"], + "display_name": "Total Revenue" + }, + { + "id": "m-002", + "alias": "order_count", + "sql": ["COUNT(orders.order_id)"], + "display_name": "Order Count" + } + ] + }, + "sql_functions": [ + { + "id": "sf-001", + "identifier": "main.analytics.calculate_churn" + } + ] + }, + "benchmarks": { + "questions": [ + { + "id": "bq-001", + "question": ["What is the monthly revenue trend?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " DATE_TRUNC('month', order_date) AS month,\n", + " SUM(amount) AS revenue\n", + "FROM main.sales.orders\n", + "GROUP BY 1\n", + "ORDER BY 1" + ] + } + ] + }, + { + "id": "bq-002", + "question": ["Which customers have the highest average order value?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " customer_id,\n", + " AVG(amount) AS avg_order_value\n", + "FROM main.sales.orders\n", + "GROUP BY customer_id\n", + "ORDER BY avg_order_value DESC\n", + "LIMIT 10" + ] + } + ] + } + ] + } +} diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml new file mode 100644 index 00000000000..d560f1de043 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/output.txt b/acceptance/bundle/validate/genie_space_complex/output.txt new file mode 100644 index 00000000000..2fe58959fc8 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/output.txt @@ -0,0 +1,18 @@ +{ + "full_featured": { + "title": "Full Featured Genie Space", + "warehouse_id": "my-warehouse-1234", + "serialized_space_is_string": true + }, + "inline_yaml": { + "title": "Inline YAML Genie Space", + "serialized_space_type": "object", + "tables_count": 1, + "has_column_configs": true, + "has_text_instructions": true + }, + "minimal_valid": { + "title": "Minimal Valid", + "tables_count": 0 + } +} diff --git a/acceptance/bundle/validate/genie_space_complex/script b/acceptance/bundle/validate/genie_space_complex/script new file mode 100644 index 00000000000..01e1fed72bc --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/script @@ -0,0 +1,19 @@ +# Validate complex genie spaces and check the serialized_space structure is preserved +$CLI bundle validate -o json | jq '{ + full_featured: .resources.genie_spaces.full_featured | { + title, + warehouse_id, + serialized_space_is_string: (.serialized_space | type == "string") + }, + inline_yaml: .resources.genie_spaces.inline_yaml | { + title, + serialized_space_type: (.serialized_space | type), + tables_count: (.serialized_space.data_sources.tables | length), + has_column_configs: ((.serialized_space.data_sources.tables[0].column_configs | length) > 0), + has_text_instructions: ((.serialized_space.instructions.text_instructions | length) > 0) + }, + minimal_valid: .resources.genie_spaces.minimal_valid | { + title, + tables_count: (.serialized_space.data_sources.tables | length) + } +}' diff --git a/acceptance/bundle/validate/genie_space_defaults/databricks.yml b/acceptance/bundle/validate/genie_space_defaults/databricks.yml new file mode 100644 index 00000000000..7e4b87d35e6 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/databricks.yml @@ -0,0 +1,30 @@ +bundle: + name: test-bundle + +workspace: + resource_path: /foo/bar + +resources: + genie_spaces: + empty_string: + warehouse_id: "my-warehouse-1234" + title: "empty-string" + serialized_space: "{}" + + # unchanged + parent_path: "" + + non_empty_string: + warehouse_id: "my-warehouse-1234" + title: "non-empty-string" + serialized_space: "{}" + + # unchanged + parent_path: "already-set" + + default_parent_path: + warehouse_id: "my-warehouse-1234" + title: "default-parent-path" + serialized_space: "{}" + + # parent_path set to default diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml new file mode 100644 index 00000000000..d560f1de043 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/output.txt b/acceptance/bundle/validate/genie_space_defaults/output.txt new file mode 100644 index 00000000000..13105c91b76 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/output.txt @@ -0,0 +1,14 @@ +{ + "default_parent_path": { + "title": "default-parent-path", + "parent_path": "/Workspace/foo/bar" + }, + "empty_string": { + "title": "empty-string", + "parent_path": "/Workspace" + }, + "non_empty_string": { + "title": "non-empty-string", + "parent_path": "/Workspace/already-set" + } +} diff --git a/acceptance/bundle/validate/genie_space_defaults/script b/acceptance/bundle/validate/genie_space_defaults/script new file mode 100644 index 00000000000..eebc582f136 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/script @@ -0,0 +1 @@ +$CLI bundle validate -o json | jq '.resources.genie_spaces | map_values({title: .title, parent_path: .parent_path})' diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index a83e0676fe8..ca3e909582a 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, clusters, dashboards, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, clusters, dashboards, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses Exit code: 1 @@ -20,6 +20,7 @@ apps clusters dashboards experiments +genie_spaces jobs model_serving_endpoints models diff --git a/bundle/config/mutator/paths/genie_space_paths_visitor.go b/bundle/config/mutator/paths/genie_space_paths_visitor.go new file mode 100644 index 00000000000..edd6ff2d8df --- /dev/null +++ b/bundle/config/mutator/paths/genie_space_paths_visitor.go @@ -0,0 +1,18 @@ +package paths + +import ( + "github.com/databricks/cli/libs/dyn" +) + +func VisitGenieSpacePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("genie_spaces"), + dyn.AnyKey(), + dyn.Key("file_path"), + ) + + return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return fn(path, TranslateModeLocalRelative, value) + }) +} diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index fd019479d77..cbb05d7d622 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -47,6 +47,10 @@ var ( permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_READ", }, + "genie_spaces": { + permissions.CAN_MANAGE: "CAN_MANAGE", + permissions.CAN_VIEW: "CAN_READ", + }, "apps": { permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_USE", diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 83d512ca518..2a0996e6b7d 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -237,6 +237,14 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos dashboard.DisplayName = prefix + dashboard.DisplayName } + // Genie Spaces: Prefix + for _, genieSpace := range r.GenieSpaces { + if genieSpace == nil { + continue + } + genieSpace.Title = prefix + genieSpace.Title + } + // Apps: No presets // Alerts: Prefix, TriggerPauseStatus diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 927c7a19132..7dcf21a479b 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -154,6 +154,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "geniespace1": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "geniespace1", + }, + }, + }, Apps: map[string]*resources.App{ "app1": { App: apps.App{ @@ -330,6 +337,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Dashboards assert.Equal(t, "[dev lennart] dashboard1", b.Config.Resources.Dashboards["dashboard1"].DisplayName) + // Genie Spaces + assert.Equal(t, "[dev lennart] geniespace1", b.Config.Resources.GenieSpaces["geniespace1"].Title) + // Alert 1: has schedule without pause status set - should be paused assert.Equal(t, "[dev lennart] alert1", b.Config.Resources.Alerts["alert1"].DisplayName) assert.Equal(t, sql.SchedulePauseStatusPaused, b.Config.Resources.Alerts["alert1"].Schedule.PauseStatus) diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go new file mode 100644 index 00000000000..bf129b4060d --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -0,0 +1,53 @@ +package resourcemutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const serializedSpaceFieldName = "serialized_space" + +type configureGenieSpaceSerializedSpace struct{} + +func ConfigureGenieSpaceSerializedSpace() bundle.Mutator { + return &configureGenieSpaceSerializedSpace{} +} + +func (c configureGenieSpaceSerializedSpace) Name() string { + return "ConfigureGenieSpaceSerializedSpace" +} + +func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("genie_spaces"), + dyn.AnyKey(), + ) + + // Configure serialized_space field for all genie spaces. + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + // Include "serialized_space" field if "file_path" is set. + path, ok := v.Get(filePathFieldName).AsString() + if !ok { + return v, nil + } + + contents, err := b.SyncRoot.ReadFile(path) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", path, err) + } + + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} diff --git a/bundle/config/mutator/resourcemutator/genie_space_fixups.go b/bundle/config/mutator/resourcemutator/genie_space_fixups.go new file mode 100644 index 00000000000..85e1bb7e745 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/genie_space_fixups.go @@ -0,0 +1,30 @@ +package resourcemutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type genieSpaceFixups struct{} + +func GenieSpaceFixups() bundle.Mutator { + return &genieSpaceFixups{} +} + +func (m *genieSpaceFixups) Name() string { + return "GenieSpaceFixups" +} + +func (m *genieSpaceFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, genieSpace := range b.Config.Resources.GenieSpaces { + if genieSpace == nil { + continue + } + + genieSpace.ParentPath = ensureWorkspacePrefix(genieSpace.ParentPath) + } + + return nil +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 2eb292cfbb0..f0b259e9001 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -52,6 +52,7 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { }{ {"resources.dashboards.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.dashboards.*.embed_credentials", false}, + {"resources.genie_spaces.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.volumes.*.volume_type", "MANAGED"}, {"resources.alerts.*.parent_path", b.Config.Workspace.ResourcePath}, @@ -115,6 +116,11 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { // Ensures dashboard parent paths have the required /Workspace prefix DashboardFixups(), + // Reads (typed): b.Config.Resources.GenieSpaces (checks genie space configurations) + // Updates (typed): b.Config.Resources.GenieSpaces[].ParentPath (ensures /Workspace prefix is present) + // Ensures genie space parent paths have the required /Workspace prefix + GenieSpaceFixups(), + // Reads (typed): b.Config.Permissions (validates permission levels) // Reads (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (reads existing permissions) // Updates (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (adds permissions from bundle-level configuration) @@ -182,6 +188,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Drops (dynamic): resources.dashboards.*.file_path ConfigureDashboardSerializedDashboard(), + // Reads (dynamic): resources.genie_spaces.*.file_path + // Updates (dynamic): resources.genie_spaces.*.serialized_space + ConfigureGenieSpaceSerializedSpace(), + // Reads (typed): resources.alerts.*.file_path // Updates (typed): resources.alerts.* (loads alert configuration from .dbalert.json file) mutator.LoadDBAlertFiles(), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 58b27113a52..5f67452e981 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -41,6 +41,7 @@ func allResourceTypes(t *testing.T) []string { "database_instances", "experiments", "external_locations", + "genie_spaces", "jobs", "model_serving_endpoints", "models", @@ -180,6 +181,7 @@ var allowList = []string{ "postgres_projects", "registered_models", "experiments", + "genie_spaces", "schemas", "secret_scopes", "sql_warehouses", diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 99dd75dd787..0cee3165288 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -369,6 +369,7 @@ func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ t.applyDashboardTranslations, + t.applyGenieSpaceTranslations, }) } diff --git a/bundle/config/mutator/translate_paths_genie_spaces.go b/bundle/config/mutator/translate_paths_genie_spaces.go new file mode 100644 index 00000000000..b6a3a652427 --- /dev/null +++ b/bundle/config/mutator/translate_paths_genie_spaces.go @@ -0,0 +1,22 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle/config/mutator/paths" + + "github.com/databricks/cli/libs/dyn" +) + +func (t *translateContext) applyGenieSpaceTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { + // Convert the `file_path` field to a local absolute path. + // We load the file at this path and use its contents for the genie space contents. + + return paths.VisitGenieSpacePaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + opts := translateOptions{ + Mode: mode, + } + + return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + }) +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 30a7f4e95e5..668cd19d918 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -25,6 +25,7 @@ type Resources struct { ExternalLocations map[string]*resources.ExternalLocation `json:"external_locations,omitempty"` Clusters map[string]*resources.Cluster `json:"clusters,omitempty"` Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"` + GenieSpaces map[string]*resources.GenieSpace `json:"genie_spaces,omitempty"` Apps map[string]*resources.App `json:"apps,omitempty"` SecretScopes map[string]*resources.SecretScope `json:"secret_scopes,omitempty"` Alerts map[string]*resources.Alert `json:"alerts,omitempty"` @@ -102,6 +103,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["external_locations"], r.ExternalLocations), collectResourceMap(descriptions["clusters"], r.Clusters), collectResourceMap(descriptions["dashboards"], r.Dashboards), + collectResourceMap(descriptions["genie_spaces"], r.GenieSpaces), collectResourceMap(descriptions["volumes"], r.Volumes), collectResourceMap(descriptions["apps"], r.Apps), collectResourceMap(descriptions["alerts"], r.Alerts), @@ -158,6 +160,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), "clusters": (&resources.Cluster{}).ResourceDescription(), "dashboards": (&resources.Dashboard{}).ResourceDescription(), + "genie_spaces": (&resources.GenieSpace{}).ResourceDescription(), "volumes": (&resources.Volume{}).ResourceDescription(), "apps": (&resources.App{}).ResourceDescription(), "secret_scopes": (&resources.SecretScope{}).ResourceDescription(), diff --git a/bundle/config/resources/genie_space.go b/bundle/config/resources/genie_space.go new file mode 100644 index 00000000000..276ab12b17d --- /dev/null +++ b/bundle/config/resources/genie_space.go @@ -0,0 +1,94 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +type GenieSpaceConfig struct { + // Description of the Genie Space + Description string `json:"description,omitempty"` + // Genie space ID + SpaceId string `json:"space_id,omitempty"` + // Title of the Genie Space + Title string `json:"title,omitempty"` + // Warehouse associated with the Genie Space + WarehouseId string `json:"warehouse_id,omitempty"` + // Parent folder path where the space will be registered + ParentPath string `json:"parent_path,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` + + // ============================================== + // === overrides over [dashboards.GenieSpace] === + // ============================================== + + // SerializedSpace holds the contents of the Genie Space in serialized JSON form. + // Even though the SDK represents this as a string, we override it as any to allow for inlining as YAML. + // If the value is a string, it is used as is. + // If it is not a string, its contents is marshalled as JSON. + SerializedSpace any `json:"serialized_space,omitempty"` +} + +func (c *GenieSpaceConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c GenieSpaceConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type GenieSpace struct { + BaseResource + GenieSpaceConfig + + Permissions []Permission `json:"permissions,omitempty"` + + // FilePath points to the local `.genie.json` file containing the Genie Space definition. + // This is inlined into serialized_space during deployment. The file_path is kept around + // as metadata which is needed for `databricks bundle generate genie-space --resource ` to work. + // This is not part of GenieSpaceConfig because we don't need to store this in the resource state. + FilePath string `json:"file_path,omitempty"` +} + +func (*GenieSpace) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: id, + }) + if err != nil { + log.Debugf(ctx, "genie space %s does not exist", id) + return false, err + } + return true, nil +} + +func (*GenieSpace) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "genie_space", + PluralName: "genie_spaces", + SingularTitle: "Genie Space", + PluralTitle: "Genie Spaces", + } +} + +func (r *GenieSpace) InitializeURL(baseURL url.URL) { + if r.ID == "" { + return + } + + r.URL = workspaceurls.ResourceURL(baseURL, "genie_spaces", r.ID) +} + +func (r *GenieSpace) GetName() string { + return r.Title +} + +func (r *GenieSpace) GetURL() string { + return r.URL +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 626fac07983..b23865cb319 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -194,6 +194,9 @@ func TestResourcesBindSupport(t *testing.T) { Dashboards: map[string]*resources.Dashboard{ "my_dashboard": {}, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "my_genie_space": {}, + }, Volumes: map[string]*resources.Volume{ "my_volume": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{}, @@ -304,6 +307,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockGenieAPI().EXPECT().GetSpace(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVolumesAPI().EXPECT().Read(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAppsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAlertsV2API().EXPECT().GetAlertById(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 7f56248bb44..f438d9b14cf 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -17,6 +17,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { ignoredResources := []string{ "catalogs", "external_locations", + "genie_spaces", "vector_search_endpoints", } diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 497bf9bcb7a..d3a3cc878e8 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -28,6 +28,7 @@ var SupportedResources = map[string]any{ "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), "dashboards": (*ResourceDashboard)(nil), + "genie_spaces": (*ResourceGenieSpace)(nil), "secret_scopes": (*ResourceSecretScope)(nil), "model_serving_endpoints": (*ResourceModelServingEndpoint)(nil), "quality_monitors": (*ResourceQualityMonitor)(nil), @@ -47,6 +48,7 @@ var SupportedResources = map[string]any{ "secret_scopes.permissions": (*ResourceSecretScopeAcls)(nil), "model_serving_endpoints.permissions": (*ResourcePermissions)(nil), "dashboards.permissions": (*ResourcePermissions)(nil), + "genie_spaces.permissions": (*ResourcePermissions)(nil), "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), // Grants diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f130afc6194..bc62dbd2dfd 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -461,6 +461,25 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "genie_spaces.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + resp, err := client.Genie.CreateSpace(ctx, dashboards.GenieCreateSpaceRequest{ + Title: "genie-space-permissions", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }) + if err != nil { + return nil, err + } + + return &PermissionsState{ + ObjectID: "/genie/spaces/" + resp.SpaceId, + EmbeddedSlice: []StatePermission{{ + Level: "CAN_MANAGE", + UserName: "user@example.com", + }}, + }, nil + }, + "model_serving_endpoints.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { waiter, err := client.ServingEndpoints.Create(ctx, serving.CreateServingEndpoint{ Name: "endpoint-permissions", diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index c37dfbccbb1..1bf5039160f 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -4,6 +4,8 @@ # Set a value to null to remove a type: # jobs: null +genie_spaces: dashboards.GenieSpace + postgres_branches: postgres.BranchSpec postgres_endpoints: postgres.EndpointSpec diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go new file mode 100644 index 00000000000..490849cfbdf --- /dev/null +++ b/bundle/direct/dresources/genie_space.go @@ -0,0 +1,164 @@ +package dresources + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +type ResourceGenieSpace struct { + client *databricks.WorkspaceClient +} + +func (*ResourceGenieSpace) New(client *databricks.WorkspaceClient) *ResourceGenieSpace { + return &ResourceGenieSpace{client: client} +} + +func (*ResourceGenieSpace) PrepareState(input *resources.GenieSpace) *resources.GenieSpaceConfig { + return &input.GenieSpaceConfig +} + +func (r *ResourceGenieSpace) RemapState(state *resources.GenieSpaceConfig) *resources.GenieSpaceConfig { + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](state.ForceSendFields, []string{ + "SpaceId", + "SerializedSpace", + }...) + + return &resources.GenieSpaceConfig{ + Description: state.Description, + Title: state.Title, + WarehouseId: state.WarehouseId, + ParentPath: state.ParentPath, + SerializedSpace: state.SerializedSpace, + + ForceSendFields: forceSendFields, + + // Clear output only fields. They should not show up on remote diff computation. + SpaceId: "", + } +} + +func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources.GenieSpaceConfig, error) { + space, err := r.client.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: id, + IncludeSerializedSpace: true, + ForceSendFields: nil, + }) + if err != nil { + return nil, err + } + + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) + + return &resources.GenieSpaceConfig{ + Description: space.Description, + Title: space.Title, + WarehouseId: space.WarehouseId, + ParentPath: "", + SerializedSpace: space.SerializedSpace, + + // Output only fields + SpaceId: space.SpaceId, + ForceSendFields: forceSendFields, + }, nil +} + +func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error) { + v := config.SerializedSpace + if serializedSpace, ok := v.(string); ok { + return serializedSpace, nil + } else if v != nil { + b, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("failed to marshal serialized_space: %w", err) + } + return string(b), nil + } + return "", nil +} + +func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace string) *resources.GenieSpaceConfig { + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) + + return &resources.GenieSpaceConfig{ + Description: space.Description, + Title: space.Title, + WarehouseId: space.WarehouseId, + ParentPath: "", + SerializedSpace: serializedSpace, + + // Output only fields + SpaceId: space.SpaceId, + ForceSendFields: forceSendFields, + } +} + +func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.GenieSpaceConfig) (string, *resources.GenieSpaceConfig, error) { + serializedSpace, err := prepareGenieSpaceRequest(config) + if err != nil { + return "", nil, err + } + + req := dashboards.GenieCreateSpaceRequest{ + Description: config.Description, + Title: config.Title, + WarehouseId: config.WarehouseId, + ParentPath: config.ParentPath, + SerializedSpace: serializedSpace, + + ForceSendFields: utils.FilterFields[dashboards.GenieCreateSpaceRequest](config.ForceSendFields), + } + + createResp, err := r.client.Genie.CreateSpace(ctx, req) + + // The API returns 404 if the parent directory doesn't exist. + // Create it and retry once. + if err != nil && apierr.IsMissing(err) { + err = r.client.Workspace.MkdirsByPath(ctx, config.ParentPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + return "", nil, fmt.Errorf("failed to create parent directory: %w", err) + } + createResp, err = r.client.Genie.CreateSpace(ctx, req) + } + if err != nil { + return "", nil, err + } + + return createResp.SpaceId, responseToGenieSpaceConfig(createResp, serializedSpace), nil +} + +func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *resources.GenieSpaceConfig, _ *PlanEntry) (*resources.GenieSpaceConfig, error) { + serializedSpace, err := prepareGenieSpaceRequest(config) + if err != nil { + return nil, err + } + + updateResp, err := r.client.Genie.UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: id, + Description: config.Description, + Title: config.Title, + WarehouseId: config.WarehouseId, + SerializedSpace: serializedSpace, + // Etag is for optimistic concurrency; we apply updates unconditionally. + Etag: "", + + ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields), + }) + if err != nil { + return nil, err + } + + return responseToGenieSpaceConfig(updateResp, serializedSpace), nil +} + +func (r *ResourceGenieSpace) DoDelete(ctx context.Context, id string) error { + return r.client.Genie.TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{ + SpaceId: id, + }) +} diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index eac5e2dcdbc..0c436eb249a 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -18,6 +18,7 @@ var permissionResourceToObjectType = map[string]string{ "apps": "/apps/", "clusters": "/clusters/", "dashboards": "/dashboards/", + "genie_spaces": "/genie/spaces/", "database_instances": "/database-instances/", "postgres_projects": "/database-projects/", "jobs": "/jobs/", diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 8c8a6ce07c5..bf2f2f9d61e 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -359,6 +359,19 @@ resources: - field: dataset_schema reason: input_only + genie_spaces: + recreate_on_changes: + - field: parent_path + reason: immutable + ignore_remote_changes: + # serialized_space locally (structured YAML) and remotely (JSON string) will differ + # textually, so we cannot meaningfully compare them for drift. + - field: serialized_space + reason: input_only + # parent_path is not reliably returned by the GET Genie space API. + - field: parent_path + reason: input_only + apps: recreate_on_changes: - field: name diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index f47d9ae2b47..b504f201115 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -76,6 +76,9 @@ var knownMissingInStateType = map[string][]string{ "dashboards": { "file_path", }, + "genie_spaces": { + "file_path", + }, "secret_scopes": { "backend_type", "keyvault_metadata", diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 9d41a86792d..5f8df77dfe9 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -475,6 +475,10 @@ resources: - Map - See [\_](#resourcesexternal_locations). +- - `genie_spaces` + - Map + - See [\_](#resourcesgenie_spaces). + - - `jobs` - Map - The job definitions for the bundle, where each key is the name of the job. See [\_](/dev-tools/bundles/resources.md#jobs). @@ -1478,6 +1482,118 @@ external_locations: ::: +### resources.genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - + +- - `file_path` + - String + - + +- - `lifecycle` + - Map + - See [\_](#resourcesgenie_spacesnamelifecycle). + +- - `parent_path` + - String + - + +- - `permissions` + - Sequence + - See [\_](#resourcesgenie_spacesnamepermissions). + +- - `serialized_space` + - Any + - + +- - `space_id` + - String + - + +- - `title` + - String + - + +- - `warehouse_id` + - String + - + +::: + + +### resources.genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### resources.genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ### resources.postgres_branches **`Type: Map`** @@ -2553,6 +2669,10 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcesexternal_locations). +- - `genie_spaces` + - Map + - See [\_](#targetsnameresourcesgenie_spaces). + - - `jobs` - Map - The job definitions for the bundle, where each key is the name of the job. See [\_](/dev-tools/bundles/resources.md#jobs). @@ -3556,6 +3676,118 @@ external_locations: ::: +### targets._name_.resources.genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - + +- - `file_path` + - String + - + +- - `lifecycle` + - Map + - See [\_](#targetsnameresourcesgenie_spacesnamelifecycle). + +- - `parent_path` + - String + - + +- - `permissions` + - Sequence + - See [\_](#targetsnameresourcesgenie_spacesnamepermissions). + +- - `serialized_space` + - Any + - + +- - `space_id` + - String + - + +- - `title` + - String + - + +- - `warehouse_id` + - String + - + +::: + + +### targets._name_.resources.genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### targets._name_.resources.genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ### targets._name_.resources.postgres_branches **`Type: Map`** diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index 392e574dcd3..eda0da5005f 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -3033,6 +3033,118 @@ The privileges assigned to the principal. ::: +## genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - + +- - `file_path` + - String + - + +- - `lifecycle` + - Map + - See [\_](#genie_spacesnamelifecycle). + +- - `parent_path` + - String + - + +- - `permissions` + - Sequence + - See [\_](#genie_spacesnamepermissions). + +- - `serialized_space` + - Any + - + +- - `space_id` + - String + - + +- - `title` + - String + - + +- - `warehouse_id` + - String + - + +::: + + +### genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ## jobs **`Type: Map`** diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go new file mode 100644 index 00000000000..ebb1098b2b1 --- /dev/null +++ b/bundle/generate/genie_space.go @@ -0,0 +1,22 @@ +package generate + +import ( + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string) (dyn.Value, error) { + // The majority of fields of the genie space struct are read-only. + // We copy the relevant fields manually. + dv := map[string]dyn.Value{ + "title": dyn.NewValue(genieSpace.Title, []dyn.Location{{Line: 1}}), + "warehouse_id": dyn.NewValue(genieSpace.WarehouseId, []dyn.Location{{Line: 2}}), + "file_path": dyn.NewValue(filePath, []dyn.Location{{Line: 3}}), + } + + if genieSpace.Description != "" { + dv["description"] = dyn.NewValue(genieSpace.Description, []dyn.Location{{Line: 4}}) + } + + return dyn.V(dv), nil +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 0f2e1b0c798..18270c760c1 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -192,6 +192,9 @@ github.com/databricks/cli/bundle/config.Resources: "external_locations": "description": |- PLACEHOLDER + "genie_spaces": + "description": |- + PLACEHOLDER "jobs": "description": |- The job definitions for the bundle, where each key is the name of the job. @@ -653,6 +656,34 @@ github.com/databricks/cli/bundle/config/resources.ExternalLocation: "url": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.GenieSpace: + "description": + "description": |- + PLACEHOLDER + "file_path": + "description": |- + PLACEHOLDER + "lifecycle": + "description": |- + PLACEHOLDER + "parent_path": + "description": |- + PLACEHOLDER + "permissions": + "description": |- + PLACEHOLDER + "serialized_space": + "description": |- + PLACEHOLDER + "space_id": + "description": |- + PLACEHOLDER + "title": + "description": |- + PLACEHOLDER + "warehouse_id": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.JobPermission: "group_name": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 389efd115bf..cc23a8200ce 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -750,6 +750,47 @@ } ] }, + "resources.GenieSpace": { + "oneOf": [ + { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/string" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "parent_path": { + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_space": { + "$ref": "#/$defs/interface" + }, + "space_id": { + "$ref": "#/$defs/string" + }, + "title": { + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { @@ -2538,6 +2579,9 @@ "external_locations": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation" }, + "genie_spaces": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -11703,6 +11747,20 @@ } ] }, + "resources.GenieSpace": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.GenieSpace" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index ab06243d227..e7548afd48b 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -731,6 +731,39 @@ "url" ] }, + "resources.GenieSpace": { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/string" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "parent_path": { + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_space": { + "$ref": "#/$defs/interface" + }, + "space_id": { + "$ref": "#/$defs/string" + }, + "title": { + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, "resources.Job": { "type": "object", "properties": { @@ -2483,6 +2516,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation", "x-since-version": "v0.289.0" }, + "genie_spaces": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -9709,6 +9745,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ExternalLocation" } }, + "resources.GenieSpace": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.GenieSpace" + } + }, "resources.Job": { "type": "object", "additionalProperties": { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 097b708e62d..229c8fd3f23 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -39,6 +39,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.volumes.test_volume": {ID: "1"}, "resources.clusters.test_cluster": {ID: "1"}, "resources.dashboards.test_dashboard": {ID: "1"}, + "resources.genie_spaces.test_genie_space": {ID: "1"}, "resources.apps.test_app": {ID: "app1"}, "resources.secret_scopes.test_secret_scope": {ID: "secret_scope1"}, "resources.sql_warehouses.test_sql_warehouse": {ID: "1"}, @@ -94,6 +95,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "app1", config.Resources.Apps["test_app"].ID) assert.Equal(t, "", config.Resources.Apps["test_app"].Name) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app"].ModifiedStatus) @@ -223,6 +227,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "test_genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space", + }, + }, + }, Apps: map[string]*resources.App{ "test_app": { App: apps.App{ @@ -355,6 +366,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.Dashboards["test_dashboard"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Equal(t, "", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Apps["test_app"].Name) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app"].ModifiedStatus) @@ -549,6 +563,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "test_genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space", + }, + }, + "test_genie_space_new": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space_new", + }, + }, + }, Apps: map[string]*resources.App{ "test_app": { App: apps.App{ @@ -726,6 +752,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.clusters.test_cluster_old": {ID: "2"}, "resources.dashboards.test_dashboard": {ID: "1"}, "resources.dashboards.test_dashboard_old": {ID: "2"}, + "resources.genie_spaces.test_genie_space": {ID: "1"}, + "resources.genie_spaces.test_genie_space_old": {ID: "2"}, "resources.apps.test_app": {ID: "test_app"}, "resources.apps.test_app_old": {ID: "test_app_old"}, "resources.secret_scopes.test_secret_scope": {ID: "test_secret_scope"}, @@ -834,6 +862,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.Dashboards["test_dashboard_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, "", config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.GenieSpaces["test_genie_space_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.GenieSpaces["test_genie_space_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space_new"].ModifiedStatus) + assert.Equal(t, "test_app", config.Resources.Apps["test_app"].Name) assert.Equal(t, "", config.Resources.Apps["test_app"].ModifiedStatus) assert.Equal(t, "test_app_old", config.Resources.Apps["test_app_old"].ID) diff --git a/cmd/bundle/generate.go b/cmd/bundle/generate.go index 9caf7fa1e37..12452293bc6 100644 --- a/cmd/bundle/generate.go +++ b/cmd/bundle/generate.go @@ -40,6 +40,7 @@ Use --bind to automatically bind the generated resource to the existing workspac cmd.AddCommand(generate.NewGenerateDashboardCommand()) cmd.AddCommand(generate.NewGenerateAlertCommand()) cmd.AddCommand(generate.NewGenerateAppCommand()) + cmd.AddCommand(generate.NewGenerateGenieSpaceCommand()) cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`) return cmd } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go new file mode 100644 index 00000000000..c6fb9e0651b --- /dev/null +++ b/cmd/bundle/generate/genie_space.go @@ -0,0 +1,503 @@ +package generate + +import ( + "context" + "errors" + "fmt" + "io" + "maps" + "os" + "path" + "path/filepath" + "slices" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/generate" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/cmd/bundle/deployment" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" +) + +type genieSpace struct { + // Lookup flags for one-time generate. + existingPath string + existingID string + + // Lookup flag for existing bundle resource. + resource string + + // Where to write the configuration and genie space representation. + resourceDir string + genieSpaceDir string + + // Force overwrite of existing files. + force bool + + // Watch for changes to the genie space. + watch bool + + // Relative path from the resource directory to the genie space directory. + relativeGenieSpaceDir string + + // Command. + cmd *cobra.Command + + // Automatically bind the generated resource to the existing resource. + bind bool + + // Output and error streams. + out io.Writer + err io.Writer +} + +func (g *genieSpace) resolveID(ctx context.Context, b *bundle.Bundle) string { + switch { + case g.existingPath != "": + return g.resolveFromPath(ctx, b) + case g.existingID != "": + return g.resolveFromID(ctx, b) + } + + logdiag.LogError(ctx, errors.New("expected one of --existing-path, --existing-id")) + return "" +} + +func (g *genieSpace) resolveFromPath(ctx context.Context, b *bundle.Bundle) string { + w := b.WorkspaceClient(ctx) + obj, err := w.Workspace.GetStatusByPath(ctx, g.existingPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + if apierr.IsMissing(err) { + logdiag.LogError(ctx, fmt.Errorf("genie space %q not found", path.Base(g.existingPath))) + return "" + } + + logdiag.LogError(ctx, err) + return "" + } + + if obj.ResourceId == "" { + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Error, + Summary: "expected a non-empty genie space resource ID", + }) + return "" + } + + return obj.ResourceId +} + +func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string { + w := b.WorkspaceClient(ctx) + obj, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: g.existingID, + }) + if err != nil { + if apierr.IsMissing(err) { + logdiag.LogError(ctx, fmt.Errorf("genie space with ID %s not found", g.existingID)) + return "" + } + logdiag.LogError(ctx, err) + return "" + } + + return obj.SpaceId +} + +func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, filename string) error { + // Unmarshal and remarshal the serialized genie space to ensure it is formatted correctly. + // The result will have alphabetically sorted keys and be indented. + data, err := remarshalJSON([]byte(genieSpace.SerializedSpace)) + if err != nil { + return err + } + + // Make sure the output directory exists. + if err := os.MkdirAll(filepath.Dir(filename), 0o755); err != nil { + return err + } + + // Clean the filename to ensure it is a valid path (and can be used on this OS). + filename = filepath.Clean(filename) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, filename) + if err != nil { + rel = filename + } + + // Verify that the file does not already exist. + info, err := os.Stat(filename) + if err == nil { + if info.IsDir() { + return fmt.Errorf("%s is a directory", filepath.ToSlash(rel)) + } + if !g.force { + return fmt.Errorf("%s already exists. Use --force to overwrite", filepath.ToSlash(rel)) + } + } + + cmdio.LogString(ctx, "Writing genie space to "+filepath.ToSlash(rel)) + return os.WriteFile(filename, data, 0o644) +} + +func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, key string) error { + // Save serialized genie space definition to the genie space directory. + genieSpaceBasename := key + ".genie.json" + genieSpacePath := filepath.Join(g.genieSpaceDir, genieSpaceBasename) + err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) + if err != nil { + return err + } + + // Synthesize resource configuration. + v, err := generate.ConvertGenieSpaceToValue(genieSpace, path.Join(g.relativeGenieSpaceDir, genieSpaceBasename)) + if err != nil { + return err + } + + result := map[string]dyn.Value{ + "resources": dyn.V(map[string]dyn.Value{ + "genie_spaces": dyn.V(map[string]dyn.Value{ + key: v, + }), + }), + } + + // Make sure the output directory exists. + if err := os.MkdirAll(g.resourceDir, 0o755); err != nil { + return err + } + + // Save the configuration to the resource directory. + resourcePath := filepath.Join(g.resourceDir, key+".genie_space.yml") + saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + "title": yaml.DoubleQuotedStyle, + }) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, resourcePath) + if err != nil { + rel = resourcePath + } + + cmdio.LogString(ctx, "Writing configuration to "+filepath.ToSlash(rel)) + err = saver.SaveAsYAML(result, resourcePath, g.force) + if err != nil { + return err + } + + return nil +} + +func waitForGenieSpaceChanges(ctx context.Context, w *databricks.WorkspaceClient, genieSpacePath string, lastModified int64) { + for { + obj, err := w.Workspace.GetStatusByPath(ctx, genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + logdiag.LogError(ctx, err) + return + } + + if obj.ModifiedAt > lastModified { + break + } + + time.Sleep(1 * time.Second) + } +} + +func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle.Bundle) { + resource, ok := b.Config.Resources.GenieSpaces[g.resource] + if !ok { + logdiag.LogError(ctx, fmt.Errorf("genie space resource %q is not defined", g.resource)) + return + } + + if resource.FilePath == "" { + logdiag.LogError(ctx, fmt.Errorf("genie space resource %q has no file path defined", g.resource)) + return + } + + genieSpaceID := resource.ID + genieSpacePath := resource.FilePath + + w := b.WorkspaceClient(ctx) + + var lastModified int64 + for { + genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: genieSpaceID, + IncludeSerializedSpace: true, + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + // Get workspace status to check modification time. + obj, err := w.Workspace.GetStatusByPath(ctx, "/Workspace"+genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil && !apierr.IsMissing(err) { + obj, err = w.Workspace.GetStatusByPath(ctx, genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + } + + var currentModified int64 + if err == nil { + currentModified = obj.ModifiedAt + } + + if lastModified == 0 || currentModified > lastModified { + err = g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) + if err != nil { + logdiag.LogError(ctx, err) + return + } + } + + if !g.watch { + return + } + + lastModified = currentModified + + if obj != nil { + waitForGenieSpaceChanges(ctx, w, obj.Path, lastModified) + } else { + time.Sleep(1 * time.Second) + } + } +} + +func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, genieSpaceID string) { + w := b.WorkspaceClient(ctx) + genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: genieSpaceID, + IncludeSerializedSpace: true, + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + key := textutil.NormalizeString(genieSpace.Title) + err = g.saveConfiguration(ctx, b, genieSpace, key) + if err != nil { + logdiag.LogError(ctx, err) + } + + if g.bind { + err = deployment.BindResource(g.cmd, key, genieSpaceID, true, false, true) + if err != nil { + logdiag.LogError(ctx, err) + return + } + cmdio.LogString(ctx, fmt.Sprintf("Successfully bound genie space with an id '%s'", genieSpaceID)) + } +} + +func (g *genieSpace) initialize(ctx context.Context, b *bundle.Bundle) { + // Make the paths absolute if they aren't already. + if !filepath.IsAbs(g.resourceDir) { + g.resourceDir = filepath.Join(b.BundleRootPath, g.resourceDir) + } + if !filepath.IsAbs(g.genieSpaceDir) { + g.genieSpaceDir = filepath.Join(b.BundleRootPath, g.genieSpaceDir) + } + + // Make sure we know how the genie space path is relative to the resource path. + rel, err := filepath.Rel(g.resourceDir, g.genieSpaceDir) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + g.relativeGenieSpaceDir = filepath.ToSlash(rel) +} + +func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { + phases.Initialize(ctx, b) + if logdiag.HasError(ctx) { + return + } + + requiredEngine, err := utils.ResolveEngineSetting(ctx, b) + if err != nil { + logdiag.LogError(ctx, err) + return + } + ctx, stateDesc := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true), requiredEngine) + if logdiag.HasError(ctx) { + return + } + + if stateDesc.Engine.IsDirect() { + _, localPath := b.StateFilenameDirect(ctx) + if err := b.DeploymentBundle.StateDB.Open(localPath); err != nil { + logdiag.LogError(ctx, err) + return + } + } + + bundle.ApplySeqContext(ctx, b, + statemgmt.Load(stateDesc.Engine), + ) + if logdiag.HasError(ctx) { + return + } + + g.updateGenieSpaceForResource(ctx, b) +} + +func (g *genieSpace) runForExisting(ctx context.Context, b *bundle.Bundle) { + // Resolve the ID of the genie space to generate configuration for. + genieSpaceID := g.resolveID(ctx, b) + if logdiag.HasError(ctx) { + return + } + + g.generateForExisting(ctx, b, genieSpaceID) +} + +func (g *genieSpace) RunE(cmd *cobra.Command, args []string) error { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + + b := root.MustConfigureBundle(cmd) + if b == nil || logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + g.initialize(ctx, b) + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + if g.resource != "" { + g.runForResource(ctx, b) + } else { + g.runForExisting(ctx, b) + } + + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + return nil +} + +// filterGenieSpaces returns a filter that only includes genie spaces. +func filterGenieSpaces(ref resources.Reference) bool { + return ref.Description.SingularName == "genie_space" +} + +// genieSpaceResourceCompletion executes to autocomplete the argument to the resource flag. +func genieSpaceResourceCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + b := root.MustConfigureBundle(cmd) + if logdiag.HasError(cmd.Context()) { + return nil, cobra.ShellCompDirectiveError + } + + if b == nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + return slices.Collect(maps.Keys(resources.Completions(b, filterGenieSpaces))), cobra.ShellCompDirectiveNoFileComp +} + +func NewGenerateGenieSpaceCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "genie-space", + Short: "Generate configuration for a Genie space", + Long: `Generate bundle configuration for an existing Databricks Genie space. + +This command downloads an existing Genie space and creates bundle files +that you can use to deploy the Genie space to other environments or manage it as code. + +Examples: + # Import Genie space by workspace path + databricks bundle generate genie-space --existing-path /Users/me/my-genie-space + + # Import Genie space by ID + databricks bundle generate genie-space --existing-id abc123 + + # Watch for changes to keep bundle in sync with UI modifications + databricks bundle generate genie-space --resource my_genie_space --watch --force + +What gets generated: +- Genie space configuration YAML file with settings and a reference to the Genie space definition +- Genie space definition (.genie.json) file with the serialized space content + +Sync workflow for Genie space development: +When developing Genie spaces, you can modify them in the Databricks UI and sync +changes back to your bundle: + +1. Make changes to Genie space in the Databricks UI +2. Run: databricks bundle generate genie-space --resource my_genie_space --force +3. Commit changes to version control +4. Deploy to other environments with: databricks bundle deploy --target prod + +The --watch flag continuously polls for remote changes and updates your local +bundle files automatically, useful during active Genie space development.`, + } + + g := &genieSpace{ + out: cmd.OutOrStdout(), + err: cmd.ErrOrStderr(), + } + + // Lookup flags. + cmd.Flags().StringVar(&g.existingPath, "existing-path", "", `workspace path of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.existingID, "existing-id", "", `ID of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.resource, "resource", "", `resource key of Genie space to watch for changes`) + + // Alias lookup flags that include the resource type name. + cmd.Flags().StringVar(&g.existingPath, "existing-genie-space-path", "", `workspace path of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.existingID, "existing-genie-space-id", "", `ID of the Genie space to generate configuration for`) + cmd.Flags().MarkHidden("existing-genie-space-path") + cmd.Flags().MarkHidden("existing-genie-space-id") + + // Output flags. + cmd.Flags().StringVarP(&g.resourceDir, "resource-dir", "d", "resources", `directory to write the configuration to`) + cmd.Flags().StringVarP(&g.genieSpaceDir, "genie-space-dir", "s", "src", `directory to write the Genie space representation to`) + cmd.Flags().BoolVarP(&g.force, "force", "f", false, `force overwrite existing files in the output directory`) + + cmd.Flags().BoolVarP(&g.bind, "bind", "b", false, `automatically bind the generated Genie space config to the existing Genie space`) + cmd.Flags().MarkHidden("bind") + + // Exactly one of the lookup flags must be provided. + cmd.MarkFlagsOneRequired( + "existing-path", + "existing-id", + "resource", + ) + + // Watch flag. This is relevant only in combination with the resource flag. + cmd.Flags().BoolVar(&g.watch, "watch", false, `watch for changes to the Genie space and update the configuration`) + + // Make sure the watch flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("watch", "existing-path") + cmd.MarkFlagsMutuallyExclusive("watch", "existing-id") + + // Make sure the bind flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("bind", "resource") + + // Completion for the resource flag. + cmd.RegisterFlagCompletionFunc("resource", genieSpaceResourceCompletion) + + cmd.RunE = g.RunE + g.cmd = cmd + return cmd +} diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 7ec8e937ece..c53eb6fd465 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", 0) assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, clusters, dashboards, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses") + assert.ErrorContains(t, err, "alerts, apps, clusters, dashboards, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -111,6 +111,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { assert.Contains(t, completions, "clusters") assert.Contains(t, completions, "dashboards") assert.Contains(t, completions, "experiments") + assert.Contains(t, completions, "genie_spaces") assert.Contains(t, completions, "jobs") assert.Contains(t, completions, "models") assert.Contains(t, completions, "model_serving_endpoints") @@ -119,7 +120,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { assert.Contains(t, completions, "queries") assert.Contains(t, completions, "registered_models") assert.Contains(t, completions, "warehouses") - assert.Len(t, completions, 13) + assert.Len(t, completions, 14) } func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { @@ -133,7 +134,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, clusters, dashboards, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, clusters, dashboards, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index d00a5039b8c..d19c2867798 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -141,6 +141,7 @@ type FakeWorkspace struct { Volumes map[string]catalog.VolumeInfo Dashboards map[string]fakeDashboard PublishedDashboards map[string]dashboards.PublishedDashboard + GenieSpaces map[string]dashboards.GenieSpace SqlWarehouses map[string]sql.GetWarehouseResponse Alerts map[string]sql.AlertV2 Experiments map[string]ml.GetExperimentResponse @@ -286,6 +287,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Volumes: map[string]catalog.VolumeInfo{}, Dashboards: map[string]fakeDashboard{}, PublishedDashboards: map[string]dashboards.PublishedDashboard{}, + GenieSpaces: map[string]dashboards.GenieSpace{}, SqlWarehouses: map[string]sql.GetWarehouseResponse{ TestDefaultWarehouseId: { Id: TestDefaultWarehouseId, diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go new file mode 100644 index 00000000000..7bd242912d3 --- /dev/null +++ b/libs/testserver/genie_spaces.go @@ -0,0 +1,162 @@ +package testserver + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "path" + "strings" + + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// generateGenieSpaceId returns a random 32-character hex string. +func generateGenieSpaceId() (string, error) { + randomBytes := make([]byte, 16) + _, err := rand.Read(randomBytes) + if err != nil { + return "", err + } + return hex.EncodeToString(randomBytes), nil +} + +func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { + defer s.LockUnlock()() + + var createReq dashboards.GenieCreateSpaceRequest + if err := json.Unmarshal(req.Body, &createReq); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "message": "Invalid request body: " + err.Error(), + }, + } + } + + spaceId, err := generateGenieSpaceId() + if err != nil { + return Response{ + StatusCode: 500, + Body: map[string]string{ + "message": "Failed to generate genie space ID", + }, + } + } + + genieSpace := dashboards.GenieSpace{ + SpaceId: spaceId, + Title: createReq.Title, + Description: createReq.Description, + WarehouseId: createReq.WarehouseId, + SerializedSpace: createReq.SerializedSpace, + } + + s.GenieSpaces[spaceId] = genieSpace + + // Register in the workspace files for path lookup. + if createReq.ParentPath != "" { + workspacePath := createReq.ParentPath + if !strings.HasPrefix(workspacePath, "/Workspace") { + workspacePath = path.Join("/Workspace", workspacePath) + } + workspacePath = path.Join(workspacePath, createReq.Title+".genie") + + s.files[workspacePath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "FILE", + Path: workspacePath, + ResourceId: spaceId, + }, + Data: []byte(createReq.SerializedSpace), + } + } + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceGet(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + var updateReq dashboards.GenieUpdateSpaceRequest + if err := json.Unmarshal(req.Body, &updateReq); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "message": "Invalid request body: " + err.Error(), + }, + } + } + + if updateReq.Title != "" { + genieSpace.Title = updateReq.Title + } + if updateReq.Description != "" { + genieSpace.Description = updateReq.Description + } + if updateReq.WarehouseId != "" { + genieSpace.WarehouseId = updateReq.WarehouseId + } + if updateReq.SerializedSpace != "" { + genieSpace.SerializedSpace = updateReq.SerializedSpace + } + + s.GenieSpaces[spaceId] = genieSpace + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + _, ok := s.GenieSpaces[spaceId] + if !ok { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + delete(s.GenieSpaces, spaceId) + + return Response{ + StatusCode: 200, + } +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index c78fd2b3ad1..20270d51697 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -309,6 +309,20 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.DashboardUnpublish(req) }) + // Genie Spaces: + server.Handle("GET", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceGet(req) + }) + server.Handle("POST", "/api/2.0/genie/spaces", func(req Request) any { + return req.Workspace.GenieSpaceCreate(req) + }) + server.Handle("PATCH", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceUpdate(req) + }) + server.Handle("DELETE", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceTrash(req) + }) + // Pipelines: server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req Request) any { diff --git a/libs/testserver/permissions.go b/libs/testserver/permissions.go index e7983b1afa7..962825dd037 100644 --- a/libs/testserver/permissions.go +++ b/libs/testserver/permissions.go @@ -25,6 +25,7 @@ var requestObjectTypeToObjectType = map[string]string{ "sql/alerts": "alert", "sql/queries": "query", "dashboards": "dashboard", + "genie/spaces": "genie-space", "experiments": "mlflowExperiment", "registered-models": "registered-model", "serving-endpoints": "serving-endpoint", diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index c6ac8fdd7cd..502ac472e9d 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -14,6 +14,7 @@ var resourceURLPatterns = map[string]string{ "clusters": "compute/clusters/%s", "dashboards": "dashboardsv3/%s/published", "experiments": "ml/experiments/%s", + "genie_spaces": "genie/rooms/%s", "jobs": "jobs/%s", "models": "ml/models/%s", "model_serving_endpoints": "ml/endpoints/%s", From f5a9b286e239e9246648e7fab0b917f2e8bf24b5 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:10:24 +0200 Subject: [PATCH 02/20] bundle: retry genie_space create on 400 INVALID_PARAMETER_VALUE missing-parent-path errors Genie surfaces a missing parent folder inconsistently across environments: some workspaces return a standard 404 missing-resource error, while others return 400 INVALID_PARAMETER_VALUE with a NOT_FOUND "Tree node ... does not exist" message embedded in the text. Treat both forms as "create the parent directory and retry once". Co-authored-by: Isaac --- bundle/direct/dresources/genie_space.go | 30 +++++- bundle/direct/dresources/genie_space_test.go | 105 +++++++++++++++++++ 2 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 bundle/direct/dresources/genie_space_test.go diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 490849cfbdf..8cd26897322 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -3,7 +3,9 @@ package dresources import ( "context" "encoding/json" + "errors" "fmt" + "strings" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/utils" @@ -99,6 +101,26 @@ func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace st } } +func isMissingGenieParentPathError(err error) bool { + if apierr.IsMissing(err) { + return true + } + + var apiErr *apierr.APIError + if !errors.As(err, &apiErr) { + return false + } + + // Genie reports a missing parent folder inconsistently across environments. + // Some workspaces return a standard missing-resource error, while others + // return INVALID_PARAMETER_VALUE with a NOT_FOUND message embedded in the + // text. Treat both forms as "create the parent directory and retry once". + return apiErr.StatusCode == 400 && + apiErr.ErrorCode == "INVALID_PARAMETER_VALUE" && + strings.Contains(apiErr.Message, "Tree node with path") && + strings.Contains(apiErr.Message, "does not exist") +} + func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.GenieSpaceConfig) (string, *resources.GenieSpaceConfig, error) { serializedSpace, err := prepareGenieSpaceRequest(config) if err != nil { @@ -117,9 +139,11 @@ func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.Gen createResp, err := r.client.Genie.CreateSpace(ctx, req) - // The API returns 404 if the parent directory doesn't exist. - // Create it and retry once. - if err != nil && apierr.IsMissing(err) { + // Retry once after creating the parent directory when the workspace folder + // is missing. Genie can surface this either as a standard missing-resource + // error or as INVALID_PARAMETER_VALUE with a "Tree node ... does not exist" + // message depending on the backend. + if err != nil && isMissingGenieParentPathError(err) { err = r.client.Workspace.MkdirsByPath(ctx, config.ParentPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. if err != nil { return "", nil, fmt.Errorf("failed to create parent directory: %w", err) diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go new file mode 100644 index 00000000000..c731322ca8e --- /dev/null +++ b/bundle/direct/dresources/genie_space_test.go @@ -0,0 +1,105 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsMissingGenieParentPathError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "standard missing error", + err: &apierr.APIError{ + StatusCode: 404, + ErrorCode: "NOT_FOUND", + Message: "not found", + }, + want: true, + }, + { + name: "invalid parameter tree node missing error", + err: &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "NOT_FOUND: Tree node with path /Workspace/foo does not exist", + }, + want: true, + }, + { + name: "other invalid parameter error", + err: &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "some other validation failure", + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isMissingGenieParentPathError(tt.err)) + }) + } +} + +func TestGenieSpaceDoCreateRetriesWhenParentPathLooksMissing(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + req := dashboards.GenieCreateSpaceRequest{ + Title: "test genie space", + Description: "test description", + ParentPath: "/Workspace/test-parent", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + } + + m.GetMockGenieAPI().EXPECT(). + CreateSpace(ctx, req). + Return(nil, &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "NOT_FOUND: Tree node with path /Workspace/test-parent does not exist", + }). + Once() + + m.GetMockWorkspaceAPI().EXPECT(). + MkdirsByPath(ctx, "/Workspace/test-parent"). + Return(nil). + Once() + + m.GetMockGenieAPI().EXPECT(). + CreateSpace(ctx, req). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "test genie space", + Description: "test description", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }, nil). + Once() + + id, state, err := r.DoCreate(ctx, &resources.GenieSpaceConfig{ + Title: "test genie space", + Description: "test description", + ParentPath: "/Workspace/test-parent", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }) + require.NoError(t, err) + assert.Equal(t, "space-id", id) + require.NotNil(t, state) + assert.Equal(t, "test genie space", state.Title) +} From 3fe9d104c1e3c189854215c0c7396161fe47d02c Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:10:38 +0200 Subject: [PATCH 03/20] bundle: register genie_space file_path translation in unified path visitor VisitGenieSpacePaths existed but was never called by VisitPaths, so NormalizePaths did not rewrite genie_space file_path values from "relative to YAML location" to "relative to bundle root" before applyGenieSpaceTranslations resolved them. The result was that generator output like "../src/.genie.json" failed on deploy with "path ... is not contained in sync root path". Co-authored-by: Isaac --- bundle/config/mutator/paths/visitor.go | 1 + .../mutator/translate_paths_genie_spaces.go | 1 - .../translate_paths_genie_spaces_test.go | 52 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 bundle/config/mutator/translate_paths_genie_spaces_test.go diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index 0e3d59d43f5..bdf42188fde 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -15,6 +15,7 @@ func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { VisitArtifactPaths, VisitAlertPaths, VisitDashboardPaths, + VisitGenieSpacePaths, VisitPipelinePaths, VisitPipelineLibrariesPaths, } diff --git a/bundle/config/mutator/translate_paths_genie_spaces.go b/bundle/config/mutator/translate_paths_genie_spaces.go index b6a3a652427..279d97be01c 100644 --- a/bundle/config/mutator/translate_paths_genie_spaces.go +++ b/bundle/config/mutator/translate_paths_genie_spaces.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle/config/mutator/paths" - "github.com/databricks/cli/libs/dyn" ) diff --git a/bundle/config/mutator/translate_paths_genie_spaces_test.go b/bundle/config/mutator/translate_paths_genie_spaces_test.go new file mode 100644 index 00000000000..a5295400905 --- /dev/null +++ b/bundle/config/mutator/translate_paths_genie_spaces_test.go @@ -0,0 +1,52 @@ +package mutator_test + +import ( + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "src", "my_space.genie.json")) + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Genie Space", + }, + FilePath: "../src/my_space.genie.json", + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.genie_spaces", []dyn.Location{{ + File: filepath.Join(dir, "resources", "genie_space.yml"), + }}) + + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePathsDashboards()) + require.NoError(t, diags.Error()) + + assert.Equal( + t, + filepath.ToSlash(filepath.Join("src", "my_space.genie.json")), + b.Config.Resources.GenieSpaces["genie_space"].FilePath, + ) +} From 1706660ba5c986e705c210d15e737133f3bdeef6 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:25:14 +0200 Subject: [PATCH 04/20] bundle: normalize inline serialized_space to JSON string before plan Inline YAML serialized_space stayed as a structured value (map[string]any with int leaves) in the config struct, while state held the JSON string that was sent to the API. structdiff compared an `any` field with reflect.DeepEqual, which reports map != string, so every plan after deploy showed a false update for the genie_space. Marshal inline serialized_space to its JSON string in ConfigureGenieSpaceSerializedSpace, mirroring the file_path code path, so config-side and state-side carry the same type. The genie_space_complex validate test is updated to reflect that serialized_space is now a string regardless of input form, and a new acceptance test under resources/genie_spaces/inline asserts that a deploy + plan cycle is drift-free for inline serialized_space. Co-authored-by: Isaac --- .../genie_spaces/inline/databricks.yml.tmpl | 22 +++++++++++++ .../genie_spaces/inline/out.plan.json | 28 +++++++++++++++++ .../genie_spaces/inline/out.test.toml | 6 ++++ .../resources/genie_spaces/inline/output.txt | 17 ++++++++++ .../resources/genie_spaces/inline/script | 18 +++++++++++ .../resources/genie_spaces/inline/test.toml | 10 ++++++ .../validate/genie_space_complex/output.txt | 14 ++++++--- .../validate/genie_space_complex/script | 17 +++++++--- .../configure_genie_space_serialized_space.go | 31 +++++++++++++------ 9 files changed, 143 insertions(+), 20 deletions(-) create mode 100644 acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/genie_spaces/inline/out.plan.json create mode 100644 acceptance/bundle/resources/genie_spaces/inline/out.test.toml create mode 100644 acceptance/bundle/resources/genie_spaces/inline/output.txt create mode 100644 acceptance/bundle/resources/genie_spaces/inline/script create mode 100644 acceptance/bundle/resources/genie_spaces/inline/test.toml diff --git a/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl new file mode 100644 index 00000000000..ea045097468 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: deploy-genie-space-inline-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Sales Analytics Inline Genie" + description: "Inline serialized_space test" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + serialized_space: + version: 1 + config: + sample_questions: + - id: "sq-001" + question: ["What is the total revenue?"] + data_sources: + tables: + - identifier: "main.sales.orders" + column_configs: + - column_name: "amount" + get_example_values: true diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json new file mode 100644 index 00000000000..49a077a1d4f --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -0,0 +1,28 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.genie_spaces.sales_analytics": { + "action": "skip", + "remote_state": { + "description": "Inline serialized_space test", + "parent_path": "", + "serialized_space": "{\"config\":{\"sample_questions\":[{\"id\":\"sq-001\",\"question\":[\"What is the total revenue?\"]}]},\"data_sources\":{\"tables\":[{\"column_configs\":[{\"column_name\":\"amount\",\"get_example_values\":true}],\"identifier\":\"main.sales.orders\"}]},\"version\":1}", + "space_id": "[GENIE_SPACE_ID]", + "title": "Sales Analytics Inline Genie", + "warehouse_id": "test-warehouse-id" + }, + "changes": { + "parent_path": { + "action": "skip", + "reason": "input_only", + "old": "/Workspace/Users/[USERNAME]", + "new": "/Workspace/Users/[USERNAME]", + "remote": "" + } + } + } + } +} diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml new file mode 100644 index 00000000000..496668716fa --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] + Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/output.txt b/acceptance/bundle/resources/genie_spaces/inline/output.txt new file mode 100644 index 00000000000..99fef197dd0 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/output.txt @@ -0,0 +1,17 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-inline-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.sales_analytics + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-inline-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/inline/script b/acceptance/bundle/resources/genie_spaces/inline/script new file mode 100644 index 00000000000..8f2f625b7b0 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') + +# Capture the genie space ID as a replacement. +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +# Plan after deploy must be drift-free aside from input_only fields. +# Without normalization the inline serialized_space leaves a map in the +# config struct while state holds a string, and structdiff reports false +# drift on every plan. +trace $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/inline/test.toml b/acceptance/bundle/resources/genie_spaces/inline/test.toml new file mode 100644 index 00000000000..18a07a1b00b --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/test.toml @@ -0,0 +1,10 @@ +Local = true +RecordRequests = false + +# Genie spaces only support direct deployment engine (no Terraform provider yet) +[EnvMatrix] +DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/validate/genie_space_complex/output.txt b/acceptance/bundle/validate/genie_space_complex/output.txt index 2fe58959fc8..4a0747210cc 100644 --- a/acceptance/bundle/validate/genie_space_complex/output.txt +++ b/acceptance/bundle/validate/genie_space_complex/output.txt @@ -6,13 +6,17 @@ }, "inline_yaml": { "title": "Inline YAML Genie Space", - "serialized_space_type": "object", - "tables_count": 1, - "has_column_configs": true, - "has_text_instructions": true + "serialized_space_type": "string", + "parsed": { + "tables_count": 1, + "has_column_configs": true, + "has_text_instructions": true + } }, "minimal_valid": { "title": "Minimal Valid", - "tables_count": 0 + "parsed": { + "tables_count": 0 + } } } diff --git a/acceptance/bundle/validate/genie_space_complex/script b/acceptance/bundle/validate/genie_space_complex/script index 01e1fed72bc..4feab1e373b 100644 --- a/acceptance/bundle/validate/genie_space_complex/script +++ b/acceptance/bundle/validate/genie_space_complex/script @@ -1,4 +1,7 @@ -# Validate complex genie spaces and check the serialized_space structure is preserved +# Validate complex genie spaces. ConfigureGenieSpaceSerializedSpace +# normalizes inline serialized_space YAML to a JSON string so the field has +# the same shape as the file_path code path; the script parses the JSON +# back to verify that the original structure is preserved. $CLI bundle validate -o json | jq '{ full_featured: .resources.genie_spaces.full_featured | { title, @@ -8,12 +11,16 @@ $CLI bundle validate -o json | jq '{ inline_yaml: .resources.genie_spaces.inline_yaml | { title, serialized_space_type: (.serialized_space | type), - tables_count: (.serialized_space.data_sources.tables | length), - has_column_configs: ((.serialized_space.data_sources.tables[0].column_configs | length) > 0), - has_text_instructions: ((.serialized_space.instructions.text_instructions | length) > 0) + parsed: (.serialized_space | fromjson) | { + tables_count: (.data_sources.tables | length), + has_column_configs: ((.data_sources.tables[0].column_configs | length) > 0), + has_text_instructions: ((.instructions.text_instructions | length) > 0) + } }, minimal_valid: .resources.genie_spaces.minimal_valid | { title, - tables_count: (.serialized_space.data_sources.tables | length) + parsed: (.serialized_space | fromjson) | { + tables_count: (.data_sources.tables | length) + } } }' diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index bf129b4060d..b8382ca6e7c 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -2,6 +2,7 @@ package resourcemutator import ( "context" + "encoding/json" "fmt" "github.com/databricks/cli/bundle" @@ -30,21 +31,31 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B dyn.AnyKey(), ) - // Configure serialized_space field for all genie spaces. err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - // Include "serialized_space" field if "file_path" is set. - path, ok := v.Get(filePathFieldName).AsString() - if !ok { - return v, nil + if path, ok := v.Get(filePathFieldName).AsString(); ok { + contents, err := b.SyncRoot.ReadFile(path) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", path, err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) } - contents, err := b.SyncRoot.ReadFile(path) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", path, err) + // Marshal an inline structured serialized_space to a JSON string so + // both config-side and state-side carry the same plain string. + // Otherwise YAML decodes small ints as Go `int` while state JSON + // round-trip decodes them as `float64`, and structdiff reports + // false drift on every plan. + ss := v.Get(serializedSpaceFieldName) + switch ss.Kind() { + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) } - - return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) + return v, nil }) }) From 41b19bb5b271e62b10d9be11e581cfa347e15359 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:29:13 +0200 Subject: [PATCH 05/20] bundle: reject genie_space permissions during plan Databricks workspaces do not expose a permissions endpoint for Genie Spaces (PUT /permissions/genie/spaces/ returns 404 ENDPOINT_NOT_FOUND). Without an upfront check the deploy creates the space first and then errors when applying permissions, leaving partial state behind. Add ValidateGenieSpacePermissions to the PreDeployChecks pipeline so both per-resource permissions and bundle-level permissions propagated by ApplyBundlePermissions surface a clear validation error before any API call is made. Co-authored-by: Isaac --- .../databricks.yml | 21 ++++++++ .../out.test.toml | 5 ++ .../output.txt | 22 ++++++++ .../script | 1 + .../test.toml | 8 +++ .../validate_genie_space_permissions.go | 44 +++++++++++++++ .../validate_genie_space_permissions_test.go | 54 +++++++++++++++++++ bundle/phases/plan.go | 1 + 8 files changed, 156 insertions(+) create mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml create mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml create mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt create mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/script create mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml create mode 100644 bundle/config/mutator/validate_genie_space_permissions.go create mode 100644 bundle/config/mutator/validate_genie_space_permissions_test.go diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml b/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml new file mode 100644 index 00000000000..71126ef7dcb --- /dev/null +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: genie-space-permissions-unsupported + +resources: + genie_spaces: + inline_perms: + title: "Inline Perms" + warehouse_id: "test-warehouse-id" + serialized_space: "{}" + permissions: + - level: CAN_MANAGE + user_name: someone@example.com + + bundle_perms: + title: "Bundle Perms" + warehouse_id: "test-warehouse-id" + serialized_space: "{}" + +permissions: + - level: CAN_MANAGE + user_name: someone@example.com diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml b/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml new file mode 100644 index 00000000000..54146af5645 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt b/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt new file mode 100644 index 00000000000..7224e5d1fea --- /dev/null +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt @@ -0,0 +1,22 @@ +Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups +If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. + +Consider using a adding a top-level permissions section such as the following: + + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE + +See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. + in databricks.yml:20:3 + +Error: Genie Space permissions are not supported + +Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support. + +Error: Genie Space permissions are not supported + +Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support. + + +Exit code: 1 diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/script b/acceptance/bundle/validate/genie_space_permissions_unsupported/script new file mode 100644 index 00000000000..b260e836a71 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/script @@ -0,0 +1 @@ +$CLI bundle plan diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml b/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml new file mode 100644 index 00000000000..c304e3eba17 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml @@ -0,0 +1,8 @@ +Local = true +Cloud = false + +Ignore = [".databricks"] + +# Genie spaces only support direct deployment engine. +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/validate_genie_space_permissions.go b/bundle/config/mutator/validate_genie_space_permissions.go new file mode 100644 index 00000000000..e2efb517705 --- /dev/null +++ b/bundle/config/mutator/validate_genie_space_permissions.go @@ -0,0 +1,44 @@ +package mutator + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type validateGenieSpacePermissions struct{} + +// ValidateGenieSpacePermissions errors if any genie_space resource has +// permissions configured. The Databricks workspace API does not expose +// PUT /permissions/genie/spaces/, so the deploy would create the +// space and then fail when applying permissions, leaving partial state. +// Bundle-level permissions are propagated to genie_spaces by +// ApplyBundlePermissions and are caught here as well. +func ValidateGenieSpacePermissions() bundle.Mutator { + return &validateGenieSpacePermissions{} +} + +func (m *validateGenieSpacePermissions) Name() string { + return "ValidateGenieSpacePermissions" +} + +func (m *validateGenieSpacePermissions) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + for key, space := range b.Config.Resources.GenieSpaces { + if space == nil || len(space.Permissions) == 0 { + continue + } + + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "Genie Space permissions are not supported", + Detail: "Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support.", + Locations: b.Config.GetLocations(fmt.Sprintf("resources.genie_spaces.%s.permissions", key)), + }) + } + + return diags +} diff --git a/bundle/config/mutator/validate_genie_space_permissions_test.go b/bundle/config/mutator/validate_genie_space_permissions_test.go new file mode 100644 index 00000000000..0ccf143815e --- /dev/null +++ b/bundle/config/mutator/validate_genie_space_permissions_test.go @@ -0,0 +1,54 @@ +package mutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/iam" + "github.com/stretchr/testify/assert" +) + +func TestValidateGenieSpacePermissions_NoPermissions(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "my_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Space", + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateGenieSpacePermissions()) + assert.Empty(t, diags) +} + +func TestValidateGenieSpacePermissions_WithPermissionsErrors(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "my_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Space", + }, + Permissions: []resources.Permission{ + {Level: iam.PermissionLevel("CAN_MANAGE"), UserName: "user@example.com"}, + }, + }, + }, + }, + }, + } + + diags := bundle.Apply(t.Context(), b, mutator.ValidateGenieSpacePermissions()) + assert.Len(t, diags, 1) + assert.Equal(t, "Genie Space permissions are not supported", diags[0].Summary) +} diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 0af9394f243..3e63b8f607b 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -25,6 +25,7 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine deploy.StatePull(), mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), + mutator.ValidateGenieSpacePermissions(), mutator.ValidateLifecycleStarted(engine), statemgmt.CheckRunningResource(engine), ) From e5af0e60e8bf9e65f8a59c46d435713f9d43f9d7 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:33:28 +0200 Subject: [PATCH 06/20] bundle: warn on dual genie_space sources and clarify direct-only error Two minor follow-ups to the genie_spaces work: - ConfigureGenieSpaceSerializedSpace silently let file_path win when a user also set serialized_space inline. Emit a warning that points at the inline block so the user knows their YAML is being dropped on the floor. - ValidateDirectOnlyResources only mentioned the DATABRICKS_BUNDLE_ENGINE env var as a way to opt into direct mode, even though 'bundle.engine: direct' in databricks.yml is the more common entry point. Mention both. Co-authored-by: Isaac --- .../catalog_requires_direct_mode/output.txt | 2 +- .../contents.genie.json | 1 + .../databricks.yml | 11 +++++++++++ .../out.test.toml | 5 +++++ .../genie_space_file_path_and_inline/output.txt | 10 ++++++++++ .../genie_space_file_path_and_inline/script | 1 + .../genie_space_file_path_and_inline/test.toml | 6 ++++++ .../configure_genie_space_serialized_space.go | 17 +++++++++++++---- .../mutator/validate_direct_only_resources.go | 2 +- .../validate_direct_only_resources_test.go | 6 +++--- 10 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/script create mode 100644 acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml diff --git a/acceptance/bundle/validate/catalog_requires_direct_mode/output.txt b/acceptance/bundle/validate/catalog_requires_direct_mode/output.txt index 2bddbc154b0..c0b5b54bbf0 100644 --- a/acceptance/bundle/validate/catalog_requires_direct_mode/output.txt +++ b/acceptance/bundle/validate/catalog_requires_direct_mode/output.txt @@ -1,7 +1,7 @@ Error: Catalog resources are only supported with direct deployment mode in databricks.yml:6:5 -Catalog resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use catalog resources. +Catalog resources require direct deployment mode. Set 'bundle.engine: direct' in your databricks.yml or set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use catalog resources. Learn more at https://docs.databricks.com/dev-tools/bundles/direct diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json new file mode 100644 index 00000000000..cb608f6e9c4 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json @@ -0,0 +1 @@ +{"version": 1} diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml new file mode 100644 index 00000000000..dea909bf2f9 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: genie-space-file-path-and-inline + +resources: + genie_spaces: + both_set: + title: "Both set" + warehouse_id: "test-warehouse-id" + file_path: "./contents.genie.json" + serialized_space: + version: 1 diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml new file mode 100644 index 00000000000..54146af5645 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt new file mode 100644 index 00000000000..589a8333c37 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt @@ -0,0 +1,10 @@ +Warning: both file_path and serialized_space are set; file_path will be used and serialized_space will be ignored + in databricks.yml:11:9 + +Name: genie-space-file-path-and-inline +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/genie-space-file-path-and-inline/default + +Found 1 warning diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/script b/acceptance/bundle/validate/genie_space_file_path_and_inline/script new file mode 100644 index 00000000000..72555b332a4 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml new file mode 100644 index 00000000000..4fc5284af50 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +# Genie spaces only support direct deployment engine. +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index b8382ca6e7c..3144bf7b614 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -33,10 +33,20 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - if path, ok := v.Get(filePathFieldName).AsString(); ok { - contents, err := b.SyncRoot.ReadFile(path) + filePath, hasFilePath := v.Get(filePathFieldName).AsString() + ss := v.Get(serializedSpaceFieldName) + + if hasFilePath { + if ss.IsValid() && ss.Kind() != dyn.KindNil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "both file_path and serialized_space are set; file_path will be used and serialized_space will be ignored", + Locations: ss.Locations(), + }) + } + contents, err := b.SyncRoot.ReadFile(filePath) if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", path, err) + return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", filePath, err) } return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) } @@ -46,7 +56,6 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B // Otherwise YAML decodes small ints as Go `int` while state JSON // round-trip decodes them as `float64`, and structdiff reports // false drift on every plan. - ss := v.Get(serializedSpaceFieldName) switch ss.Kind() { case dyn.KindMap, dyn.KindSequence: jsonBytes, err := json.Marshal(ss.AsAny()) diff --git a/bundle/config/mutator/validate_direct_only_resources.go b/bundle/config/mutator/validate_direct_only_resources.go index 556b7b815a5..8daf35449da 100644 --- a/bundle/config/mutator/validate_direct_only_resources.go +++ b/bundle/config/mutator/validate_direct_only_resources.go @@ -51,7 +51,7 @@ func (m *validateDirectOnlyResources) Apply(ctx context.Context, b *bundle.Bundl Severity: diag.Error, Summary: group.Description.SingularTitle + " resources are only supported with direct deployment mode", Detail: fmt.Sprintf("%s resources require direct deployment mode. "+ - "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use %s resources.\n"+ + "Set 'bundle.engine: direct' in your databricks.yml or set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use %s resources.\n"+ "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", group.Description.SingularTitle, group.Description.SingularName), Locations: b.Config.GetLocations("resources." + group.Description.PluralName), diff --git a/bundle/config/mutator/validate_direct_only_resources_test.go b/bundle/config/mutator/validate_direct_only_resources_test.go index dbc184c752d..a5ef9213c9f 100644 --- a/bundle/config/mutator/validate_direct_only_resources_test.go +++ b/bundle/config/mutator/validate_direct_only_resources_test.go @@ -61,7 +61,7 @@ func TestValidateDirectOnlyResourcesTerraformEngineDirectOnlyEmitsError(t *testi }, expectedSummary: "Catalog resources are only supported with direct deployment mode", expectedDetail: "Catalog resources require direct deployment mode. " + - "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use catalog resources.\n" + + "Set 'bundle.engine: direct' in your databricks.yml or set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use catalog resources.\n" + "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", }, { @@ -77,7 +77,7 @@ func TestValidateDirectOnlyResourcesTerraformEngineDirectOnlyEmitsError(t *testi }, expectedSummary: "External Location resources are only supported with direct deployment mode", expectedDetail: "External Location resources require direct deployment mode. " + - "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use external_location resources.\n" + + "Set 'bundle.engine: direct' in your databricks.yml or set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use external_location resources.\n" + "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", }, { @@ -93,7 +93,7 @@ func TestValidateDirectOnlyResourcesTerraformEngineDirectOnlyEmitsError(t *testi }, expectedSummary: "Vector Search Endpoint resources are only supported with direct deployment mode", expectedDetail: "Vector Search Endpoint resources require direct deployment mode. " + - "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use vector_search_endpoint resources.\n" + + "Set 'bundle.engine: direct' in your databricks.yml or set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use vector_search_endpoint resources.\n" + "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", }, } From 4bb02d867bcb6d0d77365a22243671155d389969 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:40:43 +0200 Subject: [PATCH 07/20] bundle: round-trip parent_path through generate genie-space When the Genie API returns parent_path on GetSpace, propagate it through bundle generate genie-space so the produced YAML deploys back to the same workspace folder. The testserver is updated to mirror that response shape so the acceptance fixture exercises the new path. Filter ParentPath out of ForceSendFields in DoRead and responseToGenieSpaceConfig: we deliberately clear ParentPath in the returned GenieSpaceConfig because the GET API does not reliably include it, but the SDK still surfaces it in ForceSendFields when the field appeared on the wire. Without this filter, deploy state serialization force-emits parent_path: "" even though the field is logically unset, producing spurious output diffs. Co-authored-by: Isaac --- .../out/resource/test_genie_space.genie_space.yml | 1 + .../resources/genie_spaces/inline/out.plan.json | 4 +--- bundle/direct/dresources/genie_space.go | 12 ++++++++++-- bundle/generate/genie_space.go | 4 ++++ libs/testserver/genie_spaces.go | 1 + 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml index 14764931f1b..a73945364ce 100644 --- a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml +++ b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml @@ -5,3 +5,4 @@ resources: warehouse_id: test-warehouse-id file_path: ../genie_space/test_genie_space.genie.json description: test description + parent_path: /Workspace/test-[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json index 49a077a1d4f..41f6aa983de 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -8,7 +8,6 @@ "action": "skip", "remote_state": { "description": "Inline serialized_space test", - "parent_path": "", "serialized_space": "{\"config\":{\"sample_questions\":[{\"id\":\"sq-001\",\"question\":[\"What is the total revenue?\"]}]},\"data_sources\":{\"tables\":[{\"column_configs\":[{\"column_name\":\"amount\",\"get_example_values\":true}],\"identifier\":\"main.sales.orders\"}]},\"version\":1}", "space_id": "[GENIE_SPACE_ID]", "title": "Sales Analytics Inline Genie", @@ -19,8 +18,7 @@ "action": "skip", "reason": "input_only", "old": "/Workspace/Users/[USERNAME]", - "new": "/Workspace/Users/[USERNAME]", - "remote": "" + "new": "/Workspace/Users/[USERNAME]" } } } diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 8cd26897322..d2569af7a61 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -56,7 +56,11 @@ func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources. return nil, err } - forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) + // Drop ParentPath from ForceSendFields. We always clear ParentPath + // below because the GET Genie space API does not reliably return it, + // and keeping it in ForceSendFields would force-emit parent_path: "" + // in state output even though the field is logically unset. + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields, "ParentPath") return &resources.GenieSpaceConfig{ Description: space.Description, @@ -86,7 +90,11 @@ func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error } func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace string) *resources.GenieSpaceConfig { - forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) + // Drop ParentPath from ForceSendFields. We always clear ParentPath + // below because the GET Genie space API does not reliably return it, + // and keeping it in ForceSendFields would force-emit parent_path: "" + // in state output even though the field is logically unset. + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields, "ParentPath") return &resources.GenieSpaceConfig{ Description: space.Description, diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go index ebb1098b2b1..5bd11761a49 100644 --- a/bundle/generate/genie_space.go +++ b/bundle/generate/genie_space.go @@ -18,5 +18,9 @@ func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string dv["description"] = dyn.NewValue(genieSpace.Description, []dyn.Location{{Line: 4}}) } + if genieSpace.ParentPath != "" { + dv["parent_path"] = dyn.NewValue(genieSpace.ParentPath, []dyn.Location{{Line: 5}}) + } + return dyn.V(dv), nil } diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index 7bd242912d3..0b11b119fcf 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -48,6 +48,7 @@ func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { SpaceId: spaceId, Title: createReq.Title, Description: createReq.Description, + ParentPath: createReq.ParentPath, WarehouseId: createReq.WarehouseId, SerializedSpace: createReq.SerializedSpace, } From 62c7c191675505bd3598d04c28e9d82acbbd2725 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 30 Apr 2026 23:42:59 +0200 Subject: [PATCH 08/20] bundle: lint cleanups for genie_space changes - Replace switch-with-fallthrough on dyn.Kind with a guard clause to satisfy the exhaustive linter without listing every Kind variant. - Use http.StatusBadRequest in isMissingGenieParentPathError instead of a magic 400 (auto-fix from golangci-lint). Co-authored-by: Isaac --- .../configure_genie_space_serialized_space.go | 15 +++++++-------- bundle/direct/dresources/genie_space.go | 3 ++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index 3144bf7b614..fe746cbd98e 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -56,15 +56,14 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B // Otherwise YAML decodes small ints as Go `int` while state JSON // round-trip decodes them as `float64`, and structdiff reports // false drift on every plan. - switch ss.Kind() { - case dyn.KindMap, dyn.KindSequence: - jsonBytes, err := json.Marshal(ss.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) - } - return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) + if ss.Kind() != dyn.KindMap && ss.Kind() != dyn.KindSequence { + return v, nil + } + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) } - return v, nil + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) }) }) diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index d2569af7a61..e719c55b4ff 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "strings" "github.com/databricks/cli/bundle/config/resources" @@ -123,7 +124,7 @@ func isMissingGenieParentPathError(err error) bool { // Some workspaces return a standard missing-resource error, while others // return INVALID_PARAMETER_VALUE with a NOT_FOUND message embedded in the // text. Treat both forms as "create the parent directory and retry once". - return apiErr.StatusCode == 400 && + return apiErr.StatusCode == http.StatusBadRequest && apiErr.ErrorCode == "INVALID_PARAMETER_VALUE" && strings.Contains(apiErr.Message, "Tree node with path") && strings.Contains(apiErr.Message, "does not exist") From fc7a59cb5008a78e29f25c4c0f396b965c14625c Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:41:36 +0200 Subject: [PATCH 09/20] bundle: skip genie_space serialized_space in update when unchanged locally serialized_space is in ignore_remote_changes because we cannot diff a structured local YAML body against a remote JSON string. That makes UI edits invisible at plan time, but the unconditional UpdateSpace request was still sending the local body, so any later update to title or description would silently overwrite UI changes. Use the plan entry to detect whether the user actually changed serialized_space locally; only include it in the update request when the change is an Update action (not a Skip from ignore_remote_changes). Co-authored-by: Isaac --- bundle/direct/dresources/genie_space.go | 50 ++++++++++++++- bundle/direct/dresources/genie_space_test.go | 64 ++++++++++++++++++++ 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index e719c55b4ff..f8305ffac4d 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -9,12 +9,16 @@ import ( "strings" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/dashboards" ) +var pathSerializedSpace = structpath.MustParsePath("serialized_space") + type ResourceGenieSpace struct { client *databricks.WorkspaceClient } @@ -166,12 +170,23 @@ func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.Gen return createResp.SpaceId, responseToGenieSpaceConfig(createResp, serializedSpace), nil } -func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *resources.GenieSpaceConfig, _ *PlanEntry) (*resources.GenieSpaceConfig, error) { +func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *resources.GenieSpaceConfig, entry *PlanEntry) (*resources.GenieSpaceConfig, error) { serializedSpace, err := prepareGenieSpaceRequest(config) if err != nil { return nil, err } + // serialized_space is in ignore_remote_changes (we cannot diff structured + // local YAML against remote JSON), so a UI edit produces no plan entry. + // If we still sent the unchanged local body on every update, the next + // update triggered by another field would clobber the UI edit. Only + // send it when the user actually changed it locally. + excludeForceSend := []string{} + if !hasUpdate(entry, pathSerializedSpace) { + serializedSpace = "" + excludeForceSend = append(excludeForceSend, "SerializedSpace") + } + updateResp, err := r.client.Genie.UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ SpaceId: id, Description: config.Description, @@ -181,13 +196,42 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re // Etag is for optimistic concurrency; we apply updates unconditionally. Etag: "", - ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields), + ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields, excludeForceSend...), }) if err != nil { return nil, err } - return responseToGenieSpaceConfig(updateResp, serializedSpace), nil + // When the request omitted serialized_space, use the value the response + // echoes back so RemapState records the latest body. + respSerialized := serializedSpace + if respSerialized == "" { + respSerialized = updateResp.SerializedSpace + } + + return responseToGenieSpaceConfig(updateResp, respSerialized), nil +} + +// hasUpdate reports whether entry has an Update-action change at the given path. +// HasChange alone matches Skip-action changes too, which we cannot use to drive +// request shaping for fields covered by ignore_remote_changes. +func hasUpdate(entry *PlanEntry, path *structpath.PathNode) bool { + if entry == nil { + return false + } + for s, change := range entry.Changes { + if change.Action != deployplan.Update { + continue + } + node, err := structpath.ParsePath(s) + if err != nil { + continue + } + if node.HasPrefix(path) { + return true + } + } + return false } func (r *ResourceGenieSpace) DoDelete(ctx context.Context, id string) error { diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go index c731322ca8e..7db9f5c4b16 100644 --- a/bundle/direct/dresources/genie_space_test.go +++ b/bundle/direct/dresources/genie_space_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -103,3 +104,66 @@ func TestGenieSpaceDoCreateRetriesWhenParentPathLooksMissing(t *testing.T) { require.NotNil(t, state) assert.Equal(t, "test genie space", state.Title) } + +func TestGenieSpaceDoUpdateOmitsSerializedSpaceWhenUnchanged(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + // Plan entry indicates only title changed; serialized_space is absent. + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + SerializedSpace: "{\"remote\":\"edit\"}", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + SerializedSpace: "{\"local\":\"stale\"}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"remote\":\"edit\"}", state.SerializedSpace) +} + +func TestGenieSpaceDoUpdateSendsSerializedSpaceWhenChanged(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "serialized_space": {Action: deployplan.Update, Old: "{}", New: "{\"v\":1}"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + SerializedSpace: "{\"v\":1}", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + SerializedSpace: "{\"v\":1}", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + SerializedSpace: "{\"v\":1}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"v\":1}", state.SerializedSpace) +} From ba59c4e61b6b125dfd9a815e6f7ae2a6e211598c Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:43:46 +0200 Subject: [PATCH 10/20] bundle: fix watch loop in generate genie-space The previous implementation polled w.Workspace.GetStatusByPath using resource.FilePath, which is the local relative path (e.g. "src/foo.genie.json"). Both lookups (with and without the "/Workspace" prefix) were invalid for the workspace API, so currentModified stayed at 0 and the file never updated past the first iteration. Genie has no remote modification timestamp on the response, so use content comparison instead: canonicalize the just-fetched serialized_space and compare against the on-disk body, re-saving only when they differ. The first iteration still always saves, preserving the prior unconditional initial sync. Co-authored-by: Isaac --- cmd/bundle/generate/genie_space.go | 73 ++++++++++++++++-------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index c6fb9e0651b..2c2aa9d106c 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -1,6 +1,7 @@ package generate import ( + "bytes" "context" "errors" "fmt" @@ -26,13 +27,14 @@ import ( "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/textutil" - "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/dashboards" "github.com/spf13/cobra" "go.yaml.in/yaml/v3" ) +const genieSpaceWatchInterval = 1 * time.Second + type genieSpace struct { // Lookup flags for one-time generate. existingPath string @@ -204,22 +206,6 @@ func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, ge return nil } -func waitForGenieSpaceChanges(ctx context.Context, w *databricks.WorkspaceClient, genieSpacePath string, lastModified int64) { - for { - obj, err := w.Workspace.GetStatusByPath(ctx, genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. - if err != nil { - logdiag.LogError(ctx, err) - return - } - - if obj.ModifiedAt > lastModified { - break - } - - time.Sleep(1 * time.Second) - } -} - func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle.Bundle) { resource, ok := b.Config.Resources.GenieSpaces[g.resource] if !ok { @@ -237,7 +223,7 @@ func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle. w := b.WorkspaceClient(ctx) - var lastModified int64 + first := true for { genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ SpaceId: genieSpaceID, @@ -248,20 +234,22 @@ func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle. return } - // Get workspace status to check modification time. - obj, err := w.Workspace.GetStatusByPath(ctx, "/Workspace"+genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. - if err != nil && !apierr.IsMissing(err) { - obj, err = w.Workspace.GetStatusByPath(ctx, genieSpacePath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. - } - - var currentModified int64 - if err == nil { - currentModified = obj.ModifiedAt + // Genie has no remote modification timestamp we can poll. Compare + // the canonicalized remote body against the on-disk body and only + // re-save when they differ. The first iteration always saves, to + // match the prior behavior of an unconditional initial sync. + shouldSave := first + if !first { + differs, err := genieSpaceBodyDiffersFromDisk(genieSpace.SerializedSpace, genieSpacePath) + if err != nil { + logdiag.LogError(ctx, err) + return + } + shouldSave = differs } - if lastModified == 0 || currentModified > lastModified { - err = g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) - if err != nil { + if shouldSave { + if err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath); err != nil { logdiag.LogError(ctx, err) return } @@ -271,14 +259,29 @@ func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle. return } - lastModified = currentModified + first = false + time.Sleep(genieSpaceWatchInterval) + } +} - if obj != nil { - waitForGenieSpaceChanges(ctx, w, obj.Path, lastModified) - } else { - time.Sleep(1 * time.Second) +// genieSpaceBodyDiffersFromDisk reports whether the canonicalized remote +// serialized_space differs from the contents of filename. +func genieSpaceBodyDiffersFromDisk(remoteSerialized, filename string) (bool, error) { + if remoteSerialized == "" { + return false, nil + } + canonical, err := remarshalJSON([]byte(remoteSerialized)) + if err != nil { + return false, err + } + onDisk, err := os.ReadFile(filename) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return true, nil } + return false, err } + return !bytes.Equal(canonical, onDisk), nil } func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, genieSpaceID string) { From 456e4daeae80193a75740c923925201c0bbfd68d Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:44:40 +0200 Subject: [PATCH 11/20] bundle: honor --key flag in generate genie-space The parent generate command exposes --key as a persistent flag, but the genie-space subcommand was always deriving the key from the remote title. Read the flag value and fall back to the title-derived key only when not provided. Co-authored-by: Isaac --- cmd/bundle/generate/genie_space.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 2c2aa9d106c..50242169cf9 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -295,7 +295,10 @@ func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, return } - key := textutil.NormalizeString(genieSpace.Title) + key := g.cmd.Flag("key").Value.String() + if key == "" { + key = textutil.NormalizeString(genieSpace.Title) + } err = g.saveConfiguration(ctx, b, genieSpace, key) if err != nil { logdiag.LogError(ctx, err) From f91c20b01ae41749dd27164622968c3d229e2a26 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:45:31 +0200 Subject: [PATCH 12/20] bundle: guard empty serialized_space in generate genie-space Calling json.Unmarshal on an empty serialized_space surfaces a confusing "unexpected end of JSON input" error and writes nothing useful. Bail out early with a clear message that names the target file. Co-authored-by: Isaac --- cmd/bundle/generate/genie_space.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 50242169cf9..8b74ce2a057 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -121,6 +121,10 @@ func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string } func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, filename string) error { + if genieSpace.SerializedSpace == "" { + return fmt.Errorf("Genie space response did not include serialized_space; refusing to write %s", filepath.ToSlash(filename)) + } + // Unmarshal and remarshal the serialized genie space to ensure it is formatted correctly. // The result will have alphabetically sorted keys and be indented. data, err := remarshalJSON([]byte(genieSpace.SerializedSpace)) From efdb09a01a8167e070014c0fa2c6e7bedce92fcf Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:46:22 +0200 Subject: [PATCH 13/20] bundle: collapse DoRead into responseToGenieSpaceConfig for genie_space DoRead duplicated the field copy and the ParentPath-drop comment that already lives in responseToGenieSpaceConfig. Reuse the helper directly so the two stay in sync. Co-authored-by: Isaac --- bundle/direct/dresources/genie_space.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index f8305ffac4d..08ce1b95a94 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -60,24 +60,7 @@ func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources. if err != nil { return nil, err } - - // Drop ParentPath from ForceSendFields. We always clear ParentPath - // below because the GET Genie space API does not reliably return it, - // and keeping it in ForceSendFields would force-emit parent_path: "" - // in state output even though the field is logically unset. - forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields, "ParentPath") - - return &resources.GenieSpaceConfig{ - Description: space.Description, - Title: space.Title, - WarehouseId: space.WarehouseId, - ParentPath: "", - SerializedSpace: space.SerializedSpace, - - // Output only fields - SpaceId: space.SpaceId, - ForceSendFields: forceSendFields, - }, nil + return responseToGenieSpaceConfig(space, space.SerializedSpace), nil } func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error) { From e5930e4330220644082515413267f04f8f571632 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:47:45 +0200 Subject: [PATCH 14/20] bundle: fill genie_space schema annotations The user-facing fields (title, description, warehouse_id, parent_path, file_path, serialized_space) had PLACEHOLDER descriptions, leaving the generated reference and resources docs blank. Fill them in with short descriptions and regenerate the schema and docs output. Co-authored-by: Isaac --- bundle/docsgen/output/reference.md | 24 ++++++++++++------------ bundle/docsgen/output/resources.md | 12 ++++++------ bundle/internal/schema/annotations.yml | 12 ++++++------ bundle/schema/jsonschema.json | 6 ++++++ 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 5f8df77dfe9..3d89c58e8e8 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1503,11 +1503,11 @@ genie_spaces: - - `description` - String - - + - Description of the Genie space shown alongside the title in the Databricks UI. - - `file_path` - String - - + - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map @@ -1515,7 +1515,7 @@ genie_spaces: - - `parent_path` - String - - + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. - - `permissions` - Sequence @@ -1523,7 +1523,7 @@ genie_spaces: - - `serialized_space` - Any - - + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. - - `space_id` - String @@ -1531,11 +1531,11 @@ genie_spaces: - - `title` - String - - + - Title of the Genie space shown in the Databricks UI. - - `warehouse_id` - String - - + - ID of the SQL warehouse used to run queries for this Genie space. ::: @@ -3697,11 +3697,11 @@ genie_spaces: - - `description` - String - - + - Description of the Genie space shown alongside the title in the Databricks UI. - - `file_path` - String - - + - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map @@ -3709,7 +3709,7 @@ genie_spaces: - - `parent_path` - String - - + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. - - `permissions` - Sequence @@ -3717,7 +3717,7 @@ genie_spaces: - - `serialized_space` - Any - - + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. - - `space_id` - String @@ -3725,11 +3725,11 @@ genie_spaces: - - `title` - String - - + - Title of the Genie space shown in the Databricks UI. - - `warehouse_id` - String - - + - ID of the SQL warehouse used to run queries for this Genie space. ::: diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index eda0da5005f..0c7175430e0 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -3054,11 +3054,11 @@ genie_spaces: - - `description` - String - - + - Description of the Genie space shown alongside the title in the Databricks UI. - - `file_path` - String - - + - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map @@ -3066,7 +3066,7 @@ genie_spaces: - - `parent_path` - String - - + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. - - `permissions` - Sequence @@ -3074,7 +3074,7 @@ genie_spaces: - - `serialized_space` - Any - - + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. - - `space_id` - String @@ -3082,11 +3082,11 @@ genie_spaces: - - `title` - String - - + - Title of the Genie space shown in the Databricks UI. - - `warehouse_id` - String - - + - ID of the SQL warehouse used to run queries for this Genie space. ::: diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 18270c760c1..a21452278c8 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -659,31 +659,31 @@ github.com/databricks/cli/bundle/config/resources.ExternalLocation: github.com/databricks/cli/bundle/config/resources.GenieSpace: "description": "description": |- - PLACEHOLDER + Description of the Genie space shown alongside the title in the Databricks UI. "file_path": "description": |- - PLACEHOLDER + Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. "lifecycle": "description": |- PLACEHOLDER "parent_path": "description": |- - PLACEHOLDER + Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. "permissions": "description": |- PLACEHOLDER "serialized_space": "description": |- - PLACEHOLDER + Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. "space_id": "description": |- PLACEHOLDER "title": "description": |- - PLACEHOLDER + Title of the Genie space shown in the Databricks UI. "warehouse_id": "description": |- - PLACEHOLDER + ID of the SQL warehouse used to run queries for this Genie space. github.com/databricks/cli/bundle/config/resources.JobPermission: "group_name": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index cc23a8200ce..b3a83929b7a 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -756,30 +756,36 @@ "type": "object", "properties": { "description": { + "description": "Description of the Genie space shown alongside the title in the Databricks UI.", "$ref": "#/$defs/string" }, "file_path": { + "description": "Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`.", "$ref": "#/$defs/string" }, "lifecycle": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" }, "parent_path": { + "description": "Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource.", "$ref": "#/$defs/string" }, "permissions": { "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" }, "serialized_space": { + "description": "Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`.", "$ref": "#/$defs/interface" }, "space_id": { "$ref": "#/$defs/string" }, "title": { + "description": "Title of the Genie space shown in the Databricks UI.", "$ref": "#/$defs/string" }, "warehouse_id": { + "description": "ID of the SQL warehouse used to run queries for this Genie space.", "$ref": "#/$defs/string" } }, From a9268258dc5b4e1cbdb3cba769a1a24154b90c7f Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 7 May 2026 11:48:55 +0200 Subject: [PATCH 15/20] bundle: lint cleanups for genie_space review fixes Lowercase the genie_space error message to satisfy ST1005 and let the linter convert an empty []string{} to a nil slice. Co-authored-by: Isaac --- bundle/direct/dresources/genie_space.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 08ce1b95a94..1eaa1b1995c 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -164,7 +164,7 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re // If we still sent the unchanged local body on every update, the next // update triggered by another field would clobber the UI edit. Only // send it when the user actually changed it locally. - excludeForceSend := []string{} + var excludeForceSend []string if !hasUpdate(entry, pathSerializedSpace) { serializedSpace = "" excludeForceSend = append(excludeForceSend, "SerializedSpace") diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 8b74ce2a057..f4c344667e7 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -122,7 +122,7 @@ func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, filename string) error { if genieSpace.SerializedSpace == "" { - return fmt.Errorf("Genie space response did not include serialized_space; refusing to write %s", filepath.ToSlash(filename)) + return fmt.Errorf("genie space response did not include serialized_space; refusing to write %s", filepath.ToSlash(filename)) } // Unmarshal and remarshal the serialized genie space to ensure it is formatted correctly. From dfbc1c65e8757ca15afac1c9b57341e200d53d7d Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Mon, 18 May 2026 15:02:52 +0200 Subject: [PATCH 16/20] Regenerate test toml --- acceptance/bundle/generate/genie_space/out.test.toml | 4 +--- .../bundle/resources/genie_spaces/inline/out.test.toml | 6 ++---- .../bundle/resources/genie_spaces/simple/out.test.toml | 6 ++---- .../bundle/validate/genie_space_complex/out.test.toml | 4 +--- .../bundle/validate/genie_space_defaults/out.test.toml | 4 +--- .../validate/genie_space_file_path_and_inline/out.test.toml | 4 +--- .../genie_space_permissions_unsupported/out.test.toml | 4 +--- 7 files changed, 9 insertions(+), 23 deletions(-) diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml index d560f1de043..f784a183258 100644 --- a/acceptance/bundle/generate/genie_space/out.test.toml +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml index 496668716fa..03d89cd7d57 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] - Ignore = ["databricks.yml"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml index 496668716fa..03d89cd7d57 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -1,6 +1,4 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] - Ignore = ["databricks.yml"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml index d560f1de043..f784a183258 100644 --- a/acceptance/bundle/validate/genie_space_complex/out.test.toml +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml index d560f1de043..f784a183258 100644 --- a/acceptance/bundle/validate/genie_space_defaults/out.test.toml +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml index 54146af5645..e90b6d5d1ba 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml b/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml index 54146af5645..e90b6d5d1ba 100644 --- a/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml +++ b/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml @@ -1,5 +1,3 @@ Local = true Cloud = false - -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From e4809d65ad4eac442a3d3f4f1c10e555ce83b44e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 20 May 2026 12:50:24 +0200 Subject: [PATCH 17/20] Agent review --- ....json => test_genie_space.geniespace.json} | 0 .../resource/test_genie_space.genie_space.yml | 2 +- .../bundle/generate/genie_space/output.txt | 2 +- .../genie_spaces/inline/out.plan.json | 7 + .../genie_spaces/inline/out.test.toml | 1 - .../resources/genie_spaces/inline/test.toml | 4 - .../parent_path_recreate/databricks.yml.tmpl | 10 + .../parent_path_recreate}/out.test.toml | 0 .../parent_path_recreate/output.txt | 21 + .../genie_spaces/parent_path_recreate/script | 14 + .../parent_path_recreate/test.toml | 6 + .../genie_spaces/simple/databricks.yml.tmpl | 2 +- .../genie_spaces/simple/out.plan.json | 7 + .../genie_spaces/simple/out.test.toml | 1 - ...e.json => sales_analytics.geniespace.json} | 0 .../resources/genie_spaces/simple/test.toml | 4 - .../bundle/resources/genie_spaces/test.toml | 5 +- .../current_can_manage/databricks.yml | 19 + .../current_can_manage/out.plan.direct.json | 52 ++ .../out.requests.deploy.direct.json | 24 + .../out.requests.destroy.direct.json | 0 .../current_can_manage/out.test.toml | 3 + .../current_can_manage/output.txt | 35 ++ .../genie_spaces/current_can_manage/script | 18 + .../permissions/genie_spaces/test.toml | 4 + .../genie_space_complex/databricks.yml | 2 +- ...nie.json => full_featured.geniespace.json} | 0 ...ts.genie.json => contents.geniespace.json} | 0 .../databricks.yml | 2 +- .../out.test.toml | 2 +- .../test.toml | 4 +- .../databricks.yml | 21 - .../output.txt | 22 - .../script | 1 - .../test.toml | 8 - .../configure_genie_space_serialized_space.go | 21 +- .../translate_paths_genie_spaces_test.go | 9 +- .../validate_genie_space_permissions.go | 44 -- .../validate_genie_space_permissions_test.go | 54 -- bundle/config/resources/genie_space.go | 8 +- bundle/direct/dresources/all_test.go | 9 + bundle/direct/dresources/genie_space.go | 93 ++- bundle/direct/dresources/genie_space_test.go | 91 +++ bundle/direct/dstate/state.go | 11 +- bundle/docsgen/output/reference.md | 14 +- bundle/docsgen/output/resources.md | 592 +++++++++++++++--- bundle/internal/schema/annotations.yml | 5 +- bundle/phases/plan.go | 1 - bundle/schema/jsonschema.json | 5 +- cmd/bundle/generate/dashboard.go | 6 +- cmd/bundle/generate/genie_space.go | 11 +- cmd/bundle/generate/genie_space_test.go | 126 ++++ libs/testserver/genie_spaces.go | 52 +- libs/workspaceurls/urls_test.go | 1 + 54 files changed, 1181 insertions(+), 275 deletions(-) rename acceptance/bundle/generate/genie_space/out/genie_space/{test_genie_space.genie.json => test_genie_space.geniespace.json} (100%) create mode 100644 acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl rename acceptance/bundle/{validate/genie_space_permissions_unsupported => resources/genie_spaces/parent_path_recreate}/out.test.toml (100%) create mode 100644 acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt create mode 100644 acceptance/bundle/resources/genie_spaces/parent_path_recreate/script create mode 100644 acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml rename acceptance/bundle/resources/genie_spaces/simple/{sales_analytics.genie.json => sales_analytics.geniespace.json} (100%) create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script create mode 100644 acceptance/bundle/resources/permissions/genie_spaces/test.toml rename acceptance/bundle/validate/genie_space_complex/{full_featured.genie.json => full_featured.geniespace.json} (100%) rename acceptance/bundle/validate/genie_space_file_path_and_inline/{contents.genie.json => contents.geniespace.json} (100%) delete mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml delete mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt delete mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/script delete mode 100644 acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml delete mode 100644 bundle/config/mutator/validate_genie_space_permissions.go delete mode 100644 bundle/config/mutator/validate_genie_space_permissions_test.go create mode 100644 cmd/bundle/generate/genie_space_test.go diff --git a/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json similarity index 100% rename from acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.genie.json rename to acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json diff --git a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml index a73945364ce..1471a901344 100644 --- a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml +++ b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml @@ -3,6 +3,6 @@ resources: test_genie_space: title: "test genie space" warehouse_id: test-warehouse-id - file_path: ../genie_space/test_genie_space.genie.json + file_path: ../genie_space/test_genie_space.geniespace.json description: test description parent_path: /Workspace/test-[UNIQUE_NAME] diff --git a/acceptance/bundle/generate/genie_space/output.txt b/acceptance/bundle/generate/genie_space/output.txt index 985366ada68..a313a51adc3 100644 --- a/acceptance/bundle/generate/genie_space/output.txt +++ b/acceptance/bundle/generate/genie_space/output.txt @@ -2,5 +2,5 @@ >>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME] >>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --genie-space-dir out/genie_space --resource-dir out/resource -Writing genie space to out/genie_space/test_genie_space.genie.json +Writing genie space to out/genie_space/test_genie_space.geniespace.json Writing configuration to out/resource/test_genie_space.genie_space.yml diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json index 41f6aa983de..8d6f7aabc78 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -8,12 +8,19 @@ "action": "skip", "remote_state": { "description": "Inline serialized_space test", + "etag": "1", "serialized_space": "{\"config\":{\"sample_questions\":[{\"id\":\"sq-001\",\"question\":[\"What is the total revenue?\"]}]},\"data_sources\":{\"tables\":[{\"column_configs\":[{\"column_name\":\"amount\",\"get_example_values\":true}],\"identifier\":\"main.sales.orders\"}]},\"version\":1}", "space_id": "[GENIE_SPACE_ID]", "title": "Sales Analytics Inline Genie", "warehouse_id": "test-warehouse-id" }, "changes": { + "etag": { + "action": "skip", + "reason": "custom", + "old": "1", + "remote": "1" + }, "parent_path": { "action": "skip", "reason": "input_only", diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml index 03d89cd7d57..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -1,4 +1,3 @@ Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/test.toml b/acceptance/bundle/resources/genie_spaces/inline/test.toml index 18a07a1b00b..bbdf2380b2d 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/test.toml @@ -1,10 +1,6 @@ Local = true RecordRequests = false -# Genie spaces only support direct deployment engine (no Terraform provider yet) -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = ["direct"] - Ignore = [ "databricks.yml", ] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..1eea5d933e3 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: recreate-genie-space-$UNIQUE_NAME + +resources: + genie_spaces: + recreate_target: + title: "Recreate Target" + warehouse_id: "test-warehouse-id" + parent_path: PARENT_PATH_PLACEHOLDER + serialized_space: "{}" diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/out.test.toml similarity index 100% rename from acceptance/bundle/validate/genie_space_permissions_unsupported/out.test.toml rename to acceptance/bundle/resources/genie_spaces/parent_path_recreate/out.test.toml diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt new file mode 100644 index 00000000000..e90257bf971 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt @@ -0,0 +1,21 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-genie-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan after changing parent_path should show recreate +>>> [CLI] bundle plan +recreate genie_spaces.recreate_target + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.recreate_target + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-genie-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script new file mode 100644 index 00000000000..d5f04a795ab --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script @@ -0,0 +1,14 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# Deploy with the original parent_path. +envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-old|" > databricks.yml +trace $CLI bundle deploy + +# Change parent_path. parent_path is recreate_on_changes in resources.yml, +# so the plan should show a recreate (delete + create) rather than an update. +envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-new|" > databricks.yml +title "Plan after changing parent_path should show recreate" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl index 26ed410751a..f30e4991de9 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl +++ b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl @@ -8,4 +8,4 @@ resources: description: "AI assistant for sales data analysis" warehouse_id: "test-warehouse-id" parent_path: /Users/$CURRENT_USER_NAME - file_path: "sales_analytics.genie.json" + file_path: "sales_analytics.geniespace.json" diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json index 19f063bb002..b5583b47c06 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -8,12 +8,19 @@ "action": "skip", "remote_state": { "description": "AI assistant for sales data analysis", + "etag": "1", "serialized_space": "{\n \"version\": 1,\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"sq-001\",\n \"question\": [\"What is the total revenue?\"]\n },\n {\n \"id\": \"sq-002\",\n \"question\": [\"Show me the top customers\"]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"identifier\": \"main.sales.orders\",\n \"column_configs\": [\n {\n \"column_name\": \"order_id\",\n \"get_example_values\": true,\n \"build_value_dictionary\": true\n },\n {\n \"column_name\": \"customer_id\",\n \"get_example_values\": true\n },\n {\n \"column_name\": \"amount\",\n \"get_example_values\": false\n }\n ]\n },\n {\n \"identifier\": \"main.sales.customers\"\n }\n ]\n },\n \"instructions\": {\n \"text_instructions\": [\n {\n \"id\": \"ti-001\",\n \"content\": [\n \"This genie space analyzes sales data.\\n\",\n \"Always filter by date when querying orders.\\n\",\n \"Use customer_name instead of customer_id in results.\"\n ]\n }\n ],\n \"example_question_sqls\": [\n {\n \"id\": \"eq-001\",\n \"question\": [\"What are the top customers by revenue?\"],\n \"sql\": [\n \"SELECT\\n\",\n \" c.customer_name,\\n\",\n \" SUM(o.amount) AS total_revenue\\n\",\n \"FROM main.sales.orders o\\n\",\n \"JOIN main.sales.customers c ON o.customer_id = c.id\\n\",\n \"WHERE o.order_date \u003e= :start_date\\n\",\n \"GROUP BY c.customer_name\\n\",\n \"ORDER BY total_revenue DESC\\n\",\n \"LIMIT :limit\"\n ],\n \"parameters\": [\n {\n \"name\": \"start_date\",\n \"type_hint\": \"STRING\",\n \"description\": [\"Start date for the analysis period\"],\n \"default_value\": {\n \"values\": [\"2024-01-01\"]\n }\n },\n {\n \"name\": \"limit\",\n \"type_hint\": \"INTEGER\",\n \"description\": [\"Number of customers to return\"],\n \"default_value\": {\n \"values\": [\"10\"]\n }\n }\n ]\n },\n {\n \"id\": \"eq-002\",\n \"question\": [\"Calculate daily revenue\"],\n \"sql\": [\n \"SELECT\\n\",\n \" order_date,\\n\",\n \" SUM(amount) AS daily_revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY order_date\\n\",\n \"ORDER BY order_date\"\n ]\n }\n ],\n \"sql_snippets\": {\n \"measures\": [\n {\n \"id\": \"m-001\",\n \"alias\": \"total_revenue\",\n \"sql\": [\"SUM(orders.amount)\"],\n \"display_name\": \"Total Revenue\"\n }\n ]\n },\n \"sql_functions\": [\n {\n \"id\": \"sf-001\",\n \"identifier\": \"main.analytics.calculate_churn\"\n }\n ]\n },\n \"benchmarks\": {\n \"questions\": [\n {\n \"id\": \"bq-001\",\n \"question\": [\"What is the monthly revenue trend?\"],\n \"answer\": [\n {\n \"format\": \"SQL\",\n \"content\": [\n \"SELECT\\n\",\n \" DATE_TRUNC('month', order_date) AS month,\\n\",\n \" SUM(amount) AS revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY 1\\n\",\n \"ORDER BY 1\"\n ]\n }\n ]\n }\n ]\n }\n}\n", "space_id": "[GENIE_SPACE_ID]", "title": "Sales Analytics Genie", "warehouse_id": "test-warehouse-id" }, "changes": { + "etag": { + "action": "skip", + "reason": "custom", + "old": "1", + "remote": "1" + }, "parent_path": { "action": "skip", "reason": "input_only", diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml index 03d89cd7d57..e90b6d5d1ba 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -1,4 +1,3 @@ Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json similarity index 100% rename from acceptance/bundle/resources/genie_spaces/simple/sales_analytics.genie.json rename to acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json diff --git a/acceptance/bundle/resources/genie_spaces/simple/test.toml b/acceptance/bundle/resources/genie_spaces/simple/test.toml index 18a07a1b00b..bbdf2380b2d 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/test.toml @@ -1,10 +1,6 @@ Local = true RecordRequests = false -# Genie spaces only support direct deployment engine (no Terraform provider yet) -[EnvMatrix] -DATABRICKS_BUNDLE_ENGINE = ["direct"] - Ignore = [ "databricks.yml", ] diff --git a/acceptance/bundle/resources/genie_spaces/test.toml b/acceptance/bundle/resources/genie_spaces/test.toml index 2569ef7dcb1..7f397c47833 100644 --- a/acceptance/bundle/resources/genie_spaces/test.toml +++ b/acceptance/bundle/resources/genie_spaces/test.toml @@ -1,3 +1,2 @@ -# Genie spaces are only deployed via direct deployment engine -[Env] -DATABRICKS_BUNDLE_ENGINE = "direct" +# Genie spaces are only deployed via the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml new file mode 100644 index 00000000000..d9e1c56c35c --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +resources: + genie_spaces: + foo: + title: "Permissions Test Space" + warehouse_id: test-warehouse-id + parent_path: /Workspace/Users/tester@databricks.com + serialized_space: "{}" + permissions: + - level: CAN_READ + user_name: viewer@example.com + - level: CAN_MANAGE + group_name: data-team + - level: CAN_MANAGE + service_principal_name: f37d18cd-98a8-4db5-8112-12dd0a6bfe38 + - level: CAN_MANAGE + user_name: tester@databricks.com diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json new file mode 100644 index 00000000000..dba8cde6d93 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -0,0 +1,52 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.genie_spaces.foo": { + "action": "create", + "new_state": { + "value": { + "parent_path": "/Workspace/Users/[USERNAME]", + "serialized_space": "{}", + "title": "Permissions Test Space", + "warehouse_id": "test-warehouse-id" + } + } + }, + "resources.genie_spaces.foo.permissions": { + "depends_on": [ + { + "node": "resources.genie_spaces.foo", + "label": "${resources.genie_spaces.foo.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "object_id": "", + "__embed__": [ + { + "level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "level": "CAN_MANAGE", + "group_name": "data-team" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "object_id": "/genie/spaces/${resources.genie_spaces.foo.id}" + } + } + } + } +} diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json new file mode 100644 index 00000000000..c112ccb7e76 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json @@ -0,0 +1,24 @@ +{ + "method": "PUT", + "path": "/api/2.0/permissions/genie/spaces/[FOO_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" + }, + { + "permission_level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt new file mode 100644 index 00000000000..e4e8cf0189d --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] bundle validate -o json +[ + { + "level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "level": "CAN_MANAGE" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script new file mode 100644 index 00000000000..1b20af07543 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -0,0 +1,18 @@ +trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions +rm out.requests.txt + +$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +print_requests() { + jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")))' + rm out.requests.txt +} + +rm out.requests.txt +trace $CLI bundle deploy +# Genie space IDs are random; normalize them in the recorded requests. +replace_ids.py +print_requests > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json + +trace $CLI bundle destroy --auto-approve +print_requests > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/genie_spaces/test.toml b/acceptance/bundle/resources/permissions/genie_spaces/test.toml new file mode 100644 index 00000000000..390388f04aa --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/test.toml @@ -0,0 +1,4 @@ +Env.RESOURCE = "genie_spaces" # for ../_script + +# Genie spaces only support the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/databricks.yml b/acceptance/bundle/validate/genie_space_complex/databricks.yml index eae5cdc3f10..94c5fc7188c 100644 --- a/acceptance/bundle/validate/genie_space_complex/databricks.yml +++ b/acceptance/bundle/validate/genie_space_complex/databricks.yml @@ -11,7 +11,7 @@ resources: warehouse_id: "my-warehouse-1234" title: "Full Featured Genie Space" description: "A comprehensive test of all genie space features" - file_path: ./full_featured.genie.json + file_path: ./full_featured.geniespace.json # Test with inline serialized_space (YAML syntax) inline_yaml: diff --git a/acceptance/bundle/validate/genie_space_complex/full_featured.genie.json b/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json similarity index 100% rename from acceptance/bundle/validate/genie_space_complex/full_featured.genie.json rename to acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json similarity index 100% rename from acceptance/bundle/validate/genie_space_file_path_and_inline/contents.genie.json rename to acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml index dea909bf2f9..ca57e978d83 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml @@ -6,6 +6,6 @@ resources: both_set: title: "Both set" warehouse_id: "test-warehouse-id" - file_path: "./contents.genie.json" + file_path: "./contents.geniespace.json" serialized_space: version: 1 diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml index e90b6d5d1ba..f784a183258 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -1,3 +1,3 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml index 4fc5284af50..97900adac7a 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml @@ -2,5 +2,5 @@ Local = true Cloud = false # Genie spaces only support direct deployment engine. -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] +[Env] +DATABRICKS_BUNDLE_ENGINE = "direct" diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml b/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml deleted file mode 100644 index 71126ef7dcb..00000000000 --- a/acceptance/bundle/validate/genie_space_permissions_unsupported/databricks.yml +++ /dev/null @@ -1,21 +0,0 @@ -bundle: - name: genie-space-permissions-unsupported - -resources: - genie_spaces: - inline_perms: - title: "Inline Perms" - warehouse_id: "test-warehouse-id" - serialized_space: "{}" - permissions: - - level: CAN_MANAGE - user_name: someone@example.com - - bundle_perms: - title: "Bundle Perms" - warehouse_id: "test-warehouse-id" - serialized_space: "{}" - -permissions: - - level: CAN_MANAGE - user_name: someone@example.com diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt b/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt deleted file mode 100644 index 7224e5d1fea..00000000000 --- a/acceptance/bundle/validate/genie_space_permissions_unsupported/output.txt +++ /dev/null @@ -1,22 +0,0 @@ -Recommendation: permissions section should explicitly include the current deployment identity '[USERNAME]' or one of its groups -If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy. - -Consider using a adding a top-level permissions section such as the following: - - permissions: - - user_name: [USERNAME] - level: CAN_MANAGE - -See https://docs.databricks.com/dev-tools/bundles/permissions.html to learn more about permission configuration. - in databricks.yml:20:3 - -Error: Genie Space permissions are not supported - -Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support. - -Error: Genie Space permissions are not supported - -Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support. - - -Exit code: 1 diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/script b/acceptance/bundle/validate/genie_space_permissions_unsupported/script deleted file mode 100644 index b260e836a71..00000000000 --- a/acceptance/bundle/validate/genie_space_permissions_unsupported/script +++ /dev/null @@ -1 +0,0 @@ -$CLI bundle plan diff --git a/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml b/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml deleted file mode 100644 index c304e3eba17..00000000000 --- a/acceptance/bundle/validate/genie_space_permissions_unsupported/test.toml +++ /dev/null @@ -1,8 +0,0 @@ -Local = true -Cloud = false - -Ignore = [".databricks"] - -# Genie spaces only support direct deployment engine. -[EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index fe746cbd98e..a9327217c08 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -56,14 +56,23 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B // Otherwise YAML decodes small ints as Go `int` while state JSON // round-trip decodes them as `float64`, and structdiff reports // false drift on every plan. - if ss.Kind() != dyn.KindMap && ss.Kind() != dyn.KindSequence { + switch ss.Kind() { + case dyn.KindNil, dyn.KindString: + return v, nil + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("serialized_space must be a string, map, or sequence, got %s", ss.Kind()), + Locations: ss.Locations(), + }) return v, nil } - jsonBytes, err := json.Marshal(ss.AsAny()) - if err != nil { - return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) - } - return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) }) }) diff --git a/bundle/config/mutator/translate_paths_genie_spaces_test.go b/bundle/config/mutator/translate_paths_genie_spaces_test.go index a5295400905..a1ac0b160b1 100644 --- a/bundle/config/mutator/translate_paths_genie_spaces_test.go +++ b/bundle/config/mutator/translate_paths_genie_spaces_test.go @@ -17,7 +17,7 @@ import ( func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { dir := t.TempDir() - touchEmptyFile(t, filepath.Join(dir, "src", "my_space.genie.json")) + touchEmptyFile(t, filepath.Join(dir, "src", "my_space.geniespace.json")) b := &bundle.Bundle{ SyncRootPath: dir, @@ -30,7 +30,7 @@ func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { GenieSpaceConfig: resources.GenieSpaceConfig{ Title: "My Genie Space", }, - FilePath: "../src/my_space.genie.json", + FilePath: "../src/my_space.geniespace.json", }, }, }, @@ -41,12 +41,15 @@ func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { File: filepath.Join(dir, "resources", "genie_space.yml"), }}) + // Genie space paths reuse the dashboard translator; there is no separate + // genie_space mutator. The dashboard translator walks all resource types + // that need path translation, so calling it covers genie_spaces too. diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePathsDashboards()) require.NoError(t, diags.Error()) assert.Equal( t, - filepath.ToSlash(filepath.Join("src", "my_space.genie.json")), + filepath.ToSlash(filepath.Join("src", "my_space.geniespace.json")), b.Config.Resources.GenieSpaces["genie_space"].FilePath, ) } diff --git a/bundle/config/mutator/validate_genie_space_permissions.go b/bundle/config/mutator/validate_genie_space_permissions.go deleted file mode 100644 index e2efb517705..00000000000 --- a/bundle/config/mutator/validate_genie_space_permissions.go +++ /dev/null @@ -1,44 +0,0 @@ -package mutator - -import ( - "context" - "fmt" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" -) - -type validateGenieSpacePermissions struct{} - -// ValidateGenieSpacePermissions errors if any genie_space resource has -// permissions configured. The Databricks workspace API does not expose -// PUT /permissions/genie/spaces/, so the deploy would create the -// space and then fail when applying permissions, leaving partial state. -// Bundle-level permissions are propagated to genie_spaces by -// ApplyBundlePermissions and are caught here as well. -func ValidateGenieSpacePermissions() bundle.Mutator { - return &validateGenieSpacePermissions{} -} - -func (m *validateGenieSpacePermissions) Name() string { - return "ValidateGenieSpacePermissions" -} - -func (m *validateGenieSpacePermissions) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - - for key, space := range b.Config.Resources.GenieSpaces { - if space == nil || len(space.Permissions) == 0 { - continue - } - - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "Genie Space permissions are not supported", - Detail: "Databricks workspaces do not expose a permissions endpoint for Genie Spaces, so a deploy with permissions configured would create the space and then fail. Remove the permissions block, or remove the Genie Space from any bundle-level permissions, until the API adds support.", - Locations: b.Config.GetLocations(fmt.Sprintf("resources.genie_spaces.%s.permissions", key)), - }) - } - - return diags -} diff --git a/bundle/config/mutator/validate_genie_space_permissions_test.go b/bundle/config/mutator/validate_genie_space_permissions_test.go deleted file mode 100644 index 0ccf143815e..00000000000 --- a/bundle/config/mutator/validate_genie_space_permissions_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package mutator_test - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/iam" - "github.com/stretchr/testify/assert" -) - -func TestValidateGenieSpacePermissions_NoPermissions(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - GenieSpaces: map[string]*resources.GenieSpace{ - "my_space": { - GenieSpaceConfig: resources.GenieSpaceConfig{ - Title: "My Space", - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateGenieSpacePermissions()) - assert.Empty(t, diags) -} - -func TestValidateGenieSpacePermissions_WithPermissionsErrors(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - GenieSpaces: map[string]*resources.GenieSpace{ - "my_space": { - GenieSpaceConfig: resources.GenieSpaceConfig{ - Title: "My Space", - }, - Permissions: []resources.Permission{ - {Level: iam.PermissionLevel("CAN_MANAGE"), UserName: "user@example.com"}, - }, - }, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateGenieSpacePermissions()) - assert.Len(t, diags, 1) - assert.Equal(t, "Genie Space permissions are not supported", diags[0].Summary) -} diff --git a/bundle/config/resources/genie_space.go b/bundle/config/resources/genie_space.go index 276ab12b17d..b0a5efbf304 100644 --- a/bundle/config/resources/genie_space.go +++ b/bundle/config/resources/genie_space.go @@ -14,6 +14,12 @@ import ( type GenieSpaceConfig struct { // Description of the Genie Space Description string `json:"description,omitempty"` + // Etag for change detection. The bundle persists the value the backend + // returned on the last Create/Update and uses it both as an If-Match for + // the next Update and as the signal for `bundle plan` to detect remote + // drift (see OverrideChangeDesc in bundle/direct/dresources/genie_space.go). + // Mirrors dashboards.DashboardConfig.Etag. + Etag string `json:"etag,omitempty"` // Genie space ID SpaceId string `json:"space_id,omitempty"` // Title of the Genie Space @@ -50,7 +56,7 @@ type GenieSpace struct { Permissions []Permission `json:"permissions,omitempty"` - // FilePath points to the local `.genie.json` file containing the Genie Space definition. + // FilePath points to the local `.geniespace.json` file containing the Genie Space definition. // This is inlined into serialized_space during deployment. The file_path is kept around // as metadata which is needed for `databricks bundle generate genie-space --resource ` to work. // This is not part of GenieSpaceConfig because we don't need to store this in the resource state. diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index bc62dbd2dfd..a9eafa85947 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -248,6 +248,15 @@ var testConfig map[string]any = map[string]any{ }, }, + "genie_spaces": &resources.GenieSpace{ + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "my-genie-space", + WarehouseId: "test-warehouse-id", + ParentPath: "/Workspace/Users/user@example.com", + SerializedSpace: "{}", + }, + }, + "vector_search_endpoints": &resources.VectorSearchEndpoint{ CreateEndpoint: vectorsearch.CreateEndpoint{ Name: "my-endpoint", diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go index 1eaa1b1995c..395466fda2b 100644 --- a/bundle/direct/dresources/genie_space.go +++ b/bundle/direct/dresources/genie_space.go @@ -19,6 +19,24 @@ import ( var pathSerializedSpace = structpath.MustParsePath("serialized_space") +// ResourceGenieSpace mirrors the dashboard resource pattern (see dashboard.go), +// with these intentional divergences: +// - No Published wrapper: Genie spaces have no publish lifecycle, so +// PrepareState returns the config directly. +// - RemapState filters fewer fields: Genie has no LifecycleState / CreateTime / +// Path / UpdateTime output-only fields to scrub. +// - DoRead clears ParentPath: the GET API does not reliably return parent_path, +// so we drop it from ForceSendFields and zero the value rather than re-adding +// a "/Workspace" prefix the way dashboard.go does in ensureWorkspacePrefix. +// - DoUpdate omits serialized_space when unchanged: serialized_space is in +// ignore_remote_changes (see resources.yml), so a UI edit produces no plan +// entry. Sending the local body anyway would clobber the UI edit on every +// unrelated update. +// - DoCreate has expanded missing-parent-path detection: see +// isMissingGenieParentPathError below. +// +// Permissions follow the standard /permissions/genie/{id} endpoint and are wired +// up via the generic permissions adapter (permissions.go). type ResourceGenieSpace struct { client *databricks.WorkspaceClient } @@ -39,6 +57,7 @@ func (r *ResourceGenieSpace) RemapState(state *resources.GenieSpaceConfig) *reso return &resources.GenieSpaceConfig{ Description: state.Description, + Etag: state.Etag, Title: state.Title, WarehouseId: state.WarehouseId, ParentPath: state.ParentPath, @@ -86,6 +105,7 @@ func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace st return &resources.GenieSpaceConfig{ Description: space.Description, + Etag: space.Etag, Title: space.Title, WarehouseId: space.WarehouseId, ParentPath: "", @@ -97,6 +117,25 @@ func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace st } } +// isMissingGenieParentPathError reports whether the given Create error means +// "the parent workspace folder does not exist", so DoCreate can mkdir and retry. +// +// Dashboard handles the equivalent condition with a plain apierr.IsMissing +// check (see ResourceDashboard.DoCreate). Genie cannot, because it surfaces +// the same condition in two different shapes depending on the workspace's +// backend version: +// +// 1. Standard missing-resource error: HTTP 404, ErrorCode RESOURCE_DOES_NOT_EXIST. +// Caught by apierr.IsMissing. Observed on workspaces running the newer +// Genie service implementation. +// 2. HTTP 400 with ErrorCode INVALID_PARAMETER_VALUE and a message of the +// form "Tree node with path '' does not exist". Observed on +// workspaces still backed by the legacy implementation during integration +// testing in early 2026 (aws-prod-ucws and azure-prod-ucws clusters at +// the time). The string match is intentional: there is no distinct error +// code to key on. +// +// Both forms unambiguously mean "create the parent and retry once". func isMissingGenieParentPathError(err error) bool { if apierr.IsMissing(err) { return true @@ -107,10 +146,6 @@ func isMissingGenieParentPathError(err error) bool { return false } - // Genie reports a missing parent folder inconsistently across environments. - // Some workspaces return a standard missing-resource error, while others - // return INVALID_PARAMETER_VALUE with a NOT_FOUND message embedded in the - // text. Treat both forms as "create the parent directory and retry once". return apiErr.StatusCode == http.StatusBadRequest && apiErr.ErrorCode == "INVALID_PARAMETER_VALUE" && strings.Contains(apiErr.Message, "Tree node with path") && @@ -150,6 +185,12 @@ func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.Gen return "", nil, err } + // Persist the etag in state. The deploy framework saves `config` (the input + // to DoCreate) as the state record, so mutating it here is what gets the + // backend-returned etag onto disk for the next plan's drift check. + // Matches the dashboard pattern (dashboard.go DoCreate). + config.Etag = createResp.Etag + return createResp.SpaceId, responseToGenieSpaceConfig(createResp, serializedSpace), nil } @@ -165,8 +206,10 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re // update triggered by another field would clobber the UI edit. Only // send it when the user actually changed it locally. var excludeForceSend []string + sentSerialized := true if !hasUpdate(entry, pathSerializedSpace) { serializedSpace = "" + sentSerialized = false excludeForceSend = append(excludeForceSend, "SerializedSpace") } @@ -176,8 +219,10 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re Title: config.Title, WarehouseId: config.WarehouseId, SerializedSpace: serializedSpace, - // Etag is for optimistic concurrency; we apply updates unconditionally. - Etag: "", + // Send the etag we last observed. The backend uses it as an If-Match + // guard against concurrent writes, and OverrideChangeDesc uses the + // post-update etag to detect drift on subsequent plans. + Etag: config.Etag, ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields, excludeForceSend...), }) @@ -185,16 +230,46 @@ func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *re return nil, err } - // When the request omitted serialized_space, use the value the response - // echoes back so RemapState records the latest body. + // Persist the new etag in state (see DoCreate for the rationale). + config.Etag = updateResp.Etag + + // Decide what to record as the new state's serialized_space. + // - If we sent a new body, use it. + // - If we omitted it (UI-edit protection above) but the API echoed back + // a value, record that — it's the most up-to-date view we have. + // - If neither side carries a value, keep whatever was already in state. + // Otherwise RemapState would blank the field on every unrelated update. respSerialized := serializedSpace - if respSerialized == "" { + if !sentSerialized { respSerialized = updateResp.SerializedSpace + if respSerialized == "" { + if prior, ok := config.SerializedSpace.(string); ok { + respSerialized = prior + } + } } return responseToGenieSpaceConfig(updateResp, respSerialized), nil } +// OverrideChangeDesc handles the etag field. The user never sets it directly; +// we compare the stored etag against the remote one and Skip if they match. +// This mirrors ResourceDashboard.OverrideChangeDesc. +func (r *ResourceGenieSpace) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *resources.GenieSpaceConfig) error { + switch path.String() { + case "etag": + // change.New is always nil for etag because it's not present in the + // user-authored config. Compare stored etag with remote one to decide + // whether anything changed out-of-band since the last deploy. + if change.Old == change.Remote { + change.Action = deployplan.Skip + } else { + change.Action = deployplan.Update + } + } + return nil +} + // hasUpdate reports whether entry has an Update-action change at the given path. // HasChange alone matches Skip-action changes too, which we cannot use to drive // request shaping for fields covered by ignore_remote_changes. diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go index 7db9f5c4b16..742deff6cb4 100644 --- a/bundle/direct/dresources/genie_space_test.go +++ b/bundle/direct/dresources/genie_space_test.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -167,3 +168,93 @@ func TestGenieSpaceDoUpdateSendsSerializedSpaceWhenChanged(t *testing.T) { require.NotNil(t, state) assert.Equal(t, "{\"v\":1}", state.SerializedSpace) } + +func TestGenieSpaceDoUpdateRoundTripsEtag(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + Etag: "etag-7", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + Etag: "etag-8", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + Etag: "etag-7", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "etag-8", state.Etag) +} + +func TestGenieSpaceDoUpdateKeepsPriorSerializedSpaceWhenBothEmpty(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + // Only title changed; serialized_space should be omitted from the request. + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + // API also omits serialized_space; we should keep the prior local value. + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + SerializedSpace: "{\"keep\":\"me\"}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"keep\":\"me\"}", state.SerializedSpace) +} + +func TestGenieSpaceOverrideChangeDescEtag(t *testing.T) { + r := &ResourceGenieSpace{} + etagPath := structpath.MustParsePath("etag") + + t.Run("Skip when stored matches remote", func(t *testing.T) { + change := &ChangeDesc{Old: "etag-7", Remote: "etag-7"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), etagPath, change, nil)) + assert.Equal(t, deployplan.Skip, change.Action) + }) + + t.Run("Update when stored differs from remote", func(t *testing.T) { + change := &ChangeDesc{Old: "etag-7", Remote: "etag-8"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), etagPath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) + + t.Run("Other paths are untouched", func(t *testing.T) { + titlePath := structpath.MustParsePath("title") + change := &ChangeDesc{Action: deployplan.Update, Old: "a", Remote: "b"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), titlePath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 5b2a70adbb3..7f719674ebb 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -439,9 +439,14 @@ func ExportStateFromData(data Database) resourcestate.ExportedResourcesMap { result := make(resourcestate.ExportedResourcesMap) for key, entry := range data.State { var etag string - // Extract etag for dashboards. - // covered by test case: bundle/deploy/dashboard/detect-change - if strings.Contains(key, ".dashboards.") && len(entry.State) > 0 { + // Extract etag for resources that use it for drift detection + // (dashboards and genie_spaces). Both follow the same pattern of + // persisting the backend-returned etag in state and comparing it + // against the remote on the next plan via OverrideChangeDesc. + // covered by test cases: + // - bundle/deploy/dashboard/detect-change + // - bundle/resources/genie_spaces/simple + if (strings.Contains(key, ".dashboards.") || strings.Contains(key, ".genie_spaces.")) && len(entry.State) > 0 { var holder struct { Etag string `json:"etag"` } diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 3d89c58e8e8..b3e86d6f305 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,7 +1,7 @@ --- description: 'Configuration reference for databricks.yml' last_update: - date: 2026-05-11 + date: 2026-05-20 --- @@ -1505,9 +1505,13 @@ genie_spaces: - String - Description of the Genie space shown alongside the title in the Databricks UI. +- - `etag` + - String + - + - - `file_path` - String - - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map @@ -3699,9 +3703,13 @@ genie_spaces: - String - Description of the Genie space shown alongside the title in the Databricks UI. +- - `etag` + - String + - + - - `file_path` - String - - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index 0c7175430e0..4d740961a6a 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -1,7 +1,7 @@ --- description: 'Learn about resources supported by Declarative Automation Bundles and how to configure them.' last_update: - date: 2026-05-11 + date: 2026-05-20 --- @@ -544,6 +544,10 @@ apps: - String - The description of the app. +- - `git_repository` + - Map + - Git repository configuration for app deployments. When specified, deployments can reference code from this repository by providing only the git reference (branch, tag, or commit). See [\_](#appsnamegit_repository). + - - `git_source` - Map - Git source configuration for app deployments. Specifies which git reference (branch, tag, or commit) to use when deploying the app. Used in conjunction with git_repository to deploy code directly from git. The source_code_path within git_source specifies the relative path to the app code within the repository. See [\_](#appsnamegit_source). @@ -572,10 +576,6 @@ apps: - Sequence - See [\_](#appsnametelemetry_export_destinations). -- - `thumbnail_url` - - String - - - - - `usage_policy_id` - String - @@ -641,6 +641,32 @@ apps: ::: +### apps._name_.git_repository + +**`Type: Map`** + +Git repository configuration for app deployments. When specified, deployments can +reference code from this repository by providing only the git reference (branch, tag, or commit). + + + +:::list-table + +- - Key + - Type + - Description + +- - `provider` + - String + - Git provider. Case insensitive. Supported values: gitHub, gitHubEnterprise, bitbucketCloud, bitbucketServer, azureDevOpsServices, gitLab, gitLabEnterpriseEdition, awsCodeCommit. + +- - `url` + - String + - URL of the Git repository. + +::: + + ### apps._name_.git_source **`Type: Map`** @@ -1154,6 +1180,10 @@ catalogs: - Map - See [\_](#catalogsnamelifecycle). +- - `managed_encryption_settings` + - Map + - Control CMK encryption for managed catalog data. See [\_](#catalogsnamemanaged_encryption_settings). + - - `name` - String - @@ -1234,6 +1264,64 @@ The privileges assigned to the principal. ::: +### catalogs._name_.managed_encryption_settings + +**`Type: Map`** + +Control CMK encryption for managed catalog data + + + +:::list-table + +- - Key + - Type + - Description + +- - `azure_encryption_settings` + - Map + - optional Azure settings - only required if an Azure CMK is used. See [\_](#catalogsnamemanaged_encryption_settingsazure_encryption_settings). + +- - `azure_key_vault_key_id` + - String + - the AKV URL in Azure, null otherwise. + +- - `customer_managed_key_id` + - String + - the CMK uuid in AWS and GCP, null otherwise. + +::: + + +### catalogs._name_.managed_encryption_settings.azure_encryption_settings + +**`Type: Map`** + +optional Azure settings - only required if an Azure CMK is used. + + + +:::list-table + +- - Key + - Type + - Description + +- - `azure_cmk_access_connector_id` + - String + - + +- - `azure_cmk_managed_identity_id` + - String + - + +- - `azure_tenant_id` + - String + - + +::: + + ## clusters **`Type: Map`** @@ -1790,10 +1878,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -3056,9 +3140,13 @@ genie_spaces: - String - Description of the Genie space shown alongside the title in the Databricks UI. +- - `etag` + - String + - + - - `file_path` - String - - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. - - `lifecycle` - Map @@ -3401,7 +3489,7 @@ For other serverless tasks, the task environment is required to be specified usi - - `spec` - Map - - The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines. In this minimal environment spec, only pip and java dependencies are supported. See [\_](#jobsnameenvironmentsspec). + - The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and SDP's environment for classic and serverless pipelines. In this minimal environment spec, only pip and java dependencies are supported. See [\_](#jobsnameenvironmentsspec). ::: @@ -3410,7 +3498,7 @@ For other serverless tasks, the task environment is required to be specified usi **`Type: Map`** -The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines. +The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and SDP's environment for classic and serverless pipelines. In this minimal environment spec, only pip and java dependencies are supported. @@ -4125,10 +4213,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -5820,10 +5904,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -6196,7 +6276,7 @@ The task triggers a pipeline update when the `pipeline_task` field is present. O - - `full_refresh` - Boolean - - If true, triggers a full refresh on the delta live table. + - If true, triggers a full refresh on the spark declarative pipeline. - - `pipeline_id` - String @@ -6390,7 +6470,7 @@ Controls whether the pipeline should perform a full refresh - - `full_refresh` - Boolean - - If true, triggers a full refresh on the delta live table. + - If true, triggers a full refresh on the spark declarative pipeline. ::: @@ -7457,7 +7537,7 @@ The core config of the serving endpoint. - - `auto_capture_config` - Map - - Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. Note: this field is deprecated for creating new provisioned throughput endpoints, or updating existing provisioned throughput endpoints that never have inference table configured; in these cases please use AI Gateway to manage inference tables. See [\_](#model_serving_endpointsnameconfigauto_capture_config). + - This field is deprecated - - `served_entities` - Sequence @@ -7474,42 +7554,6 @@ The core config of the serving endpoint. ::: -### model_serving_endpoints._name_.config.auto_capture_config - -**`Type: Map`** - -Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. -Note: this field is deprecated for creating new provisioned throughput endpoints, -or updating existing provisioned throughput endpoints that never have inference table configured; -in these cases please use AI Gateway to manage inference tables. - - - -:::list-table - -- - Key - - Type - - Description - -- - `catalog_name` - - String - - The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled. - -- - `enabled` - - Boolean - - Indicates whether the inference table is enabled. - -- - `schema_name` - - String - - The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled. - -- - `table_name_prefix` - - String - - The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled. - -::: - - ### model_serving_endpoints._name_.config.served_entities **`Type: Sequence`** @@ -8390,7 +8434,7 @@ pipelines: - - `channel` - String - - DLT Release Channel that specifies which version to use. + - SDP Release Channel that specifies which version to use. - - `clusters` - Sequence @@ -8909,10 +8953,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -9260,6 +9300,10 @@ The configuration for a managed ingestion pipeline. These settings cannot be use - Map - (Optional) A window that specifies a set of time ranges for snapshot queries in CDC. See [\_](#pipelinesnameingestion_definitionfull_refresh_window). +- - `ingest_from_uc_foreign_catalog` + - Boolean + - Immutable. If set to true, the pipeline will ingest tables from the UC foreign catalogs directly without the need to specify a UC connection or ingestion gateway. The `source_catalog` fields in objects of IngestionConfig are interpreted as the UC foreign catalogs to ingest from. + - - `ingestion_gateway_id` - String - Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with CDC connectors to databases like SQL Server using a gateway pipeline (connector_type = CDC). Under certain conditions, this can be replaced with connection_name to change the connector to Combined Cdc Managed Ingestion Pipeline. @@ -9412,6 +9456,18 @@ Configuration settings to control the ingestion of tables. These settings overri - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitionobjectsreporttable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9454,6 +9510,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.objects.report.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.ingestion_definition.objects.schema **`Type: Map`** @@ -9468,6 +9553,10 @@ Select all tables from a specific source schema. - Type - Description +- - `connector_options` + - Map + - (Optional) Source Specific Connector Options. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_options). + - - `destination_catalog` - String - Required. Destination catalog to store tables. @@ -9491,6 +9580,126 @@ Select all tables from a specific source schema. ::: +### pipelines._name_.ingestion_definition.objects.schema.connector_options + +**`Type: Map`** + +(Optional) Source Specific Connector Options + + + +:::list-table + +- - Key + - Type + - Description + +- - `confluence_options` + - Map + - Confluence specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_optionsconfluence_options). + +- - `jira_options` + - Map + - Jira specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_optionsjira_options). + +- - `meta_ads_options` + - Map + - Meta Marketing (Meta Ads) specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_optionsmeta_ads_options). + +::: + + +### pipelines._name_.ingestion_definition.objects.schema.connector_options.confluence_options + +**`Type: Map`** + +Confluence specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_confluence_spaces` + - Sequence + - (Optional) Spaces to filter Confluence data on + +::: + + +### pipelines._name_.ingestion_definition.objects.schema.connector_options.jira_options + +**`Type: Map`** + +Jira specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_jira_spaces` + - Sequence + - (Optional) Projects to filter Jira data on + +::: + + +### pipelines._name_.ingestion_definition.objects.schema.connector_options.meta_ads_options + +**`Type: Map`** + +Meta Marketing (Meta Ads) specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `action_attribution_windows` + - Sequence + - (Optional) Action attribution windows for insights reporting (e.g. "28d_click", "1d_view") + +- - `action_breakdowns` + - Sequence + - (Optional) Action breakdowns to configure for data aggregation + +- - `action_report_time` + - String + - (Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime) + +- - `breakdowns` + - Sequence + - (Optional) Breakdowns to configure for data aggregation + +- - `custom_insights_lookback_window` + - Integer + - (Optional) Window in days to revisit data during sync to capture updated conversion data from the API. + +- - `level` + - String + - (Optional) Granularity of data to pull (account, ad, adset, campaign) + +- - `start_date` + - String + - (Optional) Start date in yyyy-MM-dd format (e.g. 2025-01-15). Data added after this date will be ingested + +- - `time_increment` + - String + - (Optional) Value in string by which to aggregate statistics (can take all_days, monthly or number of days) + +::: + + ### pipelines._name_.ingestion_definition.objects.schema.table_configuration **`Type: Map`** @@ -9521,6 +9730,18 @@ Configuration settings to control the ingestion of tables. These settings are ap - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitionobjectsschematable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9563,6 +9784,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.objects.schema.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.ingestion_definition.objects.table **`Type: Map`** @@ -9577,6 +9827,10 @@ Select a specific source table. - Type - Description +- - `connector_options` + - Map + - (Optional) Source Specific Connector Options. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_options). + - - `destination_catalog` - String - Required. Destination catalog to store table. @@ -9608,6 +9862,126 @@ Select a specific source table. ::: +### pipelines._name_.ingestion_definition.objects.table.connector_options + +**`Type: Map`** + +(Optional) Source Specific Connector Options + + + +:::list-table + +- - Key + - Type + - Description + +- - `confluence_options` + - Map + - Confluence specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_optionsconfluence_options). + +- - `jira_options` + - Map + - Jira specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_optionsjira_options). + +- - `meta_ads_options` + - Map + - Meta Marketing (Meta Ads) specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_optionsmeta_ads_options). + +::: + + +### pipelines._name_.ingestion_definition.objects.table.connector_options.confluence_options + +**`Type: Map`** + +Confluence specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_confluence_spaces` + - Sequence + - (Optional) Spaces to filter Confluence data on + +::: + + +### pipelines._name_.ingestion_definition.objects.table.connector_options.jira_options + +**`Type: Map`** + +Jira specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_jira_spaces` + - Sequence + - (Optional) Projects to filter Jira data on + +::: + + +### pipelines._name_.ingestion_definition.objects.table.connector_options.meta_ads_options + +**`Type: Map`** + +Meta Marketing (Meta Ads) specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `action_attribution_windows` + - Sequence + - (Optional) Action attribution windows for insights reporting (e.g. "28d_click", "1d_view") + +- - `action_breakdowns` + - Sequence + - (Optional) Action breakdowns to configure for data aggregation + +- - `action_report_time` + - String + - (Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime) + +- - `breakdowns` + - Sequence + - (Optional) Breakdowns to configure for data aggregation + +- - `custom_insights_lookback_window` + - Integer + - (Optional) Window in days to revisit data during sync to capture updated conversion data from the API. + +- - `level` + - String + - (Optional) Granularity of data to pull (account, ad, adset, campaign) + +- - `start_date` + - String + - (Optional) Start date in yyyy-MM-dd format (e.g. 2025-01-15). Data added after this date will be ingested + +- - `time_increment` + - String + - (Optional) Value in string by which to aggregate statistics (can take all_days, monthly or number of days) + +::: + + ### pipelines._name_.ingestion_definition.objects.table.table_configuration **`Type: Map`** @@ -9638,6 +10012,18 @@ Configuration settings to control the ingestion of tables. These settings overri - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitionobjectstabletable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9680,6 +10066,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.objects.table.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.ingestion_definition.source_configurations **`Type: Sequence`** @@ -9698,10 +10113,6 @@ Top-level source configurations - Map - Catalog-level source configuration parameters. See [\_](#pipelinesnameingestion_definitionsource_configurationscatalog). -- - `google_ads_config` - - Map - - See [\_](#pipelinesnameingestion_definitionsource_configurationsgoogle_ads_config). - ::: @@ -9806,6 +10217,18 @@ Configuration settings to control the ingestion of tables. These settings are ap - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitiontable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9848,6 +10271,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.libraries **`Type: Sequence`** @@ -10387,7 +10839,7 @@ A collection of settings for a compute endpoint. - - `no_suspension` - Boolean - - When set to true, explicitly disables automatic suspension (never suspend). Should be set to true when provided. + - When set to true, explicitly disables automatic suspension (never suspend). Should be set to true when provided. Mutually exclusive with `suspend_timeout_duration`. When updating, use `spec.project_default_settings.suspension` in the update_mask. - - `pg_settings` - Map @@ -10395,7 +10847,7 @@ A collection of settings for a compute endpoint. - - `suspend_timeout_duration` - String - - Duration of inactivity after which the compute endpoint is automatically suspended. If specified should be between 60s and 604800s (1 minute to 1 week). + - Duration of inactivity after which the compute endpoint is automatically suspended. If specified should be between 60s and 604800s (1 minute to 1 week). Mutually exclusive with `no_suspension`. When updating, use `spec.project_default_settings.suspension` in the update_mask. ::: diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index a21452278c8..832fb90a52a 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -660,9 +660,12 @@ github.com/databricks/cli/bundle/config/resources.GenieSpace: "description": "description": |- Description of the Genie space shown alongside the title in the Databricks UI. + "etag": + "description": |- + PLACEHOLDER "file_path": "description": |- - Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. "lifecycle": "description": |- PLACEHOLDER diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 3e63b8f607b..0af9394f243 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -25,7 +25,6 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine deploy.StatePull(), mutator.ValidateGitDetails(), mutator.ValidateDirectOnlyResources(engine), - mutator.ValidateGenieSpacePermissions(), mutator.ValidateLifecycleStarted(engine), statemgmt.CheckRunningResource(engine), ) diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index b3a83929b7a..36dac7f9f7e 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -759,8 +759,11 @@ "description": "Description of the Genie space shown alongside the title in the Databricks UI.", "$ref": "#/$defs/string" }, + "etag": { + "$ref": "#/$defs/string" + }, "file_path": { - "description": "Local path to a `.genie.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`.", + "description": "Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`.", "$ref": "#/$defs/string" }, "lifecycle": { diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 7af4e01e92f..250d05e3693 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -275,7 +275,11 @@ func waitForChanges(ctx context.Context, w *databricks.WorkspaceClient, dashboar break } - time.Sleep(1 * time.Second) + select { + case <-ctx.Done(): + return + case <-time.After(1 * time.Second): + } } } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index f4c344667e7..9cc415ba43d 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -163,7 +163,7 @@ func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bun func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, key string) error { // Save serialized genie space definition to the genie space directory. - genieSpaceBasename := key + ".genie.json" + genieSpaceBasename := key + ".geniespace.json" genieSpacePath := filepath.Join(g.genieSpaceDir, genieSpaceBasename) err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) if err != nil { @@ -264,7 +264,11 @@ func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle. } first = false - time.Sleep(genieSpaceWatchInterval) + select { + case <-ctx.Done(): + return + case <-time.After(genieSpaceWatchInterval): + } } } @@ -306,6 +310,7 @@ func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, err = g.saveConfiguration(ctx, b, genieSpace, key) if err != nil { logdiag.LogError(ctx, err) + return } if g.bind { @@ -448,7 +453,7 @@ Examples: What gets generated: - Genie space configuration YAML file with settings and a reference to the Genie space definition -- Genie space definition (.genie.json) file with the serialized space content +- Genie space definition (.geniespace.json) file with the serialized space content Sync workflow for Genie space development: When developing Genie spaces, you can modify them in the Databricks UI and sync diff --git a/cmd/bundle/generate/genie_space_test.go b/cmd/bundle/generate/genie_space_test.go new file mode 100644 index 00000000000..746b2b1c3af --- /dev/null +++ b/cmd/bundle/generate/genie_space_test.go @@ -0,0 +1,126 @@ +package generate + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func newGenieSpaceTestBundle(t *testing.T, m *mocks.MockWorkspaceClient, filePath string) *bundle.Bundle { + t.Helper() + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "my_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Space", + }, + FilePath: filePath, + }, + }, + }, + }, + } + b.Config.Resources.GenieSpaces["my_space"].ID = "space-id-1" + b.SetWorkpaceClient(m.WorkspaceClient) + return b +} + +func TestGenieSpace_UpdateForResource_WritesFileWhenNotWatching(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "my_space.geniespace.json") + + m := mocks.NewMockWorkspaceClient(t) + m.GetMockGenieAPI().EXPECT(). + GetSpace(mock.Anything, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id-1", + IncludeSerializedSpace: true, + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id-1", + Title: "My Space", + SerializedSpace: `{"version":1}`, + }, nil). + Once() + + g := &genieSpace{ + resource: "my_space", + force: true, + } + b := newGenieSpaceTestBundle(t, m, filePath) + + ctx, _ := cmdio.NewTestContextWithStdout(t.Context()) + ctx = logdiag.InitContext(ctx) + logdiag.SetCollect(ctx, true) + g.updateGenieSpaceForResource(ctx, b) + + require.Empty(t, logdiag.FlushCollected(ctx)) + + contents, err := os.ReadFile(filePath) + require.NoError(t, err) + assert.Contains(t, string(contents), `"version"`) +} + +func TestGenieSpace_UpdateForResource_WatchExitsOnCancel(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "my_space.geniespace.json") + + m := mocks.NewMockWorkspaceClient(t) + // Allow any number of GetSpace calls; we don't know how many fire before cancel. + m.GetMockGenieAPI().EXPECT(). + GetSpace(mock.Anything, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id-1", + IncludeSerializedSpace: true, + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id-1", + Title: "My Space", + SerializedSpace: `{"version":1}`, + }, nil). + Maybe() + + g := &genieSpace{ + resource: "my_space", + force: true, + watch: true, + } + b := newGenieSpaceTestBundle(t, m, filePath) + + base, _ := cmdio.NewTestContextWithStdout(t.Context()) + ctx, cancel := context.WithCancel(logdiag.InitContext(base)) + logdiag.SetCollect(ctx, true) + + done := make(chan struct{}) + go func() { + g.updateGenieSpaceForResource(ctx, b) + close(done) + }() + + // First iteration always saves. Wait until the file lands, then cancel. + require.Eventually(t, func() bool { + _, err := os.Stat(filePath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "expected initial save to write file") + + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watch loop did not exit promptly after ctx cancel") + } +} diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go index 0b11b119fcf..98f5cdb381a 100644 --- a/libs/testserver/genie_spaces.go +++ b/libs/testserver/genie_spaces.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "path" + "strconv" "strings" "github.com/databricks/databricks-sdk-go/service/dashboards" @@ -51,6 +52,9 @@ func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { ParentPath: createReq.ParentPath, WarehouseId: createReq.WarehouseId, SerializedSpace: createReq.SerializedSpace, + // Mirror libs/testserver/dashboards.go: initialize etag to a numeric + // string so each subsequent update can bump it monotonically. + Etag: "1", } s.GenieSpaces[spaceId] = genieSpace @@ -61,7 +65,7 @@ func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { if !strings.HasPrefix(workspacePath, "/Workspace") { workspacePath = path.Join("/Workspace", workspacePath) } - workspacePath = path.Join(workspacePath, createReq.Title+".genie") + workspacePath = path.Join(workspacePath, createReq.Title+".geniespace") s.files[workspacePath] = FileEntry{ Info: workspace.ObjectInfo{ @@ -121,6 +125,18 @@ func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { } } + // Optimistic concurrency: if the caller sent an etag, it must match the + // current one. Empty etag means apply unconditionally. + if updateReq.Etag != "" && updateReq.Etag != genieSpace.Etag { + return Response{ + StatusCode: 409, + Body: map[string]string{ + "message": "Etag mismatch: expected " + genieSpace.Etag + ", got " + updateReq.Etag, + }, + } + } + + prev := genieSpace if updateReq.Title != "" { genieSpace.Title = updateReq.Title } @@ -134,6 +150,26 @@ func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { genieSpace.SerializedSpace = updateReq.SerializedSpace } + // Bump the etag only when the update actually changes user-visible state. + // Matches dashboard's behavior (libs/testserver/dashboards.go) and keeps + // no-op updates idempotent so TestAll can pass the same config to Create + // and Update without observing spurious drift. + if prev.Title != genieSpace.Title || + prev.Description != genieSpace.Description || + prev.WarehouseId != genieSpace.WarehouseId || + prev.SerializedSpace != genieSpace.SerializedSpace { + prevEtag, err := strconv.Atoi(genieSpace.Etag) + if err != nil { + return Response{ + StatusCode: 500, + Body: map[string]string{ + "message": "Invalid stored etag: " + genieSpace.Etag, + }, + } + } + genieSpace.Etag = strconv.Itoa(prevEtag + 1) + } + s.GenieSpaces[spaceId] = genieSpace return Response{ @@ -145,7 +181,7 @@ func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { defer s.LockUnlock()() spaceId := req.Vars["space_id"] - _, ok := s.GenieSpaces[spaceId] + genieSpace, ok := s.GenieSpaces[spaceId] if !ok { return Response{ StatusCode: 404, @@ -157,6 +193,18 @@ func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { delete(s.GenieSpaces, spaceId) + // Also remove the synthetic workspace file entry registered by + // GenieSpaceCreate, so a trash+recreate flow does not resolve to stale + // state via the workspace path index. + if genieSpace.ParentPath != "" { + workspacePath := genieSpace.ParentPath + if !strings.HasPrefix(workspacePath, "/Workspace") { + workspacePath = path.Join("/Workspace", workspacePath) + } + workspacePath = path.Join(workspacePath, genieSpace.Title+".geniespace") + delete(s.files, workspacePath) + } + return Response{ StatusCode: 200, } diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index fd28ff44c2d..9526cdda022 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -110,6 +110,7 @@ func TestResourceURL(t *testing.T) { {"jobs", "jobs", "123", "https://host.com/jobs/123"}, {"experiments", "experiments", "exp-1", "https://host.com/ml/experiments/exp-1"}, {"dashboards", "dashboards", "d-1", "https://host.com/dashboardsv3/d-1/published"}, + {"genie_spaces", "genie_spaces", "space-1", "https://host.com/genie/rooms/space-1"}, {"notebooks", "notebooks", "12345", "https://host.com/#notebook/12345"}, {"notebooks with path", "notebooks", "/Users/u/nb", "https://host.com/#notebook//Users/u/nb"}, {"registered_models normalizes dots", "registered_models", "cat.sch.model", "https://host.com/explore/data/models/cat/sch/model"}, From 273cb87a7805474472ee44087315196a08bc2c8e Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 20 May 2026 15:08:41 +0200 Subject: [PATCH 18/20] bundle: update genie_space test fixture to v2 export format The simple acceptance test fixture was a v1 serialized_space sample that the Genie backend now rejects with 409 ABORTED ("The export format has changed since this export was taken"). Bumps version to 2 and replaces get_example_values / build_value_dictionary with the v2-equivalent enable_format_assistance / enable_entity_matching, matching the format that bundle generate genie-space now produces. Co-authored-by: Isaac --- .../genie_spaces/simple/out.plan.json | 4 +- .../simple/sales_analytics.geniespace.json | 155 ++++++------------ 2 files changed, 55 insertions(+), 104 deletions(-) diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json index b5583b47c06..0e97968436e 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -2,14 +2,14 @@ "plan_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", - "serial": 1, + "serial": 6, "plan": { "resources.genie_spaces.sales_analytics": { "action": "skip", "remote_state": { "description": "AI assistant for sales data analysis", "etag": "1", - "serialized_space": "{\n \"version\": 1,\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"sq-001\",\n \"question\": [\"What is the total revenue?\"]\n },\n {\n \"id\": \"sq-002\",\n \"question\": [\"Show me the top customers\"]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"identifier\": \"main.sales.orders\",\n \"column_configs\": [\n {\n \"column_name\": \"order_id\",\n \"get_example_values\": true,\n \"build_value_dictionary\": true\n },\n {\n \"column_name\": \"customer_id\",\n \"get_example_values\": true\n },\n {\n \"column_name\": \"amount\",\n \"get_example_values\": false\n }\n ]\n },\n {\n \"identifier\": \"main.sales.customers\"\n }\n ]\n },\n \"instructions\": {\n \"text_instructions\": [\n {\n \"id\": \"ti-001\",\n \"content\": [\n \"This genie space analyzes sales data.\\n\",\n \"Always filter by date when querying orders.\\n\",\n \"Use customer_name instead of customer_id in results.\"\n ]\n }\n ],\n \"example_question_sqls\": [\n {\n \"id\": \"eq-001\",\n \"question\": [\"What are the top customers by revenue?\"],\n \"sql\": [\n \"SELECT\\n\",\n \" c.customer_name,\\n\",\n \" SUM(o.amount) AS total_revenue\\n\",\n \"FROM main.sales.orders o\\n\",\n \"JOIN main.sales.customers c ON o.customer_id = c.id\\n\",\n \"WHERE o.order_date \u003e= :start_date\\n\",\n \"GROUP BY c.customer_name\\n\",\n \"ORDER BY total_revenue DESC\\n\",\n \"LIMIT :limit\"\n ],\n \"parameters\": [\n {\n \"name\": \"start_date\",\n \"type_hint\": \"STRING\",\n \"description\": [\"Start date for the analysis period\"],\n \"default_value\": {\n \"values\": [\"2024-01-01\"]\n }\n },\n {\n \"name\": \"limit\",\n \"type_hint\": \"INTEGER\",\n \"description\": [\"Number of customers to return\"],\n \"default_value\": {\n \"values\": [\"10\"]\n }\n }\n ]\n },\n {\n \"id\": \"eq-002\",\n \"question\": [\"Calculate daily revenue\"],\n \"sql\": [\n \"SELECT\\n\",\n \" order_date,\\n\",\n \" SUM(amount) AS daily_revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY order_date\\n\",\n \"ORDER BY order_date\"\n ]\n }\n ],\n \"sql_snippets\": {\n \"measures\": [\n {\n \"id\": \"m-001\",\n \"alias\": \"total_revenue\",\n \"sql\": [\"SUM(orders.amount)\"],\n \"display_name\": \"Total Revenue\"\n }\n ]\n },\n \"sql_functions\": [\n {\n \"id\": \"sf-001\",\n \"identifier\": \"main.analytics.calculate_churn\"\n }\n ]\n },\n \"benchmarks\": {\n \"questions\": [\n {\n \"id\": \"bq-001\",\n \"question\": [\"What is the monthly revenue trend?\"],\n \"answer\": [\n {\n \"format\": \"SQL\",\n \"content\": [\n \"SELECT\\n\",\n \" DATE_TRUNC('month', order_date) AS month,\\n\",\n \" SUM(amount) AS revenue\\n\",\n \"FROM main.sales.orders\\n\",\n \"GROUP BY 1\\n\",\n \"ORDER BY 1\"\n ]\n }\n ]\n }\n ]\n }\n}\n", + "serialized_space": "{\n \"benchmarks\": {\n \"questions\": [\n {\n \"answer\": [\n {\n \"content\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ],\n \"format\": \"SQL\"\n }\n ],\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Show all names and countries\"\n ]\n }\n ]\n },\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ]\n },\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Which names are in Canada?\"\n ]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"column_configs\": [\n {\n \"column_name\": \"country\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n },\n {\n \"column_name\": \"name\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n }\n ],\n \"identifier\": \"main.default.countries\"\n }\n ]\n },\n \"instructions\": {\n \"example_question_sqls\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ],\n \"sql\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ]\n }\n ],\n \"text_instructions\": [\n {\n \"content\": [\n \"This genie space answers simple questions about people and their countries.\\n\",\n \"Use only the main.default.countries table.\\n\",\n \"Prefer returning the name and country columns directly.\"\n ],\n \"id\": \"[NUMID]\"\n }\n ]\n },\n \"version\": 2\n}\n", "space_id": "[GENIE_SPACE_ID]", "title": "Sales Analytics Genie", "warehouse_id": "test-warehouse-id" diff --git a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json index 5d59dff96d5..fb62b7c4859 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json +++ b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json @@ -1,136 +1,87 @@ { - "version": 1, + "benchmarks": { + "questions": [ + { + "answer": [ + { + "content": [ + "SELECT\n", + " name,\n", + " country\n", + "FROM main.default.countries\n", + "ORDER BY name" + ], + "format": "SQL" + } + ], + "id": "88888888888888888888888888888888", + "question": [ + "Show all names and countries" + ] + } + ] + }, "config": { "sample_questions": [ { - "id": "sq-001", - "question": ["What is the total revenue?"] + "id": "11111111111111111111111111111111", + "question": [ + "List the names and countries" + ] }, { - "id": "sq-002", - "question": ["Show me the top customers"] + "id": "22222222222222222222222222222222", + "question": [ + "Which names are in Canada?" + ] } ] }, "data_sources": { "tables": [ { - "identifier": "main.sales.orders", "column_configs": [ { - "column_name": "order_id", - "get_example_values": true, - "build_value_dictionary": true - }, - { - "column_name": "customer_id", - "get_example_values": true + "column_name": "country", + "enable_entity_matching": true, + "enable_format_assistance": true }, { - "column_name": "amount", - "get_example_values": false + "column_name": "name", + "enable_entity_matching": true, + "enable_format_assistance": true } - ] - }, - { - "identifier": "main.sales.customers" + ], + "identifier": "main.default.countries" } ] }, "instructions": { - "text_instructions": [ - { - "id": "ti-001", - "content": [ - "This genie space analyzes sales data.\n", - "Always filter by date when querying orders.\n", - "Use customer_name instead of customer_id in results." - ] - } - ], "example_question_sqls": [ { - "id": "eq-001", - "question": ["What are the top customers by revenue?"], - "sql": [ - "SELECT\n", - " c.customer_name,\n", - " SUM(o.amount) AS total_revenue\n", - "FROM main.sales.orders o\n", - "JOIN main.sales.customers c ON o.customer_id = c.id\n", - "WHERE o.order_date >= :start_date\n", - "GROUP BY c.customer_name\n", - "ORDER BY total_revenue DESC\n", - "LIMIT :limit" + "id": "44444444444444444444444444444444", + "question": [ + "List the names and countries" ], - "parameters": [ - { - "name": "start_date", - "type_hint": "STRING", - "description": ["Start date for the analysis period"], - "default_value": { - "values": ["2024-01-01"] - } - }, - { - "name": "limit", - "type_hint": "INTEGER", - "description": ["Number of customers to return"], - "default_value": { - "values": ["10"] - } - } - ] - }, - { - "id": "eq-002", - "question": ["Calculate daily revenue"], "sql": [ "SELECT\n", - " order_date,\n", - " SUM(amount) AS daily_revenue\n", - "FROM main.sales.orders\n", - "GROUP BY order_date\n", - "ORDER BY order_date" + " name,\n", + " country\n", + "FROM main.default.countries\n", + "ORDER BY name" ] } ], - "sql_snippets": { - "measures": [ - { - "id": "m-001", - "alias": "total_revenue", - "sql": ["SUM(orders.amount)"], - "display_name": "Total Revenue" - } - ] - }, - "sql_functions": [ + "text_instructions": [ { - "id": "sf-001", - "identifier": "main.analytics.calculate_churn" + "content": [ + "This genie space answers simple questions about people and their countries.\n", + "Use only the main.default.countries table.\n", + "Prefer returning the name and country columns directly." + ], + "id": "33333333333333333333333333333333" } ] }, - "benchmarks": { - "questions": [ - { - "id": "bq-001", - "question": ["What is the monthly revenue trend?"], - "answer": [ - { - "format": "SQL", - "content": [ - "SELECT\n", - " DATE_TRUNC('month', order_date) AS month,\n", - " SUM(amount) AS revenue\n", - "FROM main.sales.orders\n", - "GROUP BY 1\n", - "ORDER BY 1" - ] - } - ] - } - ] - } + "version": 2 } From b64a95022a473b7a21ce4e2a727a41d6098c3f80 Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Wed, 20 May 2026 15:11:17 +0200 Subject: [PATCH 19/20] bundle: align genie_space generate with new dstate.DB Open signature The state DB API gained context, withRecovery and withWrite arguments on origin/main; mirror the dashboard generate command and use the same arguments. Also regenerates the simple acceptance plan output to pick up the WAL-implementation serial increment. Co-authored-by: Isaac --- .../resources/genie_spaces/simple/out.plan.json | 2 +- cmd/bundle/generate/genie_space.go | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json index 0e97968436e..5a64ae32bd2 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -2,7 +2,7 @@ "plan_version": 2, "cli_version": "[DEV_VERSION]", "lineage": "[UUID]", - "serial": 6, + "serial": 7, "plan": { "resources.genie_spaces.sales_analytics": { "action": "skip", diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 9cc415ba43d..4d67db210c2 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -14,6 +14,8 @@ import ( "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/generate" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/resources" @@ -358,16 +360,25 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { return } + var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(localPath); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + logdiag.LogError(ctx, err) + return + } + state = b.DeploymentBundle.ExportState(ctx) + } else { + var err error + state, err = terraform.ParseResourcesState(ctx, b) + if err != nil { logdiag.LogError(ctx, err) return } } bundle.ApplySeqContext(ctx, b, - statemgmt.Load(stateDesc.Engine), + statemgmt.Load(state), ) if logdiag.HasError(ctx) { return From d633f4a411fb8081e177ddc773954846bac82c8a Mon Sep 17 00:00:00 2001 From: Jan Rose Date: Thu, 21 May 2026 11:55:25 +0200 Subject: [PATCH 20/20] bundle: revert non-genie direct-only error message tweak Commit e5af0e60e bundled an unrelated change to ValidateDirectOnlyResources alongside the genie_space dual-source warning. The error-message tweak doesn't belong in this PR; reverting those three files (incl. the catalog acceptance output that reflected the old text) so the PR stays scoped to genie work. The tweak can land in a separate PR if still desired. Co-authored-by: Isaac --- .../bundle/validate/catalog_requires_direct_mode/output.txt | 2 +- bundle/config/mutator/validate_direct_only_resources.go | 2 +- .../config/mutator/validate_direct_only_resources_test.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/validate/catalog_requires_direct_mode/output.txt b/acceptance/bundle/validate/catalog_requires_direct_mode/output.txt index c0b5b54bbf0..2bddbc154b0 100644 --- a/acceptance/bundle/validate/catalog_requires_direct_mode/output.txt +++ b/acceptance/bundle/validate/catalog_requires_direct_mode/output.txt @@ -1,7 +1,7 @@ Error: Catalog resources are only supported with direct deployment mode in databricks.yml:6:5 -Catalog resources require direct deployment mode. Set 'bundle.engine: direct' in your databricks.yml or set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use catalog resources. +Catalog resources require direct deployment mode. Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use catalog resources. Learn more at https://docs.databricks.com/dev-tools/bundles/direct diff --git a/bundle/config/mutator/validate_direct_only_resources.go b/bundle/config/mutator/validate_direct_only_resources.go index 8daf35449da..556b7b815a5 100644 --- a/bundle/config/mutator/validate_direct_only_resources.go +++ b/bundle/config/mutator/validate_direct_only_resources.go @@ -51,7 +51,7 @@ func (m *validateDirectOnlyResources) Apply(ctx context.Context, b *bundle.Bundl Severity: diag.Error, Summary: group.Description.SingularTitle + " resources are only supported with direct deployment mode", Detail: fmt.Sprintf("%s resources require direct deployment mode. "+ - "Set 'bundle.engine: direct' in your databricks.yml or set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use %s resources.\n"+ + "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use %s resources.\n"+ "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", group.Description.SingularTitle, group.Description.SingularName), Locations: b.Config.GetLocations("resources." + group.Description.PluralName), diff --git a/bundle/config/mutator/validate_direct_only_resources_test.go b/bundle/config/mutator/validate_direct_only_resources_test.go index a5ef9213c9f..dbc184c752d 100644 --- a/bundle/config/mutator/validate_direct_only_resources_test.go +++ b/bundle/config/mutator/validate_direct_only_resources_test.go @@ -61,7 +61,7 @@ func TestValidateDirectOnlyResourcesTerraformEngineDirectOnlyEmitsError(t *testi }, expectedSummary: "Catalog resources are only supported with direct deployment mode", expectedDetail: "Catalog resources require direct deployment mode. " + - "Set 'bundle.engine: direct' in your databricks.yml or set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use catalog resources.\n" + + "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use catalog resources.\n" + "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", }, { @@ -77,7 +77,7 @@ func TestValidateDirectOnlyResourcesTerraformEngineDirectOnlyEmitsError(t *testi }, expectedSummary: "External Location resources are only supported with direct deployment mode", expectedDetail: "External Location resources require direct deployment mode. " + - "Set 'bundle.engine: direct' in your databricks.yml or set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use external_location resources.\n" + + "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use external_location resources.\n" + "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", }, { @@ -93,7 +93,7 @@ func TestValidateDirectOnlyResourcesTerraformEngineDirectOnlyEmitsError(t *testi }, expectedSummary: "Vector Search Endpoint resources are only supported with direct deployment mode", expectedDetail: "Vector Search Endpoint resources require direct deployment mode. " + - "Set 'bundle.engine: direct' in your databricks.yml or set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use vector_search_endpoint resources.\n" + + "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' to use vector_search_endpoint resources.\n" + "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", }, }