diff --git a/acceptance/bundle/generate/genie_space/databricks.yml b/acceptance/bundle/generate/genie_space/databricks.yml new file mode 100644 index 00000000000..c533b2b6f98 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: genie-space-generate diff --git a/acceptance/bundle/generate/genie_space/genie_space.json.tmpl b/acceptance/bundle/generate/genie_space/genie_space.json.tmpl new file mode 100644 index 00000000000..2056f3061ef --- /dev/null +++ b/acceptance/bundle/generate/genie_space/genie_space.json.tmpl @@ -0,0 +1,7 @@ +{ + "title": "test genie space", + "description": "test description", + "parent_path": "/Workspace/test-$UNIQUE_NAME", + "warehouse_id": "test-warehouse-id", + "serialized_space": "{\"tables\":[],\"questions\":[]}" +} diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json new file mode 100644 index 00000000000..2c12b3c032c --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json @@ -0,0 +1,4 @@ +{ + "questions": [], + "tables": [] +} diff --git a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml new file mode 100644 index 00000000000..1471a901344 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml @@ -0,0 +1,8 @@ +resources: + genie_spaces: + test_genie_space: + title: "test genie space" + warehouse_id: test-warehouse-id + file_path: ../genie_space/test_genie_space.geniespace.json + description: test description + parent_path: /Workspace/test-[UNIQUE_NAME] diff --git a/acceptance/bundle/generate/genie_space/output.txt b/acceptance/bundle/generate/genie_space/output.txt new file mode 100644 index 00000000000..a313a51adc3 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME] + +>>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --genie-space-dir out/genie_space --resource-dir out/resource +Writing genie space to out/genie_space/test_genie_space.geniespace.json +Writing configuration to out/resource/test_genie_space.genie_space.yml diff --git a/acceptance/bundle/generate/genie_space/script b/acceptance/bundle/generate/genie_space/script new file mode 100644 index 00000000000..8e2fa32170a --- /dev/null +++ b/acceptance/bundle/generate/genie_space/script @@ -0,0 +1,8 @@ +trace $CLI workspace mkdirs /Workspace/test-$UNIQUE_NAME + +# create a genie space to import +envsubst < genie_space.json.tmpl > genie_space.json +genie_space_id=$($CLI genie create-space --json @genie_space.json | jq -r '.space_id') +rm genie_space.json + +trace $CLI bundle generate genie-space --existing-id $genie_space_id --genie-space-dir out/genie_space --resource-dir out/resource diff --git a/acceptance/bundle/generate/genie_space/test.toml b/acceptance/bundle/generate/genie_space/test.toml new file mode 100644 index 00000000000..e389c33c277 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/test.toml @@ -0,0 +1,10 @@ +[[Repls]] +Old = '\\\\' +New = '/' + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[GENIE_SPACE_ID]" + +[Env] +MSYS_NO_PATHCONV = "1" diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 97e8667ac78..39ce9293539 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -30,6 +30,7 @@ Available Commands: alert Generate configuration for an alert app Generate bundle configuration for a Databricks app dashboard Generate configuration for a dashboard + genie-space Generate configuration for a Genie space job Generate bundle configuration for a job pipeline Generate bundle configuration for a pipeline diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d50035a048f..ad5c9316dc3 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -729,6 +729,24 @@ resources.external_locations.*.grants[*] catalog.PrivilegeAssignment ALL resources.external_locations.*.grants[*].principal string ALL resources.external_locations.*.grants[*].privileges []catalog.Privilege ALL resources.external_locations.*.grants[*].privileges[*] catalog.Privilege ALL +resources.genie_spaces.*.description string ALL +resources.genie_spaces.*.file_path string INPUT +resources.genie_spaces.*.id string INPUT +resources.genie_spaces.*.lifecycle resources.Lifecycle INPUT +resources.genie_spaces.*.lifecycle.prevent_destroy bool INPUT +resources.genie_spaces.*.modified_status string INPUT +resources.genie_spaces.*.parent_path string ALL +resources.genie_spaces.*.serialized_space any ALL +resources.genie_spaces.*.space_id string ALL +resources.genie_spaces.*.title string ALL +resources.genie_spaces.*.url string INPUT +resources.genie_spaces.*.warehouse_id string ALL +resources.genie_spaces.*.permissions.object_id string ALL +resources.genie_spaces.*.permissions[*] dresources.StatePermission ALL +resources.genie_spaces.*.permissions[*].group_name string ALL +resources.genie_spaces.*.permissions[*].level iam.PermissionLevel ALL +resources.genie_spaces.*.permissions[*].service_principal_name string ALL +resources.genie_spaces.*.permissions[*].user_name string ALL resources.jobs.*.budget_policy_id string ALL resources.jobs.*.continuous *jobs.Continuous ALL resources.jobs.*.continuous.pause_status jobs.PauseStatus ALL diff --git a/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl new file mode 100644 index 00000000000..ea045097468 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: deploy-genie-space-inline-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Sales Analytics Inline Genie" + description: "Inline serialized_space test" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + serialized_space: + version: 1 + config: + sample_questions: + - id: "sq-001" + question: ["What is the total revenue?"] + data_sources: + tables: + - identifier: "main.sales.orders" + column_configs: + - column_name: "amount" + get_example_values: true diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json new file mode 100644 index 00000000000..8d6f7aabc78 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -0,0 +1,33 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.genie_spaces.sales_analytics": { + "action": "skip", + "remote_state": { + "description": "Inline serialized_space test", + "etag": "1", + "serialized_space": "{\"config\":{\"sample_questions\":[{\"id\":\"sq-001\",\"question\":[\"What is the total revenue?\"]}]},\"data_sources\":{\"tables\":[{\"column_configs\":[{\"column_name\":\"amount\",\"get_example_values\":true}],\"identifier\":\"main.sales.orders\"}]},\"version\":1}", + "space_id": "[GENIE_SPACE_ID]", + "title": "Sales Analytics Inline Genie", + "warehouse_id": "test-warehouse-id" + }, + "changes": { + "etag": { + "action": "skip", + "reason": "custom", + "old": "1", + "remote": "1" + }, + "parent_path": { + "action": "skip", + "reason": "input_only", + "old": "/Workspace/Users/[USERNAME]", + "new": "/Workspace/Users/[USERNAME]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/output.txt b/acceptance/bundle/resources/genie_spaces/inline/output.txt new file mode 100644 index 00000000000..99fef197dd0 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/output.txt @@ -0,0 +1,17 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-inline-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.sales_analytics + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-inline-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/inline/script b/acceptance/bundle/resources/genie_spaces/inline/script new file mode 100644 index 00000000000..8f2f625b7b0 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') + +# Capture the genie space ID as a replacement. +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +# Plan after deploy must be drift-free aside from input_only fields. +# Without normalization the inline serialized_space leaves a map in the +# config struct while state holds a string, and structdiff reports false +# drift on every plan. +trace $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/inline/test.toml b/acceptance/bundle/resources/genie_spaces/inline/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..1eea5d933e3 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: recreate-genie-space-$UNIQUE_NAME + +resources: + genie_spaces: + recreate_target: + title: "Recreate Target" + warehouse_id: "test-warehouse-id" + parent_path: PARENT_PATH_PLACEHOLDER + serialized_space: "{}" diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt new file mode 100644 index 00000000000..e90257bf971 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/output.txt @@ -0,0 +1,21 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-genie-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan after changing parent_path should show recreate +>>> [CLI] bundle plan +recreate genie_spaces.recreate_target + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.recreate_target + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-genie-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script new file mode 100644 index 00000000000..d5f04a795ab --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/script @@ -0,0 +1,14 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# Deploy with the original parent_path. +envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-old|" > databricks.yml +trace $CLI bundle deploy + +# Change parent_path. parent_path is recreate_on_changes in resources.yml, +# so the plan should show a recreate (delete + create) rather than an update. +envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-new|" > databricks.yml +title "Plan after changing parent_path should show recreate" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_recreate/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl new file mode 100644 index 00000000000..f30e4991de9 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: deploy-genie-space-test-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Sales Analytics Genie" + description: "AI assistant for sales data analysis" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + file_path: "sales_analytics.geniespace.json" diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json new file mode 100644 index 00000000000..5a64ae32bd2 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -0,0 +1,33 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 7, + "plan": { + "resources.genie_spaces.sales_analytics": { + "action": "skip", + "remote_state": { + "description": "AI assistant for sales data analysis", + "etag": "1", + "serialized_space": "{\n \"benchmarks\": {\n \"questions\": [\n {\n \"answer\": [\n {\n \"content\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ],\n \"format\": \"SQL\"\n }\n ],\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Show all names and countries\"\n ]\n }\n ]\n },\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ]\n },\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Which names are in Canada?\"\n ]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"column_configs\": [\n {\n \"column_name\": \"country\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n },\n {\n \"column_name\": \"name\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n }\n ],\n \"identifier\": \"main.default.countries\"\n }\n ]\n },\n \"instructions\": {\n \"example_question_sqls\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ],\n \"sql\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ]\n }\n ],\n \"text_instructions\": [\n {\n \"content\": [\n \"This genie space answers simple questions about people and their countries.\\n\",\n \"Use only the main.default.countries table.\\n\",\n \"Prefer returning the name and country columns directly.\"\n ],\n \"id\": \"[NUMID]\"\n }\n ]\n },\n \"version\": 2\n}\n", + "space_id": "[GENIE_SPACE_ID]", + "title": "Sales Analytics Genie", + "warehouse_id": "test-warehouse-id" + }, + "changes": { + "etag": { + "action": "skip", + "reason": "custom", + "old": "1", + "remote": "1" + }, + "parent_path": { + "action": "skip", + "reason": "input_only", + "old": "/Workspace/Users/[USERNAME]", + "new": "/Workspace/Users/[USERNAME]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/output.txt b/acceptance/bundle/resources/genie_spaces/simple/output.txt new file mode 100644 index 00000000000..f65d9583e18 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-test-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] genie get-space [GENIE_SPACE_ID] +{ + "title": "Sales Analytics Genie", + "description": "AI assistant for sales data analysis", + "warehouse_id": "test-warehouse-id" +} + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.sales_analytics + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-test-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json new file mode 100644 index 00000000000..fb62b7c4859 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json @@ -0,0 +1,87 @@ +{ + "benchmarks": { + "questions": [ + { + "answer": [ + { + "content": [ + "SELECT\n", + " name,\n", + " country\n", + "FROM main.default.countries\n", + "ORDER BY name" + ], + "format": "SQL" + } + ], + "id": "88888888888888888888888888888888", + "question": [ + "Show all names and countries" + ] + } + ] + }, + "config": { + "sample_questions": [ + { + "id": "11111111111111111111111111111111", + "question": [ + "List the names and countries" + ] + }, + { + "id": "22222222222222222222222222222222", + "question": [ + "Which names are in Canada?" + ] + } + ] + }, + "data_sources": { + "tables": [ + { + "column_configs": [ + { + "column_name": "country", + "enable_entity_matching": true, + "enable_format_assistance": true + }, + { + "column_name": "name", + "enable_entity_matching": true, + "enable_format_assistance": true + } + ], + "identifier": "main.default.countries" + } + ] + }, + "instructions": { + "example_question_sqls": [ + { + "id": "44444444444444444444444444444444", + "question": [ + "List the names and countries" + ], + "sql": [ + "SELECT\n", + " name,\n", + " country\n", + "FROM main.default.countries\n", + "ORDER BY name" + ] + } + ], + "text_instructions": [ + { + "content": [ + "This genie space answers simple questions about people and their countries.\n", + "Use only the main.default.countries table.\n", + "Prefer returning the name and country columns directly." + ], + "id": "33333333333333333333333333333333" + } + ] + }, + "version": 2 +} diff --git a/acceptance/bundle/resources/genie_spaces/simple/script b/acceptance/bundle/resources/genie_spaces/simple/script new file mode 100644 index 00000000000..4db9ef03078 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') + +# Capture the genie space ID as a replacement. +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +trace $CLI genie get-space $GENIE_SPACE_ID | jq '{title, description, warehouse_id}' + +# Verify that there is no drift right after deploy. +trace $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/simple/test.toml b/acceptance/bundle/resources/genie_spaces/simple/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/test.toml b/acceptance/bundle/resources/genie_spaces/test.toml new file mode 100644 index 00000000000..7f397c47833 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/test.toml @@ -0,0 +1,2 @@ +# Genie spaces are only deployed via the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml new file mode 100644 index 00000000000..d9e1c56c35c --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +resources: + genie_spaces: + foo: + title: "Permissions Test Space" + warehouse_id: test-warehouse-id + parent_path: /Workspace/Users/tester@databricks.com + serialized_space: "{}" + permissions: + - level: CAN_READ + user_name: viewer@example.com + - level: CAN_MANAGE + group_name: data-team + - level: CAN_MANAGE + service_principal_name: f37d18cd-98a8-4db5-8112-12dd0a6bfe38 + - level: CAN_MANAGE + user_name: tester@databricks.com diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json new file mode 100644 index 00000000000..dba8cde6d93 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -0,0 +1,52 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.genie_spaces.foo": { + "action": "create", + "new_state": { + "value": { + "parent_path": "/Workspace/Users/[USERNAME]", + "serialized_space": "{}", + "title": "Permissions Test Space", + "warehouse_id": "test-warehouse-id" + } + } + }, + "resources.genie_spaces.foo.permissions": { + "depends_on": [ + { + "node": "resources.genie_spaces.foo", + "label": "${resources.genie_spaces.foo.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "object_id": "", + "__embed__": [ + { + "level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "level": "CAN_MANAGE", + "group_name": "data-team" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "object_id": "/genie/spaces/${resources.genie_spaces.foo.id}" + } + } + } + } +} diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json new file mode 100644 index 00000000000..c112ccb7e76 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json @@ -0,0 +1,24 @@ +{ + "method": "PUT", + "path": "/api/2.0/permissions/genie/spaces/[FOO_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" + }, + { + "permission_level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt new file mode 100644 index 00000000000..e4e8cf0189d --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] bundle validate -o json +[ + { + "level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "level": "CAN_MANAGE" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script new file mode 100644 index 00000000000..1b20af07543 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -0,0 +1,18 @@ +trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions +rm out.requests.txt + +$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +print_requests() { + jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")))' + rm out.requests.txt +} + +rm out.requests.txt +trace $CLI bundle deploy +# Genie space IDs are random; normalize them in the recorded requests. +replace_ids.py +print_requests > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json + +trace $CLI bundle destroy --auto-approve +print_requests > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/genie_spaces/test.toml b/acceptance/bundle/resources/permissions/genie_spaces/test.toml new file mode 100644 index 00000000000..390388f04aa --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/test.toml @@ -0,0 +1,4 @@ +Env.RESOURCE = "genie_spaces" # for ../_script + +# Genie spaces only support the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/databricks.yml b/acceptance/bundle/validate/genie_space_complex/databricks.yml new file mode 100644 index 00000000000..94c5fc7188c --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/databricks.yml @@ -0,0 +1,51 @@ +bundle: + name: genie-space-complex + +workspace: + resource_path: /foo/bar + +resources: + genie_spaces: + # Test with all features enabled + full_featured: + warehouse_id: "my-warehouse-1234" + title: "Full Featured Genie Space" + description: "A comprehensive test of all genie space features" + file_path: ./full_featured.geniespace.json + + # Test with inline serialized_space (YAML syntax) + inline_yaml: + warehouse_id: "my-warehouse-1234" + title: "Inline YAML Genie Space" + serialized_space: + version: 1 + data_sources: + tables: + - identifier: main.schema.table1 + column_configs: + - column_name: id + get_example_values: true + build_value_dictionary: true + - column_name: name + get_example_values: true + instructions: + text_instructions: + - id: inst-001 + content: + - "This is a text instruction.\n" + - "It spans multiple lines." + example_question_sqls: + - id: eq-001 + question: + - "How many records are there?" + sql: + - "SELECT COUNT(*) FROM main.schema.table1" + + # Test with empty but valid structure + minimal_valid: + warehouse_id: "my-warehouse-1234" + title: "Minimal Valid" + serialized_space: + version: 1 + data_sources: + tables: [] diff --git a/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json b/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json new file mode 100644 index 00000000000..9c9221d8328 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json @@ -0,0 +1,160 @@ +{ + "version": 1, + "config": { + "sample_questions": [ + { + "id": "sq-001", + "question": ["What is the total revenue?"] + }, + { + "id": "sq-002", + "question": ["Show me the top customers"] + } + ] + }, + "data_sources": { + "tables": [ + { + "identifier": "main.sales.orders", + "column_configs": [ + { + "column_name": "order_id", + "get_example_values": true, + "build_value_dictionary": true + }, + { + "column_name": "customer_id", + "get_example_values": true + }, + { + "column_name": "amount", + "get_example_values": false + } + ] + }, + { + "identifier": "main.sales.customers" + } + ] + }, + "instructions": { + "text_instructions": [ + { + "id": "ti-001", + "content": [ + "This genie space analyzes sales data.\n", + "Always filter by date when querying orders.\n", + "Use customer_name instead of customer_id in results." + ] + } + ], + "example_question_sqls": [ + { + "id": "eq-001", + "question": ["What are the top customers by revenue?"], + "sql": [ + "SELECT\n", + " c.customer_name,\n", + " SUM(o.amount) AS total_revenue\n", + "FROM main.sales.orders o\n", + "JOIN main.sales.customers c ON o.customer_id = c.id\n", + "WHERE o.order_date >= :start_date\n", + "GROUP BY c.customer_name\n", + "ORDER BY total_revenue DESC\n", + "LIMIT :limit" + ], + "parameters": [ + { + "name": "start_date", + "type_hint": "STRING", + "description": ["Start date for the analysis period"], + "default_value": { + "values": ["2024-01-01"] + } + }, + { + "name": "limit", + "type_hint": "INTEGER", + "description": ["Number of customers to return"], + "default_value": { + "values": ["10"] + } + } + ] + }, + { + "id": "eq-002", + "question": ["Calculate daily revenue"], + "sql": [ + "SELECT\n", + " order_date,\n", + " SUM(amount) AS daily_revenue\n", + "FROM main.sales.orders\n", + "GROUP BY order_date\n", + "ORDER BY order_date" + ] + } + ], + "sql_snippets": { + "measures": [ + { + "id": "m-001", + "alias": "total_revenue", + "sql": ["SUM(orders.amount)"], + "display_name": "Total Revenue" + }, + { + "id": "m-002", + "alias": "order_count", + "sql": ["COUNT(orders.order_id)"], + "display_name": "Order Count" + } + ] + }, + "sql_functions": [ + { + "id": "sf-001", + "identifier": "main.analytics.calculate_churn" + } + ] + }, + "benchmarks": { + "questions": [ + { + "id": "bq-001", + "question": ["What is the monthly revenue trend?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " DATE_TRUNC('month', order_date) AS month,\n", + " SUM(amount) AS revenue\n", + "FROM main.sales.orders\n", + "GROUP BY 1\n", + "ORDER BY 1" + ] + } + ] + }, + { + "id": "bq-002", + "question": ["Which customers have the highest average order value?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " customer_id,\n", + " AVG(amount) AS avg_order_value\n", + "FROM main.sales.orders\n", + "GROUP BY customer_id\n", + "ORDER BY avg_order_value DESC\n", + "LIMIT 10" + ] + } + ] + } + ] + } +} diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/output.txt b/acceptance/bundle/validate/genie_space_complex/output.txt new file mode 100644 index 00000000000..4a0747210cc --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/output.txt @@ -0,0 +1,22 @@ +{ + "full_featured": { + "title": "Full Featured Genie Space", + "warehouse_id": "my-warehouse-1234", + "serialized_space_is_string": true + }, + "inline_yaml": { + "title": "Inline YAML Genie Space", + "serialized_space_type": "string", + "parsed": { + "tables_count": 1, + "has_column_configs": true, + "has_text_instructions": true + } + }, + "minimal_valid": { + "title": "Minimal Valid", + "parsed": { + "tables_count": 0 + } + } +} diff --git a/acceptance/bundle/validate/genie_space_complex/script b/acceptance/bundle/validate/genie_space_complex/script new file mode 100644 index 00000000000..4feab1e373b --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/script @@ -0,0 +1,26 @@ +# Validate complex genie spaces. ConfigureGenieSpaceSerializedSpace +# normalizes inline serialized_space YAML to a JSON string so the field has +# the same shape as the file_path code path; the script parses the JSON +# back to verify that the original structure is preserved. +$CLI bundle validate -o json | jq '{ + full_featured: .resources.genie_spaces.full_featured | { + title, + warehouse_id, + serialized_space_is_string: (.serialized_space | type == "string") + }, + inline_yaml: .resources.genie_spaces.inline_yaml | { + title, + serialized_space_type: (.serialized_space | type), + parsed: (.serialized_space | fromjson) | { + tables_count: (.data_sources.tables | length), + has_column_configs: ((.data_sources.tables[0].column_configs | length) > 0), + has_text_instructions: ((.instructions.text_instructions | length) > 0) + } + }, + minimal_valid: .resources.genie_spaces.minimal_valid | { + title, + parsed: (.serialized_space | fromjson) | { + tables_count: (.data_sources.tables | length) + } + } +}' diff --git a/acceptance/bundle/validate/genie_space_defaults/databricks.yml b/acceptance/bundle/validate/genie_space_defaults/databricks.yml new file mode 100644 index 00000000000..7e4b87d35e6 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/databricks.yml @@ -0,0 +1,30 @@ +bundle: + name: test-bundle + +workspace: + resource_path: /foo/bar + +resources: + genie_spaces: + empty_string: + warehouse_id: "my-warehouse-1234" + title: "empty-string" + serialized_space: "{}" + + # unchanged + parent_path: "" + + non_empty_string: + warehouse_id: "my-warehouse-1234" + title: "non-empty-string" + serialized_space: "{}" + + # unchanged + parent_path: "already-set" + + default_parent_path: + warehouse_id: "my-warehouse-1234" + title: "default-parent-path" + serialized_space: "{}" + + # parent_path set to default diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/output.txt b/acceptance/bundle/validate/genie_space_defaults/output.txt new file mode 100644 index 00000000000..13105c91b76 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/output.txt @@ -0,0 +1,14 @@ +{ + "default_parent_path": { + "title": "default-parent-path", + "parent_path": "/Workspace/foo/bar" + }, + "empty_string": { + "title": "empty-string", + "parent_path": "/Workspace" + }, + "non_empty_string": { + "title": "non-empty-string", + "parent_path": "/Workspace/already-set" + } +} diff --git a/acceptance/bundle/validate/genie_space_defaults/script b/acceptance/bundle/validate/genie_space_defaults/script new file mode 100644 index 00000000000..eebc582f136 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/script @@ -0,0 +1 @@ +$CLI bundle validate -o json | jq '.resources.genie_spaces | map_values({title: .title, parent_path: .parent_path})' diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json new file mode 100644 index 00000000000..cb608f6e9c4 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json @@ -0,0 +1 @@ +{"version": 1} diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml new file mode 100644 index 00000000000..ca57e978d83 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: genie-space-file-path-and-inline + +resources: + genie_spaces: + both_set: + title: "Both set" + warehouse_id: "test-warehouse-id" + file_path: "./contents.geniespace.json" + serialized_space: + version: 1 diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt new file mode 100644 index 00000000000..589a8333c37 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt @@ -0,0 +1,10 @@ +Warning: both file_path and serialized_space are set; file_path will be used and serialized_space will be ignored + in databricks.yml:11:9 + +Name: genie-space-file-path-and-inline +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/genie-space-file-path-and-inline/default + +Found 1 warning diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/script b/acceptance/bundle/validate/genie_space_file_path_and_inline/script new file mode 100644 index 00000000000..72555b332a4 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml new file mode 100644 index 00000000000..97900adac7a --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +# Genie spaces only support direct deployment engine. +[Env] +DATABRICKS_BUNDLE_ENGINE = "direct" diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index a83e0676fe8..ca3e909582a 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, clusters, dashboards, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, clusters, dashboards, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses Exit code: 1 @@ -20,6 +20,7 @@ apps clusters dashboards experiments +genie_spaces jobs model_serving_endpoints models diff --git a/bundle/config/mutator/paths/genie_space_paths_visitor.go b/bundle/config/mutator/paths/genie_space_paths_visitor.go new file mode 100644 index 00000000000..edd6ff2d8df --- /dev/null +++ b/bundle/config/mutator/paths/genie_space_paths_visitor.go @@ -0,0 +1,18 @@ +package paths + +import ( + "github.com/databricks/cli/libs/dyn" +) + +func VisitGenieSpacePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("genie_spaces"), + dyn.AnyKey(), + dyn.Key("file_path"), + ) + + return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return fn(path, TranslateModeLocalRelative, value) + }) +} diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index 0e3d59d43f5..bdf42188fde 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -15,6 +15,7 @@ func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { VisitArtifactPaths, VisitAlertPaths, VisitDashboardPaths, + VisitGenieSpacePaths, VisitPipelinePaths, VisitPipelineLibrariesPaths, } diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index fd019479d77..cbb05d7d622 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -47,6 +47,10 @@ var ( permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_READ", }, + "genie_spaces": { + permissions.CAN_MANAGE: "CAN_MANAGE", + permissions.CAN_VIEW: "CAN_READ", + }, "apps": { permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_USE", diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 83d512ca518..2a0996e6b7d 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -237,6 +237,14 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos dashboard.DisplayName = prefix + dashboard.DisplayName } + // Genie Spaces: Prefix + for _, genieSpace := range r.GenieSpaces { + if genieSpace == nil { + continue + } + genieSpace.Title = prefix + genieSpace.Title + } + // Apps: No presets // Alerts: Prefix, TriggerPauseStatus diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 927c7a19132..7dcf21a479b 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -154,6 +154,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "geniespace1": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "geniespace1", + }, + }, + }, Apps: map[string]*resources.App{ "app1": { App: apps.App{ @@ -330,6 +337,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Dashboards assert.Equal(t, "[dev lennart] dashboard1", b.Config.Resources.Dashboards["dashboard1"].DisplayName) + // Genie Spaces + assert.Equal(t, "[dev lennart] geniespace1", b.Config.Resources.GenieSpaces["geniespace1"].Title) + // Alert 1: has schedule without pause status set - should be paused assert.Equal(t, "[dev lennart] alert1", b.Config.Resources.Alerts["alert1"].DisplayName) assert.Equal(t, sql.SchedulePauseStatusPaused, b.Config.Resources.Alerts["alert1"].Schedule.PauseStatus) diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go new file mode 100644 index 00000000000..a9327217c08 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -0,0 +1,81 @@ +package resourcemutator + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const serializedSpaceFieldName = "serialized_space" + +type configureGenieSpaceSerializedSpace struct{} + +func ConfigureGenieSpaceSerializedSpace() bundle.Mutator { + return &configureGenieSpaceSerializedSpace{} +} + +func (c configureGenieSpaceSerializedSpace) Name() string { + return "ConfigureGenieSpaceSerializedSpace" +} + +func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("genie_spaces"), + dyn.AnyKey(), + ) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + filePath, hasFilePath := v.Get(filePathFieldName).AsString() + ss := v.Get(serializedSpaceFieldName) + + if hasFilePath { + if ss.IsValid() && ss.Kind() != dyn.KindNil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Warning, + Summary: "both file_path and serialized_space are set; file_path will be used and serialized_space will be ignored", + Locations: ss.Locations(), + }) + } + contents, err := b.SyncRoot.ReadFile(filePath) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", filePath, err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) + } + + // Marshal an inline structured serialized_space to a JSON string so + // both config-side and state-side carry the same plain string. + // Otherwise YAML decodes small ints as Go `int` while state JSON + // round-trip decodes them as `float64`, and structdiff reports + // false drift on every plan. + switch ss.Kind() { + case dyn.KindNil, dyn.KindString: + return v, nil + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("serialized_space must be a string, map, or sequence, got %s", ss.Kind()), + Locations: ss.Locations(), + }) + return v, nil + } + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} diff --git a/bundle/config/mutator/resourcemutator/genie_space_fixups.go b/bundle/config/mutator/resourcemutator/genie_space_fixups.go new file mode 100644 index 00000000000..85e1bb7e745 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/genie_space_fixups.go @@ -0,0 +1,30 @@ +package resourcemutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type genieSpaceFixups struct{} + +func GenieSpaceFixups() bundle.Mutator { + return &genieSpaceFixups{} +} + +func (m *genieSpaceFixups) Name() string { + return "GenieSpaceFixups" +} + +func (m *genieSpaceFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, genieSpace := range b.Config.Resources.GenieSpaces { + if genieSpace == nil { + continue + } + + genieSpace.ParentPath = ensureWorkspacePrefix(genieSpace.ParentPath) + } + + return nil +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 2eb292cfbb0..f0b259e9001 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -52,6 +52,7 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { }{ {"resources.dashboards.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.dashboards.*.embed_credentials", false}, + {"resources.genie_spaces.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.volumes.*.volume_type", "MANAGED"}, {"resources.alerts.*.parent_path", b.Config.Workspace.ResourcePath}, @@ -115,6 +116,11 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { // Ensures dashboard parent paths have the required /Workspace prefix DashboardFixups(), + // Reads (typed): b.Config.Resources.GenieSpaces (checks genie space configurations) + // Updates (typed): b.Config.Resources.GenieSpaces[].ParentPath (ensures /Workspace prefix is present) + // Ensures genie space parent paths have the required /Workspace prefix + GenieSpaceFixups(), + // Reads (typed): b.Config.Permissions (validates permission levels) // Reads (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (reads existing permissions) // Updates (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (adds permissions from bundle-level configuration) @@ -182,6 +188,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Drops (dynamic): resources.dashboards.*.file_path ConfigureDashboardSerializedDashboard(), + // Reads (dynamic): resources.genie_spaces.*.file_path + // Updates (dynamic): resources.genie_spaces.*.serialized_space + ConfigureGenieSpaceSerializedSpace(), + // Reads (typed): resources.alerts.*.file_path // Updates (typed): resources.alerts.* (loads alert configuration from .dbalert.json file) mutator.LoadDBAlertFiles(), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 58b27113a52..5f67452e981 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -41,6 +41,7 @@ func allResourceTypes(t *testing.T) []string { "database_instances", "experiments", "external_locations", + "genie_spaces", "jobs", "model_serving_endpoints", "models", @@ -180,6 +181,7 @@ var allowList = []string{ "postgres_projects", "registered_models", "experiments", + "genie_spaces", "schemas", "secret_scopes", "sql_warehouses", diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index 99dd75dd787..0cee3165288 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -369,6 +369,7 @@ func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ t.applyDashboardTranslations, + t.applyGenieSpaceTranslations, }) } diff --git a/bundle/config/mutator/translate_paths_genie_spaces.go b/bundle/config/mutator/translate_paths_genie_spaces.go new file mode 100644 index 00000000000..279d97be01c --- /dev/null +++ b/bundle/config/mutator/translate_paths_genie_spaces.go @@ -0,0 +1,21 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle/config/mutator/paths" + "github.com/databricks/cli/libs/dyn" +) + +func (t *translateContext) applyGenieSpaceTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { + // Convert the `file_path` field to a local absolute path. + // We load the file at this path and use its contents for the genie space contents. + + return paths.VisitGenieSpacePaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + opts := translateOptions{ + Mode: mode, + } + + return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + }) +} diff --git a/bundle/config/mutator/translate_paths_genie_spaces_test.go b/bundle/config/mutator/translate_paths_genie_spaces_test.go new file mode 100644 index 00000000000..a1ac0b160b1 --- /dev/null +++ b/bundle/config/mutator/translate_paths_genie_spaces_test.go @@ -0,0 +1,55 @@ +package mutator_test + +import ( + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "src", "my_space.geniespace.json")) + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Genie Space", + }, + FilePath: "../src/my_space.geniespace.json", + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.genie_spaces", []dyn.Location{{ + File: filepath.Join(dir, "resources", "genie_space.yml"), + }}) + + // Genie space paths reuse the dashboard translator; there is no separate + // genie_space mutator. The dashboard translator walks all resource types + // that need path translation, so calling it covers genie_spaces too. + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePathsDashboards()) + require.NoError(t, diags.Error()) + + assert.Equal( + t, + filepath.ToSlash(filepath.Join("src", "my_space.geniespace.json")), + b.Config.Resources.GenieSpaces["genie_space"].FilePath, + ) +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 30a7f4e95e5..668cd19d918 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -25,6 +25,7 @@ type Resources struct { ExternalLocations map[string]*resources.ExternalLocation `json:"external_locations,omitempty"` Clusters map[string]*resources.Cluster `json:"clusters,omitempty"` Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"` + GenieSpaces map[string]*resources.GenieSpace `json:"genie_spaces,omitempty"` Apps map[string]*resources.App `json:"apps,omitempty"` SecretScopes map[string]*resources.SecretScope `json:"secret_scopes,omitempty"` Alerts map[string]*resources.Alert `json:"alerts,omitempty"` @@ -102,6 +103,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["external_locations"], r.ExternalLocations), collectResourceMap(descriptions["clusters"], r.Clusters), collectResourceMap(descriptions["dashboards"], r.Dashboards), + collectResourceMap(descriptions["genie_spaces"], r.GenieSpaces), collectResourceMap(descriptions["volumes"], r.Volumes), collectResourceMap(descriptions["apps"], r.Apps), collectResourceMap(descriptions["alerts"], r.Alerts), @@ -158,6 +160,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), "clusters": (&resources.Cluster{}).ResourceDescription(), "dashboards": (&resources.Dashboard{}).ResourceDescription(), + "genie_spaces": (&resources.GenieSpace{}).ResourceDescription(), "volumes": (&resources.Volume{}).ResourceDescription(), "apps": (&resources.App{}).ResourceDescription(), "secret_scopes": (&resources.SecretScope{}).ResourceDescription(), diff --git a/bundle/config/resources/genie_space.go b/bundle/config/resources/genie_space.go new file mode 100644 index 00000000000..b0a5efbf304 --- /dev/null +++ b/bundle/config/resources/genie_space.go @@ -0,0 +1,100 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +type GenieSpaceConfig struct { + // Description of the Genie Space + Description string `json:"description,omitempty"` + // Etag for change detection. The bundle persists the value the backend + // returned on the last Create/Update and uses it both as an If-Match for + // the next Update and as the signal for `bundle plan` to detect remote + // drift (see OverrideChangeDesc in bundle/direct/dresources/genie_space.go). + // Mirrors dashboards.DashboardConfig.Etag. + Etag string `json:"etag,omitempty"` + // Genie space ID + SpaceId string `json:"space_id,omitempty"` + // Title of the Genie Space + Title string `json:"title,omitempty"` + // Warehouse associated with the Genie Space + WarehouseId string `json:"warehouse_id,omitempty"` + // Parent folder path where the space will be registered + ParentPath string `json:"parent_path,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` + + // ============================================== + // === overrides over [dashboards.GenieSpace] === + // ============================================== + + // SerializedSpace holds the contents of the Genie Space in serialized JSON form. + // Even though the SDK represents this as a string, we override it as any to allow for inlining as YAML. + // If the value is a string, it is used as is. + // If it is not a string, its contents is marshalled as JSON. + SerializedSpace any `json:"serialized_space,omitempty"` +} + +func (c *GenieSpaceConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c GenieSpaceConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type GenieSpace struct { + BaseResource + GenieSpaceConfig + + Permissions []Permission `json:"permissions,omitempty"` + + // FilePath points to the local `.geniespace.json` file containing the Genie Space definition. + // This is inlined into serialized_space during deployment. The file_path is kept around + // as metadata which is needed for `databricks bundle generate genie-space --resource ` to work. + // This is not part of GenieSpaceConfig because we don't need to store this in the resource state. + FilePath string `json:"file_path,omitempty"` +} + +func (*GenieSpace) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: id, + }) + if err != nil { + log.Debugf(ctx, "genie space %s does not exist", id) + return false, err + } + return true, nil +} + +func (*GenieSpace) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "genie_space", + PluralName: "genie_spaces", + SingularTitle: "Genie Space", + PluralTitle: "Genie Spaces", + } +} + +func (r *GenieSpace) InitializeURL(baseURL url.URL) { + if r.ID == "" { + return + } + + r.URL = workspaceurls.ResourceURL(baseURL, "genie_spaces", r.ID) +} + +func (r *GenieSpace) GetName() string { + return r.Title +} + +func (r *GenieSpace) GetURL() string { + return r.URL +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 626fac07983..b23865cb319 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -194,6 +194,9 @@ func TestResourcesBindSupport(t *testing.T) { Dashboards: map[string]*resources.Dashboard{ "my_dashboard": {}, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "my_genie_space": {}, + }, Volumes: map[string]*resources.Volume{ "my_volume": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{}, @@ -304,6 +307,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockGenieAPI().EXPECT().GetSpace(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVolumesAPI().EXPECT().Read(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAppsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAlertsV2API().EXPECT().GetAlertById(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 7f56248bb44..f438d9b14cf 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -17,6 +17,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { ignoredResources := []string{ "catalogs", "external_locations", + "genie_spaces", "vector_search_endpoints", } diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 497bf9bcb7a..d3a3cc878e8 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -28,6 +28,7 @@ var SupportedResources = map[string]any{ "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), "dashboards": (*ResourceDashboard)(nil), + "genie_spaces": (*ResourceGenieSpace)(nil), "secret_scopes": (*ResourceSecretScope)(nil), "model_serving_endpoints": (*ResourceModelServingEndpoint)(nil), "quality_monitors": (*ResourceQualityMonitor)(nil), @@ -47,6 +48,7 @@ var SupportedResources = map[string]any{ "secret_scopes.permissions": (*ResourceSecretScopeAcls)(nil), "model_serving_endpoints.permissions": (*ResourcePermissions)(nil), "dashboards.permissions": (*ResourcePermissions)(nil), + "genie_spaces.permissions": (*ResourcePermissions)(nil), "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), // Grants diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f130afc6194..a9eafa85947 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -248,6 +248,15 @@ var testConfig map[string]any = map[string]any{ }, }, + "genie_spaces": &resources.GenieSpace{ + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "my-genie-space", + WarehouseId: "test-warehouse-id", + ParentPath: "/Workspace/Users/user@example.com", + SerializedSpace: "{}", + }, + }, + "vector_search_endpoints": &resources.VectorSearchEndpoint{ CreateEndpoint: vectorsearch.CreateEndpoint{ Name: "my-endpoint", @@ -461,6 +470,25 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "genie_spaces.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + resp, err := client.Genie.CreateSpace(ctx, dashboards.GenieCreateSpaceRequest{ + Title: "genie-space-permissions", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }) + if err != nil { + return nil, err + } + + return &PermissionsState{ + ObjectID: "/genie/spaces/" + resp.SpaceId, + EmbeddedSlice: []StatePermission{{ + Level: "CAN_MANAGE", + UserName: "user@example.com", + }}, + }, nil + }, + "model_serving_endpoints.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { waiter, err := client.ServingEndpoints.Create(ctx, serving.CreateServingEndpoint{ Name: "endpoint-permissions", diff --git a/bundle/direct/dresources/apitypes.yml b/bundle/direct/dresources/apitypes.yml index c37dfbccbb1..1bf5039160f 100644 --- a/bundle/direct/dresources/apitypes.yml +++ b/bundle/direct/dresources/apitypes.yml @@ -4,6 +4,8 @@ # Set a value to null to remove a type: # jobs: null +genie_spaces: dashboards.GenieSpace + postgres_branches: postgres.BranchSpec postgres_endpoints: postgres.EndpointSpec diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go new file mode 100644 index 00000000000..395466fda2b --- /dev/null +++ b/bundle/direct/dresources/genie_space.go @@ -0,0 +1,299 @@ +package dresources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +var pathSerializedSpace = structpath.MustParsePath("serialized_space") + +// ResourceGenieSpace mirrors the dashboard resource pattern (see dashboard.go), +// with these intentional divergences: +// - No Published wrapper: Genie spaces have no publish lifecycle, so +// PrepareState returns the config directly. +// - RemapState filters fewer fields: Genie has no LifecycleState / CreateTime / +// Path / UpdateTime output-only fields to scrub. +// - DoRead clears ParentPath: the GET API does not reliably return parent_path, +// so we drop it from ForceSendFields and zero the value rather than re-adding +// a "/Workspace" prefix the way dashboard.go does in ensureWorkspacePrefix. +// - DoUpdate omits serialized_space when unchanged: serialized_space is in +// ignore_remote_changes (see resources.yml), so a UI edit produces no plan +// entry. Sending the local body anyway would clobber the UI edit on every +// unrelated update. +// - DoCreate has expanded missing-parent-path detection: see +// isMissingGenieParentPathError below. +// +// Permissions follow the standard /permissions/genie/{id} endpoint and are wired +// up via the generic permissions adapter (permissions.go). +type ResourceGenieSpace struct { + client *databricks.WorkspaceClient +} + +func (*ResourceGenieSpace) New(client *databricks.WorkspaceClient) *ResourceGenieSpace { + return &ResourceGenieSpace{client: client} +} + +func (*ResourceGenieSpace) PrepareState(input *resources.GenieSpace) *resources.GenieSpaceConfig { + return &input.GenieSpaceConfig +} + +func (r *ResourceGenieSpace) RemapState(state *resources.GenieSpaceConfig) *resources.GenieSpaceConfig { + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](state.ForceSendFields, []string{ + "SpaceId", + "SerializedSpace", + }...) + + return &resources.GenieSpaceConfig{ + Description: state.Description, + Etag: state.Etag, + Title: state.Title, + WarehouseId: state.WarehouseId, + ParentPath: state.ParentPath, + SerializedSpace: state.SerializedSpace, + + ForceSendFields: forceSendFields, + + // Clear output only fields. They should not show up on remote diff computation. + SpaceId: "", + } +} + +func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources.GenieSpaceConfig, error) { + space, err := r.client.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: id, + IncludeSerializedSpace: true, + ForceSendFields: nil, + }) + if err != nil { + return nil, err + } + return responseToGenieSpaceConfig(space, space.SerializedSpace), nil +} + +func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error) { + v := config.SerializedSpace + if serializedSpace, ok := v.(string); ok { + return serializedSpace, nil + } else if v != nil { + b, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("failed to marshal serialized_space: %w", err) + } + return string(b), nil + } + return "", nil +} + +func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace string) *resources.GenieSpaceConfig { + // Drop ParentPath from ForceSendFields. We always clear ParentPath + // below because the GET Genie space API does not reliably return it, + // and keeping it in ForceSendFields would force-emit parent_path: "" + // in state output even though the field is logically unset. + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields, "ParentPath") + + return &resources.GenieSpaceConfig{ + Description: space.Description, + Etag: space.Etag, + Title: space.Title, + WarehouseId: space.WarehouseId, + ParentPath: "", + SerializedSpace: serializedSpace, + + // Output only fields + SpaceId: space.SpaceId, + ForceSendFields: forceSendFields, + } +} + +// isMissingGenieParentPathError reports whether the given Create error means +// "the parent workspace folder does not exist", so DoCreate can mkdir and retry. +// +// Dashboard handles the equivalent condition with a plain apierr.IsMissing +// check (see ResourceDashboard.DoCreate). Genie cannot, because it surfaces +// the same condition in two different shapes depending on the workspace's +// backend version: +// +// 1. Standard missing-resource error: HTTP 404, ErrorCode RESOURCE_DOES_NOT_EXIST. +// Caught by apierr.IsMissing. Observed on workspaces running the newer +// Genie service implementation. +// 2. HTTP 400 with ErrorCode INVALID_PARAMETER_VALUE and a message of the +// form "Tree node with path '' does not exist". Observed on +// workspaces still backed by the legacy implementation during integration +// testing in early 2026 (aws-prod-ucws and azure-prod-ucws clusters at +// the time). The string match is intentional: there is no distinct error +// code to key on. +// +// Both forms unambiguously mean "create the parent and retry once". +func isMissingGenieParentPathError(err error) bool { + if apierr.IsMissing(err) { + return true + } + + var apiErr *apierr.APIError + if !errors.As(err, &apiErr) { + return false + } + + return apiErr.StatusCode == http.StatusBadRequest && + apiErr.ErrorCode == "INVALID_PARAMETER_VALUE" && + strings.Contains(apiErr.Message, "Tree node with path") && + strings.Contains(apiErr.Message, "does not exist") +} + +func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.GenieSpaceConfig) (string, *resources.GenieSpaceConfig, error) { + serializedSpace, err := prepareGenieSpaceRequest(config) + if err != nil { + return "", nil, err + } + + req := dashboards.GenieCreateSpaceRequest{ + Description: config.Description, + Title: config.Title, + WarehouseId: config.WarehouseId, + ParentPath: config.ParentPath, + SerializedSpace: serializedSpace, + + ForceSendFields: utils.FilterFields[dashboards.GenieCreateSpaceRequest](config.ForceSendFields), + } + + createResp, err := r.client.Genie.CreateSpace(ctx, req) + + // Retry once after creating the parent directory when the workspace folder + // is missing. Genie can surface this either as a standard missing-resource + // error or as INVALID_PARAMETER_VALUE with a "Tree node ... does not exist" + // message depending on the backend. + if err != nil && isMissingGenieParentPathError(err) { + err = r.client.Workspace.MkdirsByPath(ctx, config.ParentPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + return "", nil, fmt.Errorf("failed to create parent directory: %w", err) + } + createResp, err = r.client.Genie.CreateSpace(ctx, req) + } + if err != nil { + return "", nil, err + } + + // Persist the etag in state. The deploy framework saves `config` (the input + // to DoCreate) as the state record, so mutating it here is what gets the + // backend-returned etag onto disk for the next plan's drift check. + // Matches the dashboard pattern (dashboard.go DoCreate). + config.Etag = createResp.Etag + + return createResp.SpaceId, responseToGenieSpaceConfig(createResp, serializedSpace), nil +} + +func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *resources.GenieSpaceConfig, entry *PlanEntry) (*resources.GenieSpaceConfig, error) { + serializedSpace, err := prepareGenieSpaceRequest(config) + if err != nil { + return nil, err + } + + // serialized_space is in ignore_remote_changes (we cannot diff structured + // local YAML against remote JSON), so a UI edit produces no plan entry. + // If we still sent the unchanged local body on every update, the next + // update triggered by another field would clobber the UI edit. Only + // send it when the user actually changed it locally. + var excludeForceSend []string + sentSerialized := true + if !hasUpdate(entry, pathSerializedSpace) { + serializedSpace = "" + sentSerialized = false + excludeForceSend = append(excludeForceSend, "SerializedSpace") + } + + updateResp, err := r.client.Genie.UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: id, + Description: config.Description, + Title: config.Title, + WarehouseId: config.WarehouseId, + SerializedSpace: serializedSpace, + // Send the etag we last observed. The backend uses it as an If-Match + // guard against concurrent writes, and OverrideChangeDesc uses the + // post-update etag to detect drift on subsequent plans. + Etag: config.Etag, + + ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields, excludeForceSend...), + }) + if err != nil { + return nil, err + } + + // Persist the new etag in state (see DoCreate for the rationale). + config.Etag = updateResp.Etag + + // Decide what to record as the new state's serialized_space. + // - If we sent a new body, use it. + // - If we omitted it (UI-edit protection above) but the API echoed back + // a value, record that — it's the most up-to-date view we have. + // - If neither side carries a value, keep whatever was already in state. + // Otherwise RemapState would blank the field on every unrelated update. + respSerialized := serializedSpace + if !sentSerialized { + respSerialized = updateResp.SerializedSpace + if respSerialized == "" { + if prior, ok := config.SerializedSpace.(string); ok { + respSerialized = prior + } + } + } + + return responseToGenieSpaceConfig(updateResp, respSerialized), nil +} + +// OverrideChangeDesc handles the etag field. The user never sets it directly; +// we compare the stored etag against the remote one and Skip if they match. +// This mirrors ResourceDashboard.OverrideChangeDesc. +func (r *ResourceGenieSpace) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *resources.GenieSpaceConfig) error { + switch path.String() { + case "etag": + // change.New is always nil for etag because it's not present in the + // user-authored config. Compare stored etag with remote one to decide + // whether anything changed out-of-band since the last deploy. + if change.Old == change.Remote { + change.Action = deployplan.Skip + } else { + change.Action = deployplan.Update + } + } + return nil +} + +// hasUpdate reports whether entry has an Update-action change at the given path. +// HasChange alone matches Skip-action changes too, which we cannot use to drive +// request shaping for fields covered by ignore_remote_changes. +func hasUpdate(entry *PlanEntry, path *structpath.PathNode) bool { + if entry == nil { + return false + } + for s, change := range entry.Changes { + if change.Action != deployplan.Update { + continue + } + node, err := structpath.ParsePath(s) + if err != nil { + continue + } + if node.HasPrefix(path) { + return true + } + } + return false +} + +func (r *ResourceGenieSpace) DoDelete(ctx context.Context, id string) error { + return r.client.Genie.TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{ + SpaceId: id, + }) +} diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go new file mode 100644 index 00000000000..742deff6cb4 --- /dev/null +++ b/bundle/direct/dresources/genie_space_test.go @@ -0,0 +1,260 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsMissingGenieParentPathError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "standard missing error", + err: &apierr.APIError{ + StatusCode: 404, + ErrorCode: "NOT_FOUND", + Message: "not found", + }, + want: true, + }, + { + name: "invalid parameter tree node missing error", + err: &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "NOT_FOUND: Tree node with path /Workspace/foo does not exist", + }, + want: true, + }, + { + name: "other invalid parameter error", + err: &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "some other validation failure", + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isMissingGenieParentPathError(tt.err)) + }) + } +} + +func TestGenieSpaceDoCreateRetriesWhenParentPathLooksMissing(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + req := dashboards.GenieCreateSpaceRequest{ + Title: "test genie space", + Description: "test description", + ParentPath: "/Workspace/test-parent", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + } + + m.GetMockGenieAPI().EXPECT(). + CreateSpace(ctx, req). + Return(nil, &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "NOT_FOUND: Tree node with path /Workspace/test-parent does not exist", + }). + Once() + + m.GetMockWorkspaceAPI().EXPECT(). + MkdirsByPath(ctx, "/Workspace/test-parent"). + Return(nil). + Once() + + m.GetMockGenieAPI().EXPECT(). + CreateSpace(ctx, req). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "test genie space", + Description: "test description", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }, nil). + Once() + + id, state, err := r.DoCreate(ctx, &resources.GenieSpaceConfig{ + Title: "test genie space", + Description: "test description", + ParentPath: "/Workspace/test-parent", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }) + require.NoError(t, err) + assert.Equal(t, "space-id", id) + require.NotNil(t, state) + assert.Equal(t, "test genie space", state.Title) +} + +func TestGenieSpaceDoUpdateOmitsSerializedSpaceWhenUnchanged(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + // Plan entry indicates only title changed; serialized_space is absent. + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + SerializedSpace: "{\"remote\":\"edit\"}", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + SerializedSpace: "{\"local\":\"stale\"}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"remote\":\"edit\"}", state.SerializedSpace) +} + +func TestGenieSpaceDoUpdateSendsSerializedSpaceWhenChanged(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "serialized_space": {Action: deployplan.Update, Old: "{}", New: "{\"v\":1}"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + SerializedSpace: "{\"v\":1}", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + SerializedSpace: "{\"v\":1}", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + SerializedSpace: "{\"v\":1}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"v\":1}", state.SerializedSpace) +} + +func TestGenieSpaceDoUpdateRoundTripsEtag(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + Etag: "etag-7", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + Etag: "etag-8", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + Etag: "etag-7", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "etag-8", state.Etag) +} + +func TestGenieSpaceDoUpdateKeepsPriorSerializedSpaceWhenBothEmpty(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + // Only title changed; serialized_space should be omitted from the request. + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + // API also omits serialized_space; we should keep the prior local value. + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + SerializedSpace: "{\"keep\":\"me\"}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"keep\":\"me\"}", state.SerializedSpace) +} + +func TestGenieSpaceOverrideChangeDescEtag(t *testing.T) { + r := &ResourceGenieSpace{} + etagPath := structpath.MustParsePath("etag") + + t.Run("Skip when stored matches remote", func(t *testing.T) { + change := &ChangeDesc{Old: "etag-7", Remote: "etag-7"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), etagPath, change, nil)) + assert.Equal(t, deployplan.Skip, change.Action) + }) + + t.Run("Update when stored differs from remote", func(t *testing.T) { + change := &ChangeDesc{Old: "etag-7", Remote: "etag-8"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), etagPath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) + + t.Run("Other paths are untouched", func(t *testing.T) { + titlePath := structpath.MustParsePath("title") + change := &ChangeDesc{Action: deployplan.Update, Old: "a", Remote: "b"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), titlePath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) +} diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index eac5e2dcdbc..0c436eb249a 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -18,6 +18,7 @@ var permissionResourceToObjectType = map[string]string{ "apps": "/apps/", "clusters": "/clusters/", "dashboards": "/dashboards/", + "genie_spaces": "/genie/spaces/", "database_instances": "/database-instances/", "postgres_projects": "/database-projects/", "jobs": "/jobs/", diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 8c8a6ce07c5..bf2f2f9d61e 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -359,6 +359,19 @@ resources: - field: dataset_schema reason: input_only + genie_spaces: + recreate_on_changes: + - field: parent_path + reason: immutable + ignore_remote_changes: + # serialized_space locally (structured YAML) and remotely (JSON string) will differ + # textually, so we cannot meaningfully compare them for drift. + - field: serialized_space + reason: input_only + # parent_path is not reliably returned by the GET Genie space API. + - field: parent_path + reason: input_only + apps: recreate_on_changes: - field: name diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index f47d9ae2b47..b504f201115 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -76,6 +76,9 @@ var knownMissingInStateType = map[string][]string{ "dashboards": { "file_path", }, + "genie_spaces": { + "file_path", + }, "secret_scopes": { "backend_type", "keyvault_metadata", diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 5b2a70adbb3..7f719674ebb 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -439,9 +439,14 @@ func ExportStateFromData(data Database) resourcestate.ExportedResourcesMap { result := make(resourcestate.ExportedResourcesMap) for key, entry := range data.State { var etag string - // Extract etag for dashboards. - // covered by test case: bundle/deploy/dashboard/detect-change - if strings.Contains(key, ".dashboards.") && len(entry.State) > 0 { + // Extract etag for resources that use it for drift detection + // (dashboards and genie_spaces). Both follow the same pattern of + // persisting the backend-returned etag in state and comparing it + // against the remote on the next plan via OverrideChangeDesc. + // covered by test cases: + // - bundle/deploy/dashboard/detect-change + // - bundle/resources/genie_spaces/simple + if (strings.Contains(key, ".dashboards.") || strings.Contains(key, ".genie_spaces.")) && len(entry.State) > 0 { var holder struct { Etag string `json:"etag"` } diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 9d41a86792d..b3e86d6f305 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,7 +1,7 @@ --- description: 'Configuration reference for databricks.yml' last_update: - date: 2026-05-11 + date: 2026-05-20 --- @@ -475,6 +475,10 @@ resources: - Map - See [\_](#resourcesexternal_locations). +- - `genie_spaces` + - Map + - See [\_](#resourcesgenie_spaces). + - - `jobs` - Map - The job definitions for the bundle, where each key is the name of the job. See [\_](/dev-tools/bundles/resources.md#jobs). @@ -1478,6 +1482,122 @@ external_locations: ::: +### resources.genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - Description of the Genie space shown alongside the title in the Databricks UI. + +- - `etag` + - String + - + +- - `file_path` + - String + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + +- - `lifecycle` + - Map + - See [\_](#resourcesgenie_spacesnamelifecycle). + +- - `parent_path` + - String + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + +- - `permissions` + - Sequence + - See [\_](#resourcesgenie_spacesnamepermissions). + +- - `serialized_space` + - Any + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + +- - `space_id` + - String + - + +- - `title` + - String + - Title of the Genie space shown in the Databricks UI. + +- - `warehouse_id` + - String + - ID of the SQL warehouse used to run queries for this Genie space. + +::: + + +### resources.genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### resources.genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ### resources.postgres_branches **`Type: Map`** @@ -2553,6 +2673,10 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcesexternal_locations). +- - `genie_spaces` + - Map + - See [\_](#targetsnameresourcesgenie_spaces). + - - `jobs` - Map - The job definitions for the bundle, where each key is the name of the job. See [\_](/dev-tools/bundles/resources.md#jobs). @@ -3556,6 +3680,122 @@ external_locations: ::: +### targets._name_.resources.genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - Description of the Genie space shown alongside the title in the Databricks UI. + +- - `etag` + - String + - + +- - `file_path` + - String + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + +- - `lifecycle` + - Map + - See [\_](#targetsnameresourcesgenie_spacesnamelifecycle). + +- - `parent_path` + - String + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + +- - `permissions` + - Sequence + - See [\_](#targetsnameresourcesgenie_spacesnamepermissions). + +- - `serialized_space` + - Any + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + +- - `space_id` + - String + - + +- - `title` + - String + - Title of the Genie space shown in the Databricks UI. + +- - `warehouse_id` + - String + - ID of the SQL warehouse used to run queries for this Genie space. + +::: + + +### targets._name_.resources.genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### targets._name_.resources.genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ### targets._name_.resources.postgres_branches **`Type: Map`** diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index 392e574dcd3..4d740961a6a 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -1,7 +1,7 @@ --- description: 'Learn about resources supported by Declarative Automation Bundles and how to configure them.' last_update: - date: 2026-05-11 + date: 2026-05-20 --- @@ -544,6 +544,10 @@ apps: - String - The description of the app. +- - `git_repository` + - Map + - Git repository configuration for app deployments. When specified, deployments can reference code from this repository by providing only the git reference (branch, tag, or commit). See [\_](#appsnamegit_repository). + - - `git_source` - Map - Git source configuration for app deployments. Specifies which git reference (branch, tag, or commit) to use when deploying the app. Used in conjunction with git_repository to deploy code directly from git. The source_code_path within git_source specifies the relative path to the app code within the repository. See [\_](#appsnamegit_source). @@ -572,10 +576,6 @@ apps: - Sequence - See [\_](#appsnametelemetry_export_destinations). -- - `thumbnail_url` - - String - - - - - `usage_policy_id` - String - @@ -641,6 +641,32 @@ apps: ::: +### apps._name_.git_repository + +**`Type: Map`** + +Git repository configuration for app deployments. When specified, deployments can +reference code from this repository by providing only the git reference (branch, tag, or commit). + + + +:::list-table + +- - Key + - Type + - Description + +- - `provider` + - String + - Git provider. Case insensitive. Supported values: gitHub, gitHubEnterprise, bitbucketCloud, bitbucketServer, azureDevOpsServices, gitLab, gitLabEnterpriseEdition, awsCodeCommit. + +- - `url` + - String + - URL of the Git repository. + +::: + + ### apps._name_.git_source **`Type: Map`** @@ -1154,6 +1180,10 @@ catalogs: - Map - See [\_](#catalogsnamelifecycle). +- - `managed_encryption_settings` + - Map + - Control CMK encryption for managed catalog data. See [\_](#catalogsnamemanaged_encryption_settings). + - - `name` - String - @@ -1234,6 +1264,64 @@ The privileges assigned to the principal. ::: +### catalogs._name_.managed_encryption_settings + +**`Type: Map`** + +Control CMK encryption for managed catalog data + + + +:::list-table + +- - Key + - Type + - Description + +- - `azure_encryption_settings` + - Map + - optional Azure settings - only required if an Azure CMK is used. See [\_](#catalogsnamemanaged_encryption_settingsazure_encryption_settings). + +- - `azure_key_vault_key_id` + - String + - the AKV URL in Azure, null otherwise. + +- - `customer_managed_key_id` + - String + - the CMK uuid in AWS and GCP, null otherwise. + +::: + + +### catalogs._name_.managed_encryption_settings.azure_encryption_settings + +**`Type: Map`** + +optional Azure settings - only required if an Azure CMK is used. + + + +:::list-table + +- - Key + - Type + - Description + +- - `azure_cmk_access_connector_id` + - String + - + +- - `azure_cmk_managed_identity_id` + - String + - + +- - `azure_tenant_id` + - String + - + +::: + + ## clusters **`Type: Map`** @@ -1790,10 +1878,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -3033,6 +3117,122 @@ The privileges assigned to the principal. ::: +## genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - Description of the Genie space shown alongside the title in the Databricks UI. + +- - `etag` + - String + - + +- - `file_path` + - String + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + +- - `lifecycle` + - Map + - See [\_](#genie_spacesnamelifecycle). + +- - `parent_path` + - String + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + +- - `permissions` + - Sequence + - See [\_](#genie_spacesnamepermissions). + +- - `serialized_space` + - Any + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + +- - `space_id` + - String + - + +- - `title` + - String + - Title of the Genie space shown in the Databricks UI. + +- - `warehouse_id` + - String + - ID of the SQL warehouse used to run queries for this Genie space. + +::: + + +### genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ## jobs **`Type: Map`** @@ -3289,7 +3489,7 @@ For other serverless tasks, the task environment is required to be specified usi - - `spec` - Map - - The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines. In this minimal environment spec, only pip and java dependencies are supported. See [\_](#jobsnameenvironmentsspec). + - The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and SDP's environment for classic and serverless pipelines. In this minimal environment spec, only pip and java dependencies are supported. See [\_](#jobsnameenvironmentsspec). ::: @@ -3298,7 +3498,7 @@ For other serverless tasks, the task environment is required to be specified usi **`Type: Map`** -The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and DLT's environment for classic and serverless pipelines. +The environment entity used to preserve serverless environment side panel, jobs' environment for non-notebook task, and SDP's environment for classic and serverless pipelines. In this minimal environment spec, only pip and java dependencies are supported. @@ -4013,10 +4213,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -5708,10 +5904,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -6084,7 +6276,7 @@ The task triggers a pipeline update when the `pipeline_task` field is present. O - - `full_refresh` - Boolean - - If true, triggers a full refresh on the delta live table. + - If true, triggers a full refresh on the spark declarative pipeline. - - `pipeline_id` - String @@ -6278,7 +6470,7 @@ Controls whether the pipeline should perform a full refresh - - `full_refresh` - Boolean - - If true, triggers a full refresh on the delta live table. + - If true, triggers a full refresh on the spark declarative pipeline. ::: @@ -7345,7 +7537,7 @@ The core config of the serving endpoint. - - `auto_capture_config` - Map - - Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. Note: this field is deprecated for creating new provisioned throughput endpoints, or updating existing provisioned throughput endpoints that never have inference table configured; in these cases please use AI Gateway to manage inference tables. See [\_](#model_serving_endpointsnameconfigauto_capture_config). + - This field is deprecated - - `served_entities` - Sequence @@ -7362,42 +7554,6 @@ The core config of the serving endpoint. ::: -### model_serving_endpoints._name_.config.auto_capture_config - -**`Type: Map`** - -Configuration for Inference Tables which automatically logs requests and responses to Unity Catalog. -Note: this field is deprecated for creating new provisioned throughput endpoints, -or updating existing provisioned throughput endpoints that never have inference table configured; -in these cases please use AI Gateway to manage inference tables. - - - -:::list-table - -- - Key - - Type - - Description - -- - `catalog_name` - - String - - The name of the catalog in Unity Catalog. NOTE: On update, you cannot change the catalog name if the inference table is already enabled. - -- - `enabled` - - Boolean - - Indicates whether the inference table is enabled. - -- - `schema_name` - - String - - The name of the schema in Unity Catalog. NOTE: On update, you cannot change the schema name if the inference table is already enabled. - -- - `table_name_prefix` - - String - - The prefix of the table in Unity Catalog. NOTE: On update, you cannot change the prefix name if the inference table is already enabled. - -::: - - ### model_serving_endpoints._name_.config.served_entities **`Type: Sequence`** @@ -8278,7 +8434,7 @@ pipelines: - - `channel` - String - - DLT Release Channel that specifies which version to use. + - SDP Release Channel that specifies which version to use. - - `clusters` - Sequence @@ -8797,10 +8953,6 @@ If not specified at cluster creation, a set of default values will be used. - Integer - Boot disk size in GB -- - `confidential_compute_type` - - String - - - - - `first_on_demand` - Integer - The first `first_on_demand` nodes of the cluster will be placed on on-demand instances. This value should be greater than 0, to make sure the cluster driver node is placed on an on-demand instance. If this value is greater than or equal to the current cluster size, all nodes will be placed on on-demand instances. If this value is less than the current cluster size, `first_on_demand` nodes will be placed on on-demand instances and the remainder will be placed on `availability` instances. Note that this value does not affect cluster size and cannot currently be mutated over the lifetime of a cluster. @@ -9148,6 +9300,10 @@ The configuration for a managed ingestion pipeline. These settings cannot be use - Map - (Optional) A window that specifies a set of time ranges for snapshot queries in CDC. See [\_](#pipelinesnameingestion_definitionfull_refresh_window). +- - `ingest_from_uc_foreign_catalog` + - Boolean + - Immutable. If set to true, the pipeline will ingest tables from the UC foreign catalogs directly without the need to specify a UC connection or ingestion gateway. The `source_catalog` fields in objects of IngestionConfig are interpreted as the UC foreign catalogs to ingest from. + - - `ingestion_gateway_id` - String - Identifier for the gateway that is used by this ingestion pipeline to communicate with the source database. This is used with CDC connectors to databases like SQL Server using a gateway pipeline (connector_type = CDC). Under certain conditions, this can be replaced with connection_name to change the connector to Combined Cdc Managed Ingestion Pipeline. @@ -9300,6 +9456,18 @@ Configuration settings to control the ingestion of tables. These settings overri - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitionobjectsreporttable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9342,6 +9510,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.objects.report.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.ingestion_definition.objects.schema **`Type: Map`** @@ -9356,6 +9553,10 @@ Select all tables from a specific source schema. - Type - Description +- - `connector_options` + - Map + - (Optional) Source Specific Connector Options. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_options). + - - `destination_catalog` - String - Required. Destination catalog to store tables. @@ -9379,6 +9580,126 @@ Select all tables from a specific source schema. ::: +### pipelines._name_.ingestion_definition.objects.schema.connector_options + +**`Type: Map`** + +(Optional) Source Specific Connector Options + + + +:::list-table + +- - Key + - Type + - Description + +- - `confluence_options` + - Map + - Confluence specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_optionsconfluence_options). + +- - `jira_options` + - Map + - Jira specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_optionsjira_options). + +- - `meta_ads_options` + - Map + - Meta Marketing (Meta Ads) specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectsschemaconnector_optionsmeta_ads_options). + +::: + + +### pipelines._name_.ingestion_definition.objects.schema.connector_options.confluence_options + +**`Type: Map`** + +Confluence specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_confluence_spaces` + - Sequence + - (Optional) Spaces to filter Confluence data on + +::: + + +### pipelines._name_.ingestion_definition.objects.schema.connector_options.jira_options + +**`Type: Map`** + +Jira specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_jira_spaces` + - Sequence + - (Optional) Projects to filter Jira data on + +::: + + +### pipelines._name_.ingestion_definition.objects.schema.connector_options.meta_ads_options + +**`Type: Map`** + +Meta Marketing (Meta Ads) specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `action_attribution_windows` + - Sequence + - (Optional) Action attribution windows for insights reporting (e.g. "28d_click", "1d_view") + +- - `action_breakdowns` + - Sequence + - (Optional) Action breakdowns to configure for data aggregation + +- - `action_report_time` + - String + - (Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime) + +- - `breakdowns` + - Sequence + - (Optional) Breakdowns to configure for data aggregation + +- - `custom_insights_lookback_window` + - Integer + - (Optional) Window in days to revisit data during sync to capture updated conversion data from the API. + +- - `level` + - String + - (Optional) Granularity of data to pull (account, ad, adset, campaign) + +- - `start_date` + - String + - (Optional) Start date in yyyy-MM-dd format (e.g. 2025-01-15). Data added after this date will be ingested + +- - `time_increment` + - String + - (Optional) Value in string by which to aggregate statistics (can take all_days, monthly or number of days) + +::: + + ### pipelines._name_.ingestion_definition.objects.schema.table_configuration **`Type: Map`** @@ -9409,6 +9730,18 @@ Configuration settings to control the ingestion of tables. These settings are ap - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitionobjectsschematable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9451,6 +9784,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.objects.schema.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.ingestion_definition.objects.table **`Type: Map`** @@ -9465,6 +9827,10 @@ Select a specific source table. - Type - Description +- - `connector_options` + - Map + - (Optional) Source Specific Connector Options. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_options). + - - `destination_catalog` - String - Required. Destination catalog to store table. @@ -9496,6 +9862,126 @@ Select a specific source table. ::: +### pipelines._name_.ingestion_definition.objects.table.connector_options + +**`Type: Map`** + +(Optional) Source Specific Connector Options + + + +:::list-table + +- - Key + - Type + - Description + +- - `confluence_options` + - Map + - Confluence specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_optionsconfluence_options). + +- - `jira_options` + - Map + - Jira specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_optionsjira_options). + +- - `meta_ads_options` + - Map + - Meta Marketing (Meta Ads) specific options for ingestion. See [\_](#pipelinesnameingestion_definitionobjectstableconnector_optionsmeta_ads_options). + +::: + + +### pipelines._name_.ingestion_definition.objects.table.connector_options.confluence_options + +**`Type: Map`** + +Confluence specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_confluence_spaces` + - Sequence + - (Optional) Spaces to filter Confluence data on + +::: + + +### pipelines._name_.ingestion_definition.objects.table.connector_options.jira_options + +**`Type: Map`** + +Jira specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `include_jira_spaces` + - Sequence + - (Optional) Projects to filter Jira data on + +::: + + +### pipelines._name_.ingestion_definition.objects.table.connector_options.meta_ads_options + +**`Type: Map`** + +Meta Marketing (Meta Ads) specific options for ingestion + + + +:::list-table + +- - Key + - Type + - Description + +- - `action_attribution_windows` + - Sequence + - (Optional) Action attribution windows for insights reporting (e.g. "28d_click", "1d_view") + +- - `action_breakdowns` + - Sequence + - (Optional) Action breakdowns to configure for data aggregation + +- - `action_report_time` + - String + - (Optional) Timing used to report action statistics (impression, conversion, mixed, or lifetime) + +- - `breakdowns` + - Sequence + - (Optional) Breakdowns to configure for data aggregation + +- - `custom_insights_lookback_window` + - Integer + - (Optional) Window in days to revisit data during sync to capture updated conversion data from the API. + +- - `level` + - String + - (Optional) Granularity of data to pull (account, ad, adset, campaign) + +- - `start_date` + - String + - (Optional) Start date in yyyy-MM-dd format (e.g. 2025-01-15). Data added after this date will be ingested + +- - `time_increment` + - String + - (Optional) Value in string by which to aggregate statistics (can take all_days, monthly or number of days) + +::: + + ### pipelines._name_.ingestion_definition.objects.table.table_configuration **`Type: Map`** @@ -9526,6 +10012,18 @@ Configuration settings to control the ingestion of tables. These settings overri - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitionobjectstabletable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9568,6 +10066,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.objects.table.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.ingestion_definition.source_configurations **`Type: Sequence`** @@ -9586,10 +10113,6 @@ Top-level source configurations - Map - Catalog-level source configuration parameters. See [\_](#pipelinesnameingestion_definitionsource_configurationscatalog). -- - `google_ads_config` - - Map - - See [\_](#pipelinesnameingestion_definitionsource_configurationsgoogle_ads_config). - ::: @@ -9694,6 +10217,18 @@ Configuration settings to control the ingestion of tables. These settings are ap - Sequence - The primary key of the table used to apply changes. +- - `query_based_connector_config` + - Map + - Configurations that are only applicable for query-based ingestion connectors. See [\_](#pipelinesnameingestion_definitiontable_configurationquery_based_connector_config). + +- - `row_filter` + - String + - (Optional, Immutable) The row filter condition to be applied to the table. It must not contain the WHERE keyword, only the actual filter condition. It must be in DBSQL format. + +- - `scd_type` + - String + - The SCD type to use to ingest the table. + - - `sequence_by` - Sequence - The column names specifying the logical order of events in the source data. Spark Declarative Pipelines uses this sequencing to handle change events that arrive out of order. @@ -9736,6 +10271,35 @@ If unspecified, auto full refresh is disabled. ::: +### pipelines._name_.ingestion_definition.table_configuration.query_based_connector_config + +**`Type: Map`** + +Configurations that are only applicable for query-based ingestion connectors. + + + +:::list-table + +- - Key + - Type + - Description + +- - `cursor_columns` + - Sequence + - The names of the monotonically increasing columns in the source table that are used to enable the table to be read and ingested incrementally through structured streaming. The columns are allowed to have repeated values but have to be non-decreasing. If the source data is merged into the destination (e.g., using SCD Type 1 or Type 2), these columns will implicitly define the `sequence_by` behavior. You can still explicitly set `sequence_by` to override this default. + +- - `deletion_condition` + - String + - Specifies a SQL WHERE condition that specifies that the source row has been deleted. This is sometimes referred to as "soft-deletes". For example: "Operation = 'DELETE'" or "is_deleted = true". This field is orthogonal to `hard_deletion_sync_interval_in_seconds`, one for soft-deletes and the other for hard-deletes. See also the hard_deletion_sync_min_interval_in_seconds field for handling of "hard deletes" where the source rows are physically removed from the table. + +- - `hard_deletion_sync_min_interval_in_seconds` + - Integer + - Specifies the minimum interval (in seconds) between snapshots on primary keys for detecting and synchronizing hard deletions—i.e., rows that have been physically removed from the source table. This interval acts as a lower bound. If ingestion runs less frequently than this value, hard deletion synchronization will align with the actual ingestion frequency instead of happening more often. If not set, hard deletion synchronization via snapshots is disabled. This field is mutable and can be updated without triggering a full snapshot. + +::: + + ### pipelines._name_.libraries **`Type: Sequence`** @@ -10275,7 +10839,7 @@ A collection of settings for a compute endpoint. - - `no_suspension` - Boolean - - When set to true, explicitly disables automatic suspension (never suspend). Should be set to true when provided. + - When set to true, explicitly disables automatic suspension (never suspend). Should be set to true when provided. Mutually exclusive with `suspend_timeout_duration`. When updating, use `spec.project_default_settings.suspension` in the update_mask. - - `pg_settings` - Map @@ -10283,7 +10847,7 @@ A collection of settings for a compute endpoint. - - `suspend_timeout_duration` - String - - Duration of inactivity after which the compute endpoint is automatically suspended. If specified should be between 60s and 604800s (1 minute to 1 week). + - Duration of inactivity after which the compute endpoint is automatically suspended. If specified should be between 60s and 604800s (1 minute to 1 week). Mutually exclusive with `no_suspension`. When updating, use `spec.project_default_settings.suspension` in the update_mask. ::: diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go new file mode 100644 index 00000000000..5bd11761a49 --- /dev/null +++ b/bundle/generate/genie_space.go @@ -0,0 +1,26 @@ +package generate + +import ( + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string) (dyn.Value, error) { + // The majority of fields of the genie space struct are read-only. + // We copy the relevant fields manually. + dv := map[string]dyn.Value{ + "title": dyn.NewValue(genieSpace.Title, []dyn.Location{{Line: 1}}), + "warehouse_id": dyn.NewValue(genieSpace.WarehouseId, []dyn.Location{{Line: 2}}), + "file_path": dyn.NewValue(filePath, []dyn.Location{{Line: 3}}), + } + + if genieSpace.Description != "" { + dv["description"] = dyn.NewValue(genieSpace.Description, []dyn.Location{{Line: 4}}) + } + + if genieSpace.ParentPath != "" { + dv["parent_path"] = dyn.NewValue(genieSpace.ParentPath, []dyn.Location{{Line: 5}}) + } + + return dyn.V(dv), nil +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 0f2e1b0c798..832fb90a52a 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -192,6 +192,9 @@ github.com/databricks/cli/bundle/config.Resources: "external_locations": "description": |- PLACEHOLDER + "genie_spaces": + "description": |- + PLACEHOLDER "jobs": "description": |- The job definitions for the bundle, where each key is the name of the job. @@ -653,6 +656,37 @@ github.com/databricks/cli/bundle/config/resources.ExternalLocation: "url": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.GenieSpace: + "description": + "description": |- + Description of the Genie space shown alongside the title in the Databricks UI. + "etag": + "description": |- + PLACEHOLDER + "file_path": + "description": |- + Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + "lifecycle": + "description": |- + PLACEHOLDER + "parent_path": + "description": |- + Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + "permissions": + "description": |- + PLACEHOLDER + "serialized_space": + "description": |- + Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + "space_id": + "description": |- + PLACEHOLDER + "title": + "description": |- + Title of the Genie space shown in the Databricks UI. + "warehouse_id": + "description": |- + ID of the SQL warehouse used to run queries for this Genie space. github.com/databricks/cli/bundle/config/resources.JobPermission: "group_name": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 389efd115bf..36dac7f9f7e 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -750,6 +750,56 @@ } ] }, + "resources.GenieSpace": { + "oneOf": [ + { + "type": "object", + "properties": { + "description": { + "description": "Description of the Genie space shown alongside the title in the Databricks UI.", + "$ref": "#/$defs/string" + }, + "etag": { + "$ref": "#/$defs/string" + }, + "file_path": { + "description": "Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`.", + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "parent_path": { + "description": "Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_space": { + "description": "Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`.", + "$ref": "#/$defs/interface" + }, + "space_id": { + "$ref": "#/$defs/string" + }, + "title": { + "description": "Title of the Genie space shown in the Databricks UI.", + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "description": "ID of the SQL warehouse used to run queries for this Genie space.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { @@ -2538,6 +2588,9 @@ "external_locations": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation" }, + "genie_spaces": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -11703,6 +11756,20 @@ } ] }, + "resources.GenieSpace": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.GenieSpace" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index ab06243d227..e7548afd48b 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -731,6 +731,39 @@ "url" ] }, + "resources.GenieSpace": { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/string" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "parent_path": { + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_space": { + "$ref": "#/$defs/interface" + }, + "space_id": { + "$ref": "#/$defs/string" + }, + "title": { + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, "resources.Job": { "type": "object", "properties": { @@ -2483,6 +2516,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation", "x-since-version": "v0.289.0" }, + "genie_spaces": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -9709,6 +9745,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ExternalLocation" } }, + "resources.GenieSpace": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.GenieSpace" + } + }, "resources.Job": { "type": "object", "additionalProperties": { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 097b708e62d..229c8fd3f23 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -39,6 +39,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.volumes.test_volume": {ID: "1"}, "resources.clusters.test_cluster": {ID: "1"}, "resources.dashboards.test_dashboard": {ID: "1"}, + "resources.genie_spaces.test_genie_space": {ID: "1"}, "resources.apps.test_app": {ID: "app1"}, "resources.secret_scopes.test_secret_scope": {ID: "secret_scope1"}, "resources.sql_warehouses.test_sql_warehouse": {ID: "1"}, @@ -94,6 +95,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "app1", config.Resources.Apps["test_app"].ID) assert.Equal(t, "", config.Resources.Apps["test_app"].Name) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app"].ModifiedStatus) @@ -223,6 +227,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "test_genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space", + }, + }, + }, Apps: map[string]*resources.App{ "test_app": { App: apps.App{ @@ -355,6 +366,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.Dashboards["test_dashboard"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Equal(t, "", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "", config.Resources.Apps["test_app"].Name) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app"].ModifiedStatus) @@ -549,6 +563,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "test_genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space", + }, + }, + "test_genie_space_new": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space_new", + }, + }, + }, Apps: map[string]*resources.App{ "test_app": { App: apps.App{ @@ -726,6 +752,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.clusters.test_cluster_old": {ID: "2"}, "resources.dashboards.test_dashboard": {ID: "1"}, "resources.dashboards.test_dashboard_old": {ID: "2"}, + "resources.genie_spaces.test_genie_space": {ID: "1"}, + "resources.genie_spaces.test_genie_space_old": {ID: "2"}, "resources.apps.test_app": {ID: "test_app"}, "resources.apps.test_app_old": {ID: "test_app_old"}, "resources.secret_scopes.test_secret_scope": {ID: "test_secret_scope"}, @@ -834,6 +862,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.Dashboards["test_dashboard_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, "", config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.GenieSpaces["test_genie_space_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.GenieSpaces["test_genie_space_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space_new"].ModifiedStatus) + assert.Equal(t, "test_app", config.Resources.Apps["test_app"].Name) assert.Equal(t, "", config.Resources.Apps["test_app"].ModifiedStatus) assert.Equal(t, "test_app_old", config.Resources.Apps["test_app_old"].ID) diff --git a/cmd/bundle/generate.go b/cmd/bundle/generate.go index 9caf7fa1e37..12452293bc6 100644 --- a/cmd/bundle/generate.go +++ b/cmd/bundle/generate.go @@ -40,6 +40,7 @@ Use --bind to automatically bind the generated resource to the existing workspac cmd.AddCommand(generate.NewGenerateDashboardCommand()) cmd.AddCommand(generate.NewGenerateAlertCommand()) cmd.AddCommand(generate.NewGenerateAppCommand()) + cmd.AddCommand(generate.NewGenerateGenieSpaceCommand()) cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`) return cmd } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 7af4e01e92f..250d05e3693 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -275,7 +275,11 @@ func waitForChanges(ctx context.Context, w *databricks.WorkspaceClient, dashboar break } - time.Sleep(1 * time.Second) + select { + case <-ctx.Done(): + return + case <-time.After(1 * time.Second): + } } } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go new file mode 100644 index 00000000000..4d67db210c2 --- /dev/null +++ b/cmd/bundle/generate/genie_space.go @@ -0,0 +1,529 @@ +package generate + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "maps" + "os" + "path" + "path/filepath" + "slices" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/generate" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/cmd/bundle/deployment" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" +) + +const genieSpaceWatchInterval = 1 * time.Second + +type genieSpace struct { + // Lookup flags for one-time generate. + existingPath string + existingID string + + // Lookup flag for existing bundle resource. + resource string + + // Where to write the configuration and genie space representation. + resourceDir string + genieSpaceDir string + + // Force overwrite of existing files. + force bool + + // Watch for changes to the genie space. + watch bool + + // Relative path from the resource directory to the genie space directory. + relativeGenieSpaceDir string + + // Command. + cmd *cobra.Command + + // Automatically bind the generated resource to the existing resource. + bind bool + + // Output and error streams. + out io.Writer + err io.Writer +} + +func (g *genieSpace) resolveID(ctx context.Context, b *bundle.Bundle) string { + switch { + case g.existingPath != "": + return g.resolveFromPath(ctx, b) + case g.existingID != "": + return g.resolveFromID(ctx, b) + } + + logdiag.LogError(ctx, errors.New("expected one of --existing-path, --existing-id")) + return "" +} + +func (g *genieSpace) resolveFromPath(ctx context.Context, b *bundle.Bundle) string { + w := b.WorkspaceClient(ctx) + obj, err := w.Workspace.GetStatusByPath(ctx, g.existingPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + if apierr.IsMissing(err) { + logdiag.LogError(ctx, fmt.Errorf("genie space %q not found", path.Base(g.existingPath))) + return "" + } + + logdiag.LogError(ctx, err) + return "" + } + + if obj.ResourceId == "" { + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Error, + Summary: "expected a non-empty genie space resource ID", + }) + return "" + } + + return obj.ResourceId +} + +func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string { + w := b.WorkspaceClient(ctx) + obj, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: g.existingID, + }) + if err != nil { + if apierr.IsMissing(err) { + logdiag.LogError(ctx, fmt.Errorf("genie space with ID %s not found", g.existingID)) + return "" + } + logdiag.LogError(ctx, err) + return "" + } + + return obj.SpaceId +} + +func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, filename string) error { + if genieSpace.SerializedSpace == "" { + return fmt.Errorf("genie space response did not include serialized_space; refusing to write %s", filepath.ToSlash(filename)) + } + + // Unmarshal and remarshal the serialized genie space to ensure it is formatted correctly. + // The result will have alphabetically sorted keys and be indented. + data, err := remarshalJSON([]byte(genieSpace.SerializedSpace)) + if err != nil { + return err + } + + // Make sure the output directory exists. + if err := os.MkdirAll(filepath.Dir(filename), 0o755); err != nil { + return err + } + + // Clean the filename to ensure it is a valid path (and can be used on this OS). + filename = filepath.Clean(filename) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, filename) + if err != nil { + rel = filename + } + + // Verify that the file does not already exist. + info, err := os.Stat(filename) + if err == nil { + if info.IsDir() { + return fmt.Errorf("%s is a directory", filepath.ToSlash(rel)) + } + if !g.force { + return fmt.Errorf("%s already exists. Use --force to overwrite", filepath.ToSlash(rel)) + } + } + + cmdio.LogString(ctx, "Writing genie space to "+filepath.ToSlash(rel)) + return os.WriteFile(filename, data, 0o644) +} + +func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, key string) error { + // Save serialized genie space definition to the genie space directory. + genieSpaceBasename := key + ".geniespace.json" + genieSpacePath := filepath.Join(g.genieSpaceDir, genieSpaceBasename) + err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) + if err != nil { + return err + } + + // Synthesize resource configuration. + v, err := generate.ConvertGenieSpaceToValue(genieSpace, path.Join(g.relativeGenieSpaceDir, genieSpaceBasename)) + if err != nil { + return err + } + + result := map[string]dyn.Value{ + "resources": dyn.V(map[string]dyn.Value{ + "genie_spaces": dyn.V(map[string]dyn.Value{ + key: v, + }), + }), + } + + // Make sure the output directory exists. + if err := os.MkdirAll(g.resourceDir, 0o755); err != nil { + return err + } + + // Save the configuration to the resource directory. + resourcePath := filepath.Join(g.resourceDir, key+".genie_space.yml") + saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + "title": yaml.DoubleQuotedStyle, + }) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, resourcePath) + if err != nil { + rel = resourcePath + } + + cmdio.LogString(ctx, "Writing configuration to "+filepath.ToSlash(rel)) + err = saver.SaveAsYAML(result, resourcePath, g.force) + if err != nil { + return err + } + + return nil +} + +func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle.Bundle) { + resource, ok := b.Config.Resources.GenieSpaces[g.resource] + if !ok { + logdiag.LogError(ctx, fmt.Errorf("genie space resource %q is not defined", g.resource)) + return + } + + if resource.FilePath == "" { + logdiag.LogError(ctx, fmt.Errorf("genie space resource %q has no file path defined", g.resource)) + return + } + + genieSpaceID := resource.ID + genieSpacePath := resource.FilePath + + w := b.WorkspaceClient(ctx) + + first := true + for { + genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: genieSpaceID, + IncludeSerializedSpace: true, + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + // Genie has no remote modification timestamp we can poll. Compare + // the canonicalized remote body against the on-disk body and only + // re-save when they differ. The first iteration always saves, to + // match the prior behavior of an unconditional initial sync. + shouldSave := first + if !first { + differs, err := genieSpaceBodyDiffersFromDisk(genieSpace.SerializedSpace, genieSpacePath) + if err != nil { + logdiag.LogError(ctx, err) + return + } + shouldSave = differs + } + + if shouldSave { + if err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath); err != nil { + logdiag.LogError(ctx, err) + return + } + } + + if !g.watch { + return + } + + first = false + select { + case <-ctx.Done(): + return + case <-time.After(genieSpaceWatchInterval): + } + } +} + +// genieSpaceBodyDiffersFromDisk reports whether the canonicalized remote +// serialized_space differs from the contents of filename. +func genieSpaceBodyDiffersFromDisk(remoteSerialized, filename string) (bool, error) { + if remoteSerialized == "" { + return false, nil + } + canonical, err := remarshalJSON([]byte(remoteSerialized)) + if err != nil { + return false, err + } + onDisk, err := os.ReadFile(filename) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return true, nil + } + return false, err + } + return !bytes.Equal(canonical, onDisk), nil +} + +func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, genieSpaceID string) { + w := b.WorkspaceClient(ctx) + genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: genieSpaceID, + IncludeSerializedSpace: true, + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + key := g.cmd.Flag("key").Value.String() + if key == "" { + key = textutil.NormalizeString(genieSpace.Title) + } + err = g.saveConfiguration(ctx, b, genieSpace, key) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + if g.bind { + err = deployment.BindResource(g.cmd, key, genieSpaceID, true, false, true) + if err != nil { + logdiag.LogError(ctx, err) + return + } + cmdio.LogString(ctx, fmt.Sprintf("Successfully bound genie space with an id '%s'", genieSpaceID)) + } +} + +func (g *genieSpace) initialize(ctx context.Context, b *bundle.Bundle) { + // Make the paths absolute if they aren't already. + if !filepath.IsAbs(g.resourceDir) { + g.resourceDir = filepath.Join(b.BundleRootPath, g.resourceDir) + } + if !filepath.IsAbs(g.genieSpaceDir) { + g.genieSpaceDir = filepath.Join(b.BundleRootPath, g.genieSpaceDir) + } + + // Make sure we know how the genie space path is relative to the resource path. + rel, err := filepath.Rel(g.resourceDir, g.genieSpaceDir) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + g.relativeGenieSpaceDir = filepath.ToSlash(rel) +} + +func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { + phases.Initialize(ctx, b) + if logdiag.HasError(ctx) { + return + } + + requiredEngine, err := utils.ResolveEngineSetting(ctx, b) + if err != nil { + logdiag.LogError(ctx, err) + return + } + ctx, stateDesc := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true), requiredEngine) + if logdiag.HasError(ctx) { + return + } + + var state statemgmt.ExportedResourcesMap + if stateDesc.Engine.IsDirect() { + _, localPath := b.StateFilenameDirect(ctx) + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + logdiag.LogError(ctx, err) + return + } + state = b.DeploymentBundle.ExportState(ctx) + } else { + var err error + state, err = terraform.ParseResourcesState(ctx, b) + if err != nil { + logdiag.LogError(ctx, err) + return + } + } + + bundle.ApplySeqContext(ctx, b, + statemgmt.Load(state), + ) + if logdiag.HasError(ctx) { + return + } + + g.updateGenieSpaceForResource(ctx, b) +} + +func (g *genieSpace) runForExisting(ctx context.Context, b *bundle.Bundle) { + // Resolve the ID of the genie space to generate configuration for. + genieSpaceID := g.resolveID(ctx, b) + if logdiag.HasError(ctx) { + return + } + + g.generateForExisting(ctx, b, genieSpaceID) +} + +func (g *genieSpace) RunE(cmd *cobra.Command, args []string) error { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + + b := root.MustConfigureBundle(cmd) + if b == nil || logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + g.initialize(ctx, b) + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + if g.resource != "" { + g.runForResource(ctx, b) + } else { + g.runForExisting(ctx, b) + } + + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + return nil +} + +// filterGenieSpaces returns a filter that only includes genie spaces. +func filterGenieSpaces(ref resources.Reference) bool { + return ref.Description.SingularName == "genie_space" +} + +// genieSpaceResourceCompletion executes to autocomplete the argument to the resource flag. +func genieSpaceResourceCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + b := root.MustConfigureBundle(cmd) + if logdiag.HasError(cmd.Context()) { + return nil, cobra.ShellCompDirectiveError + } + + if b == nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + return slices.Collect(maps.Keys(resources.Completions(b, filterGenieSpaces))), cobra.ShellCompDirectiveNoFileComp +} + +func NewGenerateGenieSpaceCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "genie-space", + Short: "Generate configuration for a Genie space", + Long: `Generate bundle configuration for an existing Databricks Genie space. + +This command downloads an existing Genie space and creates bundle files +that you can use to deploy the Genie space to other environments or manage it as code. + +Examples: + # Import Genie space by workspace path + databricks bundle generate genie-space --existing-path /Users/me/my-genie-space + + # Import Genie space by ID + databricks bundle generate genie-space --existing-id abc123 + + # Watch for changes to keep bundle in sync with UI modifications + databricks bundle generate genie-space --resource my_genie_space --watch --force + +What gets generated: +- Genie space configuration YAML file with settings and a reference to the Genie space definition +- Genie space definition (.geniespace.json) file with the serialized space content + +Sync workflow for Genie space development: +When developing Genie spaces, you can modify them in the Databricks UI and sync +changes back to your bundle: + +1. Make changes to Genie space in the Databricks UI +2. Run: databricks bundle generate genie-space --resource my_genie_space --force +3. Commit changes to version control +4. Deploy to other environments with: databricks bundle deploy --target prod + +The --watch flag continuously polls for remote changes and updates your local +bundle files automatically, useful during active Genie space development.`, + } + + g := &genieSpace{ + out: cmd.OutOrStdout(), + err: cmd.ErrOrStderr(), + } + + // Lookup flags. + cmd.Flags().StringVar(&g.existingPath, "existing-path", "", `workspace path of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.existingID, "existing-id", "", `ID of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.resource, "resource", "", `resource key of Genie space to watch for changes`) + + // Alias lookup flags that include the resource type name. + cmd.Flags().StringVar(&g.existingPath, "existing-genie-space-path", "", `workspace path of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.existingID, "existing-genie-space-id", "", `ID of the Genie space to generate configuration for`) + cmd.Flags().MarkHidden("existing-genie-space-path") + cmd.Flags().MarkHidden("existing-genie-space-id") + + // Output flags. + cmd.Flags().StringVarP(&g.resourceDir, "resource-dir", "d", "resources", `directory to write the configuration to`) + cmd.Flags().StringVarP(&g.genieSpaceDir, "genie-space-dir", "s", "src", `directory to write the Genie space representation to`) + cmd.Flags().BoolVarP(&g.force, "force", "f", false, `force overwrite existing files in the output directory`) + + cmd.Flags().BoolVarP(&g.bind, "bind", "b", false, `automatically bind the generated Genie space config to the existing Genie space`) + cmd.Flags().MarkHidden("bind") + + // Exactly one of the lookup flags must be provided. + cmd.MarkFlagsOneRequired( + "existing-path", + "existing-id", + "resource", + ) + + // Watch flag. This is relevant only in combination with the resource flag. + cmd.Flags().BoolVar(&g.watch, "watch", false, `watch for changes to the Genie space and update the configuration`) + + // Make sure the watch flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("watch", "existing-path") + cmd.MarkFlagsMutuallyExclusive("watch", "existing-id") + + // Make sure the bind flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("bind", "resource") + + // Completion for the resource flag. + cmd.RegisterFlagCompletionFunc("resource", genieSpaceResourceCompletion) + + cmd.RunE = g.RunE + g.cmd = cmd + return cmd +} diff --git a/cmd/bundle/generate/genie_space_test.go b/cmd/bundle/generate/genie_space_test.go new file mode 100644 index 00000000000..746b2b1c3af --- /dev/null +++ b/cmd/bundle/generate/genie_space_test.go @@ -0,0 +1,126 @@ +package generate + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func newGenieSpaceTestBundle(t *testing.T, m *mocks.MockWorkspaceClient, filePath string) *bundle.Bundle { + t.Helper() + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "my_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Space", + }, + FilePath: filePath, + }, + }, + }, + }, + } + b.Config.Resources.GenieSpaces["my_space"].ID = "space-id-1" + b.SetWorkpaceClient(m.WorkspaceClient) + return b +} + +func TestGenieSpace_UpdateForResource_WritesFileWhenNotWatching(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "my_space.geniespace.json") + + m := mocks.NewMockWorkspaceClient(t) + m.GetMockGenieAPI().EXPECT(). + GetSpace(mock.Anything, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id-1", + IncludeSerializedSpace: true, + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id-1", + Title: "My Space", + SerializedSpace: `{"version":1}`, + }, nil). + Once() + + g := &genieSpace{ + resource: "my_space", + force: true, + } + b := newGenieSpaceTestBundle(t, m, filePath) + + ctx, _ := cmdio.NewTestContextWithStdout(t.Context()) + ctx = logdiag.InitContext(ctx) + logdiag.SetCollect(ctx, true) + g.updateGenieSpaceForResource(ctx, b) + + require.Empty(t, logdiag.FlushCollected(ctx)) + + contents, err := os.ReadFile(filePath) + require.NoError(t, err) + assert.Contains(t, string(contents), `"version"`) +} + +func TestGenieSpace_UpdateForResource_WatchExitsOnCancel(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "my_space.geniespace.json") + + m := mocks.NewMockWorkspaceClient(t) + // Allow any number of GetSpace calls; we don't know how many fire before cancel. + m.GetMockGenieAPI().EXPECT(). + GetSpace(mock.Anything, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id-1", + IncludeSerializedSpace: true, + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id-1", + Title: "My Space", + SerializedSpace: `{"version":1}`, + }, nil). + Maybe() + + g := &genieSpace{ + resource: "my_space", + force: true, + watch: true, + } + b := newGenieSpaceTestBundle(t, m, filePath) + + base, _ := cmdio.NewTestContextWithStdout(t.Context()) + ctx, cancel := context.WithCancel(logdiag.InitContext(base)) + logdiag.SetCollect(ctx, true) + + done := make(chan struct{}) + go func() { + g.updateGenieSpaceForResource(ctx, b) + close(done) + }() + + // First iteration always saves. Wait until the file lands, then cancel. + require.Eventually(t, func() bool { + _, err := os.Stat(filePath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "expected initial save to write file") + + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watch loop did not exit promptly after ctx cancel") + } +} diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index 7ec8e937ece..c53eb6fd465 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", 0) assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, clusters, dashboards, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses") + assert.ErrorContains(t, err, "alerts, apps, clusters, dashboards, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -111,6 +111,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { assert.Contains(t, completions, "clusters") assert.Contains(t, completions, "dashboards") assert.Contains(t, completions, "experiments") + assert.Contains(t, completions, "genie_spaces") assert.Contains(t, completions, "jobs") assert.Contains(t, completions, "models") assert.Contains(t, completions, "model_serving_endpoints") @@ -119,7 +120,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { assert.Contains(t, completions, "queries") assert.Contains(t, completions, "registered_models") assert.Contains(t, completions, "warehouses") - assert.Len(t, completions, 13) + assert.Len(t, completions, 14) } func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { @@ -133,7 +134,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, clusters, dashboards, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, clusters, dashboards, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, queries, registered_models, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index d00a5039b8c..d19c2867798 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -141,6 +141,7 @@ type FakeWorkspace struct { Volumes map[string]catalog.VolumeInfo Dashboards map[string]fakeDashboard PublishedDashboards map[string]dashboards.PublishedDashboard + GenieSpaces map[string]dashboards.GenieSpace SqlWarehouses map[string]sql.GetWarehouseResponse Alerts map[string]sql.AlertV2 Experiments map[string]ml.GetExperimentResponse @@ -286,6 +287,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Volumes: map[string]catalog.VolumeInfo{}, Dashboards: map[string]fakeDashboard{}, PublishedDashboards: map[string]dashboards.PublishedDashboard{}, + GenieSpaces: map[string]dashboards.GenieSpace{}, SqlWarehouses: map[string]sql.GetWarehouseResponse{ TestDefaultWarehouseId: { Id: TestDefaultWarehouseId, diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go new file mode 100644 index 00000000000..98f5cdb381a --- /dev/null +++ b/libs/testserver/genie_spaces.go @@ -0,0 +1,211 @@ +package testserver + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "path" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// generateGenieSpaceId returns a random 32-character hex string. +func generateGenieSpaceId() (string, error) { + randomBytes := make([]byte, 16) + _, err := rand.Read(randomBytes) + if err != nil { + return "", err + } + return hex.EncodeToString(randomBytes), nil +} + +func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { + defer s.LockUnlock()() + + var createReq dashboards.GenieCreateSpaceRequest + if err := json.Unmarshal(req.Body, &createReq); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "message": "Invalid request body: " + err.Error(), + }, + } + } + + spaceId, err := generateGenieSpaceId() + if err != nil { + return Response{ + StatusCode: 500, + Body: map[string]string{ + "message": "Failed to generate genie space ID", + }, + } + } + + genieSpace := dashboards.GenieSpace{ + SpaceId: spaceId, + Title: createReq.Title, + Description: createReq.Description, + ParentPath: createReq.ParentPath, + WarehouseId: createReq.WarehouseId, + SerializedSpace: createReq.SerializedSpace, + // Mirror libs/testserver/dashboards.go: initialize etag to a numeric + // string so each subsequent update can bump it monotonically. + Etag: "1", + } + + s.GenieSpaces[spaceId] = genieSpace + + // Register in the workspace files for path lookup. + if createReq.ParentPath != "" { + workspacePath := createReq.ParentPath + if !strings.HasPrefix(workspacePath, "/Workspace") { + workspacePath = path.Join("/Workspace", workspacePath) + } + workspacePath = path.Join(workspacePath, createReq.Title+".geniespace") + + s.files[workspacePath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "FILE", + Path: workspacePath, + ResourceId: spaceId, + }, + Data: []byte(createReq.SerializedSpace), + } + } + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceGet(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + var updateReq dashboards.GenieUpdateSpaceRequest + if err := json.Unmarshal(req.Body, &updateReq); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "message": "Invalid request body: " + err.Error(), + }, + } + } + + // Optimistic concurrency: if the caller sent an etag, it must match the + // current one. Empty etag means apply unconditionally. + if updateReq.Etag != "" && updateReq.Etag != genieSpace.Etag { + return Response{ + StatusCode: 409, + Body: map[string]string{ + "message": "Etag mismatch: expected " + genieSpace.Etag + ", got " + updateReq.Etag, + }, + } + } + + prev := genieSpace + if updateReq.Title != "" { + genieSpace.Title = updateReq.Title + } + if updateReq.Description != "" { + genieSpace.Description = updateReq.Description + } + if updateReq.WarehouseId != "" { + genieSpace.WarehouseId = updateReq.WarehouseId + } + if updateReq.SerializedSpace != "" { + genieSpace.SerializedSpace = updateReq.SerializedSpace + } + + // Bump the etag only when the update actually changes user-visible state. + // Matches dashboard's behavior (libs/testserver/dashboards.go) and keeps + // no-op updates idempotent so TestAll can pass the same config to Create + // and Update without observing spurious drift. + if prev.Title != genieSpace.Title || + prev.Description != genieSpace.Description || + prev.WarehouseId != genieSpace.WarehouseId || + prev.SerializedSpace != genieSpace.SerializedSpace { + prevEtag, err := strconv.Atoi(genieSpace.Etag) + if err != nil { + return Response{ + StatusCode: 500, + Body: map[string]string{ + "message": "Invalid stored etag: " + genieSpace.Etag, + }, + } + } + genieSpace.Etag = strconv.Itoa(prevEtag + 1) + } + + s.GenieSpaces[spaceId] = genieSpace + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + delete(s.GenieSpaces, spaceId) + + // Also remove the synthetic workspace file entry registered by + // GenieSpaceCreate, so a trash+recreate flow does not resolve to stale + // state via the workspace path index. + if genieSpace.ParentPath != "" { + workspacePath := genieSpace.ParentPath + if !strings.HasPrefix(workspacePath, "/Workspace") { + workspacePath = path.Join("/Workspace", workspacePath) + } + workspacePath = path.Join(workspacePath, genieSpace.Title+".geniespace") + delete(s.files, workspacePath) + } + + return Response{ + StatusCode: 200, + } +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index c78fd2b3ad1..20270d51697 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -309,6 +309,20 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.DashboardUnpublish(req) }) + // Genie Spaces: + server.Handle("GET", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceGet(req) + }) + server.Handle("POST", "/api/2.0/genie/spaces", func(req Request) any { + return req.Workspace.GenieSpaceCreate(req) + }) + server.Handle("PATCH", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceUpdate(req) + }) + server.Handle("DELETE", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceTrash(req) + }) + // Pipelines: server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req Request) any { diff --git a/libs/testserver/permissions.go b/libs/testserver/permissions.go index e7983b1afa7..962825dd037 100644 --- a/libs/testserver/permissions.go +++ b/libs/testserver/permissions.go @@ -25,6 +25,7 @@ var requestObjectTypeToObjectType = map[string]string{ "sql/alerts": "alert", "sql/queries": "query", "dashboards": "dashboard", + "genie/spaces": "genie-space", "experiments": "mlflowExperiment", "registered-models": "registered-model", "serving-endpoints": "serving-endpoint", diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index c6ac8fdd7cd..502ac472e9d 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -14,6 +14,7 @@ var resourceURLPatterns = map[string]string{ "clusters": "compute/clusters/%s", "dashboards": "dashboardsv3/%s/published", "experiments": "ml/experiments/%s", + "genie_spaces": "genie/rooms/%s", "jobs": "jobs/%s", "models": "ml/models/%s", "model_serving_endpoints": "ml/endpoints/%s", diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index fd28ff44c2d..9526cdda022 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -110,6 +110,7 @@ func TestResourceURL(t *testing.T) { {"jobs", "jobs", "123", "https://host.com/jobs/123"}, {"experiments", "experiments", "exp-1", "https://host.com/ml/experiments/exp-1"}, {"dashboards", "dashboards", "d-1", "https://host.com/dashboardsv3/d-1/published"}, + {"genie_spaces", "genie_spaces", "space-1", "https://host.com/genie/rooms/space-1"}, {"notebooks", "notebooks", "12345", "https://host.com/#notebook/12345"}, {"notebooks with path", "notebooks", "/Users/u/nb", "https://host.com/#notebook//Users/u/nb"}, {"registered_models normalizes dots", "registered_models", "cat.sch.model", "https://host.com/explore/data/models/cat/sch/model"},