diff --git a/Makefile b/Makefile index b31bb73d08..cc76b15e6e 100644 --- a/Makefile +++ b/Makefile @@ -113,6 +113,11 @@ test-update-templates: generate-out-test-toml: go test ./acceptance -run '^TestAccept$$' -only-out-test-toml -timeout=${LOCAL_TIMEOUT} +# Regenerate refschema for new resources +.PHONY: generate-refschema +generate-refschema: + go test ./acceptance -run '^TestAccept/bundle/refschema$$' -update -timeout=${LOCAL_TIMEOUT} + # Updates acceptance test output (integration tests, requires access) .PHONY: test-update-aws test-update-aws: diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 5fa1e1f15c..1dfbd77fd8 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -15,6 +15,7 @@ * Fix resource references not correctly resolved in apps config section ([#4964](https://github.com/databricks/cli/pull/4964)) * Allow run_as for dashboards with embed_credentials set to false ([#4961](https://github.com/databricks/cli/pull/4961)) * direct: Pass changed fields into update mask for apps instead of wildcard ([#4963](https://github.com/databricks/cli/pull/4963)) +* engine/direct: Added support for Vector Search Endpoints ([#4887](https://github.com/databricks/cli/pull/4887)) ### Dependency updates diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/vector_search_endpoint/databricks.yml.tmpl new file mode 100644 index 0000000000..b523fc5790 --- /dev/null +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_endpoints: + endpoint1: + name: $ENDPOINT_NAME + endpoint_type: STANDARD diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/output.txt b/acceptance/bundle/deployment/bind/vector_search_endpoint/output.txt new file mode 100644 index 0000000000..2a731b4827 --- /dev/null +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/output.txt @@ -0,0 +1,43 @@ + +>>> [CLI] vector-search-endpoints create-endpoint test-vs-endpoint-[UNIQUE_NAME] STANDARD +{ + "id": "[UUID]", + "name": "test-vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +>>> [CLI] bundle deployment bind endpoint1 test-vs-endpoint-[UNIQUE_NAME] --auto-approve +Updating deployment state... +Successfully bound vector_search_endpoint with an id 'test-vs-endpoint-[UNIQUE_NAME]' +Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] vector-search-endpoints get-endpoint test-vs-endpoint-[UNIQUE_NAME] +{ + "id": "[UUID]", + "name": "test-vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +>>> [CLI] bundle deployment unbind endpoint1 +Updating deployment state... + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! + +>>> [CLI] vector-search-endpoints get-endpoint test-vs-endpoint-[UNIQUE_NAME] +{ + "id": "[UUID]", + "name": "test-vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +>>> [CLI] vector-search-endpoints delete-endpoint test-vs-endpoint-[UNIQUE_NAME] diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/script b/acceptance/bundle/deployment/bind/vector_search_endpoint/script new file mode 100644 index 0000000000..bf45cbea78 --- /dev/null +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/script @@ -0,0 +1,23 @@ +ENDPOINT_NAME="test-vs-endpoint-$UNIQUE_NAME" +export ENDPOINT_NAME +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI vector-search-endpoints delete-endpoint "${ENDPOINT_NAME}" +} +trap cleanup EXIT + +trace $CLI vector-search-endpoints create-endpoint "${ENDPOINT_NAME}" STANDARD | jq '{id, name, endpoint_type}' + +trace $CLI bundle deployment bind endpoint1 "${ENDPOINT_NAME}" --auto-approve + +trace $CLI bundle deploy --auto-approve + +trace $CLI vector-search-endpoints get-endpoint "${ENDPOINT_NAME}" | jq '{id, name, endpoint_type}' + +trace $CLI bundle deployment unbind endpoint1 + +trace $CLI bundle destroy --auto-approve + +# Read the pre-defined endpoint again (expecting it still exists and is not deleted): +trace $CLI vector-search-endpoints get-endpoint "${ENDPOINT_NAME}" | jq '{id, name, endpoint_type}' diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/test.toml b/acceptance/bundle/deployment/bind/vector_search_endpoint/test.toml new file mode 100644 index 0000000000..bc31b13cdb --- /dev/null +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/test.toml @@ -0,0 +1,10 @@ +Local = true +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl b/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl new file mode 100644 index 0000000000..62c918f8b5 --- /dev/null +++ b/acceptance/bundle/invariant/configs/vector_search_endpoint.yml.tmpl @@ -0,0 +1,8 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + vector_search_endpoints: + foo: + name: test-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 4d44965426..cb44317795 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", "secret_scope_with_permissions.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", "secret_scope_with_permissions.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 0434791919..1b02ca219d 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -5,3 +5,6 @@ EnvMatrixExclude.no_grant_ref = ["INPUT_CONFIG=schema_grant_ref.yml.tmpl"] # Model permissions did not work until 0.297.0 https://github.com/databricks/cli/pull/4941 EnvMatrixExclude.no_model_with_permissions = ["INPUT_CONFIG=model_with_permissions.yml.tmpl"] + +# vector_search_endpoints resource is not supported on v0.293.0 +EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoint.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 4d44965426..cb44317795 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", "secret_scope_with_permissions.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", "secret_scope_with_permissions.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 5ffa32ae13..3bc78c6014 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -1,3 +1,6 @@ +# vector_search_endpoints has no terraform converter +EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoint.yml.tmpl"] + # Error: Catalog resources are only supported with direct deployment mode EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"] EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 4d44965426..cb44317795 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", "secret_scope_with_permissions.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_with_permissions.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "secret_scope_default_backend_type.yml.tmpl", "secret_scope_with_permissions.yml.tmpl", "synced_database_table.yml.tmpl", "vector_search_endpoint.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index a850fc91a1..dec3fc06d4 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -52,6 +52,7 @@ EnvMatrix.INPUT_CONFIG = [ "secret_scope_default_backend_type.yml.tmpl", "secret_scope_with_permissions.yml.tmpl", "synced_database_table.yml.tmpl", + "vector_search_endpoint.yml.tmpl", "volume.yml.tmpl", ] diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 256a5195dd..a6aa71e703 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2885,6 +2885,38 @@ resources.synced_database_tables.*.spec.source_table_full_name string ALL resources.synced_database_tables.*.spec.timeseries_key string ALL resources.synced_database_tables.*.unity_catalog_provisioning_state database.ProvisioningInfoState ALL resources.synced_database_tables.*.url string INPUT +resources.vector_search_endpoints.*.budget_policy_id string INPUT STATE +resources.vector_search_endpoints.*.creation_timestamp int64 REMOTE +resources.vector_search_endpoints.*.creator string REMOTE +resources.vector_search_endpoints.*.custom_tags []vectorsearch.CustomTag REMOTE +resources.vector_search_endpoints.*.custom_tags[*] vectorsearch.CustomTag REMOTE +resources.vector_search_endpoints.*.custom_tags[*].key string REMOTE +resources.vector_search_endpoints.*.custom_tags[*].value string REMOTE +resources.vector_search_endpoints.*.effective_budget_policy_id string REMOTE +resources.vector_search_endpoints.*.endpoint_status *vectorsearch.EndpointStatus REMOTE +resources.vector_search_endpoints.*.endpoint_status.message string REMOTE +resources.vector_search_endpoints.*.endpoint_status.state vectorsearch.EndpointStatusState REMOTE +resources.vector_search_endpoints.*.endpoint_type vectorsearch.EndpointType ALL +resources.vector_search_endpoints.*.endpoint_uuid string REMOTE +resources.vector_search_endpoints.*.id string INPUT REMOTE +resources.vector_search_endpoints.*.last_updated_timestamp int64 REMOTE +resources.vector_search_endpoints.*.last_updated_user string REMOTE +resources.vector_search_endpoints.*.lifecycle resources.Lifecycle INPUT +resources.vector_search_endpoints.*.lifecycle.prevent_destroy bool INPUT +resources.vector_search_endpoints.*.min_qps int64 INPUT STATE +resources.vector_search_endpoints.*.modified_status string INPUT +resources.vector_search_endpoints.*.name string ALL +resources.vector_search_endpoints.*.num_indexes int REMOTE +resources.vector_search_endpoints.*.scaling_info *vectorsearch.EndpointScalingInfo REMOTE +resources.vector_search_endpoints.*.scaling_info.requested_min_qps int64 REMOTE +resources.vector_search_endpoints.*.scaling_info.state vectorsearch.ScalingChangeState REMOTE +resources.vector_search_endpoints.*.url string INPUT +resources.vector_search_endpoints.*.permissions.object_id string ALL +resources.vector_search_endpoints.*.permissions[*] dresources.StatePermission ALL +resources.vector_search_endpoints.*.permissions[*].group_name string ALL +resources.vector_search_endpoints.*.permissions[*].level iam.PermissionLevel ALL +resources.vector_search_endpoints.*.permissions[*].service_principal_name string ALL +resources.vector_search_endpoints.*.permissions[*].user_name string ALL resources.volumes.*.access_point string REMOTE resources.volumes.*.browse_only bool REMOTE resources.volumes.*.catalog_name string ALL diff --git a/acceptance/bundle/resources/permissions/analyze_requests.py b/acceptance/bundle/resources/permissions/analyze_requests.py index bd540e017f..185b22df4f 100755 --- a/acceptance/bundle/resources/permissions/analyze_requests.py +++ b/acceptance/bundle/resources/permissions/analyze_requests.py @@ -3,10 +3,10 @@ Analyze all requests recorded in subtests to highlight differences between direct and terraform. """ -import os -import re import json +import re import sys +import tomllib from pathlib import Path from difflib import unified_diff @@ -91,6 +91,20 @@ def to_slash(x): return str(x).replace("\\", "/") +def load_supported_engines(path): + current = path + while True: + for name in ("out.test.toml", "test.toml"): + config_file = current / name + if config_file.exists(): + with config_file.open("rb") as fobj: + config = tomllib.load(fobj) + return set(config.get("EnvMatrix", {}).get("DATABRICKS_BUNDLE_ENGINE", [])) + if current == Path("."): + return set() + current = current.parent + + def main(): current_dir = Path(".") @@ -104,10 +118,13 @@ def main(): terraform_file = direct_file.parent / direct_file.name.replace(".direct.", ".terraform.") fname = to_slash(direct_file) + supported_engines = load_supported_engines(direct_file.parent) if terraform_file.exists(): result, diff = compare_files(direct_file, terraform_file) print(result + " " + fname + diff) + elif "terraform" not in supported_engines: + print(f"DIRECT_ONLY {fname}") else: print(f"ERROR {fname}: Missing terraform file {to_slash(terraform_file)}") diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index 32d04633f3..59038a417f 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -411,3 +411,5 @@ DIFF target_permissions/out.requests_delete.direct.json { "body": { "job_id": "[NUMID]" +DIRECT_ONLY vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json +DIRECT_ONLY vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/databricks.yml b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/databricks.yml new file mode 100644 index 0000000000..a4419ad44b --- /dev/null +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/databricks.yml @@ -0,0 +1,17 @@ +bundle: + name: test-bundle + +resources: + vector_search_endpoints: + foo: + name: vs-permissions-endpoint + endpoint_type: STANDARD + permissions: + - level: CAN_USE + user_name: viewer@example.com + - level: CAN_MANAGE + group_name: data-team + - level: CAN_MANAGE + service_principal_name: f37d18cd-98a8-4db5-8112-12dd0a6bfe38 + - level: CAN_MANAGE + user_name: tester@databricks.com diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json new file mode 100644 index 0000000000..2e15fc0556 --- /dev/null +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json @@ -0,0 +1,50 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.vector_search_endpoints.foo": { + "action": "create", + "new_state": { + "value": { + "endpoint_type": "STANDARD", + "name": "vs-permissions-endpoint" + } + } + }, + "resources.vector_search_endpoints.foo.permissions": { + "depends_on": [ + { + "node": "resources.vector_search_endpoints.foo", + "label": "${resources.vector_search_endpoints.foo.endpoint_uuid}" + } + ], + "action": "create", + "new_state": { + "value": { + "object_id": "", + "__embed__": [ + { + "level": "CAN_USE", + "user_name": "viewer@example.com" + }, + { + "level": "CAN_MANAGE", + "group_name": "data-team" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "object_id": "/vector-search-endpoints/${resources.vector_search_endpoints.foo.endpoint_uuid}" + } + } + } + } +} diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json new file mode 100644 index 0000000000..9118a4da78 --- /dev/null +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json @@ -0,0 +1,24 @@ +{ + "method": "PUT", + "path": "/api/2.0/permissions/vector-search-endpoints/[UUID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_USE", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" + }, + { + "permission_level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json new file mode 100644 index 0000000000..84c87416aa --- /dev/null +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json @@ -0,0 +1,4 @@ +{ + "method": "DELETE", + "path": "/api/2.0/vector-search/endpoints/vs-permissions-endpoint" +} diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/output.txt new file mode 100644 index 0000000000..4e848ac23e --- /dev/null +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] bundle validate -o json +[ + { + "level": "CAN_USE", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "level": "CAN_MANAGE" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/script b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/script new file mode 100644 index 0000000000..7d1e9fc8e2 --- /dev/null +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/script @@ -0,0 +1 @@ +source $TESTDIR/../../_script diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/test.toml b/acceptance/bundle/resources/permissions/vector_search_endpoints/test.toml new file mode 100644 index 0000000000..1217a2ec96 --- /dev/null +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/test.toml @@ -0,0 +1,2 @@ +Env.RESOURCE = "vector_search_endpoints" # for ../_script +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_endpoints/basic/databricks.yml.tmpl new file mode 100644 index 0000000000..05a9447fac --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: deploy-vs-endpoint-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/out.requests.direct.json b/acceptance/bundle/resources/vector_search_endpoints/basic/out.requests.direct.json new file mode 100644 index 0000000000..bcd2b5094d --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/out.requests.direct.json @@ -0,0 +1,8 @@ +{ + "method": "POST", + "path": "/api/2.0/vector-search/endpoints", + "body": { + "endpoint_type": "STANDARD", + "name": "vs-endpoint-[UNIQUE_NAME]" + } +} diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml new file mode 100644 index 0000000000..19b2c349a3 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/output.txt b/acceptance/bundle/resources/vector_search_endpoints/basic/output.txt new file mode 100644 index 0000000000..53e4194128 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/output.txt @@ -0,0 +1,56 @@ + +>>> [CLI] bundle validate +Name: deploy-vs-endpoint-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-endpoint-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: deploy-vs-endpoint-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-endpoint-[UNIQUE_NAME]/default +Resources: + Vector Search Endpoints: + my_endpoint: + Name: vs-endpoint-[UNIQUE_NAME] + URL: (not deployed) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-vs-endpoint-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] vector-search-endpoints get-endpoint vs-endpoint-[UNIQUE_NAME] +{ + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +>>> [CLI] bundle summary +Name: deploy-vs-endpoint-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-endpoint-[UNIQUE_NAME]/default +Resources: + Vector Search Endpoints: + my_endpoint: + Name: vs-endpoint-[UNIQUE_NAME] + URL: [DATABRICKS_URL]/compute/vector-search/vs-endpoint-[UNIQUE_NAME]?o=[NUMID] + +>>> print_requests.py //vector-search/endpoints + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-vs-endpoint-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/script b/acceptance/bundle/resources/vector_search_endpoints/basic/script new file mode 100644 index 0000000000..e68232aab3 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/script @@ -0,0 +1,22 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate + +trace $CLI bundle summary + +rm -f out.requests.txt +trace $CLI bundle deploy + +# Get endpoint details +endpoint_name="vs-endpoint-${UNIQUE_NAME}" +trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}' + +trace $CLI bundle summary + +trace print_requests.py //vector-search/endpoints > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/test.toml b/acceptance/bundle/resources/vector_search_endpoints/basic/test.toml new file mode 100644 index 0000000000..f8b3bbe49d --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/test.toml @@ -0,0 +1 @@ +# All configuration inherited from parent test.toml diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/databricks.yml.tmpl new file mode 100644 index 0000000000..5dce5dd5d4 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: drift-vs-endpoint-budget-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/output.txt b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/output.txt new file mode 100644 index 0000000000..da4664c38e --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/output.txt @@ -0,0 +1,26 @@ + +=== Initial deployment (no budget_policy_id) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-budget-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Simulate remote change: set effective_budget_policy_id (e.g. inherited workspace policy) +>>> [CLI] vector-search-endpoints update-endpoint-budget-policy vs-endpoint-[UNIQUE_NAME] inherited-policy +{ + "effective_budget_policy_id":"inherited-policy" +} + +=== Plan shows no drift: remote budget_policy_id changes are ignored +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-budget-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/script b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/script new file mode 100644 index 0000000000..2af3f3ed02 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Initial deployment (no budget_policy_id)" +trace $CLI bundle deploy + +endpoint_name="vs-endpoint-${UNIQUE_NAME}" + +title "Simulate remote change: set effective_budget_policy_id (e.g. inherited workspace policy)" +trace $CLI vector-search-endpoints update-endpoint-budget-policy "${endpoint_name}" "inherited-policy" + +title "Plan shows no drift: remote budget_policy_id changes are ignored" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/test.toml new file mode 100644 index 0000000000..18b1a88417 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy_ignored/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/databricks.yml.tmpl new file mode 100644 index 0000000000..7936e98b23 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: drift-vs-endpoint-min-qps-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD + min_qps: 1 diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/output.txt b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/output.txt new file mode 100644 index 0000000000..294d7061a4 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/output.txt @@ -0,0 +1,62 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-min-qps-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Simulate remote drift: change min_qps to 5 outside the bundle +>>> [CLI] vector-search-endpoints patch-endpoint vs-endpoint-[UNIQUE_NAME] --min-qps 5 +{ + "creation_timestamp":[UNIX_TIME_MILLIS][0], + "creator":"[USERNAME]", + "endpoint_status": { + "state":"ONLINE" + }, + "endpoint_type":"STANDARD", + "id":"[UUID]", + "last_updated_timestamp":[UNIX_TIME_MILLIS][1], + "last_updated_user":"[USERNAME]", + "name":"vs-endpoint-[UNIQUE_NAME]", + "scaling_info": { + "requested_min_qps":5 + } +} + +=== Plan detects drift and proposes update +>>> [CLI] bundle plan +update vector_search_endpoints.my_endpoint + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +=== Deploy restores min_qps to 1 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-min-qps-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //vector-search/endpoints +{ + "method": "PATCH", + "path": "/api/2.0/vector-search/endpoints/vs-endpoint-[UNIQUE_NAME]", + "body": { + "min_qps": 1 + } +} + +>>> [CLI] vector-search-endpoints get-endpoint vs-endpoint-[UNIQUE_NAME] +{ + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/drift-vs-endpoint-min-qps-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/script b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/script new file mode 100644 index 0000000000..ea9a46982a --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/script @@ -0,0 +1,25 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Initial deployment" +trace $CLI bundle deploy + +endpoint_name="vs-endpoint-${UNIQUE_NAME}" + +title "Simulate remote drift: change min_qps to 5 outside the bundle" +trace $CLI vector-search-endpoints patch-endpoint "${endpoint_name}" --min-qps 5 + +title "Plan detects drift and proposes update" +trace $CLI bundle plan + +title "Deploy restores min_qps to 1" +rm -f out.requests.txt +trace $CLI bundle deploy +trace print_requests.py '//vector-search/endpoints' + +trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}' diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/test.toml new file mode 100644 index 0000000000..18b1a88417 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/min_qps/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/databricks.yml.tmpl new file mode 100644 index 0000000000..b4528cc03b --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: recreate-vs-endpoint-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.requests.create.direct.json b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.requests.create.direct.json new file mode 100644 index 0000000000..bcd2b5094d --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.requests.create.direct.json @@ -0,0 +1,8 @@ +{ + "method": "POST", + "path": "/api/2.0/vector-search/endpoints", + "body": { + "endpoint_type": "STANDARD", + "name": "vs-endpoint-[UNIQUE_NAME]" + } +} diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.requests.recreate.direct.json b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.requests.recreate.direct.json new file mode 100644 index 0000000000..90e1aff4cc --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.requests.recreate.direct.json @@ -0,0 +1,12 @@ +{ + "method": "DELETE", + "path": "/api/2.0/vector-search/endpoints/vs-endpoint-[UNIQUE_NAME]" +} +{ + "method": "POST", + "path": "/api/2.0/vector-search/endpoints", + "body": { + "endpoint_type": "STORAGE_OPTIMIZED", + "name": "vs-endpoint-[UNIQUE_NAME]" + } +} diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/output.txt b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/output.txt new file mode 100644 index 0000000000..e9f05865ae --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/output.txt @@ -0,0 +1,52 @@ + +=== Initial deployment with STANDARD endpoint_type +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-vs-endpoint-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --keep //vector-search/endpoints + +=== Change endpoint_type (should trigger recreation) +>>> update_file.py databricks.yml endpoint_type: STANDARD endpoint_type: STORAGE_OPTIMIZED + +>>> [CLI] bundle plan +Warning: invalid value "STORAGE_OPTIMIZED" for enum field. Valid values are [STANDARD] + at resources.vector_search_endpoints.my_endpoint.endpoint_type + in databricks.yml:11:22 + +recreate vector_search_endpoints.my_endpoint + +Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged + +>>> [CLI] bundle deploy --auto-approve +Warning: invalid value "STORAGE_OPTIMIZED" for enum field. Valid values are [STANDARD] + at resources.vector_search_endpoints.my_endpoint.endpoint_type + in databricks.yml:11:22 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-vs-endpoint-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --keep //vector-search/endpoints + +>>> [CLI] vector-search-endpoints get-endpoint vs-endpoint-[UNIQUE_NAME] +{ + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STORAGE_OPTIMIZED" +} + +>>> [CLI] bundle destroy --auto-approve +Warning: invalid value "STORAGE_OPTIMIZED" for enum field. Valid values are [STANDARD] + at resources.vector_search_endpoints.my_endpoint.endpoint_type + in databricks.yml:11:22 + +The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-vs-endpoint-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/script b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/script new file mode 100644 index 0000000000..b3920cacbb --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/script @@ -0,0 +1,31 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +print_requests() { + local name=$1 + trace print_requests.py --keep '//vector-search/endpoints' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + rm -f out.requests.txt +} + +title "Initial deployment with STANDARD endpoint_type" +rm -f out.requests.txt +trace $CLI bundle deploy + +print_requests create + +title "Change endpoint_type (should trigger recreation)" +trace update_file.py databricks.yml "endpoint_type: STANDARD" "endpoint_type: STORAGE_OPTIMIZED" + +trace $CLI bundle plan +rm -f out.requests.txt +trace $CLI bundle deploy --auto-approve + +print_requests recreate + +endpoint_name="vs-endpoint-${UNIQUE_NAME}" +trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}' diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/test.toml new file mode 100644 index 0000000000..18b1a88417 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/bundle/resources/vector_search_endpoints/test.toml b/acceptance/bundle/resources/vector_search_endpoints/test.toml new file mode 100644 index 0000000000..0d3f0e1ca3 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/test.toml @@ -0,0 +1,10 @@ +Local = true +Cloud = true + +# Vector Search endpoints are only available in direct mode (no Terraform provider) +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +Ignore = [ + "databricks.yml", + ".databricks", +] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/databricks.yml.tmpl new file mode 100644 index 0000000000..5dfbdf6c35 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: update-vs-endpoint-budget-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.requests.create.direct.json b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.requests.create.direct.json new file mode 100644 index 0000000000..bcd2b5094d --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.requests.create.direct.json @@ -0,0 +1,8 @@ +{ + "method": "POST", + "path": "/api/2.0/vector-search/endpoints", + "body": { + "endpoint_type": "STANDARD", + "name": "vs-endpoint-[UNIQUE_NAME]" + } +} diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.requests.update.direct.json b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.requests.update.direct.json new file mode 100644 index 0000000000..3637585440 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.requests.update.direct.json @@ -0,0 +1,7 @@ +{ + "method": "PATCH", + "path": "/api/2.0/vector-search/endpoints/vs-endpoint-[UNIQUE_NAME]/budget-policy", + "body": { + "budget_policy_id": "test-policy-id" + } +} diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/output.txt b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/output.txt new file mode 100644 index 0000000000..3b2bb7bf1c --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/output.txt @@ -0,0 +1,35 @@ + +=== Initial deployment (no budget policy) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-vs-endpoint-budget-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --keep //vector-search/endpoints + +=== Add budget_policy_id +>>> update_file.py databricks.yml endpoint_type: STANDARD endpoint_type: STANDARD + budget_policy_id: test-policy-id + +>>> [CLI] bundle plan +update vector_search_endpoints.my_endpoint + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-vs-endpoint-budget-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --keep //vector-search/endpoints + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-vs-endpoint-budget-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/script b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/script new file mode 100644 index 0000000000..8d19b9f60c --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/script @@ -0,0 +1,29 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +print_requests() { + local name=$1 + trace print_requests.py --keep '//vector-search/endpoints' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + rm -f out.requests.txt +} + +title "Initial deployment (no budget policy)" +rm -f out.requests.txt +trace $CLI bundle deploy + +print_requests create + +title "Add budget_policy_id" +trace update_file.py databricks.yml "endpoint_type: STANDARD" "endpoint_type: STANDARD + budget_policy_id: test-policy-id" + +trace $CLI bundle plan +rm -f out.requests.txt +trace $CLI bundle deploy + +print_requests update diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/test.toml new file mode 100644 index 0000000000..18b1a88417 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/test.toml @@ -0,0 +1 @@ +Cloud = false diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/databricks.yml.tmpl b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/databricks.yml.tmpl new file mode 100644 index 0000000000..7c326b69d5 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/databricks.yml.tmpl @@ -0,0 +1,15 @@ +bundle: + name: update-vs-endpoint-min-qps-$UNIQUE_NAME + +sync: + paths: [] + +resources: + vector_search_endpoints: + my_endpoint: + name: vs-endpoint-$UNIQUE_NAME + endpoint_type: STANDARD + min_qps: 1 + permissions: + - level: CAN_USE + group_name: admins diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/out.requests.create.direct.json b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/out.requests.create.direct.json new file mode 100644 index 0000000000..0a1d51a351 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/out.requests.create.direct.json @@ -0,0 +1,25 @@ +{ + "method": "POST", + "path": "/api/2.0/vector-search/endpoints", + "body": { + "endpoint_type": "STANDARD", + "min_qps": 1, + "name": "vs-endpoint-[UNIQUE_NAME]" + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/vector-search-endpoints/[UUID]", + "body": { + "access_control_list": [ + { + "group_name": "admins", + "permission_level": "CAN_USE" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/out.requests.update.direct.json b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/out.requests.update.direct.json new file mode 100644 index 0000000000..24876c67be --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/out.requests.update.direct.json @@ -0,0 +1,23 @@ +{ + "method": "PATCH", + "path": "/api/2.0/vector-search/endpoints/vs-endpoint-[UNIQUE_NAME]", + "body": { + "min_qps": 2 + } +} +{ + "method": "PUT", + "path": "/api/2.0/permissions/vector-search-endpoints/[UUID]", + "body": { + "access_control_list": [ + { + "group_name": "admins", + "permission_level": "CAN_USE" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/output.txt b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/output.txt new file mode 100644 index 0000000000..b77e88de53 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/output.txt @@ -0,0 +1,47 @@ + +=== Initial deployment +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-vs-endpoint-min-qps-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --keep //vector-search/endpoints //permissions + +>>> [CLI] vector-search-endpoints get-endpoint vs-endpoint-[UNIQUE_NAME] +{ + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +=== Update min_qps +>>> update_file.py databricks.yml min_qps: 1 min_qps: 2 + +>>> [CLI] bundle plan +update vector_search_endpoints.my_endpoint +update vector_search_endpoints.my_endpoint.permissions + +Plan: 0 to add, 2 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-vs-endpoint-min-qps-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py --keep //vector-search/endpoints //permissions + +>>> [CLI] vector-search-endpoints get-endpoint vs-endpoint-[UNIQUE_NAME] +{ + "name": "vs-endpoint-[UNIQUE_NAME]", + "endpoint_type": "STANDARD" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.vector_search_endpoints.my_endpoint + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-vs-endpoint-min-qps-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/script b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/script new file mode 100644 index 0000000000..a466d7b383 --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/script @@ -0,0 +1,33 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +print_requests() { + local name=$1 + trace print_requests.py --keep '//vector-search/endpoints' '//permissions' > out.requests.${name}.$DATABRICKS_BUNDLE_ENGINE.json + rm -f out.requests.txt +} + +title "Initial deployment" +rm -f out.requests.txt +trace $CLI bundle deploy + +print_requests create + +endpoint_name="vs-endpoint-${UNIQUE_NAME}" +trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}' + +title "Update min_qps" +trace update_file.py databricks.yml "min_qps: 1" "min_qps: 2" + +trace $CLI bundle plan +rm -f out.requests.txt +trace $CLI bundle deploy + +print_requests update + +trace $CLI vector-search-endpoints get-endpoint "${endpoint_name}" | jq '{name, endpoint_type}' diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/test.toml new file mode 100644 index 0000000000..ac17c7f22f --- /dev/null +++ b/acceptance/bundle/resources/vector_search_endpoints/update/min_qps/test.toml @@ -0,0 +1,2 @@ +Cloud = false +Badness = "Updating min_qps also plans and applies permissions due to unresolved permissions object_id during planning" diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index 73ce556868..fd019479d7 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -18,7 +18,8 @@ import ( var ( allowedLevels = []string{permissions.CAN_MANAGE, permissions.CAN_VIEW, permissions.CAN_RUN} - levelsMap = map[string](map[string]string){ + // Map of allowed permission levels to the corresponding permission level of specific resources + levelsMap = map[string](map[string]string){ "jobs": { permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_VIEW", @@ -78,6 +79,11 @@ var ( permissions.CAN_VIEW: "CAN_ATTACH_TO", permissions.CAN_RUN: "CAN_RESTART", }, + "vector_search_endpoints": { + // https://docs.databricks.com/aws/en/security/auth/access-control/#vector-search-endpoint-acls + permissions.CAN_MANAGE: "CAN_MANAGE", + permissions.CAN_VIEW: "CAN_USE", + }, } ) diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index ba12113034..c347de79df 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -78,6 +78,10 @@ func TestApplyBundlePermissions(t *testing.T) { "app_1": {}, "app_2": {}, }, + VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ + "vs_1": {}, + "vs_2": {}, + }, }, }, } @@ -138,6 +142,14 @@ func TestApplyBundlePermissions(t *testing.T) { require.Len(t, b.Config.Resources.Apps["app_1"].Permissions, 2) require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.AppPermission{Level: "CAN_MANAGE", UserName: "TestUser"}) require.Contains(t, b.Config.Resources.Apps["app_1"].Permissions, resources.AppPermission{Level: "CAN_USE", GroupName: "TestGroup"}) + + require.Len(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, 2) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_1"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"}) + + require.Len(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, 2) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_MANAGE", UserName: "TestUser"}) + require.Contains(t, b.Config.Resources.VectorSearchEndpoints["vs_2"].Permissions, resources.Permission{Level: "CAN_USE", GroupName: "TestGroup"}) } func TestWarningOnOverlapPermission(t *testing.T) { diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index dd6625633c..d5c97266cd 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -290,6 +290,14 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos } } + // Vector Search Endpoints: Prefix + for _, e := range r.VectorSearchEndpoints { + if e == nil { + continue + } + e.Name = normalizePrefix(prefix) + e.Name + } + return diags } diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 804552f56a..ce299d341b 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -23,6 +23,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/postgres" "github.com/databricks/databricks-sdk-go/service/serving" "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -246,6 +247,14 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ + "vs_endpoint1": { + CreateEndpoint: vectorsearch.CreateEndpoint{ + Name: "vs_endpoint1", + EndpointType: vectorsearch.EndpointTypeStandard, + }, + }, + }, }, }, SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"), @@ -294,6 +303,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Model serving endpoint 1 assert.Equal(t, "dev_lennart_servingendpoint1", b.Config.Resources.ModelServingEndpoints["servingendpoint1"].Name) + // Vector search endpoint 1 + assert.Equal(t, "dev_lennart_vs_endpoint1", b.Config.Resources.VectorSearchEndpoints["vs_endpoint1"].Name) + // Registered model 1 assert.Equal(t, "dev_lennart_registeredmodel1", b.Config.Resources.RegisteredModels["registeredmodel1"].Name) diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 9616de202a..2eb292cfbb 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -116,8 +116,8 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { DashboardFixups(), // Reads (typed): b.Config.Permissions (validates permission levels) - // Reads (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps}.*.permissions (reads existing permissions) - // Updates (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps}.*.permissions (adds permissions from bundle-level configuration) + // Reads (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (reads existing permissions) + // Updates (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (adds permissions from bundle-level configuration) // Applies bundle-level permissions to all supported resources ApplyBundlePermissions(), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 5ed9edad54..0b7003f587 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -54,6 +54,7 @@ func allResourceTypes(t *testing.T) []string { "secret_scopes", "sql_warehouses", "synced_database_tables", + "vector_search_endpoints", "volumes", }, resourceTypes, @@ -180,6 +181,7 @@ var allowList = []string{ "schemas", "secret_scopes", "sql_warehouses", + "vector_search_endpoints", "volumes", } diff --git a/bundle/config/mutator/validate_direct_only_resources.go b/bundle/config/mutator/validate_direct_only_resources.go index 54e2924848..5717497205 100644 --- a/bundle/config/mutator/validate_direct_only_resources.go +++ b/bundle/config/mutator/validate_direct_only_resources.go @@ -42,6 +42,18 @@ var directOnlyResources = []directOnlyResource{ return result }, }, + { + resourceType: "vector_search_endpoints", + pluralName: "Vector Search Endpoint", + singularName: "vector search endpoint", + getResources: func(b *bundle.Bundle) map[string]any { + result := make(map[string]any) + for k, v := range b.Config.Resources.VectorSearchEndpoints { + result[k] = v + } + return result + }, + }, } type validateDirectOnlyResources struct { diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 4131f68695..225ec32165 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -35,6 +35,7 @@ type Resources struct { PostgresProjects map[string]*resources.PostgresProject `json:"postgres_projects,omitempty"` PostgresBranches map[string]*resources.PostgresBranch `json:"postgres_branches,omitempty"` PostgresEndpoints map[string]*resources.PostgresEndpoint `json:"postgres_endpoints,omitempty"` + VectorSearchEndpoints map[string]*resources.VectorSearchEndpoint `json:"vector_search_endpoints,omitempty"` } type ConfigResource interface { @@ -111,6 +112,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["postgres_projects"], r.PostgresProjects), collectResourceMap(descriptions["postgres_branches"], r.PostgresBranches), collectResourceMap(descriptions["postgres_endpoints"], r.PostgresEndpoints), + collectResourceMap(descriptions["vector_search_endpoints"], r.VectorSearchEndpoints), } } @@ -165,5 +167,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "postgres_projects": (&resources.PostgresProject{}).ResourceDescription(), "postgres_branches": (&resources.PostgresBranch{}).ResourceDescription(), "postgres_endpoints": (&resources.PostgresEndpoint{}).ResourceDescription(), + "vector_search_endpoints": (&resources.VectorSearchEndpoint{}).ResourceDescription(), } } diff --git a/bundle/config/resources/permission_types.go b/bundle/config/resources/permission_types.go index a40b5c8eec..3029ee40b8 100644 --- a/bundle/config/resources/permission_types.go +++ b/bundle/config/resources/permission_types.go @@ -22,6 +22,7 @@ func (p Permission) String() string { return PermissionT[iam.PermissionLevel](p).String() } +// If the SDK exposes a resource's permission level, add it here. type ( AppPermission PermissionT[apps.AppPermissionLevel] ClusterPermission PermissionT[compute.ClusterPermissionLevel] diff --git a/bundle/config/resources/vector_search_endpoint.go b/bundle/config/resources/vector_search_endpoint.go new file mode 100644 index 0000000000..4f4ee66e75 --- /dev/null +++ b/bundle/config/resources/vector_search_endpoint.go @@ -0,0 +1,64 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" +) + +type VectorSearchEndpoint struct { + BaseResource + vectorsearch.CreateEndpoint + + Permissions []Permission `json:"permissions,omitempty"` +} + +func (e *VectorSearchEndpoint) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, e) +} + +func (e VectorSearchEndpoint) MarshalJSON() ([]byte, error) { + return marshal.Marshal(e) +} + +func (e *VectorSearchEndpoint) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.VectorSearchEndpoints.GetEndpoint(ctx, vectorsearch.GetEndpointRequest{EndpointName: name}) + if err != nil { + log.Debugf(ctx, "vector search endpoint %s does not exist: %v", name, err) + if apierr.IsMissing(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (e *VectorSearchEndpoint) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "vector_search_endpoint", + PluralName: "vector_search_endpoints", + SingularTitle: "Vector Search Endpoint", + PluralTitle: "Vector Search Endpoints", + } +} + +func (e *VectorSearchEndpoint) InitializeURL(baseURL url.URL) { + if e.ID == "" { + return + } + baseURL.Path = "compute/vector-search/" + e.Name + e.URL = baseURL.String() +} + +func (e *VectorSearchEndpoint) GetName() string { + return e.Name +} + +func (e *VectorSearchEndpoint) GetURL() string { + return e.URL +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index ddc90209e8..d23b28e104 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -21,6 +21,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/ml" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/postgres" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/assert" @@ -239,6 +240,14 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ + "my_vector_search_endpoint": { + CreateEndpoint: vectorsearch.CreateEndpoint{ + Name: "my_vector_search_endpoint", + EndpointType: vectorsearch.EndpointTypeStandard, + }, + }, + }, } unbindableResources := map[string]bool{ "model": true, @@ -270,6 +279,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPostgresAPI().EXPECT().GetProject(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetBranch(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockPostgresAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockVectorSearchEndpointsAPI().EXPECT().GetEndpoint(mock.Anything, mock.Anything).Return(nil, nil) allResources := supportedResources.AllResources() for _, group := range allResources { diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index b07b488890..7f56248bb4 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -17,6 +17,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { ignoredResources := []string{ "catalogs", "external_locations", + "vector_search_endpoints", } for resourceType := range supportedResources { diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 6a7381a3fc..ddc30c41f5 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -30,6 +30,7 @@ var SupportedResources = map[string]any{ "secret_scopes": (*ResourceSecretScope)(nil), "model_serving_endpoints": (*ResourceModelServingEndpoint)(nil), "quality_monitors": (*ResourceQualityMonitor)(nil), + "vector_search_endpoints": (*ResourceVectorSearchEndpoint)(nil), // Permissions "jobs.permissions": (*ResourcePermissions)(nil), @@ -45,6 +46,7 @@ var SupportedResources = map[string]any{ "secret_scopes.permissions": (*ResourceSecretScopeAcls)(nil), "model_serving_endpoints.permissions": (*ResourcePermissions)(nil), "dashboards.permissions": (*ResourcePermissions)(nil), + "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), // Grants "catalogs.grants": (*ResourceGrants)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 63caa5cfed..2f777d7093 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -27,6 +27,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/postgres" "github.com/databricks/databricks-sdk-go/service/serving" "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" "github.com/databricks/databricks-sdk-go/service/workspace" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -239,6 +240,13 @@ var testConfig map[string]any = map[string]any{ DatasetSchema: "myschema", }, }, + + "vector_search_endpoints": &resources.VectorSearchEndpoint{ + CreateEndpoint: vectorsearch.CreateEndpoint{ + Name: "my-endpoint", + EndpointType: vectorsearch.EndpointTypeStandard, + }, + }, } type prepareWorkspace func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) @@ -473,6 +481,24 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "vector_search_endpoints.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + waiter, err := client.VectorSearchEndpoints.CreateEndpoint(ctx, vectorsearch.CreateEndpoint{ + Name: "vs-endpoint-permissions", + EndpointType: vectorsearch.EndpointTypeStandard, + }) + if err != nil { + return nil, err + } + + return &PermissionsState{ + ObjectID: "/vector-search-endpoints/" + waiter.Response.Id, + EmbeddedSlice: []StatePermission{{ + Level: "CAN_MANAGE", + UserName: "user@example.com", + }}, + }, nil + }, + "alerts.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { resp, err := client.AlertsV2.CreateAlert(ctx, sql.CreateAlertV2Request{ Alert: sql.AlertV2{ diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 8dfabd1098..101b73b6ab 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -44,4 +44,6 @@ sql_warehouses: sql.EditWarehouseRequest synced_database_tables: database.SyncedDatabaseTable +vector_search_endpoints: vectorsearch.CreateEndpoint + volumes: catalog.CreateVolumeRequestContent diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 40d4de5487..2b1ab92860 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -26,6 +26,7 @@ var permissionResourceToObjectType = map[string]string{ "model_serving_endpoints": "/serving-endpoints/", "pipelines": "/pipelines/", "sql_warehouses": "/sql/warehouses/", + "vector_search_endpoints": "/vector-search-endpoints/", } type ResourcePermissions struct { @@ -90,6 +91,11 @@ func PreparePermissionsInputConfig(inputConfig any, node string) (*structvar.Str objectIdRef = prefix + "${" + baseNode + ".model_id}" } + // Vector search endpoints use the endpoint name as deployment id; the permissions API uses endpoint UUID. + if strings.HasPrefix(baseNode, "resources.vector_search_endpoints.") { + objectIdRef = prefix + "${" + baseNode + ".endpoint_uuid}" + } + // Postgres projects store their hierarchical name ("projects/{project_id}") as the state ID, // but the permissions API expects just the project_id. if strings.HasPrefix(baseNode, "resources.postgres_projects.") { diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index 5a02c184f4..09e5568f14 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -289,4 +289,6 @@ resources: - field: unity_catalog_provisioning_state reason: spec:output_only + # vector_search_endpoints: no api field behaviors + # volumes: no api field behaviors diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 51a8f7b8a2..8b24b12320 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -500,3 +500,13 @@ resources: reason: immutable - field: endpoint_id reason: immutable + + vector_search_endpoints: + recreate_on_changes: + - field: endpoint_type + reason: immutable + ignore_remote_changes: + # The API returns effective_budget_policy_id which may include inherited workspace policies, + # not the user-set budget_policy_id. Ignore until the API exposes the user-set value directly. + - field: budget_policy_id + reason: effective_vs_requested diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 3321725049..0882844972 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -74,6 +74,10 @@ var knownMissingInRemoteType = map[string][]string{ "pg_version", "project_id", }, + "vector_search_endpoints": { + "budget_policy_id", + "min_qps", + }, } // commonMissingInStateType lists fields that are commonly missing across all resource types. diff --git a/bundle/direct/dresources/vector_search_endpoint.go b/bundle/direct/dresources/vector_search_endpoint.go new file mode 100644 index 0000000000..531c04d2c7 --- /dev/null +++ b/bundle/direct/dresources/vector_search_endpoint.go @@ -0,0 +1,125 @@ +package dresources + +import ( + "context" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" +) + +var ( + pathBudgetPolicyId = structpath.MustParsePath("budget_policy_id") + pathMinQps = structpath.MustParsePath("min_qps") +) + +// VectorSearchRefreshOutput is remote state for a vector search endpoint. It embeds API response +// fields for drift comparison and adds endpoint_uuid for permissions; deployment state id remains the endpoint name. +type VectorSearchRefreshOutput struct { + *vectorsearch.EndpointInfo + EndpointUuid string `json:"endpoint_uuid"` +} + +func newVectorSearchRefreshOutput(info *vectorsearch.EndpointInfo) *VectorSearchRefreshOutput { + if info == nil { + return nil + } + return &VectorSearchRefreshOutput{ + EndpointInfo: info, + EndpointUuid: info.Id, + } +} + +type ResourceVectorSearchEndpoint struct { + client *databricks.WorkspaceClient +} + +func (*ResourceVectorSearchEndpoint) New(client *databricks.WorkspaceClient) *ResourceVectorSearchEndpoint { + return &ResourceVectorSearchEndpoint{client: client} +} + +func (*ResourceVectorSearchEndpoint) PrepareState(input *resources.VectorSearchEndpoint) *vectorsearch.CreateEndpoint { + return &input.CreateEndpoint +} + +func (*ResourceVectorSearchEndpoint) RemapState(remote *VectorSearchRefreshOutput) *vectorsearch.CreateEndpoint { + if remote == nil || remote.EndpointInfo == nil { + return &vectorsearch.CreateEndpoint{ + BudgetPolicyId: "", + EndpointType: "", + MinQps: 0, + Name: "", + ForceSendFields: nil, + } + } + info := remote.EndpointInfo + budgetPolicyId := info.EffectiveBudgetPolicyId // TODO: use info.BudgetPolicyId when available + var minQps int64 + if info.ScalingInfo != nil { + minQps = info.ScalingInfo.RequestedMinQps + } + return &vectorsearch.CreateEndpoint{ + Name: info.Name, + EndpointType: info.EndpointType, + BudgetPolicyId: budgetPolicyId, + MinQps: minQps, + ForceSendFields: utils.FilterFields[vectorsearch.CreateEndpoint](info.ForceSendFields), + } +} + +func (r *ResourceVectorSearchEndpoint) DoRead(ctx context.Context, id string) (*VectorSearchRefreshOutput, error) { + info, err := r.client.VectorSearchEndpoints.GetEndpointByEndpointName(ctx, id) + if err != nil { + return nil, err + } + return newVectorSearchRefreshOutput(info), nil +} + +func (r *ResourceVectorSearchEndpoint) DoCreate(ctx context.Context, config *vectorsearch.CreateEndpoint) (string, *VectorSearchRefreshOutput, error) { + waiter, err := r.client.VectorSearchEndpoints.CreateEndpoint(ctx, *config) + if err != nil { + return "", nil, err + } + id := config.Name + return id, newVectorSearchRefreshOutput(waiter.Response), nil +} + +func (r *ResourceVectorSearchEndpoint) WaitAfterCreate(ctx context.Context, config *vectorsearch.CreateEndpoint) (*VectorSearchRefreshOutput, error) { + info, err := r.client.VectorSearchEndpoints.WaitGetEndpointVectorSearchEndpointOnline(ctx, config.Name, 60*time.Minute, nil) + if err != nil { + return nil, err + } + return newVectorSearchRefreshOutput(info), nil +} + +func (r *ResourceVectorSearchEndpoint) DoUpdate(ctx context.Context, id string, config *vectorsearch.CreateEndpoint, entry *PlanEntry) (*VectorSearchRefreshOutput, error) { + if entry.Changes.HasChange(pathBudgetPolicyId) { + _, err := r.client.VectorSearchEndpoints.UpdateEndpointBudgetPolicy(ctx, vectorsearch.PatchEndpointBudgetPolicyRequest{ + EndpointName: id, + BudgetPolicyId: config.BudgetPolicyId, + }) + if err != nil { + return nil, err + } + } + + if entry.Changes.HasChange(pathMinQps) { + _, err := r.client.VectorSearchEndpoints.PatchEndpoint(ctx, vectorsearch.PatchEndpointRequest{ + EndpointName: id, + MinQps: config.MinQps, + ForceSendFields: utils.FilterFields[vectorsearch.PatchEndpointRequest](config.ForceSendFields, "MinQps"), + }) + if err != nil { + return nil, err + } + } + + return nil, nil +} + +func (r *ResourceVectorSearchEndpoint) DoDelete(ctx context.Context, id string) error { + return r.client.VectorSearchEndpoints.DeleteEndpointByEndpointName(ctx, id) +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 459d2c19f6..6df092c4fb 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -249,6 +249,9 @@ github.com/databricks/cli/bundle/config.Resources: "synced_database_tables": "description": |- PLACEHOLDER + "vector_search_endpoints": + "description": |- + PLACEHOLDER "volumes": "description": |- The volume definitions for the bundle, where each key is the name of the volume. @@ -955,6 +958,25 @@ github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable: "unity_catalog_provisioning_state": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint: + "budget_policy_id": + "description": |- + PLACEHOLDER + "endpoint_type": + "description": |- + PLACEHOLDER + "lifecycle": + "description": |- + PLACEHOLDER + "min_qps": + "description": |- + PLACEHOLDER + "name": + "description": |- + PLACEHOLDER + "permissions": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/variable.Lookup: "alert": "description": |- diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index c196c7799a..845b9ddea8 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -1116,6 +1116,22 @@ github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable: may be in "PROVISIONING" as it runs asynchronously). "x-databricks-field-behaviors_output_only": |- true +github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint: + "budget_policy_id": + "description": |- + The budget policy id to be applied + "x-databricks-preview": |- + PRIVATE + "endpoint_type": + "description": |- + Type of endpoint + "min_qps": + "description": |- + Min QPS for the endpoint. Mutually exclusive with num_replicas. + The actual replica count is calculated at index creation/sync time based on this value. + "name": + "description": |- + Name of the vector search endpoint github.com/databricks/cli/bundle/config/resources.Volume: "catalog_name": "description": |- @@ -5934,6 +5950,13 @@ github.com/databricks/databricks-sdk-go/service/sql.WarehousePermissionLevel: CAN_MONITOR - |- CAN_VIEW +github.com/databricks/databricks-sdk-go/service/vectorsearch.EndpointType: + "_": + "description": |- + Type of endpoint. + "enum": + - |- + STANDARD github.com/databricks/databricks-sdk-go/service/workspace.AzureKeyVaultSecretScopeMetadata: "_": "description": |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 611289083e..4f2550db00 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -538,6 +538,10 @@ github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable: "lifecycle": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint: + "lifecycle": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Volume: "_": "markdown_description": |- diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index c1e098ed80..7de0479250 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -180,6 +180,9 @@ var EnumFields = map[string][]string{ "resources.synced_database_tables.*.spec.scheduling_policy": {"CONTINUOUS", "SNAPSHOT", "TRIGGERED"}, "resources.synced_database_tables.*.unity_catalog_provisioning_state": {"ACTIVE", "DEGRADED", "DELETING", "FAILED", "PROVISIONING", "UPDATING"}, + "resources.vector_search_endpoints.*.endpoint_type": {"STANDARD"}, + "resources.vector_search_endpoints.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, + "resources.volumes.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, "resources.volumes.*.volume_type": {"EXTERNAL", "MANAGED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index d90345f83f..331008b8c1 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -238,6 +238,9 @@ var RequiredFields = map[string][]string{ "resources.synced_database_tables.*": {"name"}, + "resources.vector_search_endpoints.*": {"endpoint_type", "name"}, + "resources.vector_search_endpoints.*.permissions[*]": {"level"}, + "resources.volumes.*": {"catalog_name", "name", "schema_name", "volume_type"}, "scripts.*": {"content"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 993adec793..087c1f7c58 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1925,6 +1925,44 @@ } ] }, + "resources.VectorSearchEndpoint": { + "oneOf": [ + { + "type": "object", + "properties": { + "budget_policy_id": { + "$ref": "#/$defs/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "endpoint_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.EndpointType" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "min_qps": { + "$ref": "#/$defs/int64" + }, + "name": { + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + } + }, + "additionalProperties": false, + "required": [ + "endpoint_type", + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Volume": { "oneOf": [ { @@ -2510,6 +2548,9 @@ "synced_database_tables": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable" }, + "vector_search_endpoints": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint" + }, "volumes": { "description": "The volume definitions for the bundle, where each key is the name of the volume.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume", @@ -10566,6 +10607,21 @@ } ] }, + "vectorsearch.EndpointType": { + "oneOf": [ + { + "type": "string", + "description": "Type of endpoint.", + "enum": [ + "STANDARD" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "workspace.AzureKeyVaultSecretScopeMetadata": { "oneOf": [ { @@ -10982,6 +11038,20 @@ } ] }, + "resources.VectorSearchEndpoint": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Volume": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index bcb6866296..0d8d9d1a4c 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -1895,6 +1895,36 @@ "name" ] }, + "resources.VectorSearchEndpoint": { + "type": "object", + "properties": { + "budget_policy_id": { + "$ref": "#/$defs/string", + "x-databricks-preview": "PRIVATE", + "doNotSuggest": true + }, + "endpoint_type": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/vectorsearch.EndpointType" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "min_qps": { + "$ref": "#/$defs/int64" + }, + "name": { + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + } + }, + "additionalProperties": false, + "required": [ + "endpoint_type", + "name" + ] + }, "resources.Volume": { "type": "object", "properties": { @@ -2466,6 +2496,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable", "x-since-version": "v0.266.0" }, + "vector_search_endpoints": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint" + }, "volumes": { "description": "The volume definitions for the bundle, where each key is the name of the volume.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume", @@ -2592,6 +2625,11 @@ "config.Workspace": { "type": "object", "properties": { + "account_id": { + "description": "The Databricks account ID.", + "$ref": "#/$defs/string", + "x-since-version": "v0.296.0" + }, "artifact_path": { "description": "The artifact path to use within the workspace for both deployments and workflow runs", "$ref": "#/$defs/string", @@ -4753,19 +4791,23 @@ "properties": { "alert_id": { "description": "The alert_id is the canonical identifier of the alert.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.296.0" }, "subscribers": { "description": "The subscribers receive alert evaluation result notifications after the alert task is completed.\nThe number of subscriptions is limited to 100.", - "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.AlertTaskSubscriber" + "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/jobs.AlertTaskSubscriber", + "x-since-version": "v0.296.0" }, "warehouse_id": { "description": "The warehouse_id identifies the warehouse settings used by the alert task.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.296.0" }, "workspace_path": { "description": "The workspace_path is the path to the alert file in the workspace. The path:\n* must start with \"/Workspace\"\n* must be a normalized path.\nUser has to select only one of alert_id or workspace_path to identify the alert.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.296.0" } }, "additionalProperties": false @@ -4775,11 +4817,13 @@ "description": "Represents a subscriber that will receive alert notifications.\nA subscriber can be either a user (via email) or a notification destination (via destination_id).", "properties": { "destination_id": { - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.296.0" }, "user_name": { "description": "A valid workspace email address.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.296.0" } }, "additionalProperties": false @@ -6159,7 +6203,8 @@ "properties": { "alert_task": { "description": "New alert v2 task", - "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.AlertTask" + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/jobs.AlertTask", + "x-since-version": "v0.296.0" }, "clean_rooms_notebook_task": { "description": "The task runs a [clean rooms](https://docs.databricks.com/clean-rooms/index.html) notebook\nwhen the `clean_rooms_notebook_task` field is present.", @@ -6637,15 +6682,18 @@ "properties": { "catalog_name": { "description": "(Required, Immutable) The name of the catalog for the connector's staging storage location.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.296.0" }, "schema_name": { "description": "(Required, Immutable) The name of the schema for the connector's staging storage location.", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.296.0" }, "volume_name": { "description": "(Optional) The Unity Catalog-compatible name for the storage location.\nThis is the volume to use for the data that is extracted by the connector.\nSpark Declarative Pipelines system will automatically create the volume under the catalog and schema.\nFor Combined Cdc Managed Ingestion pipelines default name for the volume would be :\n__databricks_ingestion_gateway_staging_data-$pipelineId", - "$ref": "#/$defs/string" + "$ref": "#/$defs/string", + "x-since-version": "v0.296.0" } }, "additionalProperties": false, @@ -6801,13 +6849,15 @@ "description": "(Optional) Connector Type for sources. Ex: CDC, Query Based.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.ConnectorType", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.296.0" }, "data_staging_options": { "description": "(Optional) Location of staged data storage. This is required for migration from Cdc Managed Ingestion Pipeline\nwith Gateway pipeline to Combined Cdc Managed Ingestion Pipeline.\nIf not specified, the volume for staged data will be created in catalog and schema/target specified in the\ntop level pipeline definition.", "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/pipelines.DataStagingOptions", "x-databricks-preview": "PRIVATE", - "doNotSuggest": true + "doNotSuggest": true, + "x-since-version": "v0.296.0" }, "full_refresh_window": { "description": "(Optional) A window that specifies a set of time ranges for snapshot queries in CDC.", @@ -8850,6 +8900,13 @@ "CAN_VIEW" ] }, + "vectorsearch.EndpointType": { + "type": "string", + "description": "Type of endpoint.", + "enum": [ + "STANDARD" + ] + }, "workspace.AzureKeyVaultSecretScopeMetadata": { "type": "object", "description": "The metadata of the Azure KeyVault for a secret scope of type `AZURE_KEYVAULT`", @@ -9028,6 +9085,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable" } }, + "resources.VectorSearchEndpoint": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.VectorSearchEndpoint" + } + }, "resources.Volume": { "type": "object", "additionalProperties": { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index d86cca7789..34c4fa4f5a 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/postgres" "github.com/databricks/databricks-sdk-go/service/serving" "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" "github.com/stretchr/testify/assert" ) @@ -25,29 +26,30 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { } state := ExportedResourcesMap{ - "resources.jobs.test_job": {ID: "1"}, - "resources.pipelines.test_pipeline": {ID: "1"}, - "resources.models.test_mlflow_model": {ID: "1"}, - "resources.experiments.test_mlflow_experiment": {ID: "1"}, - "resources.model_serving_endpoints.test_model_serving": {ID: "1"}, - "resources.registered_models.test_registered_model": {ID: "1"}, - "resources.quality_monitors.test_monitor": {ID: "1"}, - "resources.catalogs.test_catalog": {ID: "1"}, - "resources.schemas.test_schema": {ID: "1"}, - "resources.external_locations.test_external_location": {ID: "1"}, - "resources.volumes.test_volume": {ID: "1"}, - "resources.clusters.test_cluster": {ID: "1"}, - "resources.dashboards.test_dashboard": {ID: "1"}, - "resources.apps.test_app": {ID: "app1"}, - "resources.secret_scopes.test_secret_scope": {ID: "secret_scope1"}, - "resources.sql_warehouses.test_sql_warehouse": {ID: "1"}, - "resources.database_instances.test_database_instance": {ID: "1"}, - "resources.database_catalogs.test_database_catalog": {ID: "1"}, - "resources.synced_database_tables.test_synced_database_table": {ID: "1"}, - "resources.alerts.test_alert": {ID: "1"}, - "resources.postgres_projects.test_postgres_project": {ID: "projects/test-project"}, - "resources.postgres_branches.test_postgres_branch": {ID: "projects/test-project/branches/main"}, - "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, + "resources.jobs.test_job": {ID: "1"}, + "resources.pipelines.test_pipeline": {ID: "1"}, + "resources.models.test_mlflow_model": {ID: "1"}, + "resources.experiments.test_mlflow_experiment": {ID: "1"}, + "resources.model_serving_endpoints.test_model_serving": {ID: "1"}, + "resources.registered_models.test_registered_model": {ID: "1"}, + "resources.quality_monitors.test_monitor": {ID: "1"}, + "resources.catalogs.test_catalog": {ID: "1"}, + "resources.schemas.test_schema": {ID: "1"}, + "resources.external_locations.test_external_location": {ID: "1"}, + "resources.volumes.test_volume": {ID: "1"}, + "resources.clusters.test_cluster": {ID: "1"}, + "resources.dashboards.test_dashboard": {ID: "1"}, + "resources.apps.test_app": {ID: "app1"}, + "resources.secret_scopes.test_secret_scope": {ID: "secret_scope1"}, + "resources.sql_warehouses.test_sql_warehouse": {ID: "1"}, + "resources.database_instances.test_database_instance": {ID: "1"}, + "resources.database_catalogs.test_database_catalog": {ID: "1"}, + "resources.synced_database_tables.test_synced_database_table": {ID: "1"}, + "resources.alerts.test_alert": {ID: "1"}, + "resources.postgres_projects.test_postgres_project": {ID: "projects/test-project"}, + "resources.postgres_branches.test_postgres_branch": {ID: "projects/test-project/branches/main"}, + "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, + "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -116,6 +118,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "projects/test-project/branches/main/endpoints/primary", config.Resources.PostgresEndpoints["test_postgres_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.PostgresEndpoints["test_postgres_endpoint"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -287,6 +292,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ + "test_vector_search_endpoint": { + CreateEndpoint: vectorsearch.CreateEndpoint{ + Name: "test_vector_search_endpoint", + }, + }, + }, }, } @@ -362,6 +374,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.PostgresEndpoints["test_postgres_endpoint"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresEndpoints["test_postgres_endpoint"].ModifiedStatus) + assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -646,49 +661,63 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ + "test_vector_search_endpoint": { + CreateEndpoint: vectorsearch.CreateEndpoint{ + Name: "test_vector_search_endpoint", + }, + }, + "test_vector_search_endpoint_new": { + CreateEndpoint: vectorsearch.CreateEndpoint{ + Name: "test_vector_search_endpoint_new", + }, + }, + }, }, } state := ExportedResourcesMap{ - "resources.jobs.test_job": {ID: "1"}, - "resources.jobs.test_job_old": {ID: "2"}, - "resources.pipelines.test_pipeline": {ID: "1"}, - "resources.pipelines.test_pipeline_old": {ID: "2"}, - "resources.models.test_mlflow_model": {ID: "1"}, - "resources.models.test_mlflow_model_old": {ID: "2"}, - "resources.experiments.test_mlflow_experiment": {ID: "1"}, - "resources.experiments.test_mlflow_experiment_old": {ID: "2"}, - "resources.model_serving_endpoints.test_model_serving": {ID: "1"}, - "resources.model_serving_endpoints.test_model_serving_old": {ID: "2"}, - "resources.registered_models.test_registered_model": {ID: "1"}, - "resources.registered_models.test_registered_model_old": {ID: "2"}, - "resources.quality_monitors.test_monitor": {ID: "test_monitor"}, - "resources.quality_monitors.test_monitor_old": {ID: "test_monitor_old"}, - "resources.catalogs.test_catalog": {ID: "1"}, - "resources.catalogs.test_catalog_old": {ID: "2"}, - "resources.schemas.test_schema": {ID: "1"}, - "resources.schemas.test_schema_old": {ID: "2"}, - "resources.volumes.test_volume": {ID: "1"}, - "resources.volumes.test_volume_old": {ID: "2"}, - "resources.clusters.test_cluster": {ID: "1"}, - "resources.clusters.test_cluster_old": {ID: "2"}, - "resources.dashboards.test_dashboard": {ID: "1"}, - "resources.dashboards.test_dashboard_old": {ID: "2"}, - "resources.apps.test_app": {ID: "test_app"}, - "resources.apps.test_app_old": {ID: "test_app_old"}, - "resources.secret_scopes.test_secret_scope": {ID: "test_secret_scope"}, - "resources.secret_scopes.test_secret_scope_old": {ID: "test_secret_scope_old"}, - "resources.sql_warehouses.test_sql_warehouse": {ID: "1"}, - "resources.sql_warehouses.test_sql_warehouse_old": {ID: "2"}, - "resources.database_instances.test_database_instance": {ID: "1"}, - "resources.database_instances.test_database_instance_old": {ID: "2"}, - "resources.alerts.test_alert": {ID: "1"}, - "resources.alerts.test_alert_old": {ID: "2"}, - "resources.postgres_projects.test_postgres_project": {ID: "projects/test-project"}, - "resources.postgres_projects.test_postgres_project_old": {ID: "projects/test-project-old"}, - "resources.postgres_branches.test_postgres_branch": {ID: "projects/test-project/branches/main"}, - "resources.postgres_branches.test_postgres_branch_old": {ID: "projects/test-project/branches/old"}, - "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, - "resources.postgres_endpoints.test_postgres_endpoint_old": {ID: "projects/test-project/branches/main/endpoints/old"}, + "resources.jobs.test_job": {ID: "1"}, + "resources.jobs.test_job_old": {ID: "2"}, + "resources.pipelines.test_pipeline": {ID: "1"}, + "resources.pipelines.test_pipeline_old": {ID: "2"}, + "resources.models.test_mlflow_model": {ID: "1"}, + "resources.models.test_mlflow_model_old": {ID: "2"}, + "resources.experiments.test_mlflow_experiment": {ID: "1"}, + "resources.experiments.test_mlflow_experiment_old": {ID: "2"}, + "resources.model_serving_endpoints.test_model_serving": {ID: "1"}, + "resources.model_serving_endpoints.test_model_serving_old": {ID: "2"}, + "resources.registered_models.test_registered_model": {ID: "1"}, + "resources.registered_models.test_registered_model_old": {ID: "2"}, + "resources.quality_monitors.test_monitor": {ID: "test_monitor"}, + "resources.quality_monitors.test_monitor_old": {ID: "test_monitor_old"}, + "resources.catalogs.test_catalog": {ID: "1"}, + "resources.catalogs.test_catalog_old": {ID: "2"}, + "resources.schemas.test_schema": {ID: "1"}, + "resources.schemas.test_schema_old": {ID: "2"}, + "resources.volumes.test_volume": {ID: "1"}, + "resources.volumes.test_volume_old": {ID: "2"}, + "resources.clusters.test_cluster": {ID: "1"}, + "resources.clusters.test_cluster_old": {ID: "2"}, + "resources.dashboards.test_dashboard": {ID: "1"}, + "resources.dashboards.test_dashboard_old": {ID: "2"}, + "resources.apps.test_app": {ID: "test_app"}, + "resources.apps.test_app_old": {ID: "test_app_old"}, + "resources.secret_scopes.test_secret_scope": {ID: "test_secret_scope"}, + "resources.secret_scopes.test_secret_scope_old": {ID: "test_secret_scope_old"}, + "resources.sql_warehouses.test_sql_warehouse": {ID: "1"}, + "resources.sql_warehouses.test_sql_warehouse_old": {ID: "2"}, + "resources.database_instances.test_database_instance": {ID: "1"}, + "resources.database_instances.test_database_instance_old": {ID: "2"}, + "resources.alerts.test_alert": {ID: "1"}, + "resources.alerts.test_alert_old": {ID: "2"}, + "resources.postgres_projects.test_postgres_project": {ID: "projects/test-project"}, + "resources.postgres_projects.test_postgres_project_old": {ID: "projects/test-project-old"}, + "resources.postgres_branches.test_postgres_branch": {ID: "projects/test-project/branches/main"}, + "resources.postgres_branches.test_postgres_branch_old": {ID: "projects/test-project/branches/old"}, + "resources.postgres_endpoints.test_postgres_endpoint": {ID: "projects/test-project/branches/main/endpoints/primary"}, + "resources.postgres_endpoints.test_postgres_endpoint_old": {ID: "projects/test-project/branches/main/endpoints/old"}, + "resources.vector_search_endpoints.test_vector_search_endpoint": {ID: "vs-endpoint-1"}, + "resources.vector_search_endpoints.test_vector_search_endpoint_old": {ID: "vs-endpoint-old"}, } err := StateToBundle(t.Context(), state, &config) assert.NoError(t, err) @@ -835,6 +864,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Equal(t, "", config.Resources.PostgresEndpoints["test_postgres_endpoint_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.PostgresEndpoints["test_postgres_endpoint_new"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-1", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ID) + assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint"].ModifiedStatus) + assert.Equal(t, "vs-endpoint-old", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint_old"].ModifiedStatus) + assert.Equal(t, "", config.Resources.VectorSearchEndpoints["test_vector_search_endpoint_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.VectorSearchEndpoints["test_vector_search_endpoint_new"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 0ac7fe34aa..5430c68cbc 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -26,6 +26,7 @@ import ( "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/serving" "github.com/databricks/databricks-sdk-go/service/sql" + "github.com/databricks/databricks-sdk-go/service/vectorsearch" "github.com/databricks/databricks-sdk-go/service/workspace" ) @@ -150,6 +151,7 @@ type FakeWorkspace struct { ExternalLocations map[string]catalog.ExternalLocationInfo RegisteredModels map[string]catalog.RegisteredModelInfo ServingEndpoints map[string]serving.ServingEndpointDetailed + VectorSearchEndpoints map[string]vectorsearch.EndpointInfo SecretScopes map[string]workspace.SecretScope Secrets map[string]map[string]string // scope -> key -> value @@ -284,6 +286,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { }, }, ServingEndpoints: map[string]serving.ServingEndpointDetailed{}, + VectorSearchEndpoints: map[string]vectorsearch.EndpointInfo{}, Repos: map[string]workspace.RepoInfo{}, SecretScopes: map[string]workspace.SecretScope{}, Secrets: map[string]map[string]string{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index b2a95b1902..47515652c2 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -798,6 +798,32 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.ServingEndpointPatchTags(req, req.Vars["name"]) }) + // Vector Search Endpoints: + + server.Handle("POST", "/api/2.0/vector-search/endpoints", func(req Request) any { + return req.Workspace.VectorSearchEndpointCreate(req) + }) + + server.Handle("GET", "/api/2.0/vector-search/endpoints", func(req Request) any { + return MapList(req.Workspace, req.Workspace.VectorSearchEndpoints, "endpoints") + }) + + server.Handle("GET", "/api/2.0/vector-search/endpoints/{endpoint_name}", func(req Request) any { + return MapGet(req.Workspace, req.Workspace.VectorSearchEndpoints, req.Vars["endpoint_name"]) + }) + + server.Handle("PATCH", "/api/2.0/vector-search/endpoints/{endpoint_name}", func(req Request) any { + return req.Workspace.VectorSearchEndpointUpdate(req, req.Vars["endpoint_name"]) + }) + + server.Handle("DELETE", "/api/2.0/vector-search/endpoints/{endpoint_name}", func(req Request) any { + return MapDelete(req.Workspace, req.Workspace.VectorSearchEndpoints, req.Vars["endpoint_name"]) + }) + + server.Handle("PATCH", "/api/2.0/vector-search/endpoints/{endpoint_name}/budget-policy", func(req Request) any { + return req.Workspace.VectorSearchEndpointUpdateBudgetPolicy(req, req.Vars["endpoint_name"]) + }) + // Generic permissions endpoints server.Handle("GET", "/api/2.0/permissions/{object_type}/{object_id}", func(req Request) any { return req.Workspace.GetPermissions(req) diff --git a/libs/testserver/vector_search_endpoints.go b/libs/testserver/vector_search_endpoints.go new file mode 100644 index 0000000000..ec93e57dc2 --- /dev/null +++ b/libs/testserver/vector_search_endpoints.go @@ -0,0 +1,116 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/databricks/databricks-sdk-go/service/vectorsearch" +) + +func (s *FakeWorkspace) VectorSearchEndpointCreate(req Request) Response { + defer s.LockUnlock()() + + var createReq vectorsearch.CreateEndpoint + if err := json.Unmarshal(req.Body, &createReq); err != nil { + return Response{ + Body: fmt.Sprintf("cannot unmarshal request body: %s", err), + StatusCode: http.StatusBadRequest, + } + } + + if _, exists := s.VectorSearchEndpoints[createReq.Name]; exists { + return Response{ + StatusCode: http.StatusConflict, + Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": fmt.Sprintf("Vector search endpoint with name %s already exists", createReq.Name)}, + } + } + + endpoint := vectorsearch.EndpointInfo{ + EffectiveBudgetPolicyId: createReq.BudgetPolicyId, + Creator: s.CurrentUser().UserName, + CreationTimestamp: nowMilli(), + EndpointType: createReq.EndpointType, + Id: nextUUID(), + LastUpdatedUser: s.CurrentUser().UserName, + Name: createReq.Name, + EndpointStatus: &vectorsearch.EndpointStatus{ + State: vectorsearch.EndpointStatusStateOnline, // initial create is no-op, returns ONLINE immediately + }, + ScalingInfo: &vectorsearch.EndpointScalingInfo{ + RequestedMinQps: createReq.MinQps, + }, + } + endpoint.LastUpdatedTimestamp = endpoint.CreationTimestamp + + s.VectorSearchEndpoints[createReq.Name] = endpoint + + return Response{ + Body: endpoint, + } +} + +func (s *FakeWorkspace) VectorSearchEndpointUpdateBudgetPolicy(req Request, endpointName string) Response { + defer s.LockUnlock()() + + var patchReq vectorsearch.PatchEndpointBudgetPolicyRequest + if err := json.Unmarshal(req.Body, &patchReq); err != nil { + return Response{ + Body: fmt.Sprintf("cannot unmarshal request body: %s", err), + StatusCode: http.StatusBadRequest, + } + } + + endpoint, exists := s.VectorSearchEndpoints[endpointName] + if !exists { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": fmt.Sprintf("Vector search endpoint %s not found", endpointName)}, + } + } + + endpoint.EffectiveBudgetPolicyId = patchReq.BudgetPolicyId // assume it always becomes the effective policy + endpoint.LastUpdatedTimestamp = nowMilli() + endpoint.LastUpdatedUser = s.CurrentUser().UserName + + s.VectorSearchEndpoints[endpointName] = endpoint + + return Response{ + Body: vectorsearch.PatchEndpointBudgetPolicyResponse{ + EffectiveBudgetPolicyId: endpoint.EffectiveBudgetPolicyId, + }, + } +} + +func (s *FakeWorkspace) VectorSearchEndpointUpdate(req Request, endpointName string) Response { + defer s.LockUnlock()() + + var patchReq vectorsearch.PatchEndpointRequest + if err := json.Unmarshal(req.Body, &patchReq); err != nil { + return Response{ + Body: fmt.Sprintf("cannot unmarshal request body: %s", err), + StatusCode: http.StatusBadRequest, + } + } + + endpoint, exists := s.VectorSearchEndpoints[endpointName] + if !exists { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": fmt.Sprintf("Vector search endpoint %s not found", endpointName)}, + } + } + + if endpoint.ScalingInfo == nil { + endpoint.ScalingInfo = &vectorsearch.EndpointScalingInfo{} + } + endpoint.ScalingInfo.RequestedMinQps = patchReq.MinQps + endpoint.LastUpdatedTimestamp = nowMilli() + endpoint.LastUpdatedUser = s.CurrentUser().UserName + + s.VectorSearchEndpoints[endpointName] = endpoint + + return Response{ + Body: endpoint, + } +}