diff --git a/eng/pipelines/templates/jobs/live.tests.yml b/eng/pipelines/templates/jobs/live.tests.yml index c7e57c37b6bd..a73523cb0a45 100644 --- a/eng/pipelines/templates/jobs/live.tests.yml +++ b/eng/pipelines/templates/jobs/live.tests.yml @@ -69,6 +69,15 @@ parameters: - name: PersistOidcToken type: boolean default: false + # When true, skip the per-run ARM deployment (and its matching teardown) entirely. + # Intended for services whose live tests run against fixed, team-owned accounts resolved + # in BeforeTestSteps rather than provisioned per run -- see sdk/cosmos/pipeline/README.md. + # The azure-sdk live-test tenant is recreated roughly every 90 days, which invalidates the + # service principals and RBAC that New-TestResources.ps1 depends on, so per-run provisioning + # fails until a human re-onboards the tenant. + - name: DisableAzureResourceCreation + type: boolean + default: false jobs: - job: @@ -143,30 +152,31 @@ jobs: parameters: BuildTargetingString: ${{ parameters.BuildTargetingString }} - - template: /eng/common/TestResources/build-test-resource-config.yml - parameters: - SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }} - SubscriptionConfigurations: ${{ parameters.CloudConfig.SubscriptionConfigurations }} - ${{ if parameters.UseFederatedAuth }}: - SubscriptionConfigurationFilePaths: ${{ parameters.CloudConfig.SubscriptionConfigurationFilePaths }} - EnvVars: - Pool: $(Pool) - ${{ insert }}: ${{ parameters.EnvVars }} - - - ${{ each directory in coalesce(parameters.TestResourceDirectories, split(parameters.ServiceDirectory, '|')) }}: - - template: /eng/common/TestResources/deploy-test-resources.yml + - ${{ if not(parameters.DisableAzureResourceCreation) }}: + - template: /eng/common/TestResources/build-test-resource-config.yml parameters: - Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location) }} - ServiceDirectory: '${{ directory }}' - SubscriptionConfiguration: $(SubscriptionConfiguration) - ArmTemplateParameters: $(ArmTemplateParameters) - UseFederatedAuth: ${{ parameters.UseFederatedAuth }} - PersistOidcToken: ${{ parameters.PersistOidcToken }} - ServiceConnection: ${{ parameters.CloudConfig.ServiceConnection }} + SubscriptionConfiguration: ${{ parameters.CloudConfig.SubscriptionConfiguration }} + SubscriptionConfigurations: ${{ parameters.CloudConfig.SubscriptionConfigurations }} + ${{ if parameters.UseFederatedAuth }}: + SubscriptionConfigurationFilePaths: ${{ parameters.CloudConfig.SubscriptionConfigurationFilePaths }} EnvVars: Pool: $(Pool) ${{ insert }}: ${{ parameters.EnvVars }} + - ${{ each directory in coalesce(parameters.TestResourceDirectories, split(parameters.ServiceDirectory, '|')) }}: + - template: /eng/common/TestResources/deploy-test-resources.yml + parameters: + Location: ${{ coalesce(parameters.Location, parameters.CloudConfig.Location) }} + ServiceDirectory: '${{ directory }}' + SubscriptionConfiguration: $(SubscriptionConfiguration) + ArmTemplateParameters: $(ArmTemplateParameters) + UseFederatedAuth: ${{ parameters.UseFederatedAuth }} + PersistOidcToken: ${{ parameters.PersistOidcToken }} + ServiceConnection: ${{ parameters.CloudConfig.ServiceConnection }} + EnvVars: + Pool: $(Pool) + ${{ insert }}: ${{ parameters.EnvVars }} + # This is a heuristic to detect compiled requirements in the service # directory. If compiled requirements are found, install dependencies for # and configure cibuildwheel. @@ -241,13 +251,20 @@ jobs: TestProxy: ${{ parameters.TestProxy }} UseFederatedAuth: ${{ parameters.UseFederatedAuth }} ServiceConnection: ${{ parameters.CloudConfig.ServiceConnection }} + # build-test.yml runs these immediately before/after the test task, in the same job, + # so a BeforeTestSteps step can publish pipeline variables that the test task then + # picks up as environment variables. These were declared here but never forwarded, + # which silently dropped anything a service passed in via archetype-sdk-tests.yml. + BeforeTestSteps: ${{ parameters.BeforeTestSteps }} + AfterTestSteps: ${{ parameters.AfterTestSteps }} - - ${{ each directory in coalesce(parameters.TestResourceDirectories, split(parameters.ServiceDirectory, '|')) }}: - - template: /eng/common/TestResources/remove-test-resources.yml - parameters: - ServiceDirectory: '${{ directory }}' - SubscriptionConfiguration: $(SubscriptionConfiguration) - UseFederatedAuth: ${{ parameters.UseFederatedAuth }} - ServiceConnection: ${{ parameters.CloudConfig.ServiceConnection }} + - ${{ if not(parameters.DisableAzureResourceCreation) }}: + - ${{ each directory in coalesce(parameters.TestResourceDirectories, split(parameters.ServiceDirectory, '|')) }}: + - template: /eng/common/TestResources/remove-test-resources.yml + parameters: + ServiceDirectory: '${{ directory }}' + SubscriptionConfiguration: $(SubscriptionConfiguration) + UseFederatedAuth: ${{ parameters.UseFederatedAuth }} + ServiceConnection: ${{ parameters.CloudConfig.ServiceConnection }} - ${{ parameters.PostSteps }} diff --git a/eng/pipelines/templates/stages/archetype-sdk-tests.yml b/eng/pipelines/templates/stages/archetype-sdk-tests.yml index e6e84f1c8170..1c8971482eed 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-tests.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-tests.yml @@ -100,6 +100,12 @@ parameters: - name: PersistOidcToken type: boolean default: false + # Skip the per-run ARM deployment and its teardown for this stage. Use when the service's + # live tests bind to fixed, team-owned accounts (resolved in BeforeTestSteps) instead of + # provisioning per run. See sdk/cosmos/pipeline/README.md for the Cosmos implementation. + - name: DisableAzureResourceCreation + type: boolean + default: false extends: template: /eng/pipelines/templates/stages/1es-redirect.yml @@ -146,6 +152,7 @@ extends: CheckEnv: ${{ parameters.CheckEnv }} UseFederatedAuth: ${{ parameters.UseFederatedAuth }} PersistOidcToken: ${{ parameters.PersistOidcToken }} + DisableAzureResourceCreation: ${{ parameters.DisableAzureResourceCreation }} MatrixConfigs: # Enumerate platforms and additional platforms based on supported clouds (sparse platform<-->cloud matrix). - ${{ each config in parameters.MatrixConfigs }}: diff --git a/sdk/cosmos/azure-cosmos/pytest.ini b/sdk/cosmos/azure-cosmos/pytest.ini index e7facc2e9ca7..e3154a718b6a 100644 --- a/sdk/cosmos/azure-cosmos/pytest.ini +++ b/sdk/cosmos/azure-cosmos/pytest.ini @@ -8,6 +8,7 @@ markers = cosmosEmulator: marks tests as depending in Cosmos DB Emulator. cosmosLong: marks tests to be run on a Cosmos DB live account. cosmosQuery: marks tests running queries on Cosmos DB live account. + cosmosSearchQuery: marks tests running full text and vector search queries on a Cosmos DB live account. cosmosAADLong: marks AAD tests for the standard live-account lane. cosmosAADSplit: marks AAD tests for partition split scenarios. cosmosAADMultiRegion: marks AAD tests for multi-region scenarios. diff --git a/sdk/cosmos/azure-cosmos/tests/test_aad.py b/sdk/cosmos/azure-cosmos/tests/test_aad.py index 437ad59aa519..fbea0b3c5105 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_aad.py +++ b/sdk/cosmos/azure-cosmos/tests/test_aad.py @@ -138,7 +138,8 @@ def capturing_get_token(self, *scopes, **kwargs): def test_override_scope_no_fallback(self): """When override scope is provided, only that scope is used and no fallback occurs.""" override_scope = "https://my.custom.scope/.default" - os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] = override_scope + previous_env = test_config.set_environment_variables(AZURE_COSMOS_AAD_SCOPE_OVERRIDE=override_scope) + self.addCleanup(test_config.restore_environment_variables, previous_env) def action(scopes_captured): credential = CosmosEmulatorCredential() @@ -152,7 +153,6 @@ def action(scopes_captured): try: assert all(scope == override_scope for scope in scopes), f"Expected only override scope(s), got: {scopes}" finally: - del os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] try: container.delete_item(item='Item_10', partition_key='pk') except Exception: @@ -162,7 +162,8 @@ def action(scopes_captured): def test_override_scope_auth_error_no_fallback(self): """When override scope is provided and auth fails, no fallback to other scopes occurs.""" override_scope = "https://my.custom.scope/.default" - os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] = override_scope + previous_env = test_config.set_environment_variables(AZURE_COSMOS_AAD_SCOPE_OVERRIDE=override_scope) + self.addCleanup(test_config.restore_environment_variables, previous_env) class FailingCredential(CosmosEmulatorCredential): def get_token(self, *scopes, **kwargs): @@ -178,16 +179,14 @@ def action(scopes_captured): return None scopes, _ = self._run_with_scope_capture(FailingCredential, action) - try: - assert scopes == [override_scope], f"Expected only override scope, got: {scopes}" - finally: - del os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] + assert scopes == [override_scope], f"Expected only override scope, got: {scopes}" @_skip_on_non_emulator def test_account_scope_only(self): """When account scope is provided, only that scope is used.""" account_scope = "https://localhost/.default" - os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] = "" + previous_env = test_config.set_environment_variables(AZURE_COSMOS_AAD_SCOPE_OVERRIDE="") + self.addCleanup(test_config.restore_environment_variables, previous_env) def action(scopes_captured): credential = CosmosEmulatorCredential() @@ -212,7 +211,8 @@ def test_account_scope_fallback_on_error(self): """When account scope is provided and auth fails, fallback to default scope occurs.""" account_scope = "https://localhost/.default" fallback_scope = "https://cosmos.azure.com/.default" - os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] = "" + previous_env = test_config.set_environment_variables(AZURE_COSMOS_AAD_SCOPE_OVERRIDE="") + self.addCleanup(test_config.restore_environment_variables, previous_env) class FallbackCredential(CosmosEmulatorCredential): def __init__(self): diff --git a/sdk/cosmos/azure-cosmos/tests/test_aad_async.py b/sdk/cosmos/azure-cosmos/tests/test_aad_async.py index 20fec64a666b..c3c2959c2e88 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_aad_async.py +++ b/sdk/cosmos/azure-cosmos/tests/test_aad_async.py @@ -139,7 +139,8 @@ async def capturing_get_token(self, *scopes, **kwargs): async def test_override_scope_no_fallback_async(self): """When override scope is provided, only that scope is used and no fallback occurs.""" override_scope = "https://my.custom.scope/.default" - os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] = override_scope + previous_env = test_config.set_environment_variables(AZURE_COSMOS_AAD_SCOPE_OVERRIDE=override_scope) + self.addCleanup(test_config.restore_environment_variables, previous_env) async def action(scopes_captured): credential = CosmosEmulatorCredential() @@ -156,7 +157,6 @@ async def action(scopes_captured): try: assert all(scope == override_scope for scope in scopes), f"Expected only override scope, got: {scopes}" finally: - del os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] try: await container.delete_item(item='Item_20', partition_key='pk') except Exception: @@ -166,7 +166,8 @@ async def action(scopes_captured): async def test_override_scope_no_fallback_on_error_async(self): """When override scope is provided and auth fails, no fallback occurs.""" override_scope = "https://my.custom.scope/.default" - os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] = override_scope + previous_env = test_config.set_environment_variables(AZURE_COSMOS_AAD_SCOPE_OVERRIDE=override_scope) + self.addCleanup(test_config.restore_environment_variables, previous_env) class FailingCredential(CosmosEmulatorCredential): async def get_token(self, *scopes, **kwargs): @@ -190,7 +191,6 @@ async def action(scopes_captured): try: assert all(scope == override_scope for scope in scopes), f"Expected only override scope, got: {scopes}" finally: - del os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] try: await container.delete_item(item='Item_21', partition_key='pk') except Exception: @@ -200,7 +200,8 @@ async def action(scopes_captured): async def test_account_scope_only_async(self): """When account scope is provided, only that scope is used.""" account_scope = "https://localhost/.default" - os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] = "" + previous_env = test_config.set_environment_variables(AZURE_COSMOS_AAD_SCOPE_OVERRIDE="") + self.addCleanup(test_config.restore_environment_variables, previous_env) async def action(scopes_captured): credential = CosmosEmulatorCredential() @@ -227,7 +228,8 @@ async def test_account_scope_fallback_on_error_async(self): """When account scope is provided and auth fails, fallback to default scope occurs.""" account_scope = "https://localhost/.default" fallback_scope = "https://cosmos.azure.com/.default" - os.environ["AZURE_COSMOS_AAD_SCOPE_OVERRIDE"] = "" + previous_env = test_config.set_environment_variables(AZURE_COSMOS_AAD_SCOPE_OVERRIDE="") + self.addCleanup(test_config.restore_environment_variables, previous_env) class FallbackCredential(CosmosEmulatorCredential): def __init__(self): diff --git a/sdk/cosmos/azure-cosmos/tests/test_aad_inference_service_lazy_init.py b/sdk/cosmos/azure-cosmos/tests/test_aad_inference_service_lazy_init.py index d20316af6c82..26a376d9494a 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_aad_inference_service_lazy_init.py +++ b/sdk/cosmos/azure-cosmos/tests/test_aad_inference_service_lazy_init.py @@ -67,7 +67,7 @@ def get_token(self, *scopes, **kwargs): @pytest.mark.cosmosEmulator -@pytest.mark.cosmosLong +@pytest.mark.cosmosAADLong class TestAADInferenceServiceLazyInit(unittest.TestCase): """Verify AAD client construction succeeds without the semantic reranker env var. diff --git a/sdk/cosmos/azure-cosmos/tests/test_aad_inference_service_lazy_init_async.py b/sdk/cosmos/azure-cosmos/tests/test_aad_inference_service_lazy_init_async.py index 95d9b2a285b1..f94b28c0511c 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_aad_inference_service_lazy_init_async.py +++ b/sdk/cosmos/azure-cosmos/tests/test_aad_inference_service_lazy_init_async.py @@ -66,7 +66,7 @@ async def get_token(self, *scopes, **kwargs): @pytest.mark.cosmosEmulator -@pytest.mark.cosmosLong +@pytest.mark.cosmosAADLong class TestAADInferenceServiceLazyInitAsync(unittest.IsolatedAsyncioTestCase): """Verify async AAD client construction succeeds without the semantic reranker env var. diff --git a/sdk/cosmos/azure-cosmos/tests/test_auto_scale.py b/sdk/cosmos/azure-cosmos/tests/test_auto_scale.py index 634dd8c6b52b..4699372ca8b3 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_auto_scale.py +++ b/sdk/cosmos/azure-cosmos/tests/test_auto_scale.py @@ -72,7 +72,7 @@ def test_autoscale_create_container(self): self.created_database.delete_container(container_id) def test_autoscale_create_database(self): - database_id = "db_auto_scale_" + str(uuid.uuid4()) + database_id = test_config.unique_database_id("auto-scale") try: # Testing auto_scale_settings for the create_database method created_database = self.key_client.create_database(database_id, offer_throughput=ThroughputProperties( @@ -87,7 +87,7 @@ def test_autoscale_create_database(self): self.key_client.delete_database(created_database.id) # Testing auto_scale_settings for the create_database_if_not_exists method - database_id = "db_auto_scale_2_" + str(uuid.uuid4()) + database_id = test_config.unique_database_id("auto-scale-2") created_database = self.key_client.create_database_if_not_exists(database_id, offer_throughput=ThroughputProperties( auto_scale_max_throughput=9000, @@ -101,7 +101,7 @@ def test_autoscale_create_database(self): self.key_client.delete_database(database_id) def test_autoscale_replace_throughput(self): - database_id = "replace_db" + str(uuid.uuid4()) + database_id = test_config.unique_database_id("replace-db") container_id = None try: created_database = self.key_client.create_database(database_id, offer_throughput=ThroughputProperties( diff --git a/sdk/cosmos/azure-cosmos/tests/test_auto_scale_async.py b/sdk/cosmos/azure-cosmos/tests/test_auto_scale_async.py index 7fa9c36f6163..43b0c26fb658 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_auto_scale_async.py +++ b/sdk/cosmos/azure-cosmos/tests/test_auto_scale_async.py @@ -82,7 +82,7 @@ async def test_autoscale_create_database_async(self): database_id = None try: # Testing auto_scale_settings for the create_database method - database_id = "db1_" + str(uuid.uuid4()) + database_id = test_config.unique_database_id("db1") created_database = await self.key_client.create_database(database_id, offer_throughput=ThroughputProperties( auto_scale_max_throughput=5000, auto_scale_increment_percent=0)) @@ -95,7 +95,7 @@ async def test_autoscale_create_database_async(self): await self.key_client.delete_database(created_database.id) # Testing auto_scale_settings for the create_database_if_not_exists method - database_id = "db2_" + str(uuid.uuid4()) + database_id = test_config.unique_database_id("db2") created_database = await self.key_client.create_database_if_not_exists(database_id, offer_throughput=ThroughputProperties( auto_scale_max_throughput=9000, auto_scale_increment_percent=11)) @@ -108,7 +108,7 @@ async def test_autoscale_create_database_async(self): await self.key_client.delete_database(database_id) async def test_replace_throughput_async(self): - database_id = "replace_db" + str(uuid.uuid4()) + database_id = test_config.unique_database_id("replace-db") container_id = None try: created_database = await self.key_client.create_database(database_id, offer_throughput=ThroughputProperties( diff --git a/sdk/cosmos/azure-cosmos/tests/test_availability_strategy.py b/sdk/cosmos/azure-cosmos/tests/test_availability_strategy.py index 53ef61bb88a4..bbe3d1223899 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_availability_strategy.py +++ b/sdk/cosmos/azure-cosmos/tests/test_availability_strategy.py @@ -55,6 +55,9 @@ def _select_primary_and_failover_region(write_locations, read_locations): READ_ALL = "read_all" CHANGE_FEED = "change_feed" +STEADY_STATE_HEDGING_THRESHOLD_MS = 5000 +FAULT_INJECTION_DELAY_MS = 5000 + # Non-transient status codes NON_TRANSIENT_STATUS_CODES = [ (400, None), @@ -354,9 +357,10 @@ def test_invalid_thresholds(self, threshold_ms, threshold_steps_ms, error_messag @pytest.mark.parametrize("operation", [READ, QUERY, QUERY_PK, READ_ALL, CHANGE_FEED, CREATE, UPSERT, REPLACE, DELETE, PATCH, BATCH]) @pytest.mark.parametrize("client_availability_strategy, request_availability_strategy", [ - (None, {'threshold_ms':150, 'threshold_steps_ms':50}), - ({'threshold_ms':150, 'threshold_steps_ms':50}, None), - ({'threshold_ms':150, 'threshold_steps_ms':50}, {'threshold_ms':150, 'threshold_steps_ms':50}) + (None, {'threshold_ms':STEADY_STATE_HEDGING_THRESHOLD_MS, 'threshold_steps_ms':50}), + ({'threshold_ms':STEADY_STATE_HEDGING_THRESHOLD_MS, 'threshold_steps_ms':50}, None), + ({'threshold_ms':STEADY_STATE_HEDGING_THRESHOLD_MS, 'threshold_steps_ms':50}, + {'threshold_ms':STEADY_STATE_HEDGING_THRESHOLD_MS, 'threshold_steps_ms':50}) ]) def test_availability_strategy_in_steady_state( self, @@ -413,7 +417,7 @@ def test_client_availability_strategy_failover( FaultInjectionTransport.predicate_targets_region(r, uri_down)) error_lambda = lambda r: FaultInjectionTransport.error_after_delay( - 1000, # Add delay to trigger hedging + FAULT_INJECTION_DELAY_MS, CosmosHttpResponseError(status_code=400, message="Injected Error") ) custom_transport = self._get_custom_transport_with_fault_injection(predicate, error_lambda) @@ -597,7 +601,7 @@ def test_request_level_enable_override_client_disable(self, operation): FaultInjectionTransport.predicate_targets_region(r, uri_down)) error_lambda = lambda r: FaultInjectionTransport.error_after_delay( - 700, # Add delay to trigger hedging + FAULT_INJECTION_DELAY_MS, CosmosHttpResponseError(status_code=400, message="Injected Error") # using non retryable errors to verify the request will only go to the first region ) @@ -716,9 +720,11 @@ def test_no_cross_region_request_with_retry_write_disabled(self, operation): @pytest.mark.parametrize("operation", [READ, QUERY_PK, CHANGE_FEED, CREATE, UPSERT, REPLACE, DELETE, PATCH, BATCH]) def test_per_partition_circular_breaker_with_cancelled_first_future(self, operation): # QUERY, READ_ALL are not included because currently they are not targeting to a specific pkRange - os.environ["AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER"] = "True" - os.environ["AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_WRITE"] = "5" - os.environ["AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_READ"] = "5" + previous_env = test_config.set_environment_variables( + AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER="True", + AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_WRITE="5", + AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_READ="5", + ) try: """Test that when per partition circular breaker is enabled and after hitting the threshold, subsequent requests go directly to second region. @@ -799,9 +805,7 @@ def test_per_partition_circular_breaker_with_cancelled_first_future(self, operat availability_strategy=strategy) finally: - del os.environ["AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER"] - del os.environ["AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_WRITE"] - del os.environ["AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_READ"] + test_config.restore_environment_variables(previous_env) self._clean_up_container(setup_with_fault_injection['db'].id, setup_with_fault_injection['col'].id) diff --git a/sdk/cosmos/azure-cosmos/tests/test_availability_strategy_async.py b/sdk/cosmos/azure-cosmos/tests/test_availability_strategy_async.py index 2017c6d7a9dd..7fe9f10f90a4 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_availability_strategy_async.py +++ b/sdk/cosmos/azure-cosmos/tests/test_availability_strategy_async.py @@ -92,6 +92,9 @@ async def setup(): READ_ALL = "read_all" CHANGE_FEED = "change_feed" +STEADY_STATE_HEDGING_THRESHOLD_MS = 5000 +FAULT_INJECTION_DELAY_MS = 5000 + # Non-transient status codes NON_TRANSIENT_STATUS_CODES = [ (400, None), @@ -378,10 +381,10 @@ def test_invalid_thresholds_async(self, threshold_ms, threshold_steps_ms, error_ @pytest.mark.asyncio @pytest.mark.parametrize("operation", [READ, QUERY, QUERY_PK, READ_ALL, CHANGE_FEED, CREATE, UPSERT, REPLACE, DELETE, PATCH, BATCH]) @pytest.mark.parametrize("client_availability_strategy, request_availability_strategy", [ - (None, {'threshold_ms':150, 'threshold_steps_ms':50}), - ({'threshold_ms':150, 'threshold_steps_ms':50}, None), - ({'threshold_ms':150, 'threshold_steps_ms':50}, - {'threshold_ms':150, 'threshold_steps_ms':50}) + (None, {'threshold_ms':STEADY_STATE_HEDGING_THRESHOLD_MS, 'threshold_steps_ms':50}), + ({'threshold_ms':STEADY_STATE_HEDGING_THRESHOLD_MS, 'threshold_steps_ms':50}, None), + ({'threshold_ms':STEADY_STATE_HEDGING_THRESHOLD_MS, 'threshold_steps_ms':50}, + {'threshold_ms':STEADY_STATE_HEDGING_THRESHOLD_MS, 'threshold_steps_ms':50}) ]) async def test_availability_strategy_in_steady_state_async( self, @@ -446,7 +449,7 @@ async def test_client_availability_strategy_failover_async( FaultInjectionTransportAsync.predicate_targets_region(r, uri_down)) error_lambda = lambda r: FaultInjectionTransportAsync.error_after_delay( - 1000, # Add delay to trigger hedging + FAULT_INJECTION_DELAY_MS, CosmosHttpResponseError(status_code=400, message="Injected Error") ) custom_transport = self._get_custom_transport_with_fault_injection(predicate, error_lambda) @@ -847,9 +850,11 @@ async def test_no_cross_region_request_with_exclude_regions_async(self, operatio @pytest.mark.parametrize("operation", [READ, QUERY_PK, CHANGE_FEED, CREATE, UPSERT, REPLACE, DELETE, PATCH, BATCH]) async def test_per_partition_circular_breaker_with_cancelled_first_future_async(self, operation, setup): # QUERY, READ_ALL are not included because currently they are not targeting to a specific pkRange - os.environ["AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER"] = "True" - os.environ["AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_WRITE"] = "5" - os.environ["AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_READ"] = "5" + previous_env = test_config.set_environment_variables( + AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER="True", + AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_WRITE="5", + AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_READ="5", + ) try: """Test that when per partition circular breaker is enabled and after hitting the threshold, subsequent requests go directly to second region. @@ -938,9 +943,7 @@ async def test_per_partition_circular_breaker_with_cancelled_first_future_async( await setup_with_fault_injection['client'].close() await setup_without_fault['client'].close() finally: - del os.environ["AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER"] - del os.environ["AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_WRITE"] - del os.environ["AZURE_COSMOS_CONSECUTIVE_ERROR_COUNT_TOLERATED_FOR_READ"] + test_config.restore_environment_variables(previous_env) await self._clean_up_container(setup['client_without_fault'], setup_with_fault_injection['db'].id, setup_with_fault_injection['col'].id) @pytest.mark.asyncio diff --git a/sdk/cosmos/azure-cosmos/tests/test_backwards_compatibility.py b/sdk/cosmos/azure-cosmos/tests/test_backwards_compatibility.py index 4f4b2e9a1bfe..685fa22df8b2 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_backwards_compatibility.py +++ b/sdk/cosmos/azure-cosmos/tests/test_backwards_compatibility.py @@ -64,14 +64,18 @@ def test_populate_partition_key_range_statistics(self): def test_session_token_compatibility(self): # Verifying that behavior is unaffected across the board for using `session_token` on irrelevant methods # Database - database = self.client.create_database(str(uuid.uuid4()), session_token=str(uuid.uuid4())) + database = self.client.create_database(test_config.unique_database_id("backcompat"), session_token=str(uuid.uuid4())) assert database is not None - database2 = self.client.create_database_if_not_exists(str(uuid.uuid4()), session_token=str(uuid.uuid4())) + database2 = self.client.create_database_if_not_exists(test_config.unique_database_id("backcompat"), session_token=str(uuid.uuid4())) assert database2 is not None - database_list = list(self.client.list_databases(session_token=str(uuid.uuid4()))) - database_list2 = list(self.client.query_databases(query="select * from c", session_token=str(uuid.uuid4()))) - assert len(database_list) > 0 - assert len(database_list2) > 0 + # Assert this run's own database is present rather than that the account is + # non-empty: the account is shared with other runs and other language SDKs, so a + # non-zero count says nothing about the database under test. + database_ids = {db['id'] for db in self.client.list_databases(session_token=str(uuid.uuid4()))} + database_ids2 = {db['id'] for db in + self.client.query_databases(query="select * from c", session_token=str(uuid.uuid4()))} + assert database.id in database_ids + assert database.id in database_ids2 database_read = database.read(session_token=str(uuid.uuid4())) assert database_read is not None self.client.delete_database(database2.id, session_token=str(uuid.uuid4())) @@ -86,10 +90,12 @@ def test_session_token_compatibility(self): assert container is not None container2 = self.databaseForTest.create_container_if_not_exists(str(uuid.uuid4()), PartitionKey(path="/pk"), session_token=str(uuid.uuid4())) assert container2 is not None - container_list = list(self.databaseForTest.list_containers(session_token=str(uuid.uuid4()))) - container_list2 = list(self.databaseForTest.query_containers(query="select * from c", session_token=str(uuid.uuid4()))) - assert len(container_list) > 0 - assert len(container_list2) > 0 + container_ids = {c['id'] for c in self.databaseForTest.list_containers(session_token=str(uuid.uuid4()))} + container_ids2 = {c['id'] for c in + self.databaseForTest.query_containers(query="select * from c", + session_token=str(uuid.uuid4()))} + assert container.id in container_ids + assert container.id in container_ids2 container2_read = container2.read(session_token=str(uuid.uuid4())) assert container2_read is not None replace_container = self.databaseForTest.replace_container(container2, PartitionKey(path="/pk"), default_ttl=30, session_token=str(uuid.uuid4())) @@ -109,9 +115,9 @@ def test_session_token_compatibility(self): def test_etag_match_condition_compatibility(self): # Verifying that behavior is unaffected across the board for using `etag`/`match_condition` on irrelevant methods # Database - database = self.client.create_database(str(uuid.uuid4()), etag=str(uuid.uuid4()), match_condition=MatchConditions.IfModified) + database = self.client.create_database(test_config.unique_database_id("backcompat"), etag=str(uuid.uuid4()), match_condition=MatchConditions.IfModified) assert database is not None - database2 = self.client.create_database_if_not_exists(str(uuid.uuid4()), etag=str(uuid.uuid4()), match_condition=MatchConditions.IfNotModified) + database2 = self.client.create_database_if_not_exists(test_config.unique_database_id("backcompat"), etag=str(uuid.uuid4()), match_condition=MatchConditions.IfNotModified) assert database2 is not None self.client.delete_database(database2.id, etag=str(uuid.uuid4()), match_condition=MatchConditions.IfModified) try: diff --git a/sdk/cosmos/azure-cosmos/tests/test_backwards_compatibility_async.py b/sdk/cosmos/azure-cosmos/tests/test_backwards_compatibility_async.py index d9dcfd988cec..db27032af6f6 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_backwards_compatibility_async.py +++ b/sdk/cosmos/azure-cosmos/tests/test_backwards_compatibility_async.py @@ -47,14 +47,18 @@ async def asyncTearDown(self): async def test_session_token_compatibility_async(self): # Verifying that behavior is unaffected across the board for using `session_token` on irrelevant methods # Database - database = await self.client.create_database(str(uuid.uuid4()), session_token=str(uuid.uuid4())) + database = await self.client.create_database(test_config.unique_database_id("backcompat"), session_token=str(uuid.uuid4())) assert database is not None - database2 = await self.client.create_database_if_not_exists(str(uuid.uuid4()), session_token=str(uuid.uuid4())) + database2 = await self.client.create_database_if_not_exists(test_config.unique_database_id("backcompat"), session_token=str(uuid.uuid4())) assert database2 is not None - database_list = [db async for db in self.client.list_databases(session_token=str(uuid.uuid4()))] - database_list2 = [db async for db in self.client.query_databases(query="select * from c", session_token=str(uuid.uuid4()))] - assert len(database_list) > 0 - assert len(database_list2) > 0 + # Assert this run's own database is present rather than that the account is + # non-empty: the account is shared with other runs and other language SDKs, so a + # non-zero count says nothing about the database under test. + database_ids = {db['id'] async for db in self.client.list_databases(session_token=str(uuid.uuid4()))} + database_ids2 = {db['id'] async for db in + self.client.query_databases(query="select * from c", session_token=str(uuid.uuid4()))} + assert database.id in database_ids + assert database.id in database_ids2 database_read = await database.read(session_token=str(uuid.uuid4())) assert database_read is not None await self.client.delete_database(database2.id, session_token=str(uuid.uuid4())) @@ -69,10 +73,12 @@ async def test_session_token_compatibility_async(self): assert container is not None container2 = await self.created_database.create_container_if_not_exists(str(uuid.uuid4()), PartitionKey(path="/pk"), session_token=str(uuid.uuid4())) assert container2 is not None - container_list = [cont async for cont in self.created_database.list_containers(session_token=str(uuid.uuid4()))] - container_list2 = [cont async for cont in self.created_database.query_containers(query="select * from c", session_token=str(uuid.uuid4()))] - assert len(container_list) > 0 - assert len(container_list2) > 0 + container_ids = {c['id'] async for c in self.created_database.list_containers(session_token=str(uuid.uuid4()))} + container_ids2 = {c['id'] async for c in + self.created_database.query_containers(query="select * from c", + session_token=str(uuid.uuid4()))} + assert container.id in container_ids + assert container.id in container_ids2 container2_read = await container2.read(session_token=str(uuid.uuid4())) assert container2_read is not None replace_container = await self.created_database.replace_container(container2, PartitionKey(path="/pk"), default_ttl=30, session_token=str(uuid.uuid4())) @@ -93,9 +99,9 @@ async def test_session_token_compatibility_async(self): async def test_etag_match_condition_compatibility_async(self): # Verifying that behavior is unaffected across the board for using `etag`/`match_condition` on irrelevant methods # Database - database = await self.client.create_database(str(uuid.uuid4()), etag=str(uuid.uuid4()), match_condition=MatchConditions.IfModified) + database = await self.client.create_database(test_config.unique_database_id("backcompat"), etag=str(uuid.uuid4()), match_condition=MatchConditions.IfModified) assert database is not None - database2 = await self.client.create_database_if_not_exists(str(uuid.uuid4()), etag=str(uuid.uuid4()), match_condition=MatchConditions.IfNotModified) + database2 = await self.client.create_database_if_not_exists(test_config.unique_database_id("backcompat"), etag=str(uuid.uuid4()), match_condition=MatchConditions.IfNotModified) assert database2 is not None await self.client.delete_database(database2.id, etag=str(uuid.uuid4()), match_condition=MatchConditions.IfModified) try: diff --git a/sdk/cosmos/azure-cosmos/tests/test_circuit_breaker_emulator.py b/sdk/cosmos/azure-cosmos/tests/test_circuit_breaker_emulator.py index 4f7b9a6d3254..7ee7d2766a74 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_circuit_breaker_emulator.py +++ b/sdk/cosmos/azure-cosmos/tests/test_circuit_breaker_emulator.py @@ -22,9 +22,11 @@ COLLECTION = "created_collection" @pytest.fixture(scope="class", autouse=True) def setup_teardown(): - os.environ["AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER"] = "True" - yield - os.environ["AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER"] = "False" + previous_env = test_config.set_environment_variables(AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER="True") + try: + yield + finally: + test_config.restore_environment_variables(previous_env) def create_custom_transport_mm(): custom_transport = FaultInjectionTransport() @@ -217,7 +219,7 @@ def test_write_failure_rate_threshold_delete_all_items_by_pk_mm(self, setup_tear global_endpoint_manager = fault_injection_container.client_connection._global_endpoint_manager # lower minimum requests for testing _partition_health_tracker.MINIMUM_REQUESTS_FOR_FAILURE_RATE = 10 - os.environ["AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED"] = "80" + previous_env = test_config.set_environment_variables(AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED="80") try: # writes should fail but still be tracked and mark unavailable a partition after crossing threshold for i in range(10): @@ -245,7 +247,7 @@ def test_write_failure_rate_threshold_delete_all_items_by_pk_mm(self, setup_tear validate_unhealthy_partitions_mm(global_endpoint_manager, 1) finally: - os.environ["AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED"] = "90" + test_config.restore_environment_variables(previous_env) # restore minimum requests _partition_health_tracker.MINIMUM_REQUESTS_FOR_FAILURE_RATE = 100 @@ -258,7 +260,7 @@ def test_write_failure_rate_threshold_delete_all_items_by_pk_sm(self, setup_tear global_endpoint_manager = fault_injection_container.client_connection._global_endpoint_manager # lower minimum requests for testing _partition_health_tracker.MINIMUM_REQUESTS_FOR_FAILURE_RATE = 10 - os.environ["AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED"] = "80" + previous_env = test_config.set_environment_variables(AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED="80") try: # writes should fail but still be tracked and mark unavailable a partition after crossing threshold for i in range(10): @@ -286,7 +288,7 @@ def test_write_failure_rate_threshold_delete_all_items_by_pk_sm(self, setup_tear validate_unhealthy_partitions_sm_mrr(global_endpoint_manager, 0) finally: - os.environ["AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED"] = "90" + test_config.restore_environment_variables(previous_env) # restore minimum requests _partition_health_tracker.MINIMUM_REQUESTS_FOR_FAILURE_RATE = 100 diff --git a/sdk/cosmos/azure-cosmos/tests/test_circuit_breaker_emulator_async.py b/sdk/cosmos/azure-cosmos/tests/test_circuit_breaker_emulator_async.py index d04a54dcc5dd..130e45e18226 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_circuit_breaker_emulator_async.py +++ b/sdk/cosmos/azure-cosmos/tests/test_circuit_breaker_emulator_async.py @@ -25,9 +25,11 @@ COLLECTION = "created_collection" @pytest_asyncio.fixture(scope="class", loop_scope="class", autouse=True) async def setup_teardown(): - os.environ["AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER"] = "True" - yield - os.environ["AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER"] = "False" + previous_env = test_config.set_environment_variables(AZURE_COSMOS_ENABLE_CIRCUIT_BREAKER="True") + try: + yield + finally: + test_config.restore_environment_variables(previous_env) async def create_custom_transport_mm(): custom_transport = FaultInjectionTransportAsync() @@ -223,7 +225,7 @@ async def test_write_failure_rate_threshold_delete_all_items_by_pk_mm_async(self global_endpoint_manager = fault_injection_container.client_connection._global_endpoint_manager # lower minimum requests for testing _partition_health_tracker.MINIMUM_REQUESTS_FOR_FAILURE_RATE = 10 - os.environ["AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED"] = "80" + previous_env = test_config.set_environment_variables(AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED="80") try: # writes should fail but still be tracked and mark unavailable a partition after crossing threshold for i in range(10): @@ -251,7 +253,7 @@ async def test_write_failure_rate_threshold_delete_all_items_by_pk_mm_async(self validate_unhealthy_partitions_mm(global_endpoint_manager, 1) finally: - os.environ["AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED"] = "90" + test_config.restore_environment_variables(previous_env) # restore minimum requests _partition_health_tracker.MINIMUM_REQUESTS_FOR_FAILURE_RATE = 100 await cleanup_method([custom_setup, setup]) @@ -265,7 +267,7 @@ async def test_write_failure_rate_threshold_delete_all_items_by_pk_sm_async(self global_endpoint_manager = fault_injection_container.client_connection._global_endpoint_manager # lower minimum requests for testing _partition_health_tracker.MINIMUM_REQUESTS_FOR_FAILURE_RATE = 10 - os.environ["AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED"] = "80" + previous_env = test_config.set_environment_variables(AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED="80") try: # writes should fail but still be tracked and mark unavailable a partition after crossing threshold for i in range(10): @@ -293,7 +295,7 @@ async def test_write_failure_rate_threshold_delete_all_items_by_pk_sm_async(self validate_unhealthy_partitions_sm_mrr(global_endpoint_manager, 0) finally: - os.environ["AZURE_COSMOS_FAILURE_PERCENTAGE_TOLERATED"] = "90" + test_config.restore_environment_variables(previous_env) # restore minimum requests _partition_health_tracker.MINIMUM_REQUESTS_FOR_FAILURE_RATE = 100 await cleanup_method([custom_setup, setup]) diff --git a/sdk/cosmos/azure-cosmos/tests/test_config.py b/sdk/cosmos/azure-cosmos/tests/test_config.py index 5b5e9fd061e5..d078418537e7 100644 --- a/sdk/cosmos/azure-cosmos/tests/test_config.py +++ b/sdk/cosmos/azure-cosmos/tests/test_config.py @@ -36,6 +36,123 @@ SPLIT_TIMEOUT = 60*10 # timeout test at 10 minutes SLEEP_TIME = 30 # sleep for 30 seconds +# The live tests run against fixed, long-lived accounts that are shared with other +# language SDKs and with concurrent runs of this suite. Databases are the only +# account-scoped namespace we control, so every database this suite creates carries a +# prefix identifying the run that owns it. That keeps concurrent runs from colliding and +# lets cleanup delete only what a given run created. +RESOURCE_PREFIX = "PythonSDKTest" + + +def _build_run_id(): + # Build.BuildId is shared by every matrix leg of one pipeline run, which is what an + # out-of-band janitor matches on; the random suffix keeps parallel legs distinct. + build_id = os.getenv('BUILD_BUILDID') + suffix = uuid.uuid4().hex[:8] + return "{}-{}".format(build_id, suffix) if build_id else suffix + + +RUN_ID = _build_run_id() + + +def unique_database_id(label=""): + """Build a database id owned by this test run. + + Format: ``PythonSDKTest--