[v2] Modify model-validation tests to ensure unique properties

tests/functional/botocore/test_auth_config.py

@@ -74,16 +74,26 @@ def _all_test_cases():
 
 
 @pytest.mark.validates_models
-@pytest.mark.parametrize("auth_service, auth_config", AUTH_SERVICES)
-def test_all_requirements_match_for_service(auth_service, auth_config):
+@pytest.mark.parametrize(
+    "auth_service, auth_config",
+    AUTH_SERVICES
+)
+def test_all_requirements_match_for_service(
+        auth_service,
+        auth_config,
+        record_property
+):
+    record_property("aws_service", auth_service)
     # Validates that all service-level signature types have the same requirements
     message = f'Found mixed signer requirements for service: {auth_service}'
     assert_all_requirements_match(auth_config, message)
 
 
 @pytest.mark.validates_models
 @pytest.mark.parametrize("auth_service, operation_model", AUTH_OPERATIONS)
-def test_all_requirements_match_for_operation(auth_service, operation_model):
+def test_all_requirements_match_for_operation(auth_service, operation_model, record_property):
+    record_property("aws_service", operation_model.service_model.service_name)
+    record_property("aws_operation", operation_model.name)
     # Validates that all operation-level signature types have the same requirements
     message = f'Found mixed signer requirements for operation: {auth_service}.{operation_model.name}'
     auth_config = operation_model.auth

tests/functional/botocore/test_endpoints.py

@@ -105,7 +105,7 @@ def _available_services():
 
 @pytest.mark.validates_models
 @pytest.mark.parametrize("service_name", _available_services())
-def test_client_name_matches_hyphenized_service_id(service_name):
+def test_client_name_matches_hyphenized_service_id(service_name, record_property):
     """Generates tests for each service to verify that the computed service
     named based on the service id matches the service name used to
     create a client (i.e the directory name in botocore/data)
@@ -114,6 +114,7 @@ def test_client_name_matches_hyphenized_service_id(service_name):
     session = get_session()
     service_model = session.get_service_model(service_name)
     computed_name = service_model.service_id.replace(' ', '-').lower()
+    record_property('aws_service', service_name)
 
     # Handle known exceptions where we have renamed the service directory
     # for one reason or another.

tests/functional/botocore/test_socialmessaging.py

@@ -34,9 +34,11 @@ def _get_all_xform_operations():
 
 @pytest.mark.validates_models
 @pytest.mark.parametrize("operation, replacement", XFORM_OPERATIONS)
-def test_known_replacements(operation, replacement):
+def test_known_replacements(operation, replacement, record_property):
     # Validates that if a replacement shows up in the lowercased version of an
     # operation, we will keep all of those characters together in the final operation
     # name
+    record_property('aws_service', 'socialmessaging')
+    record_property('aws_operation', operation)
     assert replacement in xform_name(operation, '_')
     assert replacement in xform_name(operation, '-')

tests/functional/test_no_event_streams.py

@@ -19,36 +19,50 @@
 _ALLOWED_COMMANDS = ['s3api select-object-content']
 
 
-@pytest.mark.validates_models
-def test_no_event_stream_unless_allowed(record_property):
+def _generate_command_tests():
     driver = create_clidriver()
     help_command = driver.create_help_command()
-    errors = []
     for command_name, command_obj in help_command.command_table.items():
         sub_help = command_obj.create_help_command()
         if hasattr(sub_help, 'command_table'):
             for sub_name, sub_command in sub_help.command_table.items():
                 op_help = sub_command.create_help_command()
                 model = op_help.obj
                 if isinstance(model, OperationModel):
-                    full_command = f'{command_name} {sub_name}'
-                    if (
-                        model.has_event_stream_input
-                        or model.has_event_stream_output
-                    ):
-                        if full_command in _ALLOWED_COMMANDS:
-                            continue
-                        # Store the service and operation in
-                        # PyTest custom properties
-                        record_property(
-                            'aws_service', model.service_model.service_name
-                        )
-                        record_property('aws_operation', model.name)
-                        supported_commands = '\n'.join(_ALLOWED_COMMANDS)
-                        errors.append(
-                            f'The {full_command} command uses event streams '
-                            'which is only supported for these operations:\n'
-                            f'{supported_commands}'
-                        )
-    if errors:
-        raise AssertionError('\n' + '\n'.join(errors))
+                    # Extract the properties needed for tests to avoid
+                    # parametrizing entire model objects, which may cause
+                    # excessive memory usage.
+                    if not model.has_event_stream_input and not model.has_event_stream_output:
+                        # Only parameterize the models that actually have event streams.
+                        # This has shown to improve test execution performance.
+                        continue
+                    model_description = {
+                        'service_name': model.service_model.service_name,
+                        'name': model.name,
+                    }
+                    yield command_name, sub_name, model_description
+
+
+@pytest.mark.validates_models
+@pytest.mark.parametrize(
+    "command_name, sub_name, model", _generate_command_tests()
+)
+def test_no_event_stream_unless_allowed(
+        command_name,
+        sub_name,
+        model,
+        record_property
+):
+    full_command = f'{command_name} {sub_name}'
+    # Store the service and operation in
+    # PyTest custom properties
+    record_property(
+        'aws_service', model['service_name']
+    )
+    record_property('aws_operation', model['name'])
+    supported_commands = '\n'.join(_ALLOWED_COMMANDS)
+    assert full_command in _ALLOWED_COMMANDS, (
+        f'The {full_command} command uses event streams '
+        'which is only supported for these operations:\n'
+        f'{supported_commands}'
+    )

tests/functional/test_shadowing.py

@@ -19,21 +19,42 @@
 def _generate_command_tests():
     driver = create_clidriver()
     help_command = driver.create_help_command()
-    top_level_params = set(driver.create_help_command().arg_table)
+    top_level_params = set(driver.create_help_command().arg_table.keys())
     for command_name, command_obj in list(help_command.command_table.items()):
         sub_help = command_obj.create_help_command()
         if hasattr(sub_help, 'command_table'):
-            yield command_name, sub_help.command_table, top_level_params
+            for sub_name, sub_command in sub_help.command_table.items():
+                op_help = sub_command.create_help_command()
+                model = op_help.obj
+                arg_table = op_help.arg_table
+                if not isinstance(model, OperationModel):
+                    continue
+                yield (
+                    command_name,
+                    sub_name,
+                    model.service_model.service_name,
+                    model.name,
+                    arg_table,
+                    top_level_params
+                )
 
 
 @pytest.mark.validates_models
 @pytest.mark.parametrize(
-    "command_name, command_table, builtins", _generate_command_tests()
+    "command_name, sub_name, service_name, "
+    "operation_name, arg_table, builtins",
+    _generate_command_tests()
 )
 def test_no_shadowed_builtins(
-    command_name, command_table, builtins, record_property
+        command_name,
+        sub_name,
+        service_name,
+        operation_name,
+        arg_table,
+        builtins,
+        record_property
 ):
-    """Verify no command params are shadowed or prefixed by the built in param.
+    """Verify no command params are shadowed or prefixed by the built-in param.
 
     The CLI parses all command line options into a single namespace.
     This means that option names must be unique and cannot conflict
@@ -59,23 +80,18 @@ def test_no_shadowed_builtins(
 
     """
     errors = []
-    for sub_name, sub_command in command_table.items():
-        op_help = sub_command.create_help_command()
-        model = op_help.obj
-        arg_table = op_help.arg_table
-        for arg_name in arg_table:
-            if any(p.startswith(arg_name) for p in builtins):
-                if isinstance(model, OperationModel):
-                    # Store the service and operation in
-                    # PyTest custom properties
-                    record_property(
-                        'aws_service', model.service_model.service_name
-                    )
-                    record_property('aws_operation', model.name)
-                # Then we are shadowing or prefixing a top level argument
-                errors.append(
-                    'Shadowing/Prefixing a top level option: '
-                    f'{command_name}.{sub_name}.{arg_name}'
-                )
+    for arg_name in arg_table:
+        if any(p.startswith(arg_name) for p in builtins):
+            # Then we are shadowing or prefixing a top level argument
+            errors.append(
+                'Shadowing/Prefixing a top level option: '
+                f'{command_name}.{sub_name}.{arg_name}'
+            )
     if errors:
+        # Store the service and operation in
+        # PyTest custom properties
+        record_property(
+            'aws_service', service_name
+        )
+        record_property('aws_operation', operation_name)
         raise AssertionError('\n' + '\n'.join(errors))