diff --git a/packages/gapic-generator/gapic/schema/api.py b/packages/gapic-generator/gapic/schema/api.py index d86d4d29ea9a..824bc80383a6 100644 --- a/packages/gapic-generator/gapic/schema/api.py +++ b/packages/gapic-generator/gapic/schema/api.py @@ -258,114 +258,46 @@ def disambiguate(self, string: str) -> str: return self.disambiguate(f"_{string}") return string - def add_to_address_allowlist( + def with_selective_generation( self, *, - address_allowlist: Set["metadata.Address"], - method_allowlist: Set[str], - resource_messages: Dict[str, "wrappers.MessageType"], - ) -> None: - """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. - - This method is used to create an allowlist of addresses to be used to filter out unneeded - services, methods, messages, and enums at a later step. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to add to. Only the addresses of the allowlisted methods, the services - containing these methods, and messages/enums those methods use will be part of the - final address_allowlist. The set may be modified during this call. - method_allowlist (Set[str]): An allowlist of fully-qualified method names. - resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified - resource type name of a resource message to the corresponding MessageType object - representing that resource message. Only resources with a message representation - should be included in the dictionary. - Returns: - None - """ - # The method.operation_service for an extended LRO is not fully qualified, so we - # truncate the service names accordingly so they can be found in - # method.add_to_address_allowlist - services_in_proto = { - service.name: service for service in self.services.values() - } - for service in self.services.values(): - service.add_to_address_allowlist( - address_allowlist=address_allowlist, - method_allowlist=method_allowlist, - resource_messages=resource_messages, - services_in_proto=services_in_proto, - ) - - def prune_messages_for_selective_generation( - self, *, address_allowlist: Set["metadata.Address"] - ) -> Optional["Proto"]: - """Returns a truncated version of this Proto. - - Only the services, messages, and enums contained in the allowlist - of visited addresses are included in the returned object. If there - are no services, messages, or enums left, and no file level resources, - return None. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to filter against. Objects with addresses not the allowlist will be - removed from the returned Proto. - Returns: - Optional[Proto]: A truncated version of this proto. If there are no services, messages, - or enums left after the truncation process and there are no file level resources, - returns None. - """ - # Once the address allowlist has been created, it suffices to only - # prune items at 2 different levels to truncate the Proto object: - # - # 1. At the Proto level, we remove unnecessary services, messages, - # and enums. - # 2. For allowlisted services, at the Service level, we remove - # non-allowlisted methods. - services = { - k: v.prune_messages_for_selective_generation( - address_allowlist=address_allowlist - ) - for k, v in self.services.items() - if v.meta.address in address_allowlist - } + generate_omitted_as_internal: bool, + public_methods: Set[str], + excluded_addresses: Set["metadata.Address"], + ) -> "Proto": + services = {} + for k, v in self.services.items(): + new_v = v.with_selective_generation( + generate_omitted_as_internal=generate_omitted_as_internal, + public_methods=public_methods, + excluded_addresses=excluded_addresses) + if new_v: + services[k] = new_v + + # We only prune messages/enums from protos that are not dependencies. + # Messages and enums are excluded only if they are reachable from some RPC + # but NOT from any of the publicly allowed RPCs. all_messages = { - k: v for k, v in self.all_messages.items() if v.ident in address_allowlist + k: v for k, v in self.all_messages.items() if v.ident not in excluded_addresses } all_enums = { - k: v for k, v in self.all_enums.items() if v.ident in address_allowlist + k: v for k, v in self.all_enums.items() if v.ident not in excluded_addresses } + # If the proto becomes empty after pruning, we return None to signal + # that it should be excluded from generation. if not services and not all_messages and not all_enums: return None return dataclasses.replace( - self, services=services, all_messages=all_messages, all_enums=all_enums + self, + services=services, + all_messages=all_messages, + all_enums=all_enums, ) - def with_internal_methods(self, *, public_methods: Set[str]) -> "Proto": - """Returns a version of this Proto with some Methods marked as internal. - - The methods not in the public_methods set will be marked as internal and - services containing these methods will also be marked as internal by extension. - (See :meth:`Service.is_internal` for more details). - - Args: - public_methods (Set[str]): An allowlist of fully-qualified method names. - Methods not in this allowlist will be marked as internal. - Returns: - Proto: A version of this Proto with Method objects corresponding to methods - not in `public_methods` marked as internal. - """ - services = { - k: v.with_internal_methods(public_methods=public_methods) - for k, v in self.services.items() - } - return dataclasses.replace(self, services=services) - @dataclasses.dataclass(frozen=True) class API: @@ -529,37 +461,61 @@ def disambiguate_keyword_sanitize_fname( k: v for k, v in api.all_protos.items() if k not in api.protos } - if selective_gapic_settings.generate_omitted_as_internal: - for name, proto in api.protos.items(): - new_all_protos[name] = proto.with_internal_methods( - public_methods=selective_gapic_methods + all_resource_messages = collections.ChainMap( + *(proto.resource_messages for proto in api.all_protos.values()) + ) + + # Calculate all reachable addresses (API-wide). + # This includes all messages and enums reachable from ANY RPC + # defined in any proto of the API. + all_rpc_addresses: Set["metadata.Address"] = set([]) + all_methods = set(api.all_methods.keys()) + # Create a global map of services to support cross-proto lookup + # for extended LROs. + all_services: Dict[str, wrappers.Service] = {} + for p in api.all_protos.values(): + for s in p.services.values(): + all_services[s.meta.address.proto] = s + all_services[s.name] = s + + for proto in api.all_protos.values(): + for service in proto.services.values(): + service.add_to_address_allowlist( + address_allowlist=all_rpc_addresses, + method_allowlist=all_methods, + resource_messages=all_resource_messages, + services_in_proto=all_services, ) - else: - all_resource_messages = collections.ChainMap( - *(proto.resource_messages for proto in protos.values()) - ) - # Prepare a list of addresses to include in selective generation, - # then prune each Proto object. We look at metadata.Addresses, not objects, because - # objects that refer to the same thing in the proto are different Python objects - # in memory. - address_allowlist: Set["metadata.Address"] = set([]) - for proto in api.protos.values(): - proto.add_to_address_allowlist( - address_allowlist=address_allowlist, + # Calculate publicly reachable addresses (API-wide). + # This includes only types reachable from the allowlisted methods. + public_rpc_addresses: Set["metadata.Address"] = set([]) + for proto in api.all_protos.values(): + for service in proto.services.values(): + service.add_to_address_allowlist( + address_allowlist=public_rpc_addresses, method_allowlist=selective_gapic_methods, resource_messages=all_resource_messages, + services_in_proto=all_services, ) - # We only prune services/messages/enums from protos that are not dependencies. - for name, proto in api.protos.items(): - proto_to_generate = ( - proto.prune_messages_for_selective_generation( - address_allowlist=address_allowlist - ) - ) - if proto_to_generate: - new_all_protos[name] = proto_to_generate + # Addresses to exclude: those that ARE reachable from SOME RPC but NOT from any PUBLIC RPC. + # Types not attached to any RPC will not be in all_rpc_addresses and thus + # will NOT be in excluded_addresses, meaning they are preserved. + excluded_addresses = ( + all_rpc_addresses - public_rpc_addresses + if not selective_gapic_settings.generate_omitted_as_internal + else set([]) + ) + + for name, proto in api.protos.items(): + proto_to_generate = proto.with_selective_generation( + generate_omitted_as_internal=selective_gapic_settings.generate_omitted_as_internal, + public_methods=selective_gapic_methods, + excluded_addresses=excluded_addresses, + ) + if proto_to_generate: + new_all_protos[name] = proto_to_generate api = cls( naming=naming, diff --git a/packages/gapic-generator/gapic/schema/wrappers.py b/packages/gapic-generator/gapic/schema/wrappers.py index 5794bfb92b19..c8a730e4032f 100644 --- a/packages/gapic-generator/gapic/schema/wrappers.py +++ b/packages/gapic-generator/gapic/schema/wrappers.py @@ -2014,7 +2014,6 @@ def add_to_address_allowlist( objects to add to. Only the addresses of the allowlisted methods, the services containing these methods, and messages/enums those methods use will be part of the final address_allowlist. The set may be modified during this call. - method_allowlist (Set[str]): An allowlist of fully-qualified method names. resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified resource type name of a resource message to the corresponding MessageType object representing that resource message. Only resources with a message representation @@ -2061,26 +2060,28 @@ def add_to_address_allowlist( resource_messages=resource_messages, ) - def with_internal_methods(self, *, public_methods: Set[str]) -> "Method": - """Returns a version of this ``Method`` marked as internal - - The methods not in the public_methods set will be marked as internal and - this ``Service`` will as well by extension (see :meth:`Service.is_internal`). + def with_selective_generation( + self, + *, + generate_omitted_as_internal: bool, + public_methods: Set[str], + excluded_addresses: Set["metadata.Address"], + ) -> "Method": - Args: - public_methods (Set[str]): An allowlist of fully-qualified method names. - Methods not in this allowlist will be marked as internal. - Returns: - Service: A version of this `Service` with `Method` objects corresponding to methods - not in `public_methods` marked as internal. - """ if self.ident.proto in public_methods: return self - return dataclasses.replace( - self, - is_internal=True, - ) + # Not public. + # We mark it as internal if either generate_omitted_as_internal is set, + # or if the method is reachable from some public method (e.g. as a polling method). + if generate_omitted_as_internal or self.meta.address not in excluded_addresses: + return dataclasses.replace( + self, + is_internal=True, + ) + else: + return None + @dataclasses.dataclass(frozen=True) @@ -2467,45 +2468,24 @@ def add_to_address_allowlist( services_in_proto=services_in_proto, ) - def prune_messages_for_selective_generation( - self, *, address_allowlist: Set["metadata.Address"] + def with_selective_generation( + self, + *, + generate_omitted_as_internal: bool, + public_methods: Set[str], + excluded_addresses: Set["metadata.Address"], ) -> "Service": - """Returns a truncated version of this Service. - - Only the methods, messages, and enums contained in the address allowlist - are included in the returned object. - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to filter against. Objects with addresses not the allowlist will be - removed from the returned Proto. - Returns: - Service: A truncated version of this proto. - """ - return dataclasses.replace( - self, - methods={ - k: v for k, v in self.methods.items() if v.ident in address_allowlist - }, - ) - - def with_internal_methods(self, *, public_methods: Set[str]) -> "Service": - """Returns a version of this ``Service`` with some Methods marked as internal. + methods = {} + for k, v in self.methods.items(): + new_v = v.with_selective_generation( + generate_omitted_as_internal=generate_omitted_as_internal, + public_methods=public_methods, + excluded_addresses=excluded_addresses) + if new_v: + methods[k] = new_v - The methods not in the public_methods set will be marked as internal and - this ``Service`` will as well by extension (see :meth:`Service.is_internal`). + if not generate_omitted_as_internal and not methods: + return None - Args: - public_methods (Set[str]): An allowlist of fully-qualified method names. - Methods not in this allowlist will be marked as internal. - Returns: - Service: A version of this `Service` with `Method` objects corresponding to methods - not in `public_methods` marked as internal. - """ - return dataclasses.replace( - self, - methods={ - k: v.with_internal_methods(public_methods=public_methods) - for k, v in self.methods.items() - }, - ) + return dataclasses.replace(self, methods=methods) diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis/__init__.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis/__init__.py index 045bcae4c55c..d96b1aebac11 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis/__init__.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis/__init__.py @@ -27,6 +27,7 @@ from google.cloud.redis_v1.types.cloud_redis import Instance from google.cloud.redis_v1.types.cloud_redis import ListInstancesRequest from google.cloud.redis_v1.types.cloud_redis import ListInstancesResponse +from google.cloud.redis_v1.types.cloud_redis import LocationMetadata from google.cloud.redis_v1.types.cloud_redis import MaintenancePolicy from google.cloud.redis_v1.types.cloud_redis import MaintenanceSchedule from google.cloud.redis_v1.types.cloud_redis import NodeInfo @@ -35,6 +36,7 @@ from google.cloud.redis_v1.types.cloud_redis import TlsCertificate from google.cloud.redis_v1.types.cloud_redis import UpdateInstanceRequest from google.cloud.redis_v1.types.cloud_redis import WeeklyMaintenanceWindow +from google.cloud.redis_v1.types.cloud_redis import ZoneMetadata __all__ = ('CloudRedisClient', 'CloudRedisAsyncClient', @@ -44,6 +46,7 @@ 'Instance', 'ListInstancesRequest', 'ListInstancesResponse', + 'LocationMetadata', 'MaintenancePolicy', 'MaintenanceSchedule', 'NodeInfo', @@ -52,4 +55,5 @@ 'TlsCertificate', 'UpdateInstanceRequest', 'WeeklyMaintenanceWindow', + 'ZoneMetadata', ) diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/__init__.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/__init__.py index 1f7bad3796c9..bac280e47101 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/__init__.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/__init__.py @@ -37,6 +37,7 @@ from .types.cloud_redis import Instance from .types.cloud_redis import ListInstancesRequest from .types.cloud_redis import ListInstancesResponse +from .types.cloud_redis import LocationMetadata from .types.cloud_redis import MaintenancePolicy from .types.cloud_redis import MaintenanceSchedule from .types.cloud_redis import NodeInfo @@ -45,6 +46,7 @@ from .types.cloud_redis import TlsCertificate from .types.cloud_redis import UpdateInstanceRequest from .types.cloud_redis import WeeklyMaintenanceWindow +from .types.cloud_redis import ZoneMetadata if hasattr(api_core, "check_python_version") and hasattr(api_core, "check_dependency_versions"): # pragma: NO COVER api_core.check_python_version("google.cloud.redis_v1") # type: ignore @@ -139,6 +141,7 @@ def _get_version(dependency_name): 'Instance', 'ListInstancesRequest', 'ListInstancesResponse', +'LocationMetadata', 'MaintenancePolicy', 'MaintenanceSchedule', 'NodeInfo', @@ -147,4 +150,5 @@ def _get_version(dependency_name): 'TlsCertificate', 'UpdateInstanceRequest', 'WeeklyMaintenanceWindow', +'ZoneMetadata', ) diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/__init__.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/__init__.py index 1e420395cc1d..e15d95a60b3e 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/__init__.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/__init__.py @@ -20,6 +20,7 @@ Instance, ListInstancesRequest, ListInstancesResponse, + LocationMetadata, MaintenancePolicy, MaintenanceSchedule, NodeInfo, @@ -28,6 +29,7 @@ TlsCertificate, UpdateInstanceRequest, WeeklyMaintenanceWindow, + ZoneMetadata, ) __all__ = ( @@ -37,6 +39,7 @@ 'Instance', 'ListInstancesRequest', 'ListInstancesResponse', + 'LocationMetadata', 'MaintenancePolicy', 'MaintenanceSchedule', 'NodeInfo', @@ -45,4 +48,5 @@ 'TlsCertificate', 'UpdateInstanceRequest', 'WeeklyMaintenanceWindow', + 'ZoneMetadata', ) diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/cloud_redis.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/cloud_redis.py index 8022b120202a..96031fd810f2 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/cloud_redis.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/cloud_redis.py @@ -42,6 +42,8 @@ 'UpdateInstanceRequest', 'DeleteInstanceRequest', 'OperationMetadata', + 'LocationMetadata', + 'ZoneMetadata', 'TlsCertificate', }, ) @@ -973,6 +975,36 @@ class OperationMetadata(proto.Message): ) +class LocationMetadata(proto.Message): + r"""This location metadata represents additional configuration options + for a given location where a Redis instance may be created. All + fields are output only. It is returned as content of the + ``google.cloud.location.Location.metadata`` field. + + Attributes: + available_zones (MutableMapping[str, google.cloud.redis_v1.types.ZoneMetadata]): + Output only. The set of available zones in the location. The + map is keyed by the lowercase ID of each zone, as defined by + GCE. These keys can be specified in ``location_id`` or + ``alternative_location_id`` fields when creating a Redis + instance. + """ + + available_zones: MutableMapping[str, 'ZoneMetadata'] = proto.MapField( + proto.STRING, + proto.MESSAGE, + number=1, + message='ZoneMetadata', + ) + + +class ZoneMetadata(proto.Message): + r"""Defines specific information for a particular zone. Currently + empty and reserved for future use only. + + """ + + class TlsCertificate(proto.Message): r"""TlsCertificate Resource diff --git a/packages/gapic-generator/tests/unit/schema/test_api.py b/packages/gapic-generator/tests/unit/schema/test_api.py index 9d94e8251523..a5705b776caf 100644 --- a/packages/gapic-generator/tests/unit/schema/test_api.py +++ b/packages/gapic-generator/tests/unit/schema/test_api.py @@ -3606,8 +3606,13 @@ def test_selective_gapic_api_build_remove_unnecessary_proto_files(): name="GetBarRequest", fields=( make_field_pb2( - name="bar", + name="bar_local", number=1, + type_name=".google.example.v1.Bar", + ), + make_field_pb2( + name="bar_common", + number=2, type_name=".google.example.v1.bar_common.Bar", ), ),