From 5cdf67efebfaa96b2b5ac8b14dac221ef1821e81 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 5 May 2026 17:30:39 +0000 Subject: [PATCH 1/3] fix: restore messages not attached to rpc in selective_gapic_generation --- packages/gapic-generator/gapic/schema/api.py | 149 +------- .../gapic-generator/gapic/schema/wrappers.py | 346 ++---------------- 2 files changed, 43 insertions(+), 452 deletions(-) diff --git a/packages/gapic-generator/gapic/schema/api.py b/packages/gapic-generator/gapic/schema/api.py index d86d4d29ea9a..79fa0fd2fad0 100644 --- a/packages/gapic-generator/gapic/schema/api.py +++ b/packages/gapic-generator/gapic/schema/api.py @@ -258,112 +258,21 @@ def disambiguate(self, string: str) -> str: return self.disambiguate(f"_{string}") return string - def add_to_address_allowlist( + def with_selective_generation( self, *, - address_allowlist: Set["metadata.Address"], - method_allowlist: Set[str], - resource_messages: Dict[str, "wrappers.MessageType"], - ) -> None: - """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. - - This method is used to create an allowlist of addresses to be used to filter out unneeded - services, methods, messages, and enums at a later step. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to add to. Only the addresses of the allowlisted methods, the services - containing these methods, and messages/enums those methods use will be part of the - final address_allowlist. The set may be modified during this call. - method_allowlist (Set[str]): An allowlist of fully-qualified method names. - resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified - resource type name of a resource message to the corresponding MessageType object - representing that resource message. Only resources with a message representation - should be included in the dictionary. - Returns: - None - """ - # The method.operation_service for an extended LRO is not fully qualified, so we - # truncate the service names accordingly so they can be found in - # method.add_to_address_allowlist - services_in_proto = { - service.name: service for service in self.services.values() - } - for service in self.services.values(): - service.add_to_address_allowlist( - address_allowlist=address_allowlist, - method_allowlist=method_allowlist, - resource_messages=resource_messages, - services_in_proto=services_in_proto, - ) - - def prune_messages_for_selective_generation( - self, *, address_allowlist: Set["metadata.Address"] - ) -> Optional["Proto"]: - """Returns a truncated version of this Proto. - - Only the services, messages, and enums contained in the allowlist - of visited addresses are included in the returned object. If there - are no services, messages, or enums left, and no file level resources, - return None. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to filter against. Objects with addresses not the allowlist will be - removed from the returned Proto. - Returns: - Optional[Proto]: A truncated version of this proto. If there are no services, messages, - or enums left after the truncation process and there are no file level resources, - returns None. - """ - # Once the address allowlist has been created, it suffices to only - # prune items at 2 different levels to truncate the Proto object: - # - # 1. At the Proto level, we remove unnecessary services, messages, - # and enums. - # 2. For allowlisted services, at the Service level, we remove - # non-allowlisted methods. - services = { - k: v.prune_messages_for_selective_generation( - address_allowlist=address_allowlist - ) - for k, v in self.services.items() - if v.meta.address in address_allowlist - } - - all_messages = { - k: v for k, v in self.all_messages.items() if v.ident in address_allowlist - } - - all_enums = { - k: v for k, v in self.all_enums.items() if v.ident in address_allowlist - } - - if not services and not all_messages and not all_enums: - return None - - return dataclasses.replace( - self, services=services, all_messages=all_messages, all_enums=all_enums - ) - - def with_internal_methods(self, *, public_methods: Set[str]) -> "Proto": - """Returns a version of this Proto with some Methods marked as internal. + generate_omitted_as_internal: bool, + public_methods: Set[str], + ) -> "Proto": - The methods not in the public_methods set will be marked as internal and - services containing these methods will also be marked as internal by extension. - (See :meth:`Service.is_internal` for more details). + services = {} + for k, v in self.services.items(): + new_v = v.with_selective_generation( + generate_omitted_as_internal=generate_omitted_as_internal, + public_methods=public_methods) + if new_v: + services[k] = new_v - Args: - public_methods (Set[str]): An allowlist of fully-qualified method names. - Methods not in this allowlist will be marked as internal. - Returns: - Proto: A version of this Proto with Method objects corresponding to methods - not in `public_methods` marked as internal. - """ - services = { - k: v.with_internal_methods(public_methods=public_methods) - for k, v in self.services.items() - } return dataclasses.replace(self, services=services) @@ -529,37 +438,13 @@ def disambiguate_keyword_sanitize_fname( k: v for k, v in api.all_protos.items() if k not in api.protos } - if selective_gapic_settings.generate_omitted_as_internal: - for name, proto in api.protos.items(): - new_all_protos[name] = proto.with_internal_methods( - public_methods=selective_gapic_methods - ) - else: - all_resource_messages = collections.ChainMap( - *(proto.resource_messages for proto in protos.values()) + for name, proto in api.protos.items(): + proto_to_generate = proto.with_selective_generation( + generate_omitted_as_internal=selective_gapic_settings.generate_omitted_as_internal, + public_methods=selective_gapic_methods, ) - - # Prepare a list of addresses to include in selective generation, - # then prune each Proto object. We look at metadata.Addresses, not objects, because - # objects that refer to the same thing in the proto are different Python objects - # in memory. - address_allowlist: Set["metadata.Address"] = set([]) - for proto in api.protos.values(): - proto.add_to_address_allowlist( - address_allowlist=address_allowlist, - method_allowlist=selective_gapic_methods, - resource_messages=all_resource_messages, - ) - - # We only prune services/messages/enums from protos that are not dependencies. - for name, proto in api.protos.items(): - proto_to_generate = ( - proto.prune_messages_for_selective_generation( - address_allowlist=address_allowlist - ) - ) - if proto_to_generate: - new_all_protos[name] = proto_to_generate + if proto_to_generate: + new_all_protos[name] = proto_to_generate api = cls( naming=naming, diff --git a/packages/gapic-generator/gapic/schema/wrappers.py b/packages/gapic-generator/gapic/schema/wrappers.py index 5794bfb92b19..6950fd72ef2a 100644 --- a/packages/gapic-generator/gapic/schema/wrappers.py +++ b/packages/gapic-generator/gapic/schema/wrappers.py @@ -441,48 +441,6 @@ def with_context( meta=self.meta.with_context(collisions=collisions), ) - def add_to_address_allowlist( - self, - *, - address_allowlist: Set["metadata.Address"], - resource_messages: Dict[str, "MessageType"], - ) -> None: - """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. - - This method is used to create an allowlist of addresses to be used to filter out unneeded - services, methods, messages, and enums at a later step. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to add to. Only the addresses of the allowlisted methods, the services - containing these methods, and messages/enums those methods use will be part of the - final address_allowlist. The set may be modified during this call. - resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified - resource type name of a resource message to the corresponding MessageType object - representing that resource message. Only resources with a message representation - should be included in the dictionary. - Returns: - None - """ - if self.message: - self.message.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - ) - - if self.enum: - self.enum.add_to_address_allowlist( - address_allowlist=address_allowlist, - ) - - if self.resource_reference and self.resource_reference in resource_messages: - # The message types in resource_message are different objects, but should be - # defined the same as the MessageTypes we're traversing here. - resource_messages[self.resource_reference].add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - ) - @dataclasses.dataclass(frozen=True) class FieldHeader: @@ -860,49 +818,6 @@ def with_context( meta=self.meta.with_context(collisions=collisions), ) - def add_to_address_allowlist( - self, - *, - address_allowlist: Set["metadata.Address"], - resource_messages: Dict[str, "MessageType"], - ) -> None: - """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. - - This method is used to create an allowlist of addresses to be used to filter out unneeded - services, methods, messages, and enums at a later step. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to add to. Only the addresses of the allowlisted methods, the services - containing these methods, and messages/enums those methods use will be part of the - final address_allowlist. The set may be modified during this call. - resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified - resource type name of a resource message to the corresponding MessageType object - representing that resource message. Only resources with a message representation - should be included in the dictionary. - Returns: - None - """ - if self.ident not in address_allowlist: - address_allowlist.add(self.ident) - - for field in self.fields.values(): - field.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - ) - - for enum in self.nested_enums.values(): - enum.add_to_address_allowlist( - address_allowlist=address_allowlist, - ) - - for message in self.nested_messages.values(): - message.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - ) - @dataclasses.dataclass(frozen=True) class EnumValueType: @@ -963,24 +878,6 @@ def with_context(self, *, collisions: Set[str]) -> "EnumType": else self ) - def add_to_address_allowlist( - self, *, address_allowlist: Set["metadata.Address"] - ) -> None: - """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. - - This method is used to create an allowlist of addresses to be used to filter out unneeded - services, methods, messages, and enums at a later step. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to add to. Only the addresses of the allowlisted methods, the services - containing these methods, and messages/enums those methods use will be part of the - final address_allowlist. The set may be modified during this call. - Returns: - None - """ - address_allowlist.add(self.ident) - @property def options_dict(self) -> Dict: """Return the EnumOptions (if present) as a dict. @@ -1097,39 +994,6 @@ def with_context( ) ) - def add_to_address_allowlist( - self, - *, - address_allowlist: Set["metadata.Address"], - resource_messages: Dict[str, "MessageType"], - ) -> None: - """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. - - This method is used to create an allowlist of addresses to be used to filter out unneeded - services, methods, messages, and enums at a later step. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to add to. Only the addresses of the allowlisted methods, the services - containing these methods, and messages/enums those methods use will be part of the - final address_allowlist. The set may be modified during this call. - resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified - resource type name of a resource message to the corresponding MessageType object - representing that resource message. Only resources with a message representation - should be included in the dictionary. - Returns: - None - """ - - self.request_type.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - ) - self.operation_type.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - ) - @dataclasses.dataclass(frozen=True) class OperationInfo: @@ -1163,38 +1027,6 @@ def with_context( ), ) - def add_to_address_allowlist( - self, - *, - address_allowlist: Set["metadata.Address"], - resource_messages: Dict[str, "MessageType"], - ) -> None: - """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. - - This method is used to create an allowlist of addresses to be used to filter out unneeded - services, methods, messages, and enums at a later step. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to add to. Only the addresses of the allowlisted methods, the services - containing these methods, and messages/enums those methods use will be part of the - final address_allowlist. The set may be modified during this call. - resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified - resource type name of a resource message to the corresponding MessageType object - representing that resource message. Only resources with a message representation - should be included in the dictionary. - Returns: - None - """ - self.response_type.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - ) - - self.metadata_type.add_to_address_allowlist( - address_allowlist=address_allowlist, resource_messages=resource_messages - ) - @dataclasses.dataclass(frozen=True) class RetryInfo: @@ -1997,90 +1829,25 @@ def with_context( meta=self.meta.with_context(collisions=collisions), ) - def add_to_address_allowlist( + def with_selective_generation( self, *, - address_allowlist: Set["metadata.Address"], - resource_messages: Dict[str, "MessageType"], - services_in_proto: Dict[str, "Service"], - ) -> None: - """Adds to the allowlist of Addresses of wrapper objects to be included in selective GAPIC generation. - - This method is used to create an allowlist of addresses to be used to filter out unneeded - services, methods, messages, and enums at a later step. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to add to. Only the addresses of the allowlisted methods, the services - containing these methods, and messages/enums those methods use will be part of the - final address_allowlist. The set may be modified during this call. - method_allowlist (Set[str]): An allowlist of fully-qualified method names. - resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified - resource type name of a resource message to the corresponding MessageType object - representing that resource message. Only resources with a message representation - should be included in the dictionary. - services_in_proto (Dict[str, wrappers.Service]): A dictionary mapping the names of Service - objects in the proto containing this method to the Service objects. This is necessary - for traversing the operation service in the case of extended LROs. - Returns: - None - """ - - address_allowlist.add(self.ident) - - if self.lro: - self.lro.add_to_address_allowlist( - address_allowlist=address_allowlist, resource_messages=resource_messages - ) - - if self.extended_lro and self.operation_service: - # We need to add the service/method pointed to by self.operation_service to - # the allowlist, as it might not have been specified by - # the methods under selective_gapic_generation. - # We assume that the operation service lives in the same proto file as this one. - operation_service = services_in_proto[self.operation_service] - address_allowlist.add(operation_service.meta.address) - operation_service.operation_polling_method.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - services_in_proto=services_in_proto, - ) - - self.extended_lro.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - ) - - self.input.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - ) - - self.output.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - ) - - def with_internal_methods(self, *, public_methods: Set[str]) -> "Method": - """Returns a version of this ``Method`` marked as internal - - The methods not in the public_methods set will be marked as internal and - this ``Service`` will as well by extension (see :meth:`Service.is_internal`). + generate_omitted_as_internal: bool, + public_methods: Set[str], + ) -> "Method": - Args: - public_methods (Set[str]): An allowlist of fully-qualified method names. - Methods not in this allowlist will be marked as internal. - Returns: - Service: A version of this `Service` with `Method` objects corresponding to methods - not in `public_methods` marked as internal. - """ if self.ident.proto in public_methods: return self - return dataclasses.replace( - self, - is_internal=True, - ) + # Not public + if generate_omitted_as_internal: + return dataclasses.replace( + self, + is_internal=True, + ) + else: + return None + @dataclasses.dataclass(frozen=True) @@ -2429,83 +2196,22 @@ def with_context( meta=self.meta.with_context(collisions=collisions), ) - def add_to_address_allowlist( + def with_selective_generation( self, *, - address_allowlist: Set["metadata.Address"], - method_allowlist: Set[str], - resource_messages: Dict[str, "MessageType"], - services_in_proto: Dict[str, "Service"], - ) -> None: - """Adds to the allowlist of Addresses of wrapper objects to be included in selective GAPIC generation. - - This method is used to create an allowlist of addresses to be used to filter out unneeded - services, methods, messages, and enums at a later step. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to add to. Only the addresses of the allowlisted methods, the services - containing these methods, and messages/enums those methods use will be part of the - final address_allowlist. The set may be modified during this call. - method_allowlist (Set[str]): An allowlist of fully-qualified method names. - resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified - resource type name of a resource message to the corresponding MessageType object - representing that resource message. Only resources with a message representation - should be included in the dictionary. - services_in_proto (Dict[str, Service]): - Returns: - None - """ - - for method in self.methods.values(): - if method.ident.proto in method_allowlist: - # Include this service if there are any types/methods in selective gapic for this service. - address_allowlist.add(self.meta.address) - method.add_to_address_allowlist( - address_allowlist=address_allowlist, - resource_messages=resource_messages, - services_in_proto=services_in_proto, - ) - - def prune_messages_for_selective_generation( - self, *, address_allowlist: Set["metadata.Address"] + generate_omitted_as_internal: bool, + public_methods: Set[str], ) -> "Service": - """Returns a truncated version of this Service. - - Only the methods, messages, and enums contained in the address allowlist - are included in the returned object. - - Args: - address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address - objects to filter against. Objects with addresses not the allowlist will be - removed from the returned Proto. - Returns: - Service: A truncated version of this proto. - """ - return dataclasses.replace( - self, - methods={ - k: v for k, v in self.methods.items() if v.ident in address_allowlist - }, - ) - def with_internal_methods(self, *, public_methods: Set[str]) -> "Service": - """Returns a version of this ``Service`` with some Methods marked as internal. + methods = {} + for k, v in self.methods.items(): + new_v = v.with_selective_generation( + generate_omitted_as_internal=generate_omitted_as_internal, + public_methods=public_methods) + if new_v: + methods[k] = new_v - The methods not in the public_methods set will be marked as internal and - this ``Service`` will as well by extension (see :meth:`Service.is_internal`). + if not generate_omitted_as_internal and not methods: + return None - Args: - public_methods (Set[str]): An allowlist of fully-qualified method names. - Methods not in this allowlist will be marked as internal. - Returns: - Service: A version of this `Service` with `Method` objects corresponding to methods - not in `public_methods` marked as internal. - """ - return dataclasses.replace( - self, - methods={ - k: v.with_internal_methods(public_methods=public_methods) - for k, v in self.methods.items() - }, - ) + return dataclasses.replace(self, methods=methods) From e1c4e112dfa5119a4a512c855276984dea714cd5 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 5 May 2026 19:51:09 +0000 Subject: [PATCH 2/3] refine prototype --- packages/gapic-generator/gapic/schema/api.py | 75 ++++- .../gapic-generator/gapic/schema/wrappers.py | 280 +++++++++++++++++- .../tests/unit/schema/test_api.py | 7 +- 3 files changed, 356 insertions(+), 6 deletions(-) diff --git a/packages/gapic-generator/gapic/schema/api.py b/packages/gapic-generator/gapic/schema/api.py index 79fa0fd2fad0..824bc80383a6 100644 --- a/packages/gapic-generator/gapic/schema/api.py +++ b/packages/gapic-generator/gapic/schema/api.py @@ -263,17 +263,40 @@ def with_selective_generation( *, generate_omitted_as_internal: bool, public_methods: Set[str], + excluded_addresses: Set["metadata.Address"], ) -> "Proto": services = {} for k, v in self.services.items(): new_v = v.with_selective_generation( generate_omitted_as_internal=generate_omitted_as_internal, - public_methods=public_methods) + public_methods=public_methods, + excluded_addresses=excluded_addresses) if new_v: services[k] = new_v - return dataclasses.replace(self, services=services) + # We only prune messages/enums from protos that are not dependencies. + # Messages and enums are excluded only if they are reachable from some RPC + # but NOT from any of the publicly allowed RPCs. + all_messages = { + k: v for k, v in self.all_messages.items() if v.ident not in excluded_addresses + } + + all_enums = { + k: v for k, v in self.all_enums.items() if v.ident not in excluded_addresses + } + + # If the proto becomes empty after pruning, we return None to signal + # that it should be excluded from generation. + if not services and not all_messages and not all_enums: + return None + + return dataclasses.replace( + self, + services=services, + all_messages=all_messages, + all_enums=all_enums, + ) @dataclasses.dataclass(frozen=True) @@ -438,10 +461,58 @@ def disambiguate_keyword_sanitize_fname( k: v for k, v in api.all_protos.items() if k not in api.protos } + all_resource_messages = collections.ChainMap( + *(proto.resource_messages for proto in api.all_protos.values()) + ) + + # Calculate all reachable addresses (API-wide). + # This includes all messages and enums reachable from ANY RPC + # defined in any proto of the API. + all_rpc_addresses: Set["metadata.Address"] = set([]) + all_methods = set(api.all_methods.keys()) + # Create a global map of services to support cross-proto lookup + # for extended LROs. + all_services: Dict[str, wrappers.Service] = {} + for p in api.all_protos.values(): + for s in p.services.values(): + all_services[s.meta.address.proto] = s + all_services[s.name] = s + + for proto in api.all_protos.values(): + for service in proto.services.values(): + service.add_to_address_allowlist( + address_allowlist=all_rpc_addresses, + method_allowlist=all_methods, + resource_messages=all_resource_messages, + services_in_proto=all_services, + ) + + # Calculate publicly reachable addresses (API-wide). + # This includes only types reachable from the allowlisted methods. + public_rpc_addresses: Set["metadata.Address"] = set([]) + for proto in api.all_protos.values(): + for service in proto.services.values(): + service.add_to_address_allowlist( + address_allowlist=public_rpc_addresses, + method_allowlist=selective_gapic_methods, + resource_messages=all_resource_messages, + services_in_proto=all_services, + ) + + # Addresses to exclude: those that ARE reachable from SOME RPC but NOT from any PUBLIC RPC. + # Types not attached to any RPC will not be in all_rpc_addresses and thus + # will NOT be in excluded_addresses, meaning they are preserved. + excluded_addresses = ( + all_rpc_addresses - public_rpc_addresses + if not selective_gapic_settings.generate_omitted_as_internal + else set([]) + ) + for name, proto in api.protos.items(): proto_to_generate = proto.with_selective_generation( generate_omitted_as_internal=selective_gapic_settings.generate_omitted_as_internal, public_methods=selective_gapic_methods, + excluded_addresses=excluded_addresses, ) if proto_to_generate: new_all_protos[name] = proto_to_generate diff --git a/packages/gapic-generator/gapic/schema/wrappers.py b/packages/gapic-generator/gapic/schema/wrappers.py index 6950fd72ef2a..c8a730e4032f 100644 --- a/packages/gapic-generator/gapic/schema/wrappers.py +++ b/packages/gapic-generator/gapic/schema/wrappers.py @@ -441,6 +441,48 @@ def with_context( meta=self.meta.with_context(collisions=collisions), ) + def add_to_address_allowlist( + self, + *, + address_allowlist: Set["metadata.Address"], + resource_messages: Dict[str, "MessageType"], + ) -> None: + """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. + + This method is used to create an allowlist of addresses to be used to filter out unneeded + services, methods, messages, and enums at a later step. + + Args: + address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address + objects to add to. Only the addresses of the allowlisted methods, the services + containing these methods, and messages/enums those methods use will be part of the + final address_allowlist. The set may be modified during this call. + resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified + resource type name of a resource message to the corresponding MessageType object + representing that resource message. Only resources with a message representation + should be included in the dictionary. + Returns: + None + """ + if self.message: + self.message.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + ) + + if self.enum: + self.enum.add_to_address_allowlist( + address_allowlist=address_allowlist, + ) + + if self.resource_reference and self.resource_reference in resource_messages: + # The message types in resource_message are different objects, but should be + # defined the same as the MessageTypes we're traversing here. + resource_messages[self.resource_reference].add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + ) + @dataclasses.dataclass(frozen=True) class FieldHeader: @@ -818,6 +860,49 @@ def with_context( meta=self.meta.with_context(collisions=collisions), ) + def add_to_address_allowlist( + self, + *, + address_allowlist: Set["metadata.Address"], + resource_messages: Dict[str, "MessageType"], + ) -> None: + """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. + + This method is used to create an allowlist of addresses to be used to filter out unneeded + services, methods, messages, and enums at a later step. + + Args: + address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address + objects to add to. Only the addresses of the allowlisted methods, the services + containing these methods, and messages/enums those methods use will be part of the + final address_allowlist. The set may be modified during this call. + resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified + resource type name of a resource message to the corresponding MessageType object + representing that resource message. Only resources with a message representation + should be included in the dictionary. + Returns: + None + """ + if self.ident not in address_allowlist: + address_allowlist.add(self.ident) + + for field in self.fields.values(): + field.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + ) + + for enum in self.nested_enums.values(): + enum.add_to_address_allowlist( + address_allowlist=address_allowlist, + ) + + for message in self.nested_messages.values(): + message.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + ) + @dataclasses.dataclass(frozen=True) class EnumValueType: @@ -878,6 +963,24 @@ def with_context(self, *, collisions: Set[str]) -> "EnumType": else self ) + def add_to_address_allowlist( + self, *, address_allowlist: Set["metadata.Address"] + ) -> None: + """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. + + This method is used to create an allowlist of addresses to be used to filter out unneeded + services, methods, messages, and enums at a later step. + + Args: + address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address + objects to add to. Only the addresses of the allowlisted methods, the services + containing these methods, and messages/enums those methods use will be part of the + final address_allowlist. The set may be modified during this call. + Returns: + None + """ + address_allowlist.add(self.ident) + @property def options_dict(self) -> Dict: """Return the EnumOptions (if present) as a dict. @@ -994,6 +1097,39 @@ def with_context( ) ) + def add_to_address_allowlist( + self, + *, + address_allowlist: Set["metadata.Address"], + resource_messages: Dict[str, "MessageType"], + ) -> None: + """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. + + This method is used to create an allowlist of addresses to be used to filter out unneeded + services, methods, messages, and enums at a later step. + + Args: + address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address + objects to add to. Only the addresses of the allowlisted methods, the services + containing these methods, and messages/enums those methods use will be part of the + final address_allowlist. The set may be modified during this call. + resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified + resource type name of a resource message to the corresponding MessageType object + representing that resource message. Only resources with a message representation + should be included in the dictionary. + Returns: + None + """ + + self.request_type.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + ) + self.operation_type.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + ) + @dataclasses.dataclass(frozen=True) class OperationInfo: @@ -1027,6 +1163,38 @@ def with_context( ), ) + def add_to_address_allowlist( + self, + *, + address_allowlist: Set["metadata.Address"], + resource_messages: Dict[str, "MessageType"], + ) -> None: + """Adds to the set of Addresses of wrapper objects to be included in selective GAPIC generation. + + This method is used to create an allowlist of addresses to be used to filter out unneeded + services, methods, messages, and enums at a later step. + + Args: + address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address + objects to add to. Only the addresses of the allowlisted methods, the services + containing these methods, and messages/enums those methods use will be part of the + final address_allowlist. The set may be modified during this call. + resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified + resource type name of a resource message to the corresponding MessageType object + representing that resource message. Only resources with a message representation + should be included in the dictionary. + Returns: + None + """ + self.response_type.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + ) + + self.metadata_type.add_to_address_allowlist( + address_allowlist=address_allowlist, resource_messages=resource_messages + ) + @dataclasses.dataclass(frozen=True) class RetryInfo: @@ -1829,18 +1997,84 @@ def with_context( meta=self.meta.with_context(collisions=collisions), ) + def add_to_address_allowlist( + self, + *, + address_allowlist: Set["metadata.Address"], + resource_messages: Dict[str, "MessageType"], + services_in_proto: Dict[str, "Service"], + ) -> None: + """Adds to the allowlist of Addresses of wrapper objects to be included in selective GAPIC generation. + + This method is used to create an allowlist of addresses to be used to filter out unneeded + services, methods, messages, and enums at a later step. + + Args: + address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address + objects to add to. Only the addresses of the allowlisted methods, the services + containing these methods, and messages/enums those methods use will be part of the + final address_allowlist. The set may be modified during this call. + resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified + resource type name of a resource message to the corresponding MessageType object + representing that resource message. Only resources with a message representation + should be included in the dictionary. + services_in_proto (Dict[str, wrappers.Service]): A dictionary mapping the names of Service + objects in the proto containing this method to the Service objects. This is necessary + for traversing the operation service in the case of extended LROs. + Returns: + None + """ + + address_allowlist.add(self.ident) + + if self.lro: + self.lro.add_to_address_allowlist( + address_allowlist=address_allowlist, resource_messages=resource_messages + ) + + if self.extended_lro and self.operation_service: + # We need to add the service/method pointed to by self.operation_service to + # the allowlist, as it might not have been specified by + # the methods under selective_gapic_generation. + # We assume that the operation service lives in the same proto file as this one. + operation_service = services_in_proto[self.operation_service] + address_allowlist.add(operation_service.meta.address) + operation_service.operation_polling_method.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + services_in_proto=services_in_proto, + ) + + self.extended_lro.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + ) + + self.input.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + ) + + self.output.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + ) + def with_selective_generation( self, *, generate_omitted_as_internal: bool, public_methods: Set[str], + excluded_addresses: Set["metadata.Address"], ) -> "Method": if self.ident.proto in public_methods: return self - # Not public - if generate_omitted_as_internal: + # Not public. + # We mark it as internal if either generate_omitted_as_internal is set, + # or if the method is reachable from some public method (e.g. as a polling method). + if generate_omitted_as_internal or self.meta.address not in excluded_addresses: return dataclasses.replace( self, is_internal=True, @@ -2196,18 +2430,58 @@ def with_context( meta=self.meta.with_context(collisions=collisions), ) + def add_to_address_allowlist( + self, + *, + address_allowlist: Set["metadata.Address"], + method_allowlist: Set[str], + resource_messages: Dict[str, "MessageType"], + services_in_proto: Dict[str, "Service"], + ) -> None: + """Adds to the allowlist of Addresses of wrapper objects to be included in selective GAPIC generation. + + This method is used to create an allowlist of addresses to be used to filter out unneeded + services, methods, messages, and enums at a later step. + + Args: + address_allowlist (Set[metadata.Address]): A set of allowlisted metadata.Address + objects to add to. Only the addresses of the allowlisted methods, the services + containing these methods, and messages/enums those methods use will be part of the + final address_allowlist. The set may be modified during this call. + method_allowlist (Set[str]): An allowlist of fully-qualified method names. + resource_messages (Dict[str, wrappers.MessageType]): A dictionary mapping the unified + resource type name of a resource message to the corresponding MessageType object + representing that resource message. Only resources with a message representation + should be included in the dictionary. + services_in_proto (Dict[str, Service]): + Returns: + None + """ + + for method in self.methods.values(): + if method.ident.proto in method_allowlist: + # Include this service if there are any types/methods in selective gapic for this service. + address_allowlist.add(self.meta.address) + method.add_to_address_allowlist( + address_allowlist=address_allowlist, + resource_messages=resource_messages, + services_in_proto=services_in_proto, + ) + def with_selective_generation( self, *, generate_omitted_as_internal: bool, public_methods: Set[str], + excluded_addresses: Set["metadata.Address"], ) -> "Service": methods = {} for k, v in self.methods.items(): new_v = v.with_selective_generation( generate_omitted_as_internal=generate_omitted_as_internal, - public_methods=public_methods) + public_methods=public_methods, + excluded_addresses=excluded_addresses) if new_v: methods[k] = new_v diff --git a/packages/gapic-generator/tests/unit/schema/test_api.py b/packages/gapic-generator/tests/unit/schema/test_api.py index 9d94e8251523..a5705b776caf 100644 --- a/packages/gapic-generator/tests/unit/schema/test_api.py +++ b/packages/gapic-generator/tests/unit/schema/test_api.py @@ -3606,8 +3606,13 @@ def test_selective_gapic_api_build_remove_unnecessary_proto_files(): name="GetBarRequest", fields=( make_field_pb2( - name="bar", + name="bar_local", number=1, + type_name=".google.example.v1.Bar", + ), + make_field_pb2( + name="bar_common", + number=2, type_name=".google.example.v1.bar_common.Bar", ), ), From 5d8ec63f08060c0b3a03c3e7fe62564bb6c66972 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 5 May 2026 20:55:25 +0000 Subject: [PATCH 3/3] update goldens --- .../google/cloud/redis/__init__.py | 4 +++ .../google/cloud/redis_v1/__init__.py | 4 +++ .../google/cloud/redis_v1/types/__init__.py | 4 +++ .../cloud/redis_v1/types/cloud_redis.py | 32 +++++++++++++++++++ 4 files changed, 44 insertions(+) diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis/__init__.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis/__init__.py index 045bcae4c55c..d96b1aebac11 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis/__init__.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis/__init__.py @@ -27,6 +27,7 @@ from google.cloud.redis_v1.types.cloud_redis import Instance from google.cloud.redis_v1.types.cloud_redis import ListInstancesRequest from google.cloud.redis_v1.types.cloud_redis import ListInstancesResponse +from google.cloud.redis_v1.types.cloud_redis import LocationMetadata from google.cloud.redis_v1.types.cloud_redis import MaintenancePolicy from google.cloud.redis_v1.types.cloud_redis import MaintenanceSchedule from google.cloud.redis_v1.types.cloud_redis import NodeInfo @@ -35,6 +36,7 @@ from google.cloud.redis_v1.types.cloud_redis import TlsCertificate from google.cloud.redis_v1.types.cloud_redis import UpdateInstanceRequest from google.cloud.redis_v1.types.cloud_redis import WeeklyMaintenanceWindow +from google.cloud.redis_v1.types.cloud_redis import ZoneMetadata __all__ = ('CloudRedisClient', 'CloudRedisAsyncClient', @@ -44,6 +46,7 @@ 'Instance', 'ListInstancesRequest', 'ListInstancesResponse', + 'LocationMetadata', 'MaintenancePolicy', 'MaintenanceSchedule', 'NodeInfo', @@ -52,4 +55,5 @@ 'TlsCertificate', 'UpdateInstanceRequest', 'WeeklyMaintenanceWindow', + 'ZoneMetadata', ) diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/__init__.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/__init__.py index 1f7bad3796c9..bac280e47101 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/__init__.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/__init__.py @@ -37,6 +37,7 @@ from .types.cloud_redis import Instance from .types.cloud_redis import ListInstancesRequest from .types.cloud_redis import ListInstancesResponse +from .types.cloud_redis import LocationMetadata from .types.cloud_redis import MaintenancePolicy from .types.cloud_redis import MaintenanceSchedule from .types.cloud_redis import NodeInfo @@ -45,6 +46,7 @@ from .types.cloud_redis import TlsCertificate from .types.cloud_redis import UpdateInstanceRequest from .types.cloud_redis import WeeklyMaintenanceWindow +from .types.cloud_redis import ZoneMetadata if hasattr(api_core, "check_python_version") and hasattr(api_core, "check_dependency_versions"): # pragma: NO COVER api_core.check_python_version("google.cloud.redis_v1") # type: ignore @@ -139,6 +141,7 @@ def _get_version(dependency_name): 'Instance', 'ListInstancesRequest', 'ListInstancesResponse', +'LocationMetadata', 'MaintenancePolicy', 'MaintenanceSchedule', 'NodeInfo', @@ -147,4 +150,5 @@ def _get_version(dependency_name): 'TlsCertificate', 'UpdateInstanceRequest', 'WeeklyMaintenanceWindow', +'ZoneMetadata', ) diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/__init__.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/__init__.py index 1e420395cc1d..e15d95a60b3e 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/__init__.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/__init__.py @@ -20,6 +20,7 @@ Instance, ListInstancesRequest, ListInstancesResponse, + LocationMetadata, MaintenancePolicy, MaintenanceSchedule, NodeInfo, @@ -28,6 +29,7 @@ TlsCertificate, UpdateInstanceRequest, WeeklyMaintenanceWindow, + ZoneMetadata, ) __all__ = ( @@ -37,6 +39,7 @@ 'Instance', 'ListInstancesRequest', 'ListInstancesResponse', + 'LocationMetadata', 'MaintenancePolicy', 'MaintenanceSchedule', 'NodeInfo', @@ -45,4 +48,5 @@ 'TlsCertificate', 'UpdateInstanceRequest', 'WeeklyMaintenanceWindow', + 'ZoneMetadata', ) diff --git a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/cloud_redis.py b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/cloud_redis.py index 8022b120202a..96031fd810f2 100755 --- a/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/cloud_redis.py +++ b/packages/gapic-generator/tests/integration/goldens/redis_selective/google/cloud/redis_v1/types/cloud_redis.py @@ -42,6 +42,8 @@ 'UpdateInstanceRequest', 'DeleteInstanceRequest', 'OperationMetadata', + 'LocationMetadata', + 'ZoneMetadata', 'TlsCertificate', }, ) @@ -973,6 +975,36 @@ class OperationMetadata(proto.Message): ) +class LocationMetadata(proto.Message): + r"""This location metadata represents additional configuration options + for a given location where a Redis instance may be created. All + fields are output only. It is returned as content of the + ``google.cloud.location.Location.metadata`` field. + + Attributes: + available_zones (MutableMapping[str, google.cloud.redis_v1.types.ZoneMetadata]): + Output only. The set of available zones in the location. The + map is keyed by the lowercase ID of each zone, as defined by + GCE. These keys can be specified in ``location_id`` or + ``alternative_location_id`` fields when creating a Redis + instance. + """ + + available_zones: MutableMapping[str, 'ZoneMetadata'] = proto.MapField( + proto.STRING, + proto.MESSAGE, + number=1, + message='ZoneMetadata', + ) + + +class ZoneMetadata(proto.Message): + r"""Defines specific information for a particular zone. Currently + empty and reserved for future use only. + + """ + + class TlsCertificate(proto.Message): r"""TlsCertificate Resource