diff --git a/consul/assets/configuration/spec.yaml b/consul/assets/configuration/spec.yaml index 39448f373a7fe..afeabbee43589 100644 --- a/consul/assets/configuration/spec.yaml +++ b/consul/assets/configuration/spec.yaml @@ -109,6 +109,20 @@ files: - - + - name: allowed_service_tags + description: | + If set, only tags with keys matching this list will be sent to Datadog. + This is helpful if you have a lot of tags on services that are not + relevant to Datadog (ingress routing tags, etc). Tags should be specified + here in lowercase. Otherwise, the check will downcase tags from Consul before comparing. + value: + type: array + items: + type: string + example: + - + - + - name: max_services description: | Increase the maximum number of queried services. diff --git a/consul/changelog.d/20306.added b/consul/changelog.d/20306.added new file mode 100644 index 0000000000000..8928e4773bd36 --- /dev/null +++ b/consul/changelog.d/20306.added @@ -0,0 +1 @@ +Add a new feature to filter Consul service tags being sent to Datadog using an allow list. It can be configured using the `allowed_service_tags` option. diff --git a/consul/datadog_checks/consul/config_models/instance.py b/consul/datadog_checks/consul/config_models/instance.py index f66ead04b2efc..3020fd931fb6f 100644 --- a/consul/datadog_checks/consul/config_models/instance.py +++ b/consul/datadog_checks/consul/config_models/instance.py @@ -56,6 +56,7 @@ class InstanceConfig(BaseModel): ) acl_token: Optional[str] = None allow_redirects: Optional[bool] = None + allowed_service_tags: Optional[tuple[str, ...]] = None auth_token: Optional[AuthToken] = None auth_type: Optional[str] = None aws_host: Optional[str] = None diff --git a/consul/datadog_checks/consul/consul.py b/consul/datadog_checks/consul/consul.py index 585197fb62b48..e78ba2b45dd45 100644 --- a/consul/datadog_checks/consul/consul.py +++ b/consul/datadog_checks/consul/consul.py @@ -106,6 +106,9 @@ def __init__(self, name, init_config, instances): 'service_whitelist', self.instance.get('services_include', default_services_include) ) self.services_exclude = set(self.instance.get('services_exclude', self.init_config.get('services_exclude', []))) + self.allowed_service_tags = set( + self.instance.get("allowed_service_tags", self.init_config.get("allowed_service_tags", [])) + ) self.max_services = self.instance.get('max_services', self.init_config.get('max_services', MAX_SERVICES)) self.threads_count = self.instance.get('threads_count', self.init_config.get('threads_count', THREADS_COUNT)) if self.threads_count > 1: @@ -312,6 +315,18 @@ def _cull_services_list(self, services): return services + def _cull_services_tags_list(self, services): + if self.allowed_service_tags: + # services is a dict of {service_name: [tags]} where tags is a list + # of string having the form of "tagkey=tagvalue" + for service in services: + tags = services[service] + # get the tagkey (the part before the "=") and check it against the include list + tags = [t for t in tags if t.split("=")[0].lower() in self.allowed_service_tags] + services[service] = tags + + return services + @staticmethod def _get_service_tags(service, tags): service_tags = ['consul_service_id:{}'.format(service)] @@ -397,6 +412,7 @@ def check(self, _): self.count_all_nodes(main_tags) services = self._cull_services_list(services) + tags = self._cull_services_tags_list(services) # {node_id: {"up: 0, "passing": 0, "warning": 0, "critical": 0} nodes_to_service_status = defaultdict(lambda: defaultdict(int)) diff --git a/consul/datadog_checks/consul/data/conf.yaml.example b/consul/datadog_checks/consul/data/conf.yaml.example index a854346a77b3c..038dedcc361bd 100644 --- a/consul/datadog_checks/consul/data/conf.yaml.example +++ b/consul/datadog_checks/consul/data/conf.yaml.example @@ -126,6 +126,16 @@ instances: # - # - + ## @param allowed_service_tags - list of strings - optional + ## If set, only tags with keys matching this list will be sent to Datadog. + ## This is helpful if you have a lot of tags on services that are not + ## relevant to Datadog (ingress routing tags, etc). Tags should be specified + ## here in lowercase. Otherwise, the check will downcase tags from Consul before comparing. + # + # allowed_service_tags: + # - + # - + ## @param max_services - number - optional - default: 50 ## Increase the maximum number of queried services. # diff --git a/consul/tests/consul_mocks.py b/consul/tests/consul_mocks.py index be6cd803f221b..d97016561b5f0 100644 --- a/consul/tests/consul_mocks.py +++ b/consul/tests/consul_mocks.py @@ -93,6 +93,14 @@ def mock_get_services_in_cluster(): } +def mock_get_n_custom_tagged_services_in_cluster(n, tags): + svcs = {} + for i in range(n): + k = "service-{}".format(i) + svcs[k] = tags + return svcs + + def mock_get_n_services_in_cluster(n): dct = {} for i in range(n): diff --git a/consul/tests/test_unit.py b/consul/tests/test_unit.py index 6e93105a5734c..9f5a70b754b5a 100644 --- a/consul/tests/test_unit.py +++ b/consul/tests/test_unit.py @@ -54,6 +54,42 @@ def test_get_nodes_with_service(aggregator): aggregator.assert_metric('consul.catalog.services_count', value=1, tags=expected_tags) +def test_cull_services_tags_keys(aggregator): + consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG]) + consul_mocks.mock_check(consul_check, consul_mocks._get_consul_mocks()) + + all_tags = { + "active", + "standby", + "unwanted.tag=unwantedvalue", + "unwanted.tag.but.actually.wanted=wantedvalue", + "wanted.tag", + "unwanted.tag.noequals", + } + + include_tags = {'active', 'standby', 'unwanted.tag.but.actually.wanted', 'wanted.tag'} + + expected_tags = { + "active", + "standby", + "unwanted.tag.but.actually.wanted=wantedvalue", + "wanted.tag", + } + + unwanted_tags = { + "unwanted.tag=unwantedvalue", + "unwanted.tag.noequals", + } + + consul_check.allowed_service_tags = include_tags + services = consul_mocks.mock_get_n_custom_tagged_services_in_cluster(6, all_tags) + + services = consul_check._cull_services_tags_list(services) + for service in services: + assert unwanted_tags.isdisjoint(set(services[service])) + assert expected_tags == set(services[service]) + + def test_get_peers_in_cluster(aggregator): my_mocks = consul_mocks._get_consul_mocks() consul_check = ConsulCheck(common.CHECK_NAME, {}, [consul_mocks.MOCK_CONFIG])