From 3cbb66b9785061fe8354043ec273648447233d41 Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Tue, 10 Feb 2026 11:24:50 +0100 Subject: [PATCH 01/15] Remove abstract metric providers, left only openssf metric provider and simplied the usage of the openssf client by fetching scorecards only using a baseUrl --- .../README.md | 104 ++------ .../src/clients/OpenSSFClient.test.ts | 81 ++++-- .../src/clients/OpenSSFClient.ts | 19 +- .../src/clients/utils.ts | 41 --- .../AbstractMetricProvider.test.ts | 246 ------------------ .../DefaultOpenSSFMetricProvider.test.ts | 54 ---- .../DefaultOpenSSFMetricProvider.ts | 58 ----- .../OpenSSFMetricProvider.test.ts | 227 ++++++++++++++++ ...icProvider.ts => OpenSSFMetricProvider.ts} | 76 +++--- .../src/module.ts | 8 +- 10 files changed, 348 insertions(+), 566 deletions(-) delete mode 100644 workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/utils.ts delete mode 100644 workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/AbstractMetricProvider.test.ts delete mode 100644 workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/DefaultOpenSSFMetricProvider.test.ts delete mode 100644 workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/DefaultOpenSSFMetricProvider.ts create mode 100644 workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts rename workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/{AbstractMetricProvider.ts => OpenSSFMetricProvider.ts} (58%) diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/README.md b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/README.md index 1bbaf6fdae..79f3da5cd9 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/README.md +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/README.md @@ -1,21 +1,12 @@ # Scorecard Backend Module for OpenSSF -This is an extension module to the `backstage-plugin-scorecard-backend` plugin. It provides [OpenSSF Security Scorecard](https://securityscorecards.dev/) metrics for software components registered in the Backstage catalog. +Adds [OpenSSF Security Scorecard](https://securityscorecards.dev/) metrics to the Scorecard backend. Fetches scorecard data from the URL configured per component (`openssf/baseUrl`), so it can use the public API, a self-hosted endpoint, or any other scorecard source. Exposes 18 checks as Backstage metrics (scores 0–10). -## Overview - -The OpenSSF Security Scorecards project provides automated security assessments for open source projects hosted on GitHub. This module fetches scorecard data from the public OpenSSF API and exposes individual security check scores as metrics in Backstage. - -## Prerequisites - -Before installing this module, ensure that the Scorecard backend plugin is integrated into your Backstage instance. Follow the [Scorecard backend plugin README](../scorecard-backend/README.md) for setup instructions. +Requires the [Scorecard backend plugin](../scorecard-backend/README.md) to be installed. ## Installation -To install this backend module: - ```bash -# From your root directory yarn workspace backend add @red-hat-developer-hub/backstage-plugin-scorecard-backend-module-openssf ``` @@ -24,104 +15,41 @@ yarn workspace backend add @red-hat-developer-hub/backstage-plugin-scorecard-bac import { createBackend } from '@backstage/backend-defaults'; const backend = createBackend(); - -// Scorecard backend plugin backend.add( import('@red-hat-developer-hub/backstage-plugin-scorecard-backend'), ); - -// Install the OpenSSF module -/* highlight-add-next-line */ backend.add( import( '@red-hat-developer-hub/backstage-plugin-scorecard-backend-module-openssf' ), ); - backend.start(); ``` -## Entity Annotations +## Configuration + +### Catalog (catalog-info.yaml) -For the OpenSSF metrics to work, your catalog entities must have the required annotation: +| Annotation | Required | Description | +| ----------------- | -------- | --------------------------------------------------------------------------- | +| `openssf/baseUrl` | Yes | Full scorecard API URL for this component (e.g. public API or self-hosted). | + +Example: ```yaml -# catalog-info.yaml -apiVersion: backstage.io/v1alpha1 -kind: Component metadata: - name: my-service annotations: - # Required: GitHub repository in owner/repo format - openssf/project: owner/repo -spec: - type: service - lifecycle: production - owner: my-team + openssf/baseUrl: https://api.securityscorecards.dev/projects/github.com/owner/repo ``` -The `openssf/project` annotation should contain the GitHub repository path in `owner/repo` format (e.g., `kubernetes/kubernetes`). - -## Configuration - -This module uses the public OpenSSF Security Scorecards API (`api.securityscorecards.dev`) and does not require any additional configuration in `app-config.yaml`. - ### Thresholds -Thresholds define conditions that determine which category a metric value belongs to (`error`, `warning`, or `success`). Check out detailed explanation of [threshold configuration](../scorecard-backend/docs/thresholds.md). - -All OpenSSF metrics use the following **fixed** thresholds: - -| Category | Expression | Description | -| -------- | ---------- | --------------------------------- | -| Error | `<2` | Score less than 2 | -| Warning | `2-7` | Score between 2 and 7 (inclusive) | -| Success | `>7` | Score greater than 7 | - -> **Note:** These thresholds are not configurable via `app-config.yaml`. They are defined in the module source code. - -## Available Metrics - -This module provides 18 metrics corresponding to the [OpenSSF Security Scorecard checks](https://github.com/ossf/scorecard/blob/main/docs/checks.md). Each metric returns a score from 0 to 10. - -| Metric ID | Risk | Description | -| -------------------------------- | -------- | -------------------------------------------------------------------------------------------------------------------------- | -| `openssf.binary_artifacts` | High | Determines if the project has generated executable (binary) artifacts in the source repository. | -| `openssf.branch_protection` | High | Determines if the default and release branches are protected with GitHub's branch protection or repository rules settings. | -| `openssf.cii_best_practices` | Low | Determines if the project has an OpenSSF (formerly CII) Best Practices Badge. | -| `openssf.ci_tests` | Low | Determines if the project runs tests before pull requests are merged. | -| `openssf.code_review` | High | Determines if the project requires human code review before pull requests are merged. | -| `openssf.contributors` | Low | Determines if the project has contributors from multiple organizations. | -| `openssf.dangerous_workflow` | Critical | Determines if the project's GitHub Action workflows avoid dangerous patterns. | -| `openssf.dependency_update_tool` | High | Determines if the project uses a dependency update tool. | -| `openssf.fuzzing` | Medium | Determines if the project uses fuzzing. | -| `openssf.license` | Low | Determines if the project has defined a license. | -| `openssf.maintained` | High | Determines if the project is "actively maintained". | -| `openssf.packaging` | Medium | Determines if the project is published as a package that others can easily download, install, update, and uninstall. | -| `openssf.pinned_dependencies` | Medium | Determines if the project has declared and pinned the dependencies of its build process. | -| `openssf.sast` | Medium | Determines if the project uses static code analysis. | -| `openssf.security_policy` | Medium | Determines if the project has published a security policy. | -| `openssf.signed_releases` | High | Determines if the project cryptographically signs release artifacts. | -| `openssf.token_permissions` | High | Determines if the project's automated workflow tokens follow the principle of least privilege. | -| `openssf.vulnerabilities` | High | Determines if the project has open, unfixed vulnerabilities in its codebase or dependencies using OSV. | - -## Troubleshooting - -### Metric shows "not found" +All OpenSSF metrics use fixed thresholds: **Error** <2, **Warning** 2–7, **Success** >7. Not configurable. See [threshold docs](../scorecard-backend/docs/thresholds.md). -This can occur if: +## Metrics -- The repository has not been analyzed by OpenSSF Scorecards yet -- The repository is private (OpenSSF only analyzes public repositories) -- The repository path in the annotation is incorrect -- The metric score is lower than -1 or higher than 10. +18 metrics from [OpenSSF checks](https://github.com/ossf/scorecard/blob/main/docs/checks.md): `openssf.binary_artifacts`, `openssf.branch_protection`, `openssf.cii_best_practices`, `openssf.ci_tests`, `openssf.code_review`, `openssf.contributors`, `openssf.dangerous_workflow`, `openssf.dependency_update_tool`, `openssf.fuzzing`, `openssf.license`, `openssf.maintained`, `openssf.packaging`, `openssf.pinned_dependencies`, `openssf.sast`, `openssf.security_policy`, `openssf.signed_releases`, `openssf.token_permissions`, `openssf.vulnerabilities`. -### No data for my repository - -OpenSSF Security Scorecards only analyzes **public GitHub repositories**. Private repositories and repositories on other Git hosting services are not supported. - -To verify your repository has scorecard data, visit: +## Troubleshooting -``` -https://api.securityscorecards.dev/projects/github.com/{owner}/{repo} -``` +- **Metric "not found"**: Scorecard URL unreachable, repo not yet analyzed, or score outside 0–10. diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts index ca5f406255..997c3b6cc3 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts @@ -14,66 +14,91 @@ * limitations under the License. */ +import type { Entity } from '@backstage/catalog-model'; + import { OpenSSFClient } from './OpenSSFClient'; -import { OpenSSFResponse } from './types'; +import type { OpenSSFResponse } from './types'; -describe('OpenSSFClient', () => { - let client: OpenSSFClient; +const mockScorecardUrl = + 'https://api.securityscorecards.dev/projects/github.com/owner/repo'; - const mockOpenSSFResponse: OpenSSFResponse = { - date: '2024-01-15', - repo: { - name: 'github.com/owner/test', - commit: 'abc123', +function createEntity(baseUrl: string): Entity { + return { + apiVersion: 'backstage.io/v1beta1', + kind: 'Component', + metadata: { + name: 'my-service', + annotations: { + 'openssf/baseUrl': baseUrl, + }, }, - scorecard: { - version: '4.0.0', - commit: 'def456', + spec: {}, + } as Entity; +} + +const mockOpenSSFResponse: OpenSSFResponse = { + date: '2024-01-15', + repo: { name: 'github.com/owner/repo', commit: 'abc123' }, + scorecard: { version: '4.0.0', commit: 'def456' }, + score: 7.5, + checks: [ + { + name: 'Maintained', + score: 8, + reason: null, + details: null, + documentation: { short: '', url: '' }, }, - score: 7.5, - checks: [], - }; + ], +}; + +describe('OpenSSFClient', () => { + const entity = createEntity(mockScorecardUrl); beforeEach(() => { jest.clearAllMocks(); - client = new OpenSSFClient(); globalThis.fetch = jest.fn(); }); describe('getScorecard', () => { - it('should return the scorecard', async () => { - // mocked fetch behaviour for the test + it('fetches the scorecard from the entity baseUrl', async () => { (globalThis.fetch as jest.Mock).mockResolvedValue({ ok: true, json: jest.fn().mockResolvedValue(mockOpenSSFResponse), }); - const scorecard = await client.getScorecard('owner', 'test'); - expect(scorecard).toEqual(mockOpenSSFResponse); + const client = new OpenSSFClient(entity); + const result = await client.getScorecard(); + + expect(fetch).toHaveBeenCalledWith(mockScorecardUrl, { + method: 'GET', + headers: { Accept: 'application/json' }, + }); + expect(result).toEqual(mockOpenSSFResponse); }); - it('should throw an error if the API returns a non-ok response', async () => { - // mock response from the API + it('throws when the response is not ok', async () => { (globalThis.fetch as jest.Mock).mockResolvedValue({ ok: false, status: 404, statusText: 'Not Found', }); - await expect(client.getScorecard('wrong', 'test')).rejects.toThrow( + const client = new OpenSSFClient(entity); + + await expect(client.getScorecard()).rejects.toThrow( 'OpenSSF API request failed with status 404: Not Found', ); }); - it('should throw an error if API request fails', async () => { - // mocked fetch behaviour for the test + it('throws when fetch rejects', async () => { (globalThis.fetch as jest.Mock).mockRejectedValue( - new Error('API request failed'), + new Error('Network error'), ); - await expect(client.getScorecard('owner', 'test')).rejects.toThrow( - 'API request failed', - ); + const client = new OpenSSFClient(entity); + + await expect(client.getScorecard()).rejects.toThrow('Network error'); }); }); }); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts index efda22c59f..b58f37fa3a 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts @@ -14,24 +14,19 @@ * limitations under the License. */ +import type { Entity } from '@backstage/catalog-model'; + import { OpenSSFResponse } from './types'; export class OpenSSFClient { private readonly baseUrl: string; - private readonly gitServiceHost: string; - - constructor( - baseUrl: string = 'https://api.securityscorecards.dev/projects', - gitServiceHost: string = 'github.com', - ) { - this.baseUrl = baseUrl; - this.gitServiceHost = gitServiceHost; - } - async getScorecard(owner: string, repo: string): Promise { - const apiUrl = `${this.baseUrl}/${this.gitServiceHost}/${owner}/${repo}`; + constructor(entity: Entity) { + this.baseUrl = entity.metadata.annotations?.['openssf/baseUrl'] ?? ''; + } - const response = await fetch(apiUrl, { + async getScorecard(): Promise { + const response = await fetch(this.baseUrl, { method: 'GET', headers: { Accept: 'application/json', diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/utils.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/utils.ts deleted file mode 100644 index 03ec0cf43d..0000000000 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/utils.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Red Hat, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { type Entity, stringifyEntityRef } from '@backstage/catalog-model'; - -export const getRepositoryInformationFromEntity = ( - entity: Entity, -): { owner: string; repo: string } => { - const projectSlug = entity.metadata.annotations?.['openssf/project']; - if (!projectSlug) { - throw new Error( - `Missing annotation 'openssf/project' for entity ${stringifyEntityRef( - entity, - )}`, - ); - } - - const [owner, repo] = projectSlug.split('/'); - if (!owner || !repo) { - throw new Error( - `Invalid format of 'openssf/project' ${projectSlug} for entity ${stringifyEntityRef( - entity, - )}`, - ); - } - - return { owner, repo }; -}; diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/AbstractMetricProvider.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/AbstractMetricProvider.test.ts deleted file mode 100644 index 5d51a618c3..0000000000 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/AbstractMetricProvider.test.ts +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright Red Hat, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; -import { type Entity } from '@backstage/catalog-model'; -import { - DEFAULT_NUMBER_THRESHOLDS, - ThresholdConfig, -} from '@red-hat-developer-hub/backstage-plugin-scorecard-common'; - -import { OpenSSFClient } from '../clients/OpenSSFClient'; -import { OpenSSFResponse } from '../clients/types'; -import { AbstractMetricProvider } from './AbstractMetricProvider'; - -// Mock the OpenSSFClient module -jest.mock('../clients/OpenSSFClient'); - -// Concrete implementation for testing the abstract class -class TestMetricProvider extends AbstractMetricProvider { - getMetricName(): string { - return 'Test-Metric'; - } - - getMetricDisplayTitle(): string { - return 'Test Metric Title'; - } - - getMetricDescription(): string { - return 'Test metric description'; - } -} - -describe('AbstractMetricProvider', () => { - let provider: TestMetricProvider; - let mockGetScorecard: jest.Mock; - - const mockOpenSSFResponse: OpenSSFResponse = { - date: '2024-01-15', - repo: { - name: 'github.com/owner/test', - commit: 'abc123', - }, - scorecard: { - version: '4.0.0', - commit: 'def456', - }, - score: 7.5, - checks: [ - { - name: 'Test-Metric', - score: 8, - reason: 'Test reason', - details: null, - documentation: { - short: 'Short doc', - url: 'https://example.com', - }, - }, - { - name: 'Other-Metric', - score: 6, - reason: 'Other reason', - details: null, - documentation: { - short: 'Other doc', - url: 'https://example.com/other', - }, - }, - ], - }; - - const createMockEntity = (projectSlug?: string): Entity => ({ - apiVersion: 'backstage.io/v1alpha1', - kind: 'Component', - metadata: { - name: 'test-component', - annotations: projectSlug ? { 'openssf/project': projectSlug } : undefined, - }, - }); - - beforeEach(() => { - jest.clearAllMocks(); - - // Setup mock for OpenSSFClient - mockGetScorecard = jest.fn().mockResolvedValue(mockOpenSSFResponse); - (OpenSSFClient as jest.Mock).mockImplementation(() => ({ - getScorecard: mockGetScorecard, - })); - - provider = new TestMetricProvider(); - }); - - describe('getProviderDatasourceId', () => { - it('should return "openssf"', () => { - expect(provider.getProviderDatasourceId()).toBe('openssf'); - }); - }); - - describe('getProviderId', () => { - it('should return normalized provider ID with openssf prefix', () => { - expect(provider.getProviderId()).toBe('openssf.test_metric'); - }); - - it('should convert hyphens to underscores and lowercase', () => { - // The metric name is "Test-Metric", should become "test_metric" - expect(provider.getProviderId()).toBe('openssf.test_metric'); - }); - }); - - describe('getMetricType', () => { - it('should return "number"', () => { - expect(provider.getMetricType()).toBe('number'); - }); - }); - - describe('getMetric', () => { - it('should return metric object with correct properties', () => { - const metric = provider.getMetric(); - - expect(metric).toEqual({ - id: 'openssf.test_metric', - title: 'Test Metric Title', - description: 'Test metric description', - type: 'number', - history: true, - }); - }); - }); - - describe('getMetricThresholds', () => { - it('should return default thresholds when none provided', () => { - expect(provider.getMetricThresholds()).toEqual(DEFAULT_NUMBER_THRESHOLDS); - }); - - it('should return custom thresholds when provided', () => { - const customThresholds: ThresholdConfig = { - rules: [ - { key: 'success', expression: '>9' }, - { key: 'warning', expression: '7-9' }, - { key: 'error', expression: '<7' }, - ], - }; - const customProvider = new TestMetricProvider(customThresholds); - - expect(customProvider.getMetricThresholds()).toEqual(customThresholds); - }); - }); - - describe('getCatalogFilter', () => { - it('should return filter for openssf/project-slug annotation', () => { - expect(provider.getCatalogFilter()).toEqual({ - 'metadata.annotations.openssf/project': CATALOG_FILTER_EXISTS, - }); - }); - }); - - describe('calculateMetric', () => { - it('should call OpenSSFClient with owner and repo from entity', async () => { - const entity = createMockEntity('owner/test'); - - await provider.calculateMetric(entity); - - expect(mockGetScorecard).toHaveBeenCalledWith('owner', 'test'); - }); - - it('should return the score for the matching metric', async () => { - const entity = createMockEntity('owner/test'); - - const score = await provider.calculateMetric(entity); - - // provider has getMetricName() returning 'Test-Metric', so score should be 8 - expect(score).toBe(8); - }); - - it('should throw error when metric is not found in scorecard', async () => { - const responseWithoutMetric: OpenSSFResponse = { - ...mockOpenSSFResponse, - checks: [ - { - name: 'Different-Metric', - score: 5, - reason: 'Different reason', - details: null, - documentation: { - short: 'Different doc', - url: 'https://example.com/different', - }, - }, - ], - }; - mockGetScorecard.mockResolvedValue(responseWithoutMetric); - - const entity = createMockEntity('owner/test'); - - await expect(provider.calculateMetric(entity)).rejects.toThrow( - "OpenSSF check 'Test-Metric' not found in scorecard for owner/test", - ); - }); - - it('should throw error when entity is missing openssf/project annotation', async () => { - const entity = createMockEntity(); - - await expect(provider.calculateMetric(entity)).rejects.toThrow( - "Missing annotation 'openssf/project'", - ); - }); - - it('should throw error when project slug has invalid format', async () => { - const entity = createMockEntity('invalid-slug-without-slash'); - - await expect(provider.calculateMetric(entity)).rejects.toThrow( - "Invalid format of 'openssf/project'", - ); - }); - - it('should throw error when metric score is less than 0', async () => { - const entity = createMockEntity('owner/test'); - mockOpenSSFResponse.checks[0].score = -1; - mockOpenSSFResponse.checks[0].reason = 'Repository not found.'; - await expect(provider.calculateMetric(entity)).rejects.toThrow( - "OpenSSF check 'Test-Metric' has invalid score -1 for owner/test. Reason: Repository not found.", - ); - }); - - it('should throw error when metric score is greater than 10', async () => { - const entity = createMockEntity('owner/test'); - mockOpenSSFResponse.checks[0].score = 11; - await expect(provider.calculateMetric(entity)).rejects.toThrow( - "OpenSSF check 'Test-Metric' has invalid score 11 for owner/test", - ); - }); - }); -}); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/DefaultOpenSSFMetricProvider.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/DefaultOpenSSFMetricProvider.test.ts deleted file mode 100644 index 1a710addbd..0000000000 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/DefaultOpenSSFMetricProvider.test.ts +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright Red Hat, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - createDefaultOpenSSFMetricProviders, - DefaultOpenSSFMetricProvider, -} from './DefaultOpenSSFMetricProvider'; -import { OPENSSF_METRICS, OPENSSF_THRESHOLDS } from './OpenSSFConfig'; - -describe('DefaultOpenSSFMetricProviderTests', () => { - it('should create a default OpenSSF metric provider', () => { - const provider = new DefaultOpenSSFMetricProvider( - OPENSSF_METRICS[0], - OPENSSF_THRESHOLDS, - ); - expect(provider.getMetricDisplayTitle()).toBe( - OPENSSF_METRICS[0].displayTitle, - ); - expect(provider.getMetricDescription()).toBe( - OPENSSF_METRICS[0].description, - ); - expect(provider.getMetricThresholds()).toBe(OPENSSF_THRESHOLDS); - }); - - it('should create a default OpenSSF metric provider with custom thresholds', () => { - const provider = new DefaultOpenSSFMetricProvider( - OPENSSF_METRICS[0], - OPENSSF_THRESHOLDS, - ); - expect(provider).toBeDefined(); - }); - - it('should create all default OpenSSF metric providers', () => { - const providers = createDefaultOpenSSFMetricProviders(OPENSSF_THRESHOLDS); - expect(providers.length).toBe(OPENSSF_METRICS.length); - for (const provider of providers) { - expect(provider).toBeInstanceOf(DefaultOpenSSFMetricProvider); - expect(provider.getMetricThresholds()).toBe(OPENSSF_THRESHOLDS); - } - }); -}); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/DefaultOpenSSFMetricProvider.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/DefaultOpenSSFMetricProvider.ts deleted file mode 100644 index 968488bdbe..0000000000 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/DefaultOpenSSFMetricProvider.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Red Hat, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ThresholdConfig } from '@red-hat-developer-hub/backstage-plugin-scorecard-common'; -import { MetricProvider } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; -import { AbstractMetricProvider } from './AbstractMetricProvider'; -import { OPENSSF_METRICS, OpenSSFMetricConfig } from './OpenSSFConfig'; - -/** - * Default metric provider for OpenSSF Security Scorecards. - * Extracts a specific check from the OpenSSF scorecard response based on the provided configuration. - */ -export class DefaultOpenSSFMetricProvider extends AbstractMetricProvider { - constructor( - private readonly config: OpenSSFMetricConfig, - thresholds?: ThresholdConfig, - ) { - super(thresholds); - } - - getMetricName(): string { - return this.config.name; - } - - getMetricDisplayTitle(): string { - return this.config.displayTitle; - } - - getMetricDescription(): string { - return this.config.description; - } -} - -/** - * Creates all default OpenSSF metric providers. - * @param thresholds Optional threshold configuration to apply to all providers - * @returns Array of OpenSSF metric providers - */ -export function createDefaultOpenSSFMetricProviders( - thresholds?: ThresholdConfig, -): MetricProvider<'number'>[] { - return OPENSSF_METRICS.map( - config => new DefaultOpenSSFMetricProvider(config, thresholds), - ); -} diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts new file mode 100644 index 0000000000..95de2b81cf --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts @@ -0,0 +1,227 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; +import type { Entity } from '@backstage/catalog-model'; + +import { OpenSSFMetricProvider } from './OpenSSFMetricProvider'; +import { OPENSSF_THRESHOLDS } from './OpenSSFConfig'; + +const scorecardUrl = + 'https://api.securityscorecards.dev/projects/github.com/owner/repo'; + +function createEntity(): Entity { + return { + apiVersion: 'backstage.io/v1beta1', + kind: 'Component', + metadata: { + name: 'my-service', + annotations: { 'openssf/baseUrl': scorecardUrl }, + }, + spec: {}, + } as Entity; +} + +const maintainedConfig = { + name: 'Maintained', + displayTitle: 'OpenSSF Maintained', + description: 'Determines if the project is actively maintained.', +}; + +describe('OpenSSFMetricProvider', () => { + const entity = createEntity(); + + beforeEach(() => { + jest.clearAllMocks(); + globalThis.fetch = jest.fn(); + }); + + describe('metadata', () => { + it('returns metric name from config', () => { + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + expect(provider.getMetricName()).toBe('Maintained'); + }); + + it('returns display title and description from config', () => { + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + expect(provider.getMetricDisplayTitle()).toBe('OpenSSF Maintained'); + expect(provider.getMetricDescription()).toContain('actively maintained'); + }); + + it('returns provider id as openssf.', () => { + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + expect(provider.getProviderId()).toBe('openssf.maintained'); + }); + + it('returns openssf as provider datasource id', () => { + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + expect(provider.getProviderDatasourceId()).toBe('openssf'); + }); + + it('returns number as metric type', () => { + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + expect(provider.getMetricType()).toBe('number'); + }); + + it('returns metric descriptor with history enabled', () => { + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + const metric = provider.getMetric(); + expect(metric.id).toBe('openssf.maintained'); + expect(metric.title).toBe('OpenSSF Maintained'); + expect(metric.type).toBe('number'); + expect(metric.history).toBe(true); + }); + + it('returns configured thresholds', () => { + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + expect(provider.getMetricThresholds()).toEqual(OPENSSF_THRESHOLDS); + }); + + it('requires openssf/baseUrl annotation in catalog filter', () => { + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + expect(provider.getCatalogFilter()).toEqual({ + 'metadata.annotations.openssf/baseUrl': CATALOG_FILTER_EXISTS, + }); + }); + }); + + describe('calculateMetric', () => { + it('returns the score for the configured check', async () => { + (globalThis.fetch as jest.Mock).mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue({ + date: '2024-01-15', + repo: { name: 'github.com/owner/repo', commit: 'x' }, + scorecard: { version: '4.0.0', commit: 'y' }, + score: 7, + checks: [ + { + name: 'Maintained', + score: 8, + reason: null, + details: null, + documentation: { short: '', url: '' }, + }, + ], + }), + }); + + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + const result = await provider.calculateMetric(); + + expect(result).toBe(8); + expect(fetch).toHaveBeenCalledWith(scorecardUrl, expect.any(Object)); + }); + + it('throws when the check is not in the scorecard', async () => { + (globalThis.fetch as jest.Mock).mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue({ + date: '2024-01-15', + repo: { name: 'x', commit: 'x' }, + scorecard: { version: '4.0.0', commit: 'y' }, + score: 7, + checks: [ + { + name: 'Other-Check', + score: 5, + reason: null, + details: null, + documentation: { short: '', url: '' }, + }, + ], + }), + }); + + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + + await expect(provider.calculateMetric()).rejects.toThrow( + "OpenSSF check 'Maintained' not found in scorecard", + ); + }); + + it('throws when the check score is out of range (< 0 or > 10)', async () => { + (globalThis.fetch as jest.Mock).mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue({ + date: '2024-01-15', + repo: { name: 'x', commit: 'x' }, + scorecard: { version: '4.0.0', commit: 'y' }, + score: 7, + checks: [ + { + name: 'Maintained', + score: 11, + reason: null, + details: null, + documentation: { short: '', url: '' }, + }, + ], + }), + }); + + const provider = new OpenSSFMetricProvider( + maintainedConfig, + OPENSSF_THRESHOLDS, + entity, + ); + + await expect(provider.calculateMetric()).rejects.toThrow( + "OpenSSF check 'Maintained' has invalid score 11", + ); + }); + }); +}); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/AbstractMetricProvider.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts similarity index 58% rename from workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/AbstractMetricProvider.ts rename to workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts index cddf5c1506..a049fc45c7 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/AbstractMetricProvider.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts @@ -16,42 +16,45 @@ import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; import { type Entity } from '@backstage/catalog-model'; + import { - DEFAULT_NUMBER_THRESHOLDS, Metric, ThresholdConfig, } from '@red-hat-developer-hub/backstage-plugin-scorecard-common'; import { MetricProvider } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; import { OpenSSFClient } from '../clients/OpenSSFClient'; -import { getRepositoryInformationFromEntity } from '../clients/utils'; +import { + OPENSSF_METRICS, + OPENSSF_THRESHOLDS, + OpenSSFMetricConfig, +} from './OpenSSFConfig'; -/** - * Abstract base class for OpenSSF metric providers. - * Extracts a specific check from the OpenSSF scorecard response. - * - * Subclasses must implement: - * - getCheckName(): The name of the check to extract (e.g., "Maintained", "Code-Review") - * - getMetricName(): The metric name for the provider ID (e.g., "maintained", "code_review") - * - getMetricTitle(): Display title for the metric - * - getMetricDescription(): Description of what the metric measures - */ -export abstract class AbstractMetricProvider - implements MetricProvider<'number'> -{ +export class OpenSSFMetricProvider implements MetricProvider<'number'> { protected readonly openSSFClient: OpenSSFClient; protected readonly thresholds: ThresholdConfig; - constructor(thresholds?: ThresholdConfig) { - this.openSSFClient = new OpenSSFClient(); - this.thresholds = thresholds ?? DEFAULT_NUMBER_THRESHOLDS; + constructor( + readonly config: OpenSSFMetricConfig, + thresholds: ThresholdConfig, + entity: Entity, + ) { + this.thresholds = thresholds; + this.config = config; + this.openSSFClient = new OpenSSFClient(entity); } - abstract getMetricName(): string; + getMetricName(): string { + return this.config.name; + } - abstract getMetricDisplayTitle(): string; + getMetricDisplayTitle(): string { + return this.config.displayTitle; + } - abstract getMetricDescription(): string; + getMetricDescription(): string { + return this.config.description; + } getProviderDatasourceId(): string { return 'openssf'; @@ -84,30 +87,37 @@ export abstract class AbstractMetricProvider getCatalogFilter(): Record { return { - 'metadata.annotations.openssf/project': CATALOG_FILTER_EXISTS, + 'metadata.annotations.openssf/baseUrl': CATALOG_FILTER_EXISTS, }; } - async calculateMetric(entity: Entity): Promise { - const { owner, repo } = getRepositoryInformationFromEntity(entity); - const scorecard = await this.openSSFClient.getScorecard(owner, repo); + async calculateMetric(): Promise { + const scorecard = await this.openSSFClient.getScorecard(); const metricName = this.getMetricName(); const metric = scorecard.checks.find(c => c.name === metricName); if (!metric) { - throw new Error( - `OpenSSF check '${metricName}' not found in scorecard for ${owner}/${repo}`, - ); + throw new Error(`OpenSSF check '${metricName}' not found in scorecard`); } else if (metric.score < 0 || metric.score > 10) { throw new Error( - `OpenSSF check '${metricName}' has invalid score ${ - metric.score - } for ${owner}/${repo}. Reason: ${ - metric.reason ?? 'No reason provided' - }`, + `OpenSSF check '${metricName}' has invalid score ${metric.score}`, ); } return metric.score; } } + +/** + * Creates all default OpenSSF metric providers. + * @param clientOptions Optional base URL and git service host (from app-config) + * @returns Array of OpenSSF metric providers + */ +export function createOpenSSFMetricProvider( + clientOptions?: OpenSSFClientOptions, +): MetricProvider<'number'>[] { + return OPENSSF_METRICS.map( + config => + new OpenSSFMetricProvider(config, OPENSSF_THRESHOLDS, clientOptions), + ); +} diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts index 9ca937c14c..0e661f04dd 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts @@ -15,8 +15,7 @@ */ import { createBackendModule } from '@backstage/backend-plugin-api'; import { scorecardMetricsExtensionPoint } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; -import { createDefaultOpenSSFMetricProviders } from './metricProviders/DefaultOpenSSFMetricProvider'; -import { OPENSSF_THRESHOLDS } from './metricProviders/OpenSSFConfig'; +import { createOpenSSFMetricProvider } from './metricProviders/OpenSSFMetricProvider'; export const scorecardOpenSFFModule = createBackendModule({ pluginId: 'scorecard', @@ -27,10 +26,7 @@ export const scorecardOpenSFFModule = createBackendModule({ metrics: scorecardMetricsExtensionPoint, }, async init({ metrics }) { - // Register all default OpenSSF metric providers - metrics.addMetricProvider( - ...createDefaultOpenSSFMetricProviders(OPENSSF_THRESHOLDS), - ); + metrics.addMetricProvider(...createOpenSSFMetricProvider()); }, }); }, From 4c4db4654c087ea8e6300b2f8ff452cee37c155a Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Tue, 10 Feb 2026 13:11:53 +0100 Subject: [PATCH 02/15] fixed calculateMetric signature to receive entity as expected by the interface, refactored tests --- .../src/clients/OpenSSFClient.test.ts | 14 ++++++++------ .../src/clients/OpenSSFClient.ts | 13 ++++++------- .../OpenSSFMetricProvider.test.ts | 17 +++-------------- .../metricProviders/OpenSSFMetricProvider.ts | 14 +++++--------- 4 files changed, 22 insertions(+), 36 deletions(-) diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts index 997c3b6cc3..c5c4ebf857 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts @@ -67,8 +67,8 @@ describe('OpenSSFClient', () => { json: jest.fn().mockResolvedValue(mockOpenSSFResponse), }); - const client = new OpenSSFClient(entity); - const result = await client.getScorecard(); + const client = new OpenSSFClient(); + const result = await client.getScorecard(entity); expect(fetch).toHaveBeenCalledWith(mockScorecardUrl, { method: 'GET', @@ -84,9 +84,9 @@ describe('OpenSSFClient', () => { statusText: 'Not Found', }); - const client = new OpenSSFClient(entity); + const client = new OpenSSFClient(); - await expect(client.getScorecard()).rejects.toThrow( + await expect(client.getScorecard(entity)).rejects.toThrow( 'OpenSSF API request failed with status 404: Not Found', ); }); @@ -96,9 +96,11 @@ describe('OpenSSFClient', () => { new Error('Network error'), ); - const client = new OpenSSFClient(entity); + const client = new OpenSSFClient(); - await expect(client.getScorecard()).rejects.toThrow('Network error'); + await expect(client.getScorecard(entity)).rejects.toThrow( + 'Network error', + ); }); }); }); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts index b58f37fa3a..9606e7138a 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts @@ -19,14 +19,13 @@ import type { Entity } from '@backstage/catalog-model'; import { OpenSSFResponse } from './types'; export class OpenSSFClient { - private readonly baseUrl: string; - - constructor(entity: Entity) { - this.baseUrl = entity.metadata.annotations?.['openssf/baseUrl'] ?? ''; - } + async getScorecard(entity: Entity): Promise { + const baseUrl = entity.metadata.annotations?.['openssf/baseUrl'] ?? ''; + if (!baseUrl || baseUrl.trim() === '' || !baseUrl.startsWith('https://')) { + throw new Error(`Invalid annotation 'openssf/baseUrl' value`); + } - async getScorecard(): Promise { - const response = await fetch(this.baseUrl, { + const response = await fetch(baseUrl, { method: 'GET', headers: { Accept: 'application/json', diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts index 95de2b81cf..7143f747fd 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts @@ -54,7 +54,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); expect(provider.getMetricName()).toBe('Maintained'); }); @@ -63,7 +62,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); expect(provider.getMetricDisplayTitle()).toBe('OpenSSF Maintained'); expect(provider.getMetricDescription()).toContain('actively maintained'); @@ -73,7 +71,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); expect(provider.getProviderId()).toBe('openssf.maintained'); }); @@ -82,7 +79,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); expect(provider.getProviderDatasourceId()).toBe('openssf'); }); @@ -91,7 +87,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); expect(provider.getMetricType()).toBe('number'); }); @@ -100,7 +95,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); const metric = provider.getMetric(); expect(metric.id).toBe('openssf.maintained'); @@ -113,7 +107,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); expect(provider.getMetricThresholds()).toEqual(OPENSSF_THRESHOLDS); }); @@ -122,7 +115,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); expect(provider.getCatalogFilter()).toEqual({ 'metadata.annotations.openssf/baseUrl': CATALOG_FILTER_EXISTS, @@ -154,9 +146,8 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); - const result = await provider.calculateMetric(); + const result = await provider.calculateMetric(entity); expect(result).toBe(8); expect(fetch).toHaveBeenCalledWith(scorecardUrl, expect.any(Object)); @@ -185,10 +176,9 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); - await expect(provider.calculateMetric()).rejects.toThrow( + await expect(provider.calculateMetric(entity)).rejects.toThrow( "OpenSSF check 'Maintained' not found in scorecard", ); }); @@ -216,10 +206,9 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - entity, ); - await expect(provider.calculateMetric()).rejects.toThrow( + await expect(provider.calculateMetric(entity)).rejects.toThrow( "OpenSSF check 'Maintained' has invalid score 11", ); }); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts index a049fc45c7..1c39f12a8a 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts @@ -37,11 +37,10 @@ export class OpenSSFMetricProvider implements MetricProvider<'number'> { constructor( readonly config: OpenSSFMetricConfig, thresholds: ThresholdConfig, - entity: Entity, ) { this.thresholds = thresholds; this.config = config; - this.openSSFClient = new OpenSSFClient(entity); + this.openSSFClient = new OpenSSFClient(); } getMetricName(): string { @@ -91,8 +90,8 @@ export class OpenSSFMetricProvider implements MetricProvider<'number'> { }; } - async calculateMetric(): Promise { - const scorecard = await this.openSSFClient.getScorecard(); + async calculateMetric(entity: Entity): Promise { + const scorecard = await this.openSSFClient.getScorecard(entity); const metricName = this.getMetricName(); const metric = scorecard.checks.find(c => c.name === metricName); @@ -113,11 +112,8 @@ export class OpenSSFMetricProvider implements MetricProvider<'number'> { * @param clientOptions Optional base URL and git service host (from app-config) * @returns Array of OpenSSF metric providers */ -export function createOpenSSFMetricProvider( - clientOptions?: OpenSSFClientOptions, -): MetricProvider<'number'>[] { +export function createOpenSSFMetricProvider(): MetricProvider<'number'>[] { return OPENSSF_METRICS.map( - config => - new OpenSSFMetricProvider(config, OPENSSF_THRESHOLDS, clientOptions), + config => new OpenSSFMetricProvider(config, OPENSSF_THRESHOLDS), ); } From 8d0afbdff2a8f7c5ded7bef7ed83ead1e8d28e97 Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Wed, 11 Feb 2026 15:23:37 +0100 Subject: [PATCH 03/15] add component for testing the new baseUrl annotation for openssf scorecard --- .../scorecard/examples/openssf-scorecard-only.yaml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/workspaces/scorecard/examples/openssf-scorecard-only.yaml b/workspaces/scorecard/examples/openssf-scorecard-only.yaml index 00adb4328d..5a4bb82142 100644 --- a/workspaces/scorecard/examples/openssf-scorecard-only.yaml +++ b/workspaces/scorecard/examples/openssf-scorecard-only.yaml @@ -1,14 +1,11 @@ --- -# Component with OpenSSF Scorecard apiVersion: backstage.io/v1alpha1 kind: Component metadata: - name: openssf-scorecard-only + name: openssf-scorecard annotations: - github.com/project-slug: backstage/backstage - openssf/project: backstage/backstage - backstage.io/source-location: url:https://github.com/backstage/backstage + openssf/baseUrl: https://api.securityscorecards.dev/projects/github.com/alizard0/rhdh-plugins spec: type: service - owner: guests - lifecycle: experimental + owner: group:development/guests + lifecycle: development \ No newline at end of file From 2f2326990e521954e7062415cfd68dd73a418d07 Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Tue, 17 Feb 2026 11:41:14 +0100 Subject: [PATCH 04/15] add exclude metric for openssf --- .../examples/openssf-scorecard-only.yaml | 1 + .../src/clients/OpenSSFClient.test.ts | 40 +++++++++++++++++-- .../src/clients/OpenSSFClient.ts | 15 +++++++ .../OpenSSFMetricProvider.test.ts | 19 +++++++++ .../metricProviders/OpenSSFMetricProvider.ts | 12 ++++-- .../src/module.ts | 10 +++-- 6 files changed, 86 insertions(+), 11 deletions(-) diff --git a/workspaces/scorecard/examples/openssf-scorecard-only.yaml b/workspaces/scorecard/examples/openssf-scorecard-only.yaml index 5a4bb82142..067f4c30aa 100644 --- a/workspaces/scorecard/examples/openssf-scorecard-only.yaml +++ b/workspaces/scorecard/examples/openssf-scorecard-only.yaml @@ -5,6 +5,7 @@ metadata: name: openssf-scorecard annotations: openssf/baseUrl: https://api.securityscorecards.dev/projects/github.com/alizard0/rhdh-plugins + openssf/excludeChecks: Maintained,Code-Review spec: type: service owner: group:development/guests diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts index c5c4ebf857..02dbb4710c 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts @@ -22,7 +22,7 @@ import type { OpenSSFResponse } from './types'; const mockScorecardUrl = 'https://api.securityscorecards.dev/projects/github.com/owner/repo'; -function createEntity(baseUrl: string): Entity { +function createEntity(baseUrl: string, excludeChecks?: list): Entity { return { apiVersion: 'backstage.io/v1beta1', kind: 'Component', @@ -30,6 +30,7 @@ function createEntity(baseUrl: string): Entity { name: 'my-service', annotations: { 'openssf/baseUrl': baseUrl, + 'openssf/excludeChecks': excludeChecks, }, }, spec: {}, @@ -49,9 +50,24 @@ const mockOpenSSFResponse: OpenSSFResponse = { details: null, documentation: { short: '', url: '' }, }, + { + name: 'Code-Review', + score: 9, + reason: null, + details: null, + documentation: { short: '', url: '' }, + }, ], }; +const mockLogger = { + child: jest.fn().mockReturnThis(), + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), +}; + describe('OpenSSFClient', () => { const entity = createEntity(mockScorecardUrl); @@ -67,7 +83,7 @@ describe('OpenSSFClient', () => { json: jest.fn().mockResolvedValue(mockOpenSSFResponse), }); - const client = new OpenSSFClient(); + const client = new OpenSSFClient(mockLogger as any); const result = await client.getScorecard(entity); expect(fetch).toHaveBeenCalledWith(mockScorecardUrl, { @@ -84,7 +100,7 @@ describe('OpenSSFClient', () => { statusText: 'Not Found', }); - const client = new OpenSSFClient(); + const client = new OpenSSFClient(mockLogger as any); await expect(client.getScorecard(entity)).rejects.toThrow( 'OpenSSF API request failed with status 404: Not Found', @@ -96,11 +112,27 @@ describe('OpenSSFClient', () => { new Error('Network error'), ); - const client = new OpenSSFClient(); + const client = new OpenSSFClient(mockLogger as any); await expect(client.getScorecard(entity)).rejects.toThrow( 'Network error', ); }); + + it('excludes Maintained when excludeChecks annotation is present', async () => { + const entityWithExcludeChecks = createEntity(mockScorecardUrl, [ + 'Maintained', + ]); + (globalThis.fetch as jest.Mock).mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue(mockOpenSSFResponse), + }); + + const client = new OpenSSFClient(mockLogger as any); + const result = await client.getScorecard(entityWithExcludeChecks); + + expect(result.checks).toHaveLength(1); + expect(result.checks[0].name).toBe('Code-Review'); + }); }); }); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts index 9606e7138a..05d358c7d3 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts @@ -14,11 +14,14 @@ * limitations under the License. */ +import type { LoggerService } from '@backstage/backend-plugin-api'; import type { Entity } from '@backstage/catalog-model'; import { OpenSSFResponse } from './types'; export class OpenSSFClient { + constructor(private readonly logger: LoggerService) {} + async getScorecard(entity: Entity): Promise { const baseUrl = entity.metadata.annotations?.['openssf/baseUrl'] ?? ''; if (!baseUrl || baseUrl.trim() === '' || !baseUrl.startsWith('https://')) { @@ -40,6 +43,18 @@ export class OpenSSFClient { const data: OpenSSFResponse = await response.json(); + const excludeChecks = + entity.metadata.annotations?.['openssf/excludeChecks']; + + if (excludeChecks && excludeChecks.length > 0) { + this.logger.debug( + `Excluding checks: ${excludeChecks} for entity ${entity.metadata.name}`, + ); + data.checks = data.checks.filter( + check => !excludeChecks.includes(check.name), + ); + } + return data; } } diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts index 7143f747fd..feacaeea87 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts @@ -41,6 +41,14 @@ const maintainedConfig = { description: 'Determines if the project is actively maintained.', }; +const mockLogger = { + child: jest.fn().mockReturnThis(), + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), +}; + describe('OpenSSFMetricProvider', () => { const entity = createEntity(); @@ -54,6 +62,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); expect(provider.getMetricName()).toBe('Maintained'); }); @@ -62,6 +71,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); expect(provider.getMetricDisplayTitle()).toBe('OpenSSF Maintained'); expect(provider.getMetricDescription()).toContain('actively maintained'); @@ -71,6 +81,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); expect(provider.getProviderId()).toBe('openssf.maintained'); }); @@ -79,6 +90,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); expect(provider.getProviderDatasourceId()).toBe('openssf'); }); @@ -87,6 +99,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); expect(provider.getMetricType()).toBe('number'); }); @@ -95,6 +108,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); const metric = provider.getMetric(); expect(metric.id).toBe('openssf.maintained'); @@ -107,6 +121,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); expect(provider.getMetricThresholds()).toEqual(OPENSSF_THRESHOLDS); }); @@ -115,6 +130,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); expect(provider.getCatalogFilter()).toEqual({ 'metadata.annotations.openssf/baseUrl': CATALOG_FILTER_EXISTS, @@ -146,6 +162,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); const result = await provider.calculateMetric(entity); @@ -176,6 +193,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); await expect(provider.calculateMetric(entity)).rejects.toThrow( @@ -206,6 +224,7 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, + mockLogger as any, ); await expect(provider.calculateMetric(entity)).rejects.toThrow( diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts index 1c39f12a8a..d113b8ba07 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import type { LoggerService } from '@backstage/backend-plugin-api'; import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; import { type Entity } from '@backstage/catalog-model'; @@ -37,10 +38,11 @@ export class OpenSSFMetricProvider implements MetricProvider<'number'> { constructor( readonly config: OpenSSFMetricConfig, thresholds: ThresholdConfig, + logger: LoggerService, ) { this.thresholds = thresholds; this.config = config; - this.openSSFClient = new OpenSSFClient(); + this.openSSFClient = new OpenSSFClient(logger); } getMetricName(): string { @@ -109,11 +111,13 @@ export class OpenSSFMetricProvider implements MetricProvider<'number'> { /** * Creates all default OpenSSF metric providers. - * @param clientOptions Optional base URL and git service host (from app-config) + * @param logger Logger instance for the OpenSSF client and providers * @returns Array of OpenSSF metric providers */ -export function createOpenSSFMetricProvider(): MetricProvider<'number'>[] { +export function createOpenSSFMetricProvider( + logger: LoggerService, +): MetricProvider<'number'>[] { return OPENSSF_METRICS.map( - config => new OpenSSFMetricProvider(config, OPENSSF_THRESHOLDS), + config => new OpenSSFMetricProvider(config, OPENSSF_THRESHOLDS, logger), ); } diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts index 0e661f04dd..9ab771c7e3 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts @@ -13,7 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { createBackendModule } from '@backstage/backend-plugin-api'; +import { + coreServices, + createBackendModule, +} from '@backstage/backend-plugin-api'; import { scorecardMetricsExtensionPoint } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; import { createOpenSSFMetricProvider } from './metricProviders/OpenSSFMetricProvider'; @@ -23,10 +26,11 @@ export const scorecardOpenSFFModule = createBackendModule({ register(reg) { reg.registerInit({ deps: { + logger: coreServices.logger, metrics: scorecardMetricsExtensionPoint, }, - async init({ metrics }) { - metrics.addMetricProvider(...createOpenSSFMetricProvider()); + async init({ logger, metrics }) { + metrics.addMetricProvider(...createOpenSSFMetricProvider(logger)); }, }); }, From 0152c576acae843ed3286b943f39fa17eb192bda Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Wed, 25 Feb 2026 10:11:40 +0100 Subject: [PATCH 05/15] renamed baseUrl to scorecard-location --- workspaces/scorecard/examples/openssf-scorecard-only.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspaces/scorecard/examples/openssf-scorecard-only.yaml b/workspaces/scorecard/examples/openssf-scorecard-only.yaml index 9047ac234f..39f89d8f0a 100644 --- a/workspaces/scorecard/examples/openssf-scorecard-only.yaml +++ b/workspaces/scorecard/examples/openssf-scorecard-only.yaml @@ -4,7 +4,7 @@ kind: Component metadata: name: openssf-scorecard annotations: - openssf/baseUrl: https://api.securityscorecards.dev/projects/github.com/alizard0/rhdh-plugins + openssf/scorecard-location: https://api.securityscorecards.dev/projects/github.com/alizard0/rhdh-plugins openssf/excludeChecks: Maintained,Code-Review spec: type: service From aea0d6438ac183994cd936f6b70ebf50e86e47cb Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Wed, 25 Feb 2026 10:16:20 +0100 Subject: [PATCH 06/15] rename annotation excludeChecks to exclude-checks --- workspaces/scorecard/examples/openssf-scorecard-only.yaml | 2 +- .../src/clients/OpenSSFClient.test.ts | 7 +++++-- .../src/clients/OpenSSFClient.ts | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/workspaces/scorecard/examples/openssf-scorecard-only.yaml b/workspaces/scorecard/examples/openssf-scorecard-only.yaml index 39f89d8f0a..1614cffe20 100644 --- a/workspaces/scorecard/examples/openssf-scorecard-only.yaml +++ b/workspaces/scorecard/examples/openssf-scorecard-only.yaml @@ -5,7 +5,7 @@ metadata: name: openssf-scorecard annotations: openssf/scorecard-location: https://api.securityscorecards.dev/projects/github.com/alizard0/rhdh-plugins - openssf/excludeChecks: Maintained,Code-Review + openssf/exclude-checks: Maintained,Code-Review spec: type: service owner: group:development/guests diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts index a611191de4..843c15d8e9 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts @@ -22,7 +22,10 @@ import type { OpenSSFResponse } from './types'; const mockScorecardLocation = 'https://api.securityscorecards.dev/projects/github.com/owner/repo'; -function createEntity(scorecardLocation: string, excludeChecks?: list): Entity { +function createEntity( + scorecardLocation: string, + excludeChecks?: list, +): Entity { return { apiVersion: 'backstage.io/v1beta1', kind: 'Component', @@ -30,7 +33,7 @@ function createEntity(scorecardLocation: string, excludeChecks?: list): name: 'my-service', annotations: { 'openssf/scorecard-location': scorecardLocation, - 'openssf/excludeChecks': excludeChecks, + 'openssf/exclude-checks': excludeChecks, }, }, spec: {}, diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts index 41efd0e03a..d70a9ec106 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts @@ -21,7 +21,7 @@ import { OpenSSFResponse } from './types'; export class OpenSSFClient { constructor(private readonly logger: LoggerService) {} - + async getScorecard(entity: Entity): Promise { const scorecardLocation = entity.metadata.annotations?.['openssf/scorecard-location'] ?? ''; @@ -49,7 +49,7 @@ export class OpenSSFClient { const data: OpenSSFResponse = await response.json(); const excludeChecks = - entity.metadata.annotations?.['openssf/excludeChecks']; + entity.metadata.annotations?.['openssf/exclude-checks']; if (excludeChecks && excludeChecks.length > 0) { this.logger.debug( From b3b73b45060b13649f559452cf7427ff591a74cc Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Wed, 25 Feb 2026 10:18:21 +0100 Subject: [PATCH 07/15] rollback component name to openssf-scorecard-only --- workspaces/scorecard/examples/openssf-scorecard-only.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspaces/scorecard/examples/openssf-scorecard-only.yaml b/workspaces/scorecard/examples/openssf-scorecard-only.yaml index 1614cffe20..9a6ab6aed8 100644 --- a/workspaces/scorecard/examples/openssf-scorecard-only.yaml +++ b/workspaces/scorecard/examples/openssf-scorecard-only.yaml @@ -2,7 +2,7 @@ apiVersion: backstage.io/v1alpha1 kind: Component metadata: - name: openssf-scorecard + name: openssf-scorecard-only annotations: openssf/scorecard-location: https://api.securityscorecards.dev/projects/github.com/alizard0/rhdh-plugins openssf/exclude-checks: Maintained,Code-Review From 80a7a999e31bc9cabcd3b9692093b148e2b264e0 Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Wed, 25 Feb 2026 10:21:35 +0100 Subject: [PATCH 08/15] fix unit tests --- .../src/clients/OpenSSFClient.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts index 843c15d8e9..c0c7e037ed 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts @@ -103,7 +103,7 @@ describe('OpenSSFClient', () => { statusText: 'Not Found', }); - const client = new OpenSSFClient(); + const client = new OpenSSFClient(mockLogger as any); await expect(client.getScorecard(entity)).rejects.toThrow( 'OpenSSF API request failed with status 404: Not Found', @@ -115,14 +115,14 @@ describe('OpenSSFClient', () => { new Error('Network error'), ); - const client = new OpenSSFClient(); + const client = new OpenSSFClient(mockLogger as any); await expect(client.getScorecard(entity)).rejects.toThrow( 'Network error', ); }); - it('excludes Maintained when excludeChecks annotation is present', async () => { + it('excludes Maintained when exclude-checks annotation is present', async () => { const entityWithExcludeChecks = createEntity(mockScorecardLocation, [ 'Maintained', ]); From 84b21726c110d03aee11d930b5b4d6990d689467 Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Wed, 25 Feb 2026 10:36:20 +0100 Subject: [PATCH 09/15] fix the issue where the exclude-checks is compsoed by a string instead of list --- .../src/clients/OpenSSFClient.test.ts | 27 ++++++++++++++++--- .../src/clients/OpenSSFClient.ts | 14 +++++++--- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts index c0c7e037ed..06a64174e5 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts @@ -24,7 +24,7 @@ const mockScorecardLocation = function createEntity( scorecardLocation: string, - excludeChecks?: list, + excludeChecks?: string, ): Entity { return { apiVersion: 'backstage.io/v1beta1', @@ -33,7 +33,9 @@ function createEntity( name: 'my-service', annotations: { 'openssf/scorecard-location': scorecardLocation, - 'openssf/exclude-checks': excludeChecks, + ...(excludeChecks !== undefined && { + 'openssf/exclude-checks': excludeChecks, + }), }, }, spec: {}, @@ -123,9 +125,10 @@ describe('OpenSSFClient', () => { }); it('excludes Maintained when exclude-checks annotation is present', async () => { - const entityWithExcludeChecks = createEntity(mockScorecardLocation, [ + const entityWithExcludeChecks = createEntity( + mockScorecardLocation, 'Maintained', - ]); + ); (globalThis.fetch as jest.Mock).mockResolvedValue({ ok: true, json: jest.fn().mockResolvedValue(mockOpenSSFResponse), @@ -137,5 +140,21 @@ describe('OpenSSFClient', () => { expect(result.checks).toHaveLength(1); expect(result.checks[0].name).toBe('Code-Review'); }); + + it('excludes multiple checks when exclude-checks is comma-separated', async () => { + const entityWithExcludeChecks = createEntity( + mockScorecardLocation, + 'Maintained, Code-Review', // added space to test out the trim() + ); + (globalThis.fetch as jest.Mock).mockResolvedValue({ + ok: true, + json: jest.fn().mockResolvedValue(mockOpenSSFResponse), + }); + + const client = new OpenSSFClient(mockLogger as any); + const result = await client.getScorecard(entityWithExcludeChecks); + + expect(result.checks).toHaveLength(0); + }); }); }); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts index d70a9ec106..a8e3d74765 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts @@ -48,12 +48,20 @@ export class OpenSSFClient { const data: OpenSSFResponse = await response.json(); - const excludeChecks = + const excludeChecksRaw = entity.metadata.annotations?.['openssf/exclude-checks']; + const excludeChecks = excludeChecksRaw + ? excludeChecksRaw + .split(',') + .map(s => s.trim()) + .filter(Boolean) + : []; - if (excludeChecks && excludeChecks.length > 0) { + if (excludeChecks.length > 0) { this.logger.debug( - `Excluding checks: ${excludeChecks} for entity ${entity.metadata.name}`, + `Excluding checks: ${excludeChecks.join(', ')} for entity ${ + entity.metadata.name + }`, ); data.checks = data.checks.filter( check => !excludeChecks.includes(check.name), From 56c6abc289eb1d6a8c48614c6dc1286299be2814 Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Mon, 2 Mar 2026 15:19:27 +0100 Subject: [PATCH 10/15] exclude metrics in PullMetricsByProviderTask instead --- .../examples/openssf-scorecard-only.yaml | 2 +- .../src/clients/OpenSSFClient.test.ts | 41 +----------------- .../src/clients/OpenSSFClient.ts | 24 +---------- .../tasks/PullMetricsByProviderTask.test.ts | 42 ++++++++++++++++++- .../tasks/PullMetricsByProviderTask.ts | 16 ++++++- 5 files changed, 59 insertions(+), 66 deletions(-) diff --git a/workspaces/scorecard/examples/openssf-scorecard-only.yaml b/workspaces/scorecard/examples/openssf-scorecard-only.yaml index 9a6ab6aed8..7bc1835f94 100644 --- a/workspaces/scorecard/examples/openssf-scorecard-only.yaml +++ b/workspaces/scorecard/examples/openssf-scorecard-only.yaml @@ -5,7 +5,7 @@ metadata: name: openssf-scorecard-only annotations: openssf/scorecard-location: https://api.securityscorecards.dev/projects/github.com/alizard0/rhdh-plugins - openssf/exclude-checks: Maintained,Code-Review + exclude/metrics: openssf.maintained,openssf.code_review spec: type: service owner: group:development/guests diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts index 06a64174e5..c23c452bea 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts @@ -22,10 +22,7 @@ import type { OpenSSFResponse } from './types'; const mockScorecardLocation = 'https://api.securityscorecards.dev/projects/github.com/owner/repo'; -function createEntity( - scorecardLocation: string, - excludeChecks?: string, -): Entity { +function createEntity(scorecardLocation: string): Entity { return { apiVersion: 'backstage.io/v1beta1', kind: 'Component', @@ -33,9 +30,6 @@ function createEntity( name: 'my-service', annotations: { 'openssf/scorecard-location': scorecardLocation, - ...(excludeChecks !== undefined && { - 'openssf/exclude-checks': excludeChecks, - }), }, }, spec: {}, @@ -123,38 +117,5 @@ describe('OpenSSFClient', () => { 'Network error', ); }); - - it('excludes Maintained when exclude-checks annotation is present', async () => { - const entityWithExcludeChecks = createEntity( - mockScorecardLocation, - 'Maintained', - ); - (globalThis.fetch as jest.Mock).mockResolvedValue({ - ok: true, - json: jest.fn().mockResolvedValue(mockOpenSSFResponse), - }); - - const client = new OpenSSFClient(mockLogger as any); - const result = await client.getScorecard(entityWithExcludeChecks); - - expect(result.checks).toHaveLength(1); - expect(result.checks[0].name).toBe('Code-Review'); - }); - - it('excludes multiple checks when exclude-checks is comma-separated', async () => { - const entityWithExcludeChecks = createEntity( - mockScorecardLocation, - 'Maintained, Code-Review', // added space to test out the trim() - ); - (globalThis.fetch as jest.Mock).mockResolvedValue({ - ok: true, - json: jest.fn().mockResolvedValue(mockOpenSSFResponse), - }); - - const client = new OpenSSFClient(mockLogger as any); - const result = await client.getScorecard(entityWithExcludeChecks); - - expect(result.checks).toHaveLength(0); - }); }); }); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts index a8e3d74765..760f29b208 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts @@ -46,28 +46,6 @@ export class OpenSSFClient { ); } - const data: OpenSSFResponse = await response.json(); - - const excludeChecksRaw = - entity.metadata.annotations?.['openssf/exclude-checks']; - const excludeChecks = excludeChecksRaw - ? excludeChecksRaw - .split(',') - .map(s => s.trim()) - .filter(Boolean) - : []; - - if (excludeChecks.length > 0) { - this.logger.debug( - `Excluding checks: ${excludeChecks.join(', ')} for entity ${ - entity.metadata.name - }`, - ); - data.checks = data.checks.filter( - check => !excludeChecks.includes(check.name), - ); - } - - return data; + return await response.json(); } } diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts index 65895c2f03..a169a30aae 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts @@ -293,11 +293,51 @@ describe('PullMetricsByProviderTask', () => { await (task as any).pullProviderMetrics(mockProvider, mockLogger); expect(mockLogger.info).toHaveBeenNthCalledWith( - 2, + 4, `Completed metric pull for github.test_metric: processed 2 entities`, ); }); + it('should skip entities when exclude/metrics annotation contains the provider id', async () => { + const entityExcluded = { + apiVersion: '1.0.0', + kind: 'Component', + metadata: { + name: 'excluded-entity', + annotations: { 'exclude/metrics': 'github.test_metric' }, + }, + }; + const entityIncluded = { + apiVersion: '1.0.0', + kind: 'Component', + metadata: { name: 'included-entity' }, + }; + mockCatalog.queryEntities.mockReset().mockResolvedValueOnce({ + items: [entityExcluded, entityIncluded], + pageInfo: { nextCursor: undefined }, + totalItems: 2, + }); + + const calculateMetricSpy = jest.spyOn(mockProvider, 'calculateMetric'); + const createMetricValuesSpy = jest.spyOn( + mockDatabaseMetricValues, + 'createMetricValues', + ); + await (task as any).pullProviderMetrics(mockProvider, mockLogger); + + expect(calculateMetricSpy).toHaveBeenCalledTimes(1); + expect(calculateMetricSpy).toHaveBeenCalledWith(entityIncluded); + expect(createMetricValuesSpy).toHaveBeenCalledTimes(1); + expect(createMetricValuesSpy).toHaveBeenCalledWith([ + expect.objectContaining({ + catalog_entity_ref: 'component:default/included-entity', + metric_id: 'github.test_metric', + value: 42, + status: 'success', + }), + ]); + }); + it('should throw error if pullProviderMetrics fails', async () => { (task as any).pullProviderMetrics = jest .fn() diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts index 230b5b86ad..ee26dd143a 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts @@ -142,6 +142,20 @@ export class PullMetricsByProviderTask implements SchedulerTask { let value: MetricValue | undefined; try { + const excludedMetrics = + entity.metadata.annotations?.['exclude/metrics']; + logger.info( + `Loaded excluded/metrics annotation: ${excludedMetrics}`, + ); + if (excludedMetrics) { + const excludedMetricsList = excludedMetrics + .split(',') + .map(s => s.trim()); + if (excludedMetricsList.includes(provider.getProviderId())) { + logger.info(`Excluded metric: ${provider.getProviderId()}`); + return undefined; + } + } value = await provider.calculateMetric(entity); const thresholds = mergeEntityAndProviderThresholds( @@ -175,7 +189,7 @@ export class PullMetricsByProviderTask implements SchedulerTask { }), ).then(promises => promises.reduce((acc, curr) => { - if (curr.status === 'fulfilled') { + if (curr.status === 'fulfilled' && curr.value !== undefined) { return [...acc, curr.value]; } return acc; From 9ad23125fa9706efc297c3239e7ebcc9a8005ec0 Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Mon, 2 Mar 2026 17:34:52 +0100 Subject: [PATCH 11/15] review work cicd --- .../src/clients/OpenSSFClient.test.ts | 14 +++----------- .../src/clients/OpenSSFClient.ts | 3 --- .../OpenSSFMetricProvider.test.ts | 19 ------------------- .../metricProviders/OpenSSFMetricProvider.ts | 11 +++-------- .../src/module.ts | 10 +++------- 5 files changed, 9 insertions(+), 48 deletions(-) diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts index c23c452bea..7c5cb641f4 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.test.ts @@ -59,14 +59,6 @@ const mockOpenSSFResponse: OpenSSFResponse = { ], }; -const mockLogger = { - child: jest.fn().mockReturnThis(), - debug: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - error: jest.fn(), -}; - describe('OpenSSFClient', () => { const entity = createEntity(mockScorecardLocation); @@ -82,7 +74,7 @@ describe('OpenSSFClient', () => { json: jest.fn().mockResolvedValue(mockOpenSSFResponse), }); - const client = new OpenSSFClient(mockLogger as any); + const client = new OpenSSFClient(); const result = await client.getScorecard(entity); expect(fetch).toHaveBeenCalledWith(mockScorecardLocation, { @@ -99,7 +91,7 @@ describe('OpenSSFClient', () => { statusText: 'Not Found', }); - const client = new OpenSSFClient(mockLogger as any); + const client = new OpenSSFClient(); await expect(client.getScorecard(entity)).rejects.toThrow( 'OpenSSF API request failed with status 404: Not Found', @@ -111,7 +103,7 @@ describe('OpenSSFClient', () => { new Error('Network error'), ); - const client = new OpenSSFClient(mockLogger as any); + const client = new OpenSSFClient(); await expect(client.getScorecard(entity)).rejects.toThrow( 'Network error', diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts index 760f29b208..f17c1b618d 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/clients/OpenSSFClient.ts @@ -14,14 +14,11 @@ * limitations under the License. */ -import type { LoggerService } from '@backstage/backend-plugin-api'; import type { Entity } from '@backstage/catalog-model'; import { OpenSSFResponse } from './types'; export class OpenSSFClient { - constructor(private readonly logger: LoggerService) {} - async getScorecard(entity: Entity): Promise { const scorecardLocation = entity.metadata.annotations?.['openssf/scorecard-location'] ?? ''; diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts index 05c9660ead..c81b18c364 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.test.ts @@ -41,14 +41,6 @@ const maintainedConfig = { description: 'Determines if the project is actively maintained.', }; -const mockLogger = { - child: jest.fn().mockReturnThis(), - debug: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - error: jest.fn(), -}; - describe('OpenSSFMetricProvider', () => { const entity = createEntity(); @@ -62,7 +54,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); expect(provider.getMetricName()).toBe('Maintained'); }); @@ -71,7 +62,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); expect(provider.getMetricDisplayTitle()).toBe('OpenSSF Maintained'); expect(provider.getMetricDescription()).toContain('actively maintained'); @@ -81,7 +71,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); expect(provider.getProviderId()).toBe('openssf.maintained'); }); @@ -90,7 +79,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); expect(provider.getProviderDatasourceId()).toBe('openssf'); }); @@ -99,7 +87,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); expect(provider.getMetricType()).toBe('number'); }); @@ -108,7 +95,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); const metric = provider.getMetric(); expect(metric.id).toBe('openssf.maintained'); @@ -121,7 +107,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); expect(provider.getMetricThresholds()).toEqual(OPENSSF_THRESHOLDS); }); @@ -130,7 +115,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); expect(provider.getCatalogFilter()).toEqual({ 'metadata.annotations.openssf/scorecard-location': @@ -163,7 +147,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); const result = await provider.calculateMetric(entity); @@ -194,7 +177,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); await expect(provider.calculateMetric(entity)).rejects.toThrow( @@ -225,7 +207,6 @@ describe('OpenSSFMetricProvider', () => { const provider = new OpenSSFMetricProvider( maintainedConfig, OPENSSF_THRESHOLDS, - mockLogger as any, ); await expect(provider.calculateMetric(entity)).rejects.toThrow( diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts index 8364c65d9d..6bed221dbb 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import type { LoggerService } from '@backstage/backend-plugin-api'; import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; import { type Entity } from '@backstage/catalog-model'; @@ -38,11 +37,10 @@ export class OpenSSFMetricProvider implements MetricProvider<'number'> { constructor( readonly config: OpenSSFMetricConfig, thresholds: ThresholdConfig, - logger: LoggerService, ) { this.thresholds = thresholds; this.config = config; - this.openSSFClient = new OpenSSFClient(logger); + this.openSSFClient = new OpenSSFClient(); } getMetricName(): string { @@ -111,13 +109,10 @@ export class OpenSSFMetricProvider implements MetricProvider<'number'> { /** * Creates all default OpenSSF metric providers. - * @param logger Logger instance for the OpenSSF client and providers * @returns Array of OpenSSF metric providers */ -export function createOpenSSFMetricProvider( - logger: LoggerService, -): MetricProvider<'number'>[] { +export function createOpenSSFMetricProvider(): MetricProvider<'number'>[] { return OPENSSF_METRICS.map( - config => new OpenSSFMetricProvider(config, OPENSSF_THRESHOLDS, logger), + config => new OpenSSFMetricProvider(config, OPENSSF_THRESHOLDS), ); } diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts index 9ab771c7e3..0e661f04dd 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-openssf/src/module.ts @@ -13,10 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { - coreServices, - createBackendModule, -} from '@backstage/backend-plugin-api'; +import { createBackendModule } from '@backstage/backend-plugin-api'; import { scorecardMetricsExtensionPoint } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; import { createOpenSSFMetricProvider } from './metricProviders/OpenSSFMetricProvider'; @@ -26,11 +23,10 @@ export const scorecardOpenSFFModule = createBackendModule({ register(reg) { reg.registerInit({ deps: { - logger: coreServices.logger, metrics: scorecardMetricsExtensionPoint, }, - async init({ logger, metrics }) { - metrics.addMetricProvider(...createOpenSSFMetricProvider(logger)); + async init({ metrics }) { + metrics.addMetricProvider(...createOpenSSFMetricProvider()); }, }); }, From c9afc8db6562f9bce3cc7149acf08e65a44a9577 Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Wed, 4 Mar 2026 11:16:53 +0100 Subject: [PATCH 12/15] implemented exclude/include metrics using app-config and exclude using annotation --- .../examples/openssf-scorecard-only.yaml | 2 +- .../tasks/PullMetricsByProviderTask.test.ts | 184 +++++++++++++++++- .../tasks/PullMetricsByProviderTask.ts | 97 +++++++-- 3 files changed, 266 insertions(+), 17 deletions(-) diff --git a/workspaces/scorecard/examples/openssf-scorecard-only.yaml b/workspaces/scorecard/examples/openssf-scorecard-only.yaml index 7bc1835f94..ca1e699cd6 100644 --- a/workspaces/scorecard/examples/openssf-scorecard-only.yaml +++ b/workspaces/scorecard/examples/openssf-scorecard-only.yaml @@ -5,7 +5,7 @@ metadata: name: openssf-scorecard-only annotations: openssf/scorecard-location: https://api.securityscorecards.dev/projects/github.com/alizard0/rhdh-plugins - exclude/metrics: openssf.maintained,openssf.code_review + scorecard.io/exclude_metrics: openssf.maintained,openssf.packaging spec: type: service owner: group:development/guests diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts index a169a30aae..920c95a3be 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts @@ -156,6 +156,184 @@ describe('PullMetricsByProviderTask', () => { }); }); + describe('isMetricIdExcluded', () => { + const providerId = 'openssf.maintained'; + + function createTaskWithScorecardConfig( + scorecardOverrides: { + include_metrics?: string[]; + exclude_metrics?: string[]; + } = {}, + ) { + const config = mockServices.rootConfig({ + data: { + scorecard: { + schedule: scheduleConfig, + ...scorecardOverrides, + }, + }, + }); + return new PullMetricsByProviderTask( + { + scheduler: mockScheduler, + logger: mockLogger, + database: mockDatabaseMetricValues, + config, + catalog: mockCatalog, + auth: mockAuth, + thresholdEvaluator: mockThresholdEvaluator, + }, + mockProvider, + ); + } + + function createEntity(annotationValue?: string) { + return { + apiVersion: '1.0.0' as const, + kind: 'Component' as const, + metadata: { + name: 'test-entity', + ...(annotationValue !== undefined && { + annotations: { + 'scorecard.io/exclude_metrics': annotationValue, + }, + }), + }, + }; + } + + it('returns true when metric is in app-config exclude_metrics', () => { + const taskWithConfig = createTaskWithScorecardConfig({ + exclude_metrics: [providerId], + }); + const entity = createEntity(); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(true); + expect(mockLogger.info).toHaveBeenCalledWith( + `Excluded metric by app-config: ${providerId}`, + ); + }); + + it('returns true when metric is in app-config exclude_metrics even if also in include_metrics (exclude wins)', () => { + const taskWithConfig = createTaskWithScorecardConfig({ + include_metrics: [providerId], + exclude_metrics: [providerId], + }); + const entity = createEntity(); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(true); + expect(mockLogger.info).toHaveBeenCalledWith( + `Excluded metric by app-config: ${providerId}`, + ); + }); + + it('returns false when excluded by annotation but included by app-config (include overrides annotation)', () => { + const taskWithConfig = createTaskWithScorecardConfig({ + include_metrics: [providerId], + }); + const entity = createEntity(providerId); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(false); + expect(mockLogger.info).toHaveBeenCalledWith( + `Exclusion override: metric excluded by annotation but INCLUDED by app-config: ${providerId}`, + ); + }); + + it('returns true when excluded by annotation and not in app-config include_metrics', () => { + const taskWithConfig = createTaskWithScorecardConfig(); + const entity = createEntity(providerId); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(true); + expect(mockLogger.info).toHaveBeenCalledWith( + `Excluded metric by annotation: ${providerId}`, + ); + }); + + it('returns true when excluded by annotation and app-config include_metrics exists but does not contain this metric', () => { + const taskWithConfig = createTaskWithScorecardConfig({ + include_metrics: ['other_metric'], + }); + const entity = createEntity(providerId); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(true); + expect(mockLogger.info).toHaveBeenCalledWith( + `Excluded metric by annotation: ${providerId}`, + ); + }); + + it('parses comma-separated annotation and excludes when providerId is in the list', () => { + const taskWithConfig = createTaskWithScorecardConfig(); + const entity = createEntity('github.test_metric,openssf.maintained'); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(true); + expect(mockLogger.info).toHaveBeenCalledWith( + `Excluded metric by annotation: ${providerId}`, + ); + }); + + it('returns false when not excluded by app-config or annotation', () => { + const taskWithConfig = createTaskWithScorecardConfig(); + const entity = createEntity(); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(false); + }); + + it('returns false when no config and no annotation', () => { + const taskWithConfig = createTaskWithScorecardConfig(); + const entity = createEntity(); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(false); + }); + }); + describe('getScheduleFromConfig', () => { it('should return the default schedule if not configured', () => { const config = (task as any).getScheduleFromConfig( @@ -293,18 +471,18 @@ describe('PullMetricsByProviderTask', () => { await (task as any).pullProviderMetrics(mockProvider, mockLogger); expect(mockLogger.info).toHaveBeenNthCalledWith( - 4, + 2, `Completed metric pull for github.test_metric: processed 2 entities`, ); }); - it('should skip entities when exclude/metrics annotation contains the provider id', async () => { + it('should skip entities when scorecard.io/exclude_metrics annotation contains the provider id', async () => { const entityExcluded = { apiVersion: '1.0.0', kind: 'Component', metadata: { name: 'excluded-entity', - annotations: { 'exclude/metrics': 'github.test_metric' }, + annotations: { 'scorecard.io/exclude_metrics': 'github.test_metric' }, }, }; const entityIncluded = { diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts index ee26dd143a..e94c962aba 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts @@ -113,6 +113,81 @@ export class PullMetricsByProviderTask implements SchedulerTask { : PullMetricsByProviderTask.DEFAULT_SCHEDULE; } + /** + * Check if the metric is excluded by the app-config or the entity annotation + * 1. It loads include/exclude metrics from app-config.yaml and exclude metric from the entity annotation. + * 2. Include metric from app-config overrides the exclude metric from the entity annotation. + * 3. Exclude metric from app-config wins over include metric from app-config. + * @param providerId - The ID of the provider + * @param entity - The entity to check + * @param logger - The logger to use + * @returns true if the metric is excluded, false otherwise + */ + private isMetricIdExcluded( + providerId: string, + entity: Entity, + logger: LoggerService, + ): boolean { + const excludedMetricsFromAppConfig = this.config.getOptionalStringArray( + 'scorecard.exclude_metrics', + ); + logger.debug( + `Loaded scorecard.exclude_metrics from app-config: ${JSON.stringify( + excludedMetricsFromAppConfig, + )}`, + ); + + const isExcludedByAppConfig = + excludedMetricsFromAppConfig?.includes(providerId) ?? false; + + // if the metric is excluded by app-config, automatically returns true + if (isExcludedByAppConfig) { + logger.info(`Excluded metric by app-config: ${providerId}`); + return true; + } + + const includedMetricsFromAppConfig = this.config.getOptionalStringArray( + 'scorecard.include_metrics', + ); + logger.debug( + `Loaded scorecard.include_metrics from app-config: ${JSON.stringify( + includedMetricsFromAppConfig, + )}`, + ); + + const excludedMetricsFromComponentAnnotation = + entity.metadata.annotations?.['scorecard.io/exclude_metrics'] + ?.split(',') + .map((s: string) => s.trim()); + logger.debug( + `Loaded scorecard.io/exclude_metrics annotation: ${JSON.stringify( + excludedMetricsFromComponentAnnotation, + )}`, + ); + + const isIncludedByAppConfig = + includedMetricsFromAppConfig?.includes(providerId) ?? false; + const isExcludedByAnnotation = + excludedMetricsFromComponentAnnotation?.includes(providerId) ?? false; + + // if the metric is excluded by annotation but is included by app-config, returns false + if (isExcludedByAnnotation && isIncludedByAppConfig) { + logger.info( + `Exclusion override: metric excluded by annotation but INCLUDED by app-config: ${providerId}`, + ); + return false; + } + + // if the metric is excluded by annotation, returns true + if (isExcludedByAnnotation && !isIncludedByAppConfig) { + logger.info(`Excluded metric by annotation: ${providerId}`); + return true; + } + + // if the metric is not excluded by app-config or annotation, returns false + return false; + } + private async pullProviderMetrics( provider: MetricProvider, logger: LoggerService, @@ -142,20 +217,16 @@ export class PullMetricsByProviderTask implements SchedulerTask { let value: MetricValue | undefined; try { - const excludedMetrics = - entity.metadata.annotations?.['exclude/metrics']; - logger.info( - `Loaded excluded/metrics annotation: ${excludedMetrics}`, - ); - if (excludedMetrics) { - const excludedMetricsList = excludedMetrics - .split(',') - .map(s => s.trim()); - if (excludedMetricsList.includes(provider.getProviderId())) { - logger.info(`Excluded metric: ${provider.getProviderId()}`); - return undefined; - } + if ( + this.isMetricIdExcluded( + provider.getProviderId(), + entity, + logger, + ) + ) { + return undefined; } + value = await provider.calculateMetric(entity); const thresholds = mergeEntityAndProviderThresholds( From 05e836feccae3a24731a76026929f4b6a58524ea Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Fri, 6 Mar 2026 14:16:19 +0100 Subject: [PATCH 13/15] review work --- .../examples/openssf-scorecard-only.yaml | 2 +- .../plugins/scorecard-backend/config.d.ts | 17 +++ .../tasks/PullMetricsByProviderTask.test.ts | 106 ++++++++++++++---- .../tasks/PullMetricsByProviderTask.ts | 84 ++++++++------ 4 files changed, 153 insertions(+), 56 deletions(-) diff --git a/workspaces/scorecard/examples/openssf-scorecard-only.yaml b/workspaces/scorecard/examples/openssf-scorecard-only.yaml index ca1e699cd6..01825a999d 100644 --- a/workspaces/scorecard/examples/openssf-scorecard-only.yaml +++ b/workspaces/scorecard/examples/openssf-scorecard-only.yaml @@ -5,7 +5,7 @@ metadata: name: openssf-scorecard-only annotations: openssf/scorecard-location: https://api.securityscorecards.dev/projects/github.com/alizard0/rhdh-plugins - scorecard.io/exclude_metrics: openssf.maintained,openssf.packaging + scorecard.io/disabled-metrics: openssf.maintained spec: type: service owner: group:development/guests diff --git a/workspaces/scorecard/plugins/scorecard-backend/config.d.ts b/workspaces/scorecard/plugins/scorecard-backend/config.d.ts index 20ceedbd9d..110276e4e8 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/config.d.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/config.d.ts @@ -21,6 +21,23 @@ export interface Config { scorecard?: { /** Number of days to retain metric data in the database. Older data will be automatically cleaned up. Default: 365 days */ dataRetentionDays?: number; + /** List of metric IDs (e.g. openssf.packaging) that are disabled globally. Entity annotations cannot override this. */ + disabledMetrics?: string[]; + /** + * Control whether users can override behavior via entity annotations. + * This only affects entity annotations (e.g. scorecard.io/disabled-metrics); it does not affect scorecard.disabledMetrics or other app-config. + */ + entityOverrides?: { + /** Whether entity scorecard.io/disabled-metrics annotation can override. Only affects annotations; global disabledMetrics is unchanged. */ + disabledMetrics?: { + /** If true (default), entity can disable metrics via annotation; except list can force some to run. If false, entity list is still applied (union with disabledMetrics) but entity cannot override to re-enable anything. */ + enabled?: boolean; + /** When enabled is true: metric IDs that entity cannot disable (must run). When enabled is false: not used. */ + except?: string[]; + }; + }; + /** @deprecated Use entityOverrides.disabledMetrics.except (with enabled: true) instead. List of metric IDs that must run even when disabled via entity annotation. */ + includeMetrics?: string[]; /** Configuration for scorecard metric providers */ plugins?: { /** Configuration for datasource */ diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts index 584937720e..107388718d 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts @@ -161,8 +161,11 @@ describe('PullMetricsByProviderTask', () => { function createTaskWithScorecardConfig( scorecardOverrides: { - include_metrics?: string[]; - exclude_metrics?: string[]; + includeMetrics?: string[]; + disabledMetrics?: string[]; + entityOverrides?: { + disabledMetrics?: { enabled?: boolean; except?: string[] }; + }; } = {}, ) { const config = mockServices.rootConfig({ @@ -195,16 +198,16 @@ describe('PullMetricsByProviderTask', () => { name: 'test-entity', ...(annotationValue !== undefined && { annotations: { - 'scorecard.io/exclude_metrics': annotationValue, + 'scorecard.io/disabled-metrics': annotationValue, }, }), }, }; } - it('returns true when metric is in app-config exclude_metrics', () => { + it('returns true when metric is in app-config disabledMetrics', () => { const taskWithConfig = createTaskWithScorecardConfig({ - exclude_metrics: [providerId], + disabledMetrics: [providerId], }); const entity = createEntity(); @@ -216,14 +219,14 @@ describe('PullMetricsByProviderTask', () => { expect(result).toBe(true); expect(mockLogger.info).toHaveBeenCalledWith( - `Excluded metric by app-config: ${providerId}`, + `Disabled metric by app-config: ${providerId}`, ); }); - it('returns true when metric is in app-config exclude_metrics even if also in include_metrics (exclude wins)', () => { + it('returns true when metric is in app-config disabledMetrics even if also in includeMetrics (disabled wins)', () => { const taskWithConfig = createTaskWithScorecardConfig({ - include_metrics: [providerId], - exclude_metrics: [providerId], + includeMetrics: [providerId], + disabledMetrics: [providerId], }); const entity = createEntity(); @@ -235,13 +238,13 @@ describe('PullMetricsByProviderTask', () => { expect(result).toBe(true); expect(mockLogger.info).toHaveBeenCalledWith( - `Excluded metric by app-config: ${providerId}`, + `Disabled metric by app-config: ${providerId}`, ); }); - it('returns false when excluded by annotation but included by app-config (include overrides annotation)', () => { + it('returns false when disabled by annotation but included by app-config (include overrides annotation)', () => { const taskWithConfig = createTaskWithScorecardConfig({ - include_metrics: [providerId], + includeMetrics: [providerId], }); const entity = createEntity(providerId); @@ -253,11 +256,11 @@ describe('PullMetricsByProviderTask', () => { expect(result).toBe(false); expect(mockLogger.info).toHaveBeenCalledWith( - `Exclusion override: metric excluded by annotation but INCLUDED by app-config: ${providerId}`, + `Entity override: metric disabled by annotation but in entityOverrides.disabledMetrics.except (must run): ${providerId}`, ); }); - it('returns true when excluded by annotation and not in app-config include_metrics', () => { + it('returns true when disabled by annotation and not in app-config includeMetrics', () => { const taskWithConfig = createTaskWithScorecardConfig(); const entity = createEntity(providerId); @@ -269,13 +272,13 @@ describe('PullMetricsByProviderTask', () => { expect(result).toBe(true); expect(mockLogger.info).toHaveBeenCalledWith( - `Excluded metric by annotation: ${providerId}`, + `Disabled metric by annotation: ${providerId}`, ); }); - it('returns true when excluded by annotation and app-config include_metrics exists but does not contain this metric', () => { + it('returns true when disabled by annotation and app-config includeMetrics exists but does not contain this metric', () => { const taskWithConfig = createTaskWithScorecardConfig({ - include_metrics: ['other_metric'], + includeMetrics: ['other_metric'], }); const entity = createEntity(providerId); @@ -287,7 +290,7 @@ describe('PullMetricsByProviderTask', () => { expect(result).toBe(true); expect(mockLogger.info).toHaveBeenCalledWith( - `Excluded metric by annotation: ${providerId}`, + `Disabled metric by annotation: ${providerId}`, ); }); @@ -303,11 +306,11 @@ describe('PullMetricsByProviderTask', () => { expect(result).toBe(true); expect(mockLogger.info).toHaveBeenCalledWith( - `Excluded metric by annotation: ${providerId}`, + `Disabled metric by annotation: ${providerId}`, ); }); - it('returns false when not excluded by app-config or annotation', () => { + it('returns false when not disabled by app-config or annotation', () => { const taskWithConfig = createTaskWithScorecardConfig(); const entity = createEntity(); @@ -320,6 +323,63 @@ describe('PullMetricsByProviderTask', () => { expect(result).toBe(false); }); + it('returns true when entityOverrides.disabledMetrics.enabled is false and metric is in entity annotation (union of app-config and entity list)', () => { + const taskWithConfig = createTaskWithScorecardConfig({ + entityOverrides: { + disabledMetrics: { enabled: false }, + }, + }); + const entity = createEntity(providerId); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(true); + expect(mockLogger.info).toHaveBeenCalledWith( + `Disabled metric by annotation (entity overrides disabled): ${providerId}`, + ); + }); + + it('returns false when entityOverrides.disabledMetrics.enabled is false and metric is not in entity annotation', () => { + const taskWithConfig = createTaskWithScorecardConfig({ + entityOverrides: { + disabledMetrics: { enabled: false }, + }, + }); + const entity = createEntity(); // no annotation + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(false); + }); + + it('returns false when disabled by annotation but metric is in entityOverrides.disabledMetrics.except', () => { + const taskWithConfig = createTaskWithScorecardConfig({ + entityOverrides: { + disabledMetrics: { enabled: true, except: [providerId] }, + }, + }); + const entity = createEntity(providerId); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(false); + expect(mockLogger.info).toHaveBeenCalledWith( + `Entity override: metric disabled by annotation but in entityOverrides.disabledMetrics.except (must run): ${providerId}`, + ); + }); + it('returns false when no config and no annotation', () => { const taskWithConfig = createTaskWithScorecardConfig(); const entity = createEntity(); @@ -482,13 +542,15 @@ describe('PullMetricsByProviderTask', () => { ); }); - it('should skip entities when scorecard.io/exclude_metrics annotation contains the provider id', async () => { + it('should skip entities when scorecard.io/disabled-metrics annotation contains the provider id', async () => { const entityExcluded = { apiVersion: '1.0.0', kind: 'Component', metadata: { name: 'excluded-entity', - annotations: { 'scorecard.io/exclude_metrics': 'github.test_metric' }, + annotations: { + 'scorecard.io/disabled-metrics': 'github.test_metric', + }, }, }; const entityIncluded = { diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts index 74b4b312bd..e5958f1fb6 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts @@ -114,77 +114,95 @@ export class PullMetricsByProviderTask implements SchedulerTask { } /** - * Check if the metric is excluded by the app-config or the entity annotation - * 1. It loads include/exclude metrics from app-config.yaml and exclude metric from the entity annotation. - * 2. Include metric from app-config overrides the exclude metric from the entity annotation. - * 3. Exclude metric from app-config wins over include metric from app-config. + * Check if the metric is disabled by app-config or entity annotation. + * 1. Top-level disabledMetrics: always disabled, entity cannot override. + * 2. entityOverrides.disabledMetrics.enabled: if false, entity annotation is ignored. + * 3. entityOverrides.disabledMetrics.except (or includeMetrics): when entity override is allowed, these metrics cannot be disabled by entity. * @param providerId - The ID of the provider * @param entity - The entity to check * @param logger - The logger to use - * @returns true if the metric is excluded, false otherwise + * @returns true if the metric is disabled, false otherwise */ private isMetricIdExcluded( providerId: string, entity: Entity, logger: LoggerService, ): boolean { - const excludedMetricsFromAppConfig = this.config.getOptionalStringArray( - 'scorecard.exclude_metrics', + const disabledMetricsFromAppConfig = this.config.getOptionalStringArray( + 'scorecard.disabledMetrics', ); logger.debug( - `Loaded scorecard.exclude_metrics from app-config: ${JSON.stringify( - excludedMetricsFromAppConfig, + `Loaded scorecard.disabledMetrics from app-config: ${JSON.stringify( + disabledMetricsFromAppConfig, )}`, ); - const isExcludedByAppConfig = - excludedMetricsFromAppConfig?.includes(providerId) ?? false; + const isDisabledByAppConfig = + disabledMetricsFromAppConfig?.includes(providerId) ?? false; - // if the metric is excluded by app-config, automatically returns true - if (isExcludedByAppConfig) { - logger.info(`Excluded metric by app-config: ${providerId}`); + // if the metric is disabled by app-config, always disabled (entity cannot override) + if (isDisabledByAppConfig) { + logger.info(`Disabled metric by app-config: ${providerId}`); return true; } - const includedMetricsFromAppConfig = this.config.getOptionalStringArray( - 'scorecard.include_metrics', + // entityOverrides.disabledMetrics: when false, entity annotation is ignored + const entityOverrideEnabled = this.config.getOptionalBoolean( + 'scorecard.entityOverrides.disabledMetrics.enabled', ); + const entityOverrideExcept = this.config.getOptionalStringArray( + 'scorecard.entityOverrides.disabledMetrics.except', + ); + const includeMetricsFromAppConfig = this.config.getOptionalStringArray( + 'scorecard.includeMetrics', + ); + const exceptList = entityOverrideExcept ?? includeMetricsFromAppConfig; logger.debug( - `Loaded scorecard.include_metrics from app-config: ${JSON.stringify( - includedMetricsFromAppConfig, + `Loaded entityOverrides.disabledMetrics (enabled=${entityOverrideEnabled}, except=${JSON.stringify( + exceptList, + )}) and includeMetrics (deprecated): ${JSON.stringify( + includeMetricsFromAppConfig, )}`, ); - const excludedMetricsFromComponentAnnotation = - entity.metadata.annotations?.['scorecard.io/exclude_metrics'] + const disabledMetricsFromComponentAnnotation = + entity.metadata.annotations?.['scorecard.io/disabled-metrics'] ?.split(',') .map((s: string) => s.trim()); logger.debug( - `Loaded scorecard.io/exclude_metrics annotation: ${JSON.stringify( - excludedMetricsFromComponentAnnotation, + `Loaded scorecard.io/disabled-metrics annotation: ${JSON.stringify( + disabledMetricsFromComponentAnnotation, )}`, ); - const isIncludedByAppConfig = - includedMetricsFromAppConfig?.includes(providerId) ?? false; - const isExcludedByAnnotation = - excludedMetricsFromComponentAnnotation?.includes(providerId) ?? false; + const isInExceptList = exceptList?.includes(providerId) ?? false; + const isDisabledByAnnotation = + disabledMetricsFromComponentAnnotation?.includes(providerId) ?? false; + + // when entity overrides are disabled (enabled === false): apply both app-config and entity list (union) — entity cannot override to re-enable + if (entityOverrideEnabled === false) { + if (isDisabledByAnnotation) { + logger.info( + `Disabled metric by annotation (entity overrides disabled): ${providerId}`, + ); + return true; + } + return false; + } - // if the metric is excluded by annotation but is included by app-config, returns false - if (isExcludedByAnnotation && isIncludedByAppConfig) { + // when entity overrides are allowed (enabled !== false): except list = metrics entity cannot disable + if (isDisabledByAnnotation && isInExceptList) { logger.info( - `Exclusion override: metric excluded by annotation but INCLUDED by app-config: ${providerId}`, + `Entity override: metric disabled by annotation but in entityOverrides.disabledMetrics.except (must run): ${providerId}`, ); return false; } - // if the metric is excluded by annotation, returns true - if (isExcludedByAnnotation && !isIncludedByAppConfig) { - logger.info(`Excluded metric by annotation: ${providerId}`); + if (isDisabledByAnnotation && !isInExceptList) { + logger.info(`Disabled metric by annotation: ${providerId}`); return true; } - // if the metric is not excluded by app-config or annotation, returns false return false; } From e72289a74a49851af3586821d4d39ae0c9df8660 Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Fri, 6 Mar 2026 15:04:26 +0100 Subject: [PATCH 14/15] review work --- .../tasks/PullMetricsByProviderTask.test.ts | 23 +++++++++++++++++++ .../tasks/PullMetricsByProviderTask.ts | 13 ++++++----- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts index 107388718d..5e7c089e97 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.test.ts @@ -360,6 +360,29 @@ describe('PullMetricsByProviderTask', () => { expect(result).toBe(false); }); + it('returns true when enabled is false with nested config and entity annotation (getOptionalConfig path)', () => { + const taskWithConfig = createTaskWithScorecardConfig({ + entityOverrides: { + disabledMetrics: { + enabled: false, + except: [providerId], + }, + }, + }); + const entity = createEntity(providerId); + + const result = (taskWithConfig as any).isMetricIdExcluded( + providerId, + entity, + mockLogger, + ); + + expect(result).toBe(true); + expect(mockLogger.info).toHaveBeenCalledWith( + `Disabled metric by annotation (entity overrides disabled): ${providerId}`, + ); + }); + it('returns false when disabled by annotation but metric is in entityOverrides.disabledMetrics.except', () => { const taskWithConfig = createTaskWithScorecardConfig({ entityOverrides: { diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts index e5958f1fb6..b17f52185b 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts @@ -146,13 +146,14 @@ export class PullMetricsByProviderTask implements SchedulerTask { return true; } - // entityOverrides.disabledMetrics: when false, entity annotation is ignored - const entityOverrideEnabled = this.config.getOptionalBoolean( - 'scorecard.entityOverrides.disabledMetrics.enabled', - ); - const entityOverrideExcept = this.config.getOptionalStringArray( - 'scorecard.entityOverrides.disabledMetrics.except', + // entityOverrides.disabledMetrics: when false, entity list still applied (union) but entity cannot override to re-enable + const entityOverridesDisabledMetricsConfig = this.config.getOptionalConfig( + 'scorecard.entityOverrides.disabledMetrics', ); + const entityOverrideEnabled = + entityOverridesDisabledMetricsConfig?.getOptionalBoolean('enabled'); + const entityOverrideExcept = + entityOverridesDisabledMetricsConfig?.getOptionalStringArray('except'); const includeMetricsFromAppConfig = this.config.getOptionalStringArray( 'scorecard.includeMetrics', ); From c26593fdc9234c21fdd2490e6338b65d458a66af Mon Sep 17 00:00:00 2001 From: Andre Lizardo Date: Fri, 6 Mar 2026 15:18:31 +0100 Subject: [PATCH 15/15] review work --- .../src/scheduler/tasks/PullMetricsByProviderTask.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts index b17f52185b..1a65a4d186 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts +++ b/workspaces/scorecard/plugins/scorecard-backend/src/scheduler/tasks/PullMetricsByProviderTask.ts @@ -27,7 +27,7 @@ import { CatalogService } from '@backstage/plugin-catalog-node'; import { MetricProvider } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; import { mergeEntityAndProviderThresholds } from '../../utils/mergeEntityAndProviderThresholds'; import { v4 as uuid } from 'uuid'; -import { stringifyEntityRef } from '@backstage/catalog-model'; +import { stringifyEntityRef, Entity } from '@backstage/catalog-model'; import { DbMetricValueCreate } from '../../database/types'; import { SchedulerOptions, SchedulerTask } from '../types'; import { ThresholdEvaluator } from '../../threshold/ThresholdEvaluator';