From 210d6575522d62b1ace290e24cf413a657b37c9f Mon Sep 17 00:00:00 2001 From: Ruud Senden <8635138+rsenden@users.noreply.github.com> Date: Fri, 5 Jun 2026 20:44:50 +0200 Subject: [PATCH 01/10] ftest: Fix functional test error on fcli release versions --- .../groovy/com/fortify/cli/ftest/core/RPCServerSpec.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/core/RPCServerSpec.groovy b/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/core/RPCServerSpec.groovy index 6c5937796a0..bb6ab59a18f 100644 --- a/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/core/RPCServerSpec.groovy +++ b/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/core/RPCServerSpec.groovy @@ -266,7 +266,7 @@ class RPCServerSpec extends FcliBaseSpec { def server = RPCServerHelper.start("util rpc-server start") then: try { - def actionCmd = "action run ${globalVarsActionPath} --on-unsigned ignore" as String + def actionCmd = "action run ${globalVarsActionPath} --on-unsigned ignore --on-invalid-version ignore" as String // First invocation: set global var 'color' to 'red'; old value should be null def result1 = server.executeAndWait("fcli.execute", [command: "${actionCmd} --key color --value red" as String], 21, 22) From 018721d6829228602544e79d45bd7574b97a045f Mon Sep 17 00:00:00 2001 From: "Sangamesh V." <60685551+SangameshV@users.noreply.github.com> Date: Mon, 8 Jun 2026 14:38:44 +0530 Subject: [PATCH 02/10] chore: Add 'fcli fod app list-users' command to list users assigned to an application (#1030) feat: 'fcli fod app list-users': New command to list users assigned to an application (resolves #1008) --- .../fortify/cli/fod/_common/rest/FoDUrls.java | 1 + .../cli/fod/app/cli/cmd/FoDAppCommands.java | 3 +- .../app/cli/cmd/FoDAppUserListCommand.java | 56 +++++++++++++++++++ .../cli/fod/i18n/FoDMessages.properties | 7 +++ 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/app/cli/cmd/FoDAppUserListCommand.java diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/FoDUrls.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/FoDUrls.java index a934a1a939a..b5c2af40d90 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/FoDUrls.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/rest/FoDUrls.java @@ -40,6 +40,7 @@ public class FoDUrls { public static final String SCAN = ApiBase + "/scans/{scanId}"; public static final String V3_SCAN = "/api/v3scans/{scanId}"; public static final String APP_SCANS = APPLICATION + "/scans"; + public static final String APP_USERS = APPLICATION + "/users"; public static final String RELEASE_SCANS = RELEASE + "/scans"; public static final String STATIC_SCANS = ApiBase + "/releases/{relId}/static-scans"; public static final String STATIC_SCANS_IMPORT = STATIC_SCANS + "/import-scan"; diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/app/cli/cmd/FoDAppCommands.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/app/cli/cmd/FoDAppCommands.java index aaee2440c20..17eec3a3192 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/app/cli/cmd/FoDAppCommands.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/app/cli/cmd/FoDAppCommands.java @@ -24,7 +24,8 @@ FoDAppCreateCommand.class, FoDAppUpdateCommand.class, FoDAppDeleteCommand.class, - FoDAppScanListCommand.class + FoDAppScanListCommand.class, + FoDAppUserListCommand.class } ) @DefaultVariablePropertyName("applicationId") diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/app/cli/cmd/FoDAppUserListCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/app/cli/cmd/FoDAppUserListCommand.java new file mode 100644 index 00000000000..dfd79c5d032 --- /dev/null +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/app/cli/cmd/FoDAppUserListCommand.java @@ -0,0 +1,56 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.fod.app.cli.cmd; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fortify.cli.common.cli.util.CommandGroup; +import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; +import com.fortify.cli.common.output.transform.IInputTransformer; +import com.fortify.cli.common.variable.DefaultVariablePropertyName; +import com.fortify.cli.fod._common.output.cli.cmd.AbstractFoDBaseRequestOutputCommand; +import com.fortify.cli.fod._common.rest.FoDUrls; +import com.fortify.cli.fod.app.cli.mixin.FoDAppResolverMixin; + +import kong.unirest.HttpRequest; +import kong.unirest.UnirestInstance; +import lombok.Getter; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; + +/** + * Command to list users with access to an application. + * @author Sangamesh Vijaykumar + */ +@Command(name = "list-users", aliases = "lsu") @CommandGroup("user") +@DefaultVariablePropertyName("userId") +public class FoDAppUserListCommand extends AbstractFoDBaseRequestOutputCommand implements IInputTransformer { + @Getter @Mixin private OutputHelperMixins.TableWithQuery outputHelper; + @Mixin private FoDAppResolverMixin.RequiredOption appResolver; + + @Override + public HttpRequest getBaseRequest(UnirestInstance unirest) { + return unirest.get(FoDUrls.APP_USERS) + .routeParam("appId", appResolver.getAppId(unirest)); + } + + @Override + public JsonNode transformInput(JsonNode input) { + if ( input != null && input.has("users") ) { return input.get("users"); } + return input; + } + + @Override + public boolean isSingular() { + return false; + } +} \ No newline at end of file diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties index 6ada74811d8..9d9e2fc1649 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties @@ -284,6 +284,11 @@ fcli.fod.app.output.table.header.businessCriticalityType = Criticality fcli.fod.app.output.table.header.applicationDescription = Description fcli.fod.app.output.table.header.releaseName = Release fcli.fod.app.output.table.header.microserviceName = Microservice +fcli.fod.app.user.output.table.header.userId = Id +fcli.fod.app.user.output.table.header.firstName = First Name +fcli.fod.app.user.output.table.header.lastName = Last Name +fcli.fod.app.user.output.table.header.emailId = Email +fcli.fod.app.user.output.table.header.roleName = Role fcli.fod.app.create.usage.header = Create a new application. fcli.fod.app.create.usage.description = This command allows a new application and its first release to be created. \ @@ -329,6 +334,7 @@ fcli.fod.app.update.attrs = Set of application attribute id's or names and their fcli.fod.app.update.auto-required-attrs = For each mandatory application attribute that does not already \ have a value, read the default value from the server and set it automatically. fcli.fod.app.list-scans.usage.header = List scans for a given application. +fcli.fod.app.list-users.usage.header = List users assigned to a given application. # fcli fod microservice fcli.fod.microservice.usage.header = Manage FoD application microservices. @@ -1059,6 +1065,7 @@ fcli.fod.access-control.group.output.table.args = id,name,assignedUsersCount,ass fcli.fod.access-control.role.output.table.args = id,name fcli.fod.app.output.table.args = applicationId,applicationName,fcliApplicationType,businessCriticalityType fcli.fod.app.scan.output.table.args = scanId,scanType,analysisStatusType,applicationName,microserviceName,releaseName,startedDateTime,completedDateTime,scanMethodTypeName +fcli.fod.app.user.output.table.args = userId,firstName,lastName,emailId,roleName fcli.fod.microservice.output.table.args = microserviceId,microserviceName,applicationName fcli.fod.release.output.table.args = releaseId,releaseName,microserviceName,applicationName,sdlcStatusType fcli.fod.release.wait-for.output.table.args = releaseId,releaseName,microserviceName,applicationName,suspended From 6951f7705362edbee14a944ca3df287915bd17ca Mon Sep 17 00:00:00 2001 From: Madhur Jain <87946372+jmadhur87@users.noreply.github.com> Date: Fri, 12 Jun 2026 15:24:41 +0530 Subject: [PATCH 03/10] chore: Make user option non-mandatory in FOD issue update command (#1033) fix: `fcli fod issue update`: Make `--user` option optional Co-authored-by: mjain6 --- .../fortify/cli/fod/issue/cli/cmd/FoDIssueUpdateCommand.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueUpdateCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueUpdateCommand.java index 7aec6a3c085..a4d3cc71e10 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueUpdateCommand.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/issue/cli/cmd/FoDIssueUpdateCommand.java @@ -57,7 +57,7 @@ public class FoDIssueUpdateCommand extends AbstractFoDJsonNodeOutputCommand impl @Mixin private FoDReleaseByQualifiedNameOrIdResolverMixin.RequiredOption releaseResolver; @Mixin private FoDAttributeUpdateOptions.OptionalAttrOption issueAttrsUpdate; - @Option(names = {"--user"}, required = true) + @Option(names = {"--user"}, required = false) protected String user; @Option(names = {"--dev-status"}, required = false) protected String developerStatus; From 431274380ba817580c9db2d36b3c9d46c4e57e3a Mon Sep 17 00:00:00 2001 From: gilseara Date: Sun, 14 Jun 2026 17:36:36 +0200 Subject: [PATCH 04/10] chore: `fod sast-scan start`: Add `--in-progress-action` and `--entitlement-preference` options (#1015) feat: `fcli fod sast-scan start`: Add `--in-progress-action` and `--entitlement-preference` options Co-authored-by: gilseara --- .../scan/helper/sast/FoDScanSastHelper.java | 2 +- .../cli/cmd/FoDSastScanStartCommand.java | 65 +++++++++++++++---- .../cli/fod/i18n/FoDMessages.properties | 1 + .../fortify/cli/ftest/fod/FoDScanSpec.groovy | 36 ++++++++++ 4 files changed, 90 insertions(+), 14 deletions(-) diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/scan/helper/sast/FoDScanSastHelper.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/scan/helper/sast/FoDScanSastHelper.java index 3094ffb3180..a3531083fe6 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/scan/helper/sast/FoDScanSastHelper.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/_common/scan/helper/sast/FoDScanSastHelper.java @@ -67,7 +67,7 @@ public static final FoDScanDescriptor startScanAdvanced(UnirestInstance unirest, .queryString("remdiationScanPreferenceType", (req.getRemdiationScanPreferenceType() != null ? FoDEnums.RemediationScanPreferenceType.valueOf(req.getRemdiationScanPreferenceType()) : FoDEnums.RemediationScanPreferenceType.NonRemediationScanOnly)) .queryString("inProgressScanActionType", (req.getInProgressScanActionType() != null ? - FoDEnums.InProgressScanActionType.valueOf(req.getInProgressScanActionType()) : FoDEnums.InProgressScanActionType.DoNotStartScan)) + req.getInProgressScanActionType() : FoDEnums.InProgressScanActionType.DoNotStartScan.toString())) .queryString("scanTool", req.getScanTool()) .queryString("scanToolVersion", req.getScanToolVersion()) .queryString("scanMethodType", req.getScanMethodType()); diff --git a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/sast_scan/cli/cmd/FoDSastScanStartCommand.java b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/sast_scan/cli/cmd/FoDSastScanStartCommand.java index 85df65310af..dd714626b6c 100644 --- a/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/sast_scan/cli/cmd/FoDSastScanStartCommand.java +++ b/fcli-core/fcli-fod/src/main/java/com/fortify/cli/fod/sast_scan/cli/cmd/FoDSastScanStartCommand.java @@ -19,6 +19,7 @@ import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; import com.fortify.cli.common.progress.cli.mixin.ProgressWriterFactoryMixin; import com.fortify.cli.common.progress.helper.IProgressWriter; +import com.fortify.cli.common.rest.unirest.UnexpectedHttpResponseException; import com.fortify.cli.common.util.FcliBuildProperties; import com.fortify.cli.fod._common.scan.cli.cmd.AbstractFoDScanStartCommand; import com.fortify.cli.fod._common.scan.cli.mixin.FoDRemediationScanPreferenceTypeMixins; @@ -41,6 +42,10 @@ public class FoDSastScanStartCommand extends AbstractFoDScanStartCommand { @Option(names = {"--notes"}) private String notes; + @Option(names = {"--in-progress-action"}, descriptionKey = "fcli.fod.sast-scan.start.in-progress-action") + private FoDEnums.InProgressScanActionType inProgressScanActionType; + @Option(names = {"--entitlement-preference"}, descriptionKey = "fcli.fod.scan.entitlement-preference") + private FoDEnums.EntitlementPreferenceType entitlementPreferenceType; @Mixin private CommonOptionMixins.RequiredFile scanFileMixin; @Mixin private FoDRemediationScanPreferenceTypeMixins.OptionalOption remediationScanType; @@ -49,29 +54,63 @@ public class FoDSastScanStartCommand extends AbstractFoDScanStartCommand { @Override protected FoDScanDescriptor startScan(UnirestInstance unirest, FoDReleaseDescriptor releaseDescriptor) { String relId = releaseDescriptor.getReleaseId(); - Boolean isRemediation = false; - - // if we have requested remediation scan use it to find appropriate assessment type - if (remediationScanType != null && remediationScanType.getRemediationScanPreferenceType() != null) { - if (remediationScanType.getRemediationScanPreferenceType().equals(FoDEnums.RemediationScanPreferenceType.RemediationScanIfAvailable) || - remediationScanType.getRemediationScanPreferenceType().equals(FoDEnums.RemediationScanPreferenceType.RemediationScanOnly)) { - isRemediation = true; - } - } validateScanSetup(unirest, relId); - FoDScanSastStartRequest startScanRequest = FoDScanSastStartRequest.builder() - .isRemediationScan(isRemediation) + FoDEnums.RemediationScanPreferenceType remediationPref = remediationScanType != null + ? remediationScanType.getRemediationScanPreferenceType() : null; + + boolean useAdvanced = entitlementPreferenceType != null || inProgressScanActionType != null; + + FoDScanSastStartRequest.FoDScanSastStartRequestBuilder requestBuilder = FoDScanSastStartRequest.builder() .scanMethodType("Other") .notes(notes != null && !notes.isEmpty() ? notes : "") .scanTool(FcliBuildProperties.INSTANCE.getFcliProjectName()) - .scanToolVersion(FcliBuildProperties.INSTANCE.getFcliVersion()) - .build(); + .scanToolVersion(FcliBuildProperties.INSTANCE.getFcliVersion()); try (IProgressWriter progressWriter = progressWriterFactory.create()) { + if (useAdvanced) { + FoDEnums.InProgressScanActionType inProgressAction = inProgressScanActionType != null + ? inProgressScanActionType : FoDEnums.InProgressScanActionType.Queue; + // FoD's start-scan-advanced expects 'CancelInProgressScan' rather than the enum's 'CancelScanInProgress' + String inProgressApiValue = inProgressAction == FoDEnums.InProgressScanActionType.CancelScanInProgress + ? "CancelInProgressScan" : inProgressAction.name(); + FoDScanSastStartRequest startScanRequest = requestBuilder + .entitlementPreferenceType(entitlementPreferenceType != null ? entitlementPreferenceType.name() : null) + .purchaseEntitlement(false) + .remdiationScanPreferenceType(remediationPref != null ? remediationPref.name() : null) + .inProgressScanActionType(inProgressApiValue) + .build(); + return FoDScanSastHelper.startScanAdvanced(unirest, releaseDescriptor, startScanRequest, scanFileMixin.getFile(), progressWriter); + } + boolean isRemediation = remediationPref != null + && (remediationPref.equals(FoDEnums.RemediationScanPreferenceType.RemediationScanIfAvailable) + || remediationPref.equals(FoDEnums.RemediationScanPreferenceType.RemediationScanOnly)); + FoDScanSastStartRequest startScanRequest = requestBuilder + .isRemediationScan(isRemediation) + .build(); return FoDScanSastHelper.startScanWithDefaults(unirest, releaseDescriptor, startScanRequest, scanFileMixin.getFile(), progressWriter); + } catch (Exception e) { + throw translateScanInProgressException(e); + } + } + + // FoD returns HTTP 422 (errorCode 2001) when a scan is already in progress and the + // in-progress action prevents starting a new one. Translate that into a concise, + // actionable message instead of surfacing the raw upload/HTTP exception. + private RuntimeException translateScanInProgressException(Exception e) { + for (Throwable t = e; t != null; t = t.getCause()) { + if (t instanceof UnexpectedHttpResponseException) { + UnexpectedHttpResponseException httpException = (UnexpectedHttpResponseException) t; + if (httpException.getStatus() == 422 && httpException.getMessage() != null + && httpException.getMessage().toLowerCase().contains("another scan is in progress")) { + return new FcliSimpleException("Cannot start scan: another scan is already in progress for this release. " + + "Use '--in-progress-action=Queue' to queue this scan, or " + + "'--in-progress-action=CancelScanInProgress' to cancel the running scan and start a new one."); + } + } } + return e instanceof RuntimeException ? (RuntimeException) e : new FcliSimpleException(e); } private void validateScanSetup(UnirestInstance unirest, String relId) { diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties index 9d9e2fc1649..970563babbe 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/i18n/FoDMessages.properties @@ -554,6 +554,7 @@ fcli.fod.sast-scan.start.remediation = Identify this scan as a remediation scan. fcli.fod.sast-scan.start.skip-if-running = Check to see if static scan is already running before starting. fcli.fod.sast-scan.start.entitlement-id = The Id of the entitlement to use for the scan. fcli.fod.sast-scan.start.purchase-entitlement = Purchase an entitlement if one is not currently allocated or available. +fcli.fod.sast-scan.start.in-progress-action = The action to use if a scan is already in progress. Valid values: ${COMPLETION-CANDIDATES}. Defaults to 'Queue' when this or '--entitlement-preference' is specified; otherwise the FoD-side default applies. fcli.fod.sast-scan.start.notes = Scan notes. fcli.fod.sast-scan.start.file = Absolute path of the ScanCentral package (.Zip) file to upload. fcli.fod.sast-scan.start.validate-entitlement = Validate if an entitlement has been set and is still valid. diff --git a/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/fod/FoDScanSpec.groovy b/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/fod/FoDScanSpec.groovy index e684432d5c2..f465c3c1461 100644 --- a/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/fod/FoDScanSpec.groovy +++ b/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/fod/FoDScanSpec.groovy @@ -337,6 +337,42 @@ class FoDScanSpec extends FcliBaseSpec { } } + def "start.sast-scan-help-shows-new-options"() { + def args = "fod sast-scan start --help" + when: + def result = Fcli.run(args) + then: + verifyAll(result.stdout) { + it.any { it.contains("--in-progress-action") } + it.any { it.contains("--entitlement-preference") } + it.any { it.contains("DoNotStartScan") } + it.any { it.contains("CancelScanInProgress") } + it.any { it.contains("Queue") } + } + } + + def "start.sast-scan-in-progress-do-not-start"() { + // A scan was started above, so DoNotStartScan must be rejected with a friendly message + def args = "fod sast-scan start --release=fcli-1698140484524:v2 --file=$sastPackage --in-progress-action=DoNotStartScan" + when: + Fcli.run(args) + then: + def e = thrown(UnexpectedFcliResultException) + e.result.stderr.any { it.contains("another scan is already in progress") } + } + + def "start.sast-scan-advanced-queue"() { + // Exercises the start-scan-advanced path; queues behind the in-progress scan + def args = "fod sast-scan start --release=fcli-1698140484524:v2 --file=$sastPackage --in-progress-action=Queue" + when: + def result = Fcli.run(args) + then: + verifyAll(result.stdout) { + size()>=2 + it.last().contains("STARTED") + } + } + def "wait-for-sast"() { def args = "fod sast-scan wait-for ::sastScan:: -i 2s --until=all-match --any-state=Completed,In_Progress,Queued" when: From 8a9ffeb3610ea0d0a2c18c3394938a0945c6104e Mon Sep 17 00:00:00 2001 From: Ruud Senden <8635138+rsenden@users.noreply.github.com> Date: Fri, 19 Jun 2026 13:49:11 +0200 Subject: [PATCH 05/10] fix: `fcli * action run`: Only mask credentials in remote action URLs; do not mask plain action names --- .../common/action/cli/mixin/ActionResolverMixin.java | 6 +++--- .../action/cli/mixin/ActionSourceResolverMixin.java | 2 +- .../com/fortify/cli/common/log/LogMaskHelper.java | 12 +++++++++--- .../java/com/fortify/cli/common/log/MaskValue.java | 9 +++++++-- .../fortify/cli/common/log/MaskValueDescriptor.java | 7 ++++++- .../cli/common/rest/unirest/RemoteUrlAuthHelper.java | 6 ++++++ .../util/InitializationExecutionStrategyTest.java | 2 +- .../publickey/cli/cmd/PublicKeyImportCommand.java | 2 +- 8 files changed, 34 insertions(+), 12 deletions(-) diff --git a/fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/cli/mixin/ActionResolverMixin.java b/fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/cli/mixin/ActionResolverMixin.java index f48bbe92606..daf3db00755 100644 --- a/fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/cli/mixin/ActionResolverMixin.java +++ b/fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/cli/mixin/ActionResolverMixin.java @@ -50,17 +50,17 @@ public String loadActionContents(String type, ActionValidationHandler actionVali } public static class RequiredParameter extends AbstractActionResolverMixin { - @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN) + @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN, maskFullValueOnNoMatch = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_FULL_ON_NO_MATCH) @Getter @Parameters(arity="1", descriptionKey="fcli.action.nameOrLocation") private String action; } public static class OptionalParameter extends AbstractActionResolverMixin { - @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN) + @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN, maskFullValueOnNoMatch = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_FULL_ON_NO_MATCH) @Getter @Parameters(arity="0..1", descriptionKey="fcli.action.nameOrLocation") private String action; } private static class PublicKeyResolverMixin extends AbstractTextResolverMixin { - @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN) + @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN, maskFullValueOnNoMatch = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_FULL_ON_NO_MATCH) @Getter @Option(names={"--pubkey"}, required = false, descriptionKey = "fcli.action.resolver.pubkey", paramLabel = "source") private String textSource; } } diff --git a/fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/cli/mixin/ActionSourceResolverMixin.java b/fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/cli/mixin/ActionSourceResolverMixin.java index e06203bbc3b..d53fbcc85f3 100644 --- a/fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/cli/mixin/ActionSourceResolverMixin.java +++ b/fcli-core/fcli-common-action/src/main/java/com/fortify/cli/common/action/cli/mixin/ActionSourceResolverMixin.java @@ -37,7 +37,7 @@ public List getActionSources(String type) { } public static class OptionalOption extends AbstractActionSourceResolverMixin { - @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN) + @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN, maskFullValueOnNoMatch = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_FULL_ON_NO_MATCH) @Option(names={"--from-zip", "-z"}, required = false, descriptionKey = "fcli.action.resolver.from-zip") @Getter private String source; } diff --git a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/LogMaskHelper.java b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/LogMaskHelper.java index 014ab1553be..4888fac4ef6 100644 --- a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/LogMaskHelper.java +++ b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/LogMaskHelper.java @@ -52,7 +52,7 @@ public final class LogMaskHelper { */ public final LogMaskHelper registerValue(MaskValue maskAnnotation, LogMaskSource source, Object value) { if ( maskAnnotation!=null ) { - registerValue(maskAnnotation.sensitivity(), source, maskAnnotation.description(), value, maskAnnotation.pattern()); + doRegisterValue(maskAnnotation.sensitivity(), source, maskAnnotation.description(), value, maskAnnotation.pattern(), maskAnnotation.maskFullValueOnNoMatch()); } return this; } @@ -63,7 +63,7 @@ public final LogMaskHelper registerValue(MaskValue maskAnnotation, LogMaskSource */ public final LogMaskHelper registerValue(MaskValueDescriptor maskDescriptor, LogMaskSource source, Object value) { if ( maskDescriptor!=null ) { - registerValue(maskDescriptor.sensitivity(), source, maskDescriptor.description(), value, maskDescriptor.pattern()); + doRegisterValue(maskDescriptor.sensitivity(), source, maskDescriptor.description(), value, maskDescriptor.pattern(), maskDescriptor.maskFullValueOnNoMatch()); } return this; } @@ -73,8 +73,12 @@ public final LogMaskHelper registerValue(MaskValueDescriptor maskDescriptor, Log * each attribute of that annotation as a separate method argument. */ public final LogMaskHelper registerValue(LogSensitivityLevel sensitivityLevel, LogMaskSource source, String description, Object value, String patternString) { + return doRegisterValue(sensitivityLevel, source, description, value, patternString, true); + } + + private final LogMaskHelper doRegisterValue(LogSensitivityLevel sensitivityLevel, LogMaskSource source, String description, Object value, String patternString, boolean maskFullValueOnNoMatch) { if ( value instanceof Collection collection ) { - collection.forEach(item -> registerValue(sensitivityLevel, source, description, item, patternString)); + collection.forEach(item -> doRegisterValue(sensitivityLevel, source, description, item, patternString, maskFullValueOnNoMatch)); return this; } var valueString = valueAsString(value); @@ -85,6 +89,8 @@ public final LogMaskHelper registerValue(LogSensitivityLevel sensitivityLevel, L throw new FcliBugException("Pattern string passed to LogMaskHelper::registerValue must contain exactly one capturing group"); } valueString = matcher.group(1); + } else if ( !maskFullValueOnNoMatch ) { + return this; // Pattern didn't match; configured to skip masking on no match } } return registerValue(sensitivityLevel, valueString, String.format("", description.toUpperCase(), source)); diff --git a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/MaskValue.java b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/MaskValue.java index ab6975cee46..9acdb309934 100644 --- a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/MaskValue.java +++ b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/MaskValue.java @@ -53,7 +53,12 @@ /** Pattern to extract a substring to be masked from the original value. This pattern must * containing a single capturing group that represents the substring to be masked. Most * common usage is to extract host name from URL, for which {@link #URL_HOSTNAME_PATTERN} - * can be used. If the input value doesn't match the given pattern, the full value will - * be masked. */ + * can be used. The behavior when the input value doesn't match the given pattern is + * controlled by {@link #maskFullValueOnNoMatch()}. */ public String pattern() default ""; + /** Controls behavior when {@link #pattern()} is non-empty but does not match the input value. + * If true (the default), the full input value is masked as a fallback. + * If false, no masking is performed when the pattern does not match (i.e., the + * value is treated as containing no sensitive data). */ + public boolean maskFullValueOnNoMatch() default true; } diff --git a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/MaskValueDescriptor.java b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/MaskValueDescriptor.java index 6c94cc98d02..d1e9316d468 100644 --- a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/MaskValueDescriptor.java +++ b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/log/MaskValueDescriptor.java @@ -26,8 +26,13 @@ public class MaskValueDescriptor { private final LogSensitivityLevel sensitivity; private final String description; private final String pattern; + private final boolean maskFullValueOnNoMatch; public MaskValueDescriptor(LogSensitivityLevel sensitivity, String description) { - this(sensitivity, description, null); + this(sensitivity, description, null, true); + } + + public MaskValueDescriptor(LogSensitivityLevel sensitivity, String description, String pattern) { + this(sensitivity, description, pattern, true); } } diff --git a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/rest/unirest/RemoteUrlAuthHelper.java b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/rest/unirest/RemoteUrlAuthHelper.java index 0f51f93c8b6..b4b6b7ef06b 100644 --- a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/rest/unirest/RemoteUrlAuthHelper.java +++ b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/rest/unirest/RemoteUrlAuthHelper.java @@ -42,6 +42,12 @@ public final class RemoteUrlAuthHelper { * Captures password for basic auth URLs and token/header value payload for bearer/header(s) formats. */ public static final String URL_USERINFO_AUTH_VALUE_MASK_PATTERN = "https?://(?:(?:[^:@/]+:|bearer:|headers?:)([^@]*)@)?.*"; + /** + * To be used as {@link com.fortify.cli.common.log.MaskValue#maskFullValueOnNoMatch()} together with + * {@link #URL_USERINFO_AUTH_VALUE_MASK_PATTERN}: when the value does not match the URL pattern (e.g., + * a plain action name), no masking should occur because the value carries no embedded auth credentials. + */ + public static final boolean URL_USERINFO_AUTH_VALUE_MASK_FULL_ON_NO_MATCH = false; private static final String PREFIX_BEARER = "bearer:"; private static final String PREFIX_HEADER = "header:"; diff --git a/fcli-core/fcli-common-core/src/test/java/com/fortify/cli/common/cli/util/InitializationExecutionStrategyTest.java b/fcli-core/fcli-common-core/src/test/java/com/fortify/cli/common/cli/util/InitializationExecutionStrategyTest.java index c7de60657de..331da62d9dc 100644 --- a/fcli-core/fcli-common-core/src/test/java/com/fortify/cli/common/cli/util/InitializationExecutionStrategyTest.java +++ b/fcli-core/fcli-common-core/src/test/java/com/fortify/cli/common/cli/util/InitializationExecutionStrategyTest.java @@ -63,7 +63,7 @@ static class DummyPositionalMaskCommand extends AbstractRunnableCommand { static String maskedValueObservedInCall; @Parameters(index = "0") - @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN) + @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN, maskFullValueOnNoMatch = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_FULL_ON_NO_MATCH) private String source; @Override diff --git a/fcli-core/fcli-config/src/main/java/com/fortify/cli/config/publickey/cli/cmd/PublicKeyImportCommand.java b/fcli-core/fcli-config/src/main/java/com/fortify/cli/config/publickey/cli/cmd/PublicKeyImportCommand.java index eae2e468e3a..f51dfcf0d6c 100644 --- a/fcli-core/fcli-config/src/main/java/com/fortify/cli/config/publickey/cli/cmd/PublicKeyImportCommand.java +++ b/fcli-core/fcli-config/src/main/java/com/fortify/cli/config/publickey/cli/cmd/PublicKeyImportCommand.java @@ -53,7 +53,7 @@ public boolean isSingular() { } private static class PublicKeyResolverMixin extends AbstractTextResolverMixin { - @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN) + @MaskValue(sensitivity = LogSensitivityLevel.high, description = "REMOTE URL AUTH VALUE", pattern = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_PATTERN, maskFullValueOnNoMatch = RemoteUrlAuthHelper.URL_USERINFO_AUTH_VALUE_MASK_FULL_ON_NO_MATCH) @Getter @Parameters(arity = "1", descriptionKey = "fcli.config.public-key.resolver", paramLabel = "source") private String textSource; } } From f8cc4777ace9a5a2b60ec401fbeeee2f03a6ebe6 Mon Sep 17 00:00:00 2001 From: Madhur Jain <87946372+jmadhur87@users.noreply.github.com> Date: Fri, 19 Jun 2026 17:33:11 +0530 Subject: [PATCH 06/10] chore: Add ScanCentral Dast attribute in ssc attr list definition command (#1035) fix: `fcli ssc`: Add support for ScanCentral DAST attributes in attribute-related operations --- .../attribute/cli/cmd/SSCAttributeDefinitionListCommand.java | 4 +++- .../ssc/attribute/helper/SSCAttributeDefinitionHelper.java | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/attribute/cli/cmd/SSCAttributeDefinitionListCommand.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/attribute/cli/cmd/SSCAttributeDefinitionListCommand.java index cfda0aae17f..c29c3d6760a 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/attribute/cli/cmd/SSCAttributeDefinitionListCommand.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/attribute/cli/cmd/SSCAttributeDefinitionListCommand.java @@ -30,6 +30,7 @@ @Command(name = OutputHelperMixins.ListDefinitions.CMD_NAME) @CommandGroup("definition") public class SSCAttributeDefinitionListCommand extends AbstractSSCBaseRequestOutputCommand implements IServerSideQueryParamGeneratorSupplier { + private static final String EXTRA_ATTRIBUTE_CATEGORIES = "SCANCENTRAL_DAST"; @Getter @Mixin private OutputHelperMixins.ListDefinitions outputHelper; @Mixin private SSCFetchRangeMixin fetchRangeMixin; @Mixin private SSCQParamMixin qParamMixin; @@ -43,7 +44,8 @@ public class SSCAttributeDefinitionListCommand extends AbstractSSCBaseRequestOut @Override public HttpRequest getBaseRequest(UnirestInstance unirest) { - return unirest.get("/api/v1/attributeDefinitions?orderby=category,name"); + return unirest.get("/api/v1/attributeDefinitions?orderby=category,name") + .queryString("extraCategories", EXTRA_ATTRIBUTE_CATEGORIES); } @Override diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/attribute/helper/SSCAttributeDefinitionHelper.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/attribute/helper/SSCAttributeDefinitionHelper.java index 3dac81cf541..2ac6df0fa62 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/attribute/helper/SSCAttributeDefinitionHelper.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/attribute/helper/SSCAttributeDefinitionHelper.java @@ -40,6 +40,7 @@ */ // TODO Properly embed option handling/retrieval in SSCAttributeDefinitionDescriptor public final class SSCAttributeDefinitionHelper { + private static final String EXTRA_ATTRIBUTE_CATEGORIES = "SCANCENTRAL_DAST"; private final Set attrDuplicateNames = new HashSet<>(); private final Set requiredAttrDefWithoutDefaultValueDescriptors = new HashSet<>(); private final Map descriptorsById = new HashMap<>(); @@ -70,7 +71,8 @@ public SSCAttributeDefinitionHelper(JsonNode attrDefs) { * an SSC bulk request. */ public static final HttpRequest getAttributeDefinitionsRequest(UnirestInstance unirest) { - return unirest.get("/api/v1/attributeDefinitions?limit=-1&orderby=category,name&fields=id,guid,name,category,type,required,hidden,hasDefault,options"); + return unirest.get("/api/v1/attributeDefinitions?limit=-1&orderby=category,name&fields=id,guid,name,category,type,required,hidden,hasDefault,options") + .queryString("extraCategories", EXTRA_ATTRIBUTE_CATEGORIES); } /** From a201fd2215f89ecd603c74a0266b4d990964d161 Mon Sep 17 00:00:00 2001 From: Madhur Jain <87946372+jmadhur87@users.noreply.github.com> Date: Fri, 19 Jun 2026 17:50:50 +0530 Subject: [PATCH 07/10] chore: Adding new values to extensible list-type custom tags in SSC issue update during remediation (#994) feat: `fcli ssc issue update`: Add `--extend` option to allow for adding new values to extensible custom tags --- .../helper/SSCAppVersionCustomTagUpdater.java | 4 +- .../cli/cmd/SSCCustomTagCreateCommand.java | 45 +-- .../cli/cmd/SSCCustomTagUpdateCommand.java | 87 +----- .../cli/mixin/SSCCustomTagResolverMixin.java | 4 +- ...java => SSCCustomTagAssignmentHelper.java} | 25 +- .../helper/SSCCustomTagDefinitionHelper.java | 280 ++++++++++++++++++ .../helper/SSCCustomTagDescriptor.java | 1 + .../custom_tag/helper/SSCCustomTagHelper.java | 85 ------ .../issue/cli/cmd/SSCIssueUpdateCommand.java | 6 +- .../issue/helper/SSCIssueCustomTagHelper.java | 159 ++++++++-- ...AbstractSSCIssueTemplateUpdateCommand.java | 4 +- .../cli/ssc/i18n/SSCMessages.properties | 1 + 12 files changed, 439 insertions(+), 262 deletions(-) rename fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/{SSCCustomTagUpdateHelper.java => SSCCustomTagAssignmentHelper.java} (62%) create mode 100644 fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagDefinitionHelper.java delete mode 100644 fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagHelper.java diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/appversion/helper/SSCAppVersionCustomTagUpdater.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/appversion/helper/SSCAppVersionCustomTagUpdater.java index ce9e828d111..ce1d696748a 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/appversion/helper/SSCAppVersionCustomTagUpdater.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/appversion/helper/SSCAppVersionCustomTagUpdater.java @@ -22,8 +22,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fortify.cli.common.json.JsonHelper; import com.fortify.cli.ssc._common.rest.ssc.SSCUrls; +import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagAssignmentHelper; import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagDescriptor; -import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagUpdateHelper; import kong.unirest.HttpRequest; import kong.unirest.UnirestInstance; @@ -42,7 +42,7 @@ public HttpRequest buildRequest(String appVersionId, List addCustomTa return null; } ArrayNode current = getInitialTags(appVersionId); - SSCCustomTagUpdateHelper tagUpdateHelper = new SSCCustomTagUpdateHelper(unirest); + SSCCustomTagAssignmentHelper tagUpdateHelper = new SSCCustomTagAssignmentHelper(unirest); List currentDescriptors = JsonHelper.stream(current) .map(tag -> JsonHelper.treeToValue(tag, SSCCustomTagDescriptor.class)) .collect(Collectors.toList()); diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/cmd/SSCCustomTagCreateCommand.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/cmd/SSCCustomTagCreateCommand.java index 9c3ae812fa3..4c0f1b2a9a3 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/cmd/SSCCustomTagCreateCommand.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/cmd/SSCCustomTagCreateCommand.java @@ -13,13 +13,11 @@ package com.fortify.cli.ssc.custom_tag.cli.cmd; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fortify.cli.common.exception.FcliSimpleException; import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; import com.fortify.cli.common.output.transform.IActionCommandResultSupplier; import com.fortify.cli.ssc._common.output.cli.cmd.AbstractSSCJsonNodeOutputCommand; +import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagDefinitionHelper; import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagValueType; import kong.unirest.UnirestInstance; @@ -51,7 +49,8 @@ public class SSCCustomTagCreateCommand extends AbstractSSCJsonNodeOutputCommand @Override public JsonNode getJsonNode(UnirestInstance unirest) { - ObjectNode body = buildBody(); + ObjectNode body = new SSCCustomTagDefinitionHelper(unirest).buildCreateBody( + name, valueType, description, restriction, hidden, requiresComment, extensible, values); var response = unirest.post("/api/v1/customTags") .body(body) .asObject(JsonNode.class).getBody(); @@ -67,42 +66,4 @@ public String getActionCommandResult() { public boolean isSingular() { return true; } - - // --- Private body-building helpers below --- - private ObjectNode buildBody() { - ObjectNode body = JsonNodeFactory.instance.objectNode(); - body.put("name", name); - body.put("description", description != null ? description : ""); - body.put("valueType", valueType.name()); - body.put("restriction", restriction); - body.put("hidden", hidden); - body.put("requiresComment", requiresComment); - body.put("customTagType", "CUSTOM"); - if (valueType == SSCCustomTagValueType.LIST) { - body.put("extensible", extensible); - body.set("valueList", buildValueList()); - } - return body; - } - - private ArrayNode buildValueList() { - if (values == null || values.isBlank()) { - throw new FcliSimpleException("At least one value must be specified for LIST type using --values"); - } - var valueList = JsonNodeFactory.instance.arrayNode(); - String[] vals = values.split(","); - for (int i = 0; i < vals.length; i++) { - String val = vals[i].trim(); - ObjectNode entry = JsonNodeFactory.instance.objectNode(); - entry.put("lookupIndex", i+1); - entry.put("deletable", true); - entry.put("lookupValue", val); - entry.put("description", ""); - entry.putNull("auditAssistantTrainingLabel"); - entry.put("hidden", false); - entry.put("seqNumber", i+1); - valueList.add(entry); - } - return valueList; - } } \ No newline at end of file diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/cmd/SSCCustomTagUpdateCommand.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/cmd/SSCCustomTagUpdateCommand.java index 282ab96987b..a5f9cd47b47 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/cmd/SSCCustomTagUpdateCommand.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/cmd/SSCCustomTagUpdateCommand.java @@ -12,19 +12,14 @@ */ package com.fortify.cli.ssc.custom_tag.cli.cmd; -import java.util.LinkedHashMap; - import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fortify.cli.common.exception.FcliSimpleException; import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; import com.fortify.cli.common.output.transform.IActionCommandResultSupplier; import com.fortify.cli.ssc._common.output.cli.cmd.AbstractSSCJsonNodeOutputCommand; import com.fortify.cli.ssc.custom_tag.cli.mixin.SSCCustomTagResolverMixin; +import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagDefinitionHelper; import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagDescriptor; -import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagHelper; import kong.unirest.UnirestInstance; import lombok.Getter; @@ -57,12 +52,14 @@ public class SSCCustomTagUpdateCommand extends AbstractSSCJsonNodeOutputCommand @Override public JsonNode getJsonNode(UnirestInstance unirest) { + SSCCustomTagDefinitionHelper customTagDefinitionHelper = new SSCCustomTagDefinitionHelper(unirest); SSCCustomTagDescriptor desc = customTagResolver.getCustomTagDescriptor(unirest); - ObjectNode updateData = buildBody(desc); + ObjectNode updateData = customTagDefinitionHelper.buildUpdateBody( + desc, name, description, restriction, hidden, requiresComment, extensible, values, addValues, rmValues); unirest.put("/api/v1/customTags/{id}") .routeParam("id", desc.getId()) .body(updateData).asObject(JsonNode.class).getBody(); - return new SSCCustomTagHelper(unirest).getDescriptorByCustomTagSpec(desc.getGuid(), true).asJsonNode(); + return customTagDefinitionHelper.getDescriptorByCustomTagSpec(desc.getGuid(), true).asJsonNode(); } @Override @@ -74,78 +71,4 @@ public String getActionCommandResult() { public boolean isSingular() { return true; } - - // --- Private body-building helpers below --- - private ObjectNode buildBody(SSCCustomTagDescriptor desc) { - ObjectNode body = (ObjectNode)desc.asJsonNode().deepCopy(); - if (name != null && !name.isBlank()) { body.put("name", name); } - if (description != null) { body.put("description", description); } - if (restriction != null) { body.put("restriction", restriction); } - if (hidden != null) { body.put("hidden", hidden); } - if (requiresComment != null) { body.put("requiresComment", requiresComment); } - if (extensible != null && "LIST".equalsIgnoreCase(body.path("valueType").asText())) { body.put("extensible", extensible); } - if ("LIST".equalsIgnoreCase(body.path("valueType").asText())) { - body.set("valueList", buildValueList(body)); - } - return body; - } - - private ArrayNode buildValueList(ObjectNode body) { - LinkedHashMap valueMap = buildValueMap(body); - if (values != null) { - valueMap.clear(); - addValuesToMap(valueMap, values); - } - if (addValues != null) { - addValuesToMap(valueMap, addValues); - } - if (rmValues != null) { - removeValuesFromMap(valueMap, rmValues); - } - if (valueMap.isEmpty()) { - throw new FcliSimpleException("At least one value must be specified for LIST type"); - } - var newValueList = JsonNodeFactory.instance.arrayNode(); - int idx = 1; - for (ObjectNode entry : valueMap.values()) { - entry.put("lookupIndex", idx); - entry.put("seqNumber", idx); - newValueList.add(entry); - idx++; - } - return newValueList; - } - - private LinkedHashMap buildValueMap(ObjectNode body) { - var valueList = body.withArray("valueList"); - LinkedHashMap valueMap = new LinkedHashMap<>(); - for (JsonNode v : valueList) { - valueMap.put(v.path("lookupValue").asText(), (ObjectNode)v); - } - return valueMap; - } - - private void addValuesToMap(LinkedHashMap valueMap, String valuesStr) { - String[] vals = valuesStr.split(","); - for (String val : vals) { - val = val.trim(); - if (!valueMap.containsKey(val)) { - ObjectNode entry = JsonNodeFactory.instance.objectNode(); - entry.put("lookupValue", val); - entry.put("deletable", true); - entry.put("description", ""); - entry.putNull("auditAssistantTrainingLabel"); - entry.put("hidden", false); - valueMap.put(val, entry); - } - } - } - - private void removeValuesFromMap(LinkedHashMap valueMap, String valuesStr) { - String[] vals = valuesStr.split(","); - for (String val : vals) { - val = val.trim(); - valueMap.remove(val); - } - } } \ No newline at end of file diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/mixin/SSCCustomTagResolverMixin.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/mixin/SSCCustomTagResolverMixin.java index b98c1e3d488..9d028836461 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/mixin/SSCCustomTagResolverMixin.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/cli/mixin/SSCCustomTagResolverMixin.java @@ -15,8 +15,8 @@ import org.apache.commons.lang3.StringUtils; import com.fortify.cli.common.cli.util.EnvSuffix; +import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagDefinitionHelper; import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagDescriptor; -import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagHelper; import kong.unirest.UnirestInstance; import lombok.Getter; @@ -30,7 +30,7 @@ public SSCCustomTagDescriptor getCustomTagDescriptor(UnirestInstance unirest) { String customTagNameOrGuid = getCustomTagNameOrGuid(); return StringUtils.isBlank(customTagNameOrGuid) ? null - : new SSCCustomTagHelper(unirest).getDescriptorByCustomTagSpec(customTagNameOrGuid, true); + : new SSCCustomTagDefinitionHelper(unirest).getDescriptorByCustomTagSpec(customTagNameOrGuid, true); } } public static class OptionalOption extends AbstractSSCCustomTagResolverMixin { diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagUpdateHelper.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagAssignmentHelper.java similarity index 62% rename from fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagUpdateHelper.java rename to fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagAssignmentHelper.java index 34f3a6592a7..cdaaaed0b5f 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagUpdateHelper.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagAssignmentHelper.java @@ -21,39 +21,30 @@ import kong.unirest.UnirestInstance; -public class SSCCustomTagUpdateHelper { - private final SSCCustomTagHelper tagHelper; +public class SSCCustomTagAssignmentHelper { + private final SSCCustomTagDefinitionHelper tagDefinitionHelper; - public SSCCustomTagUpdateHelper(UnirestInstance unirest) { - this.tagHelper = new SSCCustomTagHelper(unirest); + public SSCCustomTagAssignmentHelper(UnirestInstance unirest) { + this.tagDefinitionHelper = new SSCCustomTagDefinitionHelper(unirest); } - /** - * Resolves tag specs (name, guid, id) to descriptors using SSCCustomTagHelper. - */ public Set resolveTagSpecs(List tagSpecs) { - return tagHelper.getDescriptorsByCustomTagSpec(tagSpecs, false).collect(Collectors.toSet()); + return tagDefinitionHelper.getDescriptorsByCustomTagSpec(tagSpecs, false).collect(Collectors.toSet()); } - /** - * Computes the updated stream of custom tag descriptors given current, add, and remove specs. - */ public Stream computeUpdatedTagDescriptors(List currentTags, List addSpecs, List rmSpecs) { var currentTagsStream = currentTags.stream(); - var addDescriptorsStream = tagHelper.getDescriptorsByCustomTagSpec(addSpecs, false); - var rmDescriptors = tagHelper.getDescriptorsByCustomTagSpec(rmSpecs, false).toList(); + var addDescriptorsStream = tagDefinitionHelper.getDescriptorsByCustomTagSpec(addSpecs, false); + var rmDescriptors = tagDefinitionHelper.getDescriptorsByCustomTagSpec(rmSpecs, false).toList(); return Stream.concat( currentTagsStream.filter(tag -> rmDescriptors.stream().noneMatch(rmTag -> rmTag.isEqualById(tag))), addDescriptorsStream ).distinct(); } - /** - * Overload: Accepts current custom tags as json nodes, resolves to descriptors, then computes updated descriptors. - */ public Stream computeUpdatedTagDescriptors(JsonNode currentTagsNode, List addSpecs, List rmSpecs) { return computeUpdatedTagDescriptors( - SSCCustomTagHelper.toDescriptors(currentTagsNode), + SSCCustomTagDefinitionHelper.toDescriptors(currentTagsNode), addSpecs, rmSpecs); } } \ No newline at end of file diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagDefinitionHelper.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagDefinitionHelper.java new file mode 100644 index 00000000000..d04deeb5345 --- /dev/null +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagDefinitionHelper.java @@ -0,0 +1,280 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.ssc.custom_tag.helper; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.common.exception.FcliSimpleException; +import com.fortify.cli.common.exception.FcliTechnicalException; +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.ssc._common.rest.ssc.SSCUrls; + +import kong.unirest.UnirestInstance; +import lombok.Getter; + +public class SSCCustomTagDefinitionHelper { + private final UnirestInstance unirest; + @Getter(lazy = true) private final List descriptors = loadDescriptors(); + + public SSCCustomTagDefinitionHelper(UnirestInstance unirest) { + this.unirest = unirest; + } + + public static final List toDescriptors(JsonNode tagsNode) { + if ( tagsNode!=null && tagsNode instanceof ObjectNode ) { + tagsNode = tagsNode.get("data"); + } + if ( tagsNode==null || !(tagsNode instanceof ArrayNode)) { + throw new FcliTechnicalException("Invalid custom tags data: "+tagsNode); + } + return JsonHelper.stream((ArrayNode)tagsNode) + .map(tag -> JsonHelper.treeToValue(tag, SSCCustomTagDescriptor.class)) + .toList(); + } + + private List loadDescriptors() { + return toDescriptors( + unirest.get(SSCUrls.CUSTOM_TAGS).queryString("limit", "-1").asObject(JsonNode.class).getBody() + ); + } + + public SSCCustomTagDescriptor getDescriptorByCustomTagSpec(String customTagSpec, boolean failIfNotFound) { + if (customTagSpec == null || customTagSpec.isBlank()) { + if (failIfNotFound) { + throw new FcliSimpleException("Custom tag not found: null or empty"); + } + return null; + } + return getDescriptors().stream() + .filter(desc -> customTagSpec.equalsIgnoreCase(desc.getGuid()) + || customTagSpec.equalsIgnoreCase(desc.getName()) + || customTagSpec.equalsIgnoreCase(desc.getId())) + .findFirst() + .orElseGet(() -> { + if (failIfNotFound) { + throw new FcliSimpleException("Custom tag not found: " + customTagSpec); + } + return null; + }); + } + + public Stream getDescriptorsByCustomTagSpec(List customTagSpecs, boolean failIfNotFound) { + if (customTagSpecs == null || customTagSpecs.isEmpty()) { + return Stream.empty(); + } + return customTagSpecs.stream() + .map(spec -> getDescriptorByCustomTagSpec(spec, failIfNotFound)) + .filter(Objects::nonNull) + .distinct(); + } + + public ObjectNode buildCreateBody(String name, SSCCustomTagValueType valueType, String description, + boolean restriction, boolean hidden, boolean requiresComment, boolean extensible, String values) { + ObjectNode body = JsonNodeFactory.instance.objectNode(); + body.put("name", name); + body.put("description", description != null ? description : ""); + body.put("valueType", valueType.name()); + body.put("restriction", restriction); + body.put("hidden", hidden); + body.put("requiresComment", requiresComment); + body.put("customTagType", "CUSTOM"); + if (valueType == SSCCustomTagValueType.LIST) { + body.put("extensible", extensible); + body.set("valueList", buildCreateValueList(values)); + } + return body; + } + + public ObjectNode buildUpdateBody(SSCCustomTagDescriptor desc, String name, String description, + Boolean restriction, Boolean hidden, Boolean requiresComment, Boolean extensible, + String values, String addValues, String rmValues) { + ObjectNode body = (ObjectNode)desc.asJsonNode().deepCopy(); + if (name != null && !name.isBlank()) { body.put("name", name); } + if (description != null) { body.put("description", description); } + if (restriction != null) { body.put("restriction", restriction); } + if (hidden != null) { body.put("hidden", hidden); } + if (requiresComment != null) { body.put("requiresComment", requiresComment); } + + String valueType = body.path("valueType").asText(); + boolean hasValueModifications = values != null || addValues != null || rmValues != null; + + if (hasValueModifications && !"LIST".equalsIgnoreCase(valueType)) { + throw new FcliSimpleException("Cannot modify values for custom tag '" + desc.getName() + + "': value operations (--values, --add-values, --rm-values) only apply to LIST type tags, not " + valueType); + } + + if ("LIST".equalsIgnoreCase(valueType)) { + if (extensible != null) { + body.put("extensible", extensible); + } + body.set("valueList", buildUpdatedValueList(body, values, addValues, rmValues)); + } + return body; + } + + public int addValueToListTag(String tagGuid, String newValue) { + SSCCustomTagDescriptor desc = getDescriptorByCustomTagSpec(tagGuid, true); + String tagNumericId = desc.getId(); + ObjectNode body = fetchTagBody(tagNumericId, desc.getName()); + // Return early if the value already exists + JsonNode existingList = body.get("valueList"); + int maxLookupIndex = 0; + if (existingList != null && existingList.isArray()) { + for (JsonNode v : existingList) { + if (newValue.equalsIgnoreCase(v.path("lookupValue").asText())) { + return v.path("lookupIndex").asInt(); + } + int idx = v.path("lookupIndex").asInt(0); + if (idx > maxLookupIndex) { + maxLookupIndex = idx; + } + } + } + // Add new entry with an explicit lookupIndex = max(existing) + 1. + // SSC's PUT reconciliation matches valueList entries by lookupIndex; if the new + // entry has no lookupIndex, SSC may interpret existing in-use values as deleted, + // causing HTTP 400 "cannot be deleted". A unique new index avoids that. + int newIndex = maxLookupIndex + 1; + ObjectNode newEntry = JsonNodeFactory.instance.objectNode(); + newEntry.put("lookupValue", newValue); + newEntry.put("lookupIndex", newIndex); + newEntry.put("seqNumber", newIndex); + body.withArray("valueList").add(newEntry); + unirest.put(SSCUrls.CUSTOM_TAG(tagNumericId)) + .body(body) + .asObject(JsonNode.class) + .getBody(); + return confirmValueLookupIndex(tagNumericId, newValue); + } + + private ObjectNode fetchTagBody(String tagNumericId, String tagName) { + JsonNode response = unirest.get(SSCUrls.CUSTOM_TAG(tagNumericId)) + .asObject(JsonNode.class).getBody(); + JsonNode dataNode = response == null ? null : response.get("data"); + if (!(dataNode instanceof ObjectNode)) { + throw new FcliSimpleException( + "Unexpected response from SSC when fetching custom tag '" + tagName + "'"); + } + return (ObjectNode) dataNode.deepCopy(); + } + + private int confirmValueLookupIndex(String tagNumericId, String value) { + JsonNode updated = unirest.get(SSCUrls.CUSTOM_TAG(tagNumericId)) + .asObject(JsonNode.class).getBody(); + JsonNode updatedList = updated == null ? null : updated.path("data").path("valueList"); + if (updatedList != null && updatedList.isArray()) { + for (JsonNode v : updatedList) { + if (value.equalsIgnoreCase(v.path("lookupValue").asText())) { + return v.path("lookupIndex").asInt(); + } + } + } + throw new FcliSimpleException( + "Value '" + value + "' was sent to SSC but could not be confirmed in the updated tag definition."); + } + + private ArrayNode buildCreateValueList(String values) { + if (values == null || values.isBlank()) { + throw new FcliSimpleException("At least one value must be specified for LIST type using --values"); + } + var valueList = JsonNodeFactory.instance.arrayNode(); + String[] vals = values.split(","); + int idx = 1; + for (int i = 0; i < vals.length; i++) { + String val = vals[i].trim(); + if (val.isBlank()) { + continue; + } + ObjectNode entry = newValueListEntry(val); + entry.put("lookupIndex", idx); + entry.put("seqNumber", idx); + valueList.add(entry); + idx++; + } + if (valueList.isEmpty()) { + throw new FcliSimpleException("At least one non-blank value must be specified for LIST type using --values"); + } + return valueList; + } + + private ArrayNode buildUpdatedValueList(ObjectNode body, String values, String addValues, String rmValues) { + LinkedHashMap valueMap = buildValueMap(body); + if (values != null) { + valueMap.clear(); + addValuesToMap(valueMap, values); + } + if (addValues != null) { + addValuesToMap(valueMap, addValues); + } + if (rmValues != null) { + removeValuesFromMap(valueMap, rmValues); + } + if (valueMap.isEmpty()) { + throw new FcliSimpleException("At least one value must remain after update; cannot remove all LIST values"); + } + var newValueList = JsonNodeFactory.instance.arrayNode(); + int idx = 1; + for (ObjectNode entry : valueMap.values()) { + entry.put("lookupIndex", idx); + entry.put("seqNumber", idx); + newValueList.add(entry); + idx++; + } + return newValueList; + } + + private LinkedHashMap buildValueMap(ObjectNode body) { + var valueList = body.withArray("valueList"); + LinkedHashMap valueMap = new LinkedHashMap<>(); + for (JsonNode v : valueList) { + String key = v.path("lookupValue").asText().toLowerCase(); + valueMap.put(key, (ObjectNode)v); + } + return valueMap; + } + + private void addValuesToMap(LinkedHashMap valueMap, String valuesStr) { + for (String val : valuesStr.split(",")) { + val = val.trim(); + if (!val.isBlank() && !valueMap.containsKey(val.toLowerCase())) { + valueMap.put(val.toLowerCase(), newValueListEntry(val)); + } + } + } + + private void removeValuesFromMap(LinkedHashMap valueMap, String valuesStr) { + for (String val : valuesStr.split(",")) { + val = val.trim(); + if (!val.isBlank()) { + valueMap.remove(val.toLowerCase()); + } + } + } + + private ObjectNode newValueListEntry(String value) { + ObjectNode entry = JsonNodeFactory.instance.objectNode(); + entry.put("lookupValue", value); + entry.put("deletable", true); + entry.put("description", ""); + entry.putNull("auditAssistantTrainingLabel"); + entry.put("hidden", false); + return entry; + } +} \ No newline at end of file diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagDescriptor.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagDescriptor.java index bb15af4cc31..5adc4891ac4 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagDescriptor.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagDescriptor.java @@ -24,6 +24,7 @@ public class SSCCustomTagDescriptor extends JsonNodeHolder { private String customTagType; private boolean deletable; + private boolean extensible; private String guid; private String id; private String name; diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagHelper.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagHelper.java deleted file mode 100644 index 89cffb10ca4..00000000000 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/custom_tag/helper/SSCCustomTagHelper.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2021-2026 Open Text. - * - * The only warranties for products and services of Open Text - * and its affiliates and licensors ("Open Text") are as may - * be set forth in the express warranty statements accompanying - * such products and services. Nothing herein should be construed - * as constituting an additional warranty. Open Text shall not be - * liable for technical or editorial errors or omissions contained - * herein. The information contained herein is subject to change - * without notice. - */ -package com.fortify.cli.ssc.custom_tag.helper; - -import java.util.List; -import java.util.Objects; -import java.util.stream.Stream; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fortify.cli.common.exception.FcliSimpleException; -import com.fortify.cli.common.exception.FcliTechnicalException; -import com.fortify.cli.common.json.JsonHelper; -import com.fortify.cli.ssc._common.rest.ssc.SSCUrls; - -import kong.unirest.UnirestInstance; -import lombok.Getter; - -public class SSCCustomTagHelper { - private final UnirestInstance unirest; - @Getter(lazy = true) private final List descriptors = loadDescriptors(); - - public SSCCustomTagHelper(UnirestInstance unirest) { - this.unirest = unirest; - } - - public static final List toDescriptors(JsonNode tagsNode) { - if ( tagsNode!=null && tagsNode instanceof ObjectNode ) { - tagsNode = tagsNode.get("data"); - } - if ( tagsNode==null || !(tagsNode instanceof ArrayNode)) { - throw new FcliTechnicalException("Invalid custom tags data: "+tagsNode); - } - return JsonHelper.stream((ArrayNode)tagsNode) - .map(tag -> JsonHelper.treeToValue(tag, SSCCustomTagDescriptor.class)) - .toList(); - } - - private List loadDescriptors() { - return toDescriptors( - unirest.get(SSCUrls.CUSTOM_TAGS).queryString("limit", "-1").asObject(JsonNode.class).getBody() - ); - } - - public SSCCustomTagDescriptor getDescriptorByCustomTagSpec(String customTagSpec, boolean failIfNotFound) { - if (customTagSpec == null || customTagSpec.isEmpty()) { - if (failIfNotFound) { - throw new FcliSimpleException("Custom tag not found: null or empty"); - } - return null; - } - return getDescriptors().stream() - .filter(desc -> customTagSpec.equalsIgnoreCase(desc.getGuid()) - || customTagSpec.equalsIgnoreCase(desc.getName()) - || customTagSpec.equalsIgnoreCase(desc.getId())) - .findFirst() - .orElseGet(() -> { - if (failIfNotFound) { - throw new FcliSimpleException("Custom tag not found: " + customTagSpec); - } - return null; - }); - } - - public Stream getDescriptorsByCustomTagSpec(List customTagSpecs, boolean failIfNotFound) { - if (customTagSpecs == null || customTagSpecs.isEmpty()) { - return Stream.empty(); - } - return customTagSpecs.stream() - .map(spec -> getDescriptorByCustomTagSpec(spec, failIfNotFound)) - .filter(Objects::nonNull) - .distinct(); - } -} \ No newline at end of file diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueUpdateCommand.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueUpdateCommand.java index cdbd7ddada2..05a8a7a012a 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueUpdateCommand.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/cli/cmd/SSCIssueUpdateCommand.java @@ -56,6 +56,8 @@ public class SSCIssueUpdateCommand extends AbstractSSCJsonNodeOutputCommand impl private String comment; @Option(names = {"--assign-user"}) private String assignUser; + @Option(names = {"--extend"}, defaultValue = "false") + private boolean extend; @Override public JsonNode getJsonNode(UnirestInstance unirest) { @@ -192,7 +194,9 @@ private void executeAuditRequest(UnirestInstance unirest, String appVersionId, L } if (hasCustomTags()) { var customTagHelper = new SSCIssueCustomTagHelper(unirest, appVersionId); - List processedTags = customTagHelper.processCustomTags(customTags); + List processedTags = customTagHelper.processCustomTags(customTags, + extend ? SSCIssueCustomTagHelper.ExtendPolicy.enabled() + : SSCIssueCustomTagHelper.ExtendPolicy.disabled("--extend")); request.put("customTagAudit", processedTags); } diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/helper/SSCIssueCustomTagHelper.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/helper/SSCIssueCustomTagHelper.java index b9e478d75cd..0a619811ece 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/helper/SSCIssueCustomTagHelper.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue/helper/SSCIssueCustomTagHelper.java @@ -27,6 +27,7 @@ import com.fortify.cli.common.exception.FcliSimpleException; import com.fortify.cli.common.json.JsonHelper; import com.fortify.cli.ssc._common.rest.ssc.SSCUrls; +import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagDefinitionHelper; import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagValueType; import kong.unirest.UnirestInstance; @@ -36,18 +37,46 @@ @RequiredArgsConstructor public class SSCIssueCustomTagHelper { + + public interface ExtendPolicy { + boolean canExtend(); + + void throwExtendNotAllowedException(String tagName, String validValues); + + record Enabled() implements ExtendPolicy { + public boolean canExtend() { return true; } + public void throwExtendNotAllowedException(String tagName, String validValues) {} + } + + record Disabled(String optionName) implements ExtendPolicy { + public boolean canExtend() { return false; } + public void throwExtendNotAllowedException(String tagName, String validValues) { + String suffix = " Use " + optionName + " to add new values."; + if (validValues == null || validValues.isBlank()) { + throw new FcliSimpleException("Custom tag '" + tagName + "' has no valid list values configured." + suffix); + } + throw new FcliSimpleException("Invalid value for custom tag '" + tagName + "'." + + " Supported values: " + validValues + "." + suffix); + } + } + + static ExtendPolicy enabled() { return new Enabled(); } + static ExtendPolicy disabled(String optionName) { return new Disabled(optionName); } + } + private final UnirestInstance unirest; private final String appVersionId; @Getter(lazy = true) private final Map customTagInfoMap = loadCustomTagInfo(); - public List processCustomTags(Map customTags) { + public List processCustomTags(Map customTags, ExtendPolicy extendPolicy) { if (customTags == null || customTags.isEmpty()) { return List.of(); } Map tagInfoMap = getCustomTagInfoMap(); + customTags.forEach((tagName, tagValue) -> validateCustomTagValue(tagName, tagValue, tagInfoMap, extendPolicy)); return customTags.entrySet().stream() .map(entry -> { @@ -57,11 +86,64 @@ public List processCustomTags(Map cu if (tagInfo == null) { throw new FcliSimpleException("Custom tag '" + tagName + "' is not available for this application version"); } - return createAuditValue(tagName, tagValue, tagInfo); + return createAuditValue(tagName, tagValue, tagInfo, extendPolicy); }) - .collect(Collectors.toList()); + .toList(); } - + + private void validateCustomTagValue(String tagName, String tagValue, Map tagInfoMap, ExtendPolicy extendPolicy) { + CustomTagInfo tagInfo = tagInfoMap.get(tagName.toLowerCase()); + if (tagInfo == null) { + throw new FcliSimpleException("Custom tag '" + tagName + "' is not available for this application version"); + } + + boolean isUnset = tagValue == null || tagValue.isBlank(); + if (isUnset) return; + switch (tagInfo.getValueType()) { + case TEXT: return; + case DECIMAL: validateDecimalValue(tagName, tagValue); return; + case DATE: return; + case LIST: validateListValue(tagName, tagValue, tagInfo, extendPolicy); return; + default: + throw new FcliSimpleException("Unsupported custom tag value type: " + tagInfo.getValueType()); + } + } + + private void validateDecimalValue(String tagName, String value) { + try { + Double.parseDouble(value); + } catch (NumberFormatException e) { + throw new FcliSimpleException("Invalid decimal value '" + value + "' for custom tag '" + tagName + "'"); + } + } + + private void validateListValue(String tagName, String value, CustomTagInfo tagInfo, ExtendPolicy extendPolicy) { + var valueList = tagInfo.getValueList(); + if (valueList != null) { + for (ValueListItem item : valueList) { + if (value.equalsIgnoreCase(item.getLookupValue())) { + return; + } + } + } + if (tagInfo.isExtensible() && extendPolicy.canExtend()) { + return; + } + if (!tagInfo.isExtensible()) { + String hint = " This tag is not extensible."; + String validValues = valueList == null || valueList.isEmpty() ? null + : valueList.stream().map(ValueListItem::getLookupValue).collect(Collectors.joining(", ")); + if (validValues == null) { + throw new FcliSimpleException("Custom tag '" + tagName + "' has no valid list values configured." + hint); + } + throw new FcliSimpleException("Invalid value '" + value + "' for custom tag '" + tagName + "'." + + " Supported values: " + validValues + "." + hint); + } + String validValues = valueList == null || valueList.isEmpty() ? null + : valueList.stream().map(ValueListItem::getLookupValue).collect(Collectors.joining(", ")); + extendPolicy.throwExtendNotAllowedException(tagName, validValues); + } + public void populateCustomTagUpdates(Map customTags, ArrayNode customTagsArray) { if (customTags == null || customTags.isEmpty()) { return; @@ -93,7 +175,6 @@ private String getValueGuidForTag(String value, CustomTagInfo tagInfo) { if (value == null || value.isBlank()) { return null; } - if (tagInfo.getValueType() == SSCCustomTagValueType.LIST) { if (tagInfo.getValueList() != null) { for (ValueListItem item : tagInfo.getValueList()) { @@ -103,11 +184,10 @@ private String getValueGuidForTag(String value, CustomTagInfo tagInfo) { } } } - return null; } - private SSCIssueCustomTagAuditValue createAuditValue(String tagName, String value, CustomTagInfo tagInfo) { + private SSCIssueCustomTagAuditValue createAuditValue(String tagName, String value, CustomTagInfo tagInfo, ExtendPolicy extendPolicy) { String guid = tagInfo.getGuid(); boolean isUnset = value == null || value.isBlank(); switch (tagInfo.getValueType()) { @@ -128,39 +208,61 @@ private SSCIssueCustomTagAuditValue createAuditValue(String tagName, String valu return SSCIssueCustomTagAuditValue.forDate(guid, dateValue); case LIST: if (isUnset) return SSCIssueCustomTagAuditValue.forList(guid, -1); - Integer lookupIndex = getListValueIndex(value, tagName, tagInfo); + Integer lookupIndex = getListValueIndex(value, tagName, tagInfo, extendPolicy); return SSCIssueCustomTagAuditValue.forList(guid, lookupIndex); default: throw new FcliSimpleException("Unsupported custom tag value type: " + tagInfo.getValueType()); } } - private String processDateValue(String value, String tagName) { + private void validateDateFormat(String value, String tagName) { try { - LocalDate date = LocalDate.parse(value, DateTimeFormatter.ISO_LOCAL_DATE); - return date.format(DateTimeFormatter.ISO_LOCAL_DATE); + LocalDate.parse(value, DateTimeFormatter.ISO_LOCAL_DATE); } catch (DateTimeParseException e) { throw new FcliSimpleException("Invalid date format '" + value + "' for custom tag '" + tagName + "'. Expected format: yyyy-MM-dd"); } } + + private String processDateValue(String value, String tagName) { + validateDateFormat(value, tagName); + return LocalDate.parse(value, DateTimeFormatter.ISO_LOCAL_DATE).format(DateTimeFormatter.ISO_LOCAL_DATE); + } - private Integer getListValueIndex(String value, String tagName, CustomTagInfo tagInfo) { - if (tagInfo.getValueList() == null || tagInfo.getValueList().isEmpty()) { - throw new FcliSimpleException("Custom tag '" + tagName + "' has no valid list values configured"); + private Integer getListValueIndex(String value, String tagName, CustomTagInfo tagInfo, ExtendPolicy extendPolicy) { + var valueList = tagInfo.getValueList(); + if (valueList != null) { + for (ValueListItem item : valueList) { + if (value.equalsIgnoreCase(item.getLookupValue())) { + return item.getLookupIndex(); + } + } } - - for (ValueListItem item : tagInfo.getValueList()) { - if (value.equalsIgnoreCase(item.getLookupValue())) { - return item.getLookupIndex(); + if (tagInfo.isExtensible() && extendPolicy.canExtend()) { + return extendTagWithValue(tagInfo, value); + } + if (!tagInfo.isExtensible()) { + String hint = " This tag is not extensible."; + String validValues = valueList == null || valueList.isEmpty() ? null + : valueList.stream().map(ValueListItem::getLookupValue).collect(Collectors.joining(", ")); + if (validValues == null) { + throw new FcliSimpleException("Custom tag '" + tagName + "' has no valid list values configured." + hint); } + throw new FcliSimpleException("Invalid value '" + value + "' for custom tag '" + tagName + "'." + + " Supported values: " + validValues + "." + hint); } - - String validValues = tagInfo.getValueList().stream() - .map(ValueListItem::getLookupValue) - .collect(Collectors.joining(", ")); - - throw new FcliSimpleException("Invalid value '" + value + "' for list custom tag '" + tagName + "'. " + - "Valid values are: " + validValues); + String validValues = valueList == null || valueList.isEmpty() ? null + : valueList.stream().map(ValueListItem::getLookupValue).collect(Collectors.joining(", ")); + extendPolicy.throwExtendNotAllowedException(tagName, validValues); + return -1; // unreachable; throwExtendNotAllowedException always throws + } + + private int extendTagWithValue(CustomTagInfo tagInfo, String newValue) { + int newIndex = new SSCCustomTagDefinitionHelper(unirest).addValueToListTag(tagInfo.getGuid(), newValue); + ValueListItem newItem = new ValueListItem(); + newItem.setLookupIndex(newIndex); + newItem.setLookupValue(newValue); + tagInfo.getValueList().add(newItem); + return newIndex; } private Map loadCustomTagInfo() { @@ -183,9 +285,7 @@ private Map loadCustomTagInfo() { return result; } catch (Exception e) { - if (e instanceof FcliSimpleException) { - throw e; - } + if (e instanceof FcliSimpleException fse) throw fse; throw new FcliSimpleException("Failed to load custom tag information: " + e.getMessage(), e); } } @@ -195,7 +295,7 @@ private CustomTagInfo parseCustomTagInfo(JsonNode tagNode) { tagInfo.setGuid(tagNode.get("guid").asText()); tagInfo.setName(tagNode.get("name").asText()); tagInfo.setValueType(SSCCustomTagValueType.valueOf(tagNode.get("valueType").asText())); - + tagInfo.setExtensible(tagNode.path("extensible").asBoolean(false)); JsonNode valueListNode = tagNode.get("valueList"); if (valueListNode != null && valueListNode.isArray()) { for (JsonNode valueNode : valueListNode) { @@ -214,6 +314,7 @@ public static class CustomTagInfo { private String guid; private String name; private SSCCustomTagValueType valueType; + private boolean extensible; private List valueList = new ArrayList<>(); } diff --git a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue_template/cli/cmd/AbstractSSCIssueTemplateUpdateCommand.java b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue_template/cli/cmd/AbstractSSCIssueTemplateUpdateCommand.java index 140e81c6804..1ec99fa9144 100644 --- a/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue_template/cli/cmd/AbstractSSCIssueTemplateUpdateCommand.java +++ b/fcli-core/fcli-ssc/src/main/java/com/fortify/cli/ssc/issue_template/cli/cmd/AbstractSSCIssueTemplateUpdateCommand.java @@ -23,7 +23,7 @@ import com.fortify.cli.ssc._common.output.cli.cmd.AbstractSSCJsonNodeOutputCommand; import com.fortify.cli.ssc._common.rest.ssc.SSCUrls; import com.fortify.cli.ssc.custom_tag.cli.mixin.SSCCustomTagAddRemoveMixin; -import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagUpdateHelper; +import com.fortify.cli.ssc.custom_tag.helper.SSCCustomTagAssignmentHelper; import com.fortify.cli.ssc.issue_template.cli.mixin.SSCIssueTemplateResolverMixin; import com.fortify.cli.ssc.issue_template.helper.SSCIssueTemplateDescriptor; import com.fortify.cli.ssc.issue_template.helper.SSCIssueTemplateHelper; @@ -57,7 +57,7 @@ public JsonNode getJsonNode(UnirestInstance unirest) { } protected ArrayNode getCustomTagIds(UnirestInstance unirest, SSCIssueTemplateDescriptor descriptor) { - SSCCustomTagUpdateHelper tagUpdateHelper = new SSCCustomTagUpdateHelper(unirest); + SSCCustomTagAssignmentHelper tagUpdateHelper = new SSCCustomTagAssignmentHelper(unirest); var currentTags = SSCIssueTemplateHelper.getCustomTagsRequest(unirest, descriptor.getId()).asObject(JsonNode.class).getBody(); return tagUpdateHelper.computeUpdatedTagDescriptors( currentTags, diff --git a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties index 5d798f40322..d2dd0ebe86a 100644 --- a/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties +++ b/fcli-core/fcli-ssc/src/main/resources/com/fortify/cli/ssc/i18n/SSCMessages.properties @@ -538,6 +538,7 @@ fcli.ssc.issue.update.custom-tags = Custom tag to set for the vulnerabilities. F fcli.ssc.issue.update.suppress = Set the suppression status of the vulnerability. Use true to suppress or false to unsuppress. fcli.ssc.issue.update.comment = A comment to apply to all the vulnerabilities that are updated. fcli.ssc.issue.update.assign-user = The username or user id of the user to assign the issues to. +fcli.ssc.issue.update.extend = For LIST tags, adds the value if it doesn't exist (requires extensible tag). fcli.ssc.issue.update.output.table.args = issueIds,updatesString,action fcli.ssc.issue.update.output.table.header.issueIds = Issue Id's fcli.ssc.issue.get-filter.usage.header = Get issue filter details. From 7a8fe5b78081702dd50467fe714f707c96646809 Mon Sep 17 00:00:00 2001 From: Ruud Senden <8635138+rsenden@users.noreply.github.com> Date: Tue, 23 Jun 2026 16:21:08 +0200 Subject: [PATCH 08/10] fix: RPC/MCP servers: Refresh trust manager on changed configuration (#1039) --- .../cli/cmd/AiAssistMCPStartHttpCommand.java | 2 + .../cli/cmd/AiAssistMCPStartStdioCommand.java | 2 + .../util/FortifyCLIStaticInitializer.java | 288 +--------- .../http/ssl/trust/FcliTrustManager.java | 525 ++++++++++++++++++ .../helper/TrustStoreConfigHelper.java | 5 + .../helper/TrustedUrlTrustStoreHelper.java | 12 +- .../cli/util/rpc_server/helper/RPCServer.java | 2 + 7 files changed, 549 insertions(+), 287 deletions(-) create mode 100644 fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/trust/FcliTrustManager.java diff --git a/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartHttpCommand.java b/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartHttpCommand.java index 913894c3621..0147d0f241c 100644 --- a/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartHttpCommand.java +++ b/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartHttpCommand.java @@ -39,6 +39,7 @@ import com.fortify.cli.common.cli.util.StdioHelper; import com.fortify.cli.common.concurrent.job.AsyncJobManager; import com.fortify.cli.common.exception.FcliSimpleException; +import com.fortify.cli.common.http.ssl.trust.FcliTrustManager; import com.fortify.cli.common.log.LogMaskContext; import com.fortify.cli.common.mcp.MCPExclude; import com.fortify.cli.common.session.helper.AbstractSessionHelper; @@ -198,6 +199,7 @@ private T withRequestExecutionContext(McpTransportContext transportContext, // (e.g. FoD OAuth token from the token-fetch response) are captured per-request. try (var tempFrame = FcliExecutionContextHolder.push( new FcliExecutionContext(new FcliIsolationScope(), new FcliActionState(), requestLogMaskCtx))) { + FcliTrustManager.refreshIfChanged(); var auth = authHeaderParser.parseAndRegister(transportContext); var isolationScope = sessionDescriptorResolver.getOrCreateIsolationScope(auth); // Real frame: same requestLogMaskCtx, real isolation scope. diff --git a/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartStdioCommand.java b/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartStdioCommand.java index bacebf6ba5d..0301fcf0a59 100644 --- a/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartStdioCommand.java +++ b/fcli-core/fcli-ai-assist/src/main/java/com/fortify/cli/ai_assist/mcp/cli/cmd/AiAssistMCPStartStdioCommand.java @@ -61,6 +61,7 @@ import com.fortify.cli.common.concurrent.job.cli.mixin.AsyncJobManagerMixin; import com.fortify.cli.common.exception.FcliBugException; import com.fortify.cli.common.exception.FcliSimpleException; +import com.fortify.cli.common.http.ssl.trust.FcliTrustManager; import com.fortify.cli.common.mcp.MCPExclude; import com.fortify.cli.common.util.DateTimePeriodHelper; import com.fortify.cli.common.util.DateTimePeriodHelper.Period; @@ -253,6 +254,7 @@ private SyncResourceTemplateSpecification wrapResourceTemplateSpec(SyncResourceT private T withSharedExecutionContext(Supplier supplier) { try (var frame = FcliExecutionContextHolder.push(new FcliExecutionContext(sharedIsolationScope, new FcliActionState()))) { + FcliTrustManager.refreshIfChanged(); return supplier.get(); } } diff --git a/fcli-core/fcli-app/src/main/java/com/fortify/cli/app/runner/util/FortifyCLIStaticInitializer.java b/fcli-core/fcli-app/src/main/java/com/fortify/cli/app/runner/util/FortifyCLIStaticInitializer.java index 130ec243c19..59d1ecc4d1b 100644 --- a/fcli-core/fcli-app/src/main/java/com/fortify/cli/app/runner/util/FortifyCLIStaticInitializer.java +++ b/fcli-core/fcli-app/src/main/java/com/fortify/cli/app/runner/util/FortifyCLIStaticInitializer.java @@ -12,38 +12,14 @@ */ package com.fortify.cli.app.runner.util; -import java.net.Socket; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.security.GeneralSecurityException; -import java.security.KeyStore; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import java.util.Locale; -import java.util.Objects; -import javax.net.ssl.SSLContext; -import javax.net.ssl.SSLEngine; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import javax.net.ssl.X509ExtendedTrustManager; -import javax.net.ssl.X509TrustManager; - -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.fortify.cli.common.action.runner.ActionProductContextProviders; -import com.fortify.cli.common.http.ssl.truststore.helper.TrustStoreConfigDescriptor; -import com.fortify.cli.common.http.ssl.truststore.helper.TrustStoreConfigHelper; -import com.fortify.cli.common.http.ssl.truststore.helper.TrustedUrlTrustStoreHelper; +import com.fortify.cli.common.http.ssl.trust.FcliTrustManager; import com.fortify.cli.common.i18n.helper.LanguageHelper; -import com.fortify.cli.common.util.EnvHelper; -import com.fortify.cli.common.util.PlatformHelper; import com.fortify.cli.fod.action.helper.FoDActionProductContextProvider; import com.fortify.cli.ssc.action.helper.SSCActionProductContextProvider; import com.fortify.cli.tool._common.helper.ToolUninstaller; @@ -78,269 +54,11 @@ private void initializeProductContextProviders() { } private void initializeTrustStore() { - String trustStorePropertyKey = "javax.net.ssl.trustStore"; - String trustStoreTypePropertyKey = "javax.net.ssl.trustStoreType"; - String trustStorePasswordPropertyKey = "javax.net.ssl.trustStorePassword"; - - // First clear existing configuration - System.clearProperty(trustStorePropertyKey); - System.clearProperty(trustStoreTypePropertyKey); - System.clearProperty(trustStorePasswordPropertyKey); - - TrustStoreConfigDescriptor descriptor = TrustStoreConfigHelper.getTrustStoreConfig(); - if (descriptor != null && StringUtils.isNotBlank(descriptor.getPath())) { - initializeTrustStoreFromConfig(descriptor, trustStorePropertyKey, trustStoreTypePropertyKey, - trustStorePasswordPropertyKey); - } else { - initializeTrustStoreFromEnv(trustStorePropertyKey, trustStoreTypePropertyKey, - trustStorePasswordPropertyKey); - } - log.debug("INFO: Trust store file: " + System.getProperty(trustStorePropertyKey, "NONE")); - - // Merge OS platform trust store (e.g. Windows Certificate Store / macOS Keychain) - // with the configured trust store so enterprise CAs are trusted automatically. - List trustManagers = initializePlatformTrustStore(descriptor); - logAcceptedIssuers(trustManagers); - } - - private List initializePlatformTrustStore(TrustStoreConfigDescriptor descriptor) { - List managers = new ArrayList<>(); - addTrustManagerFromKeyStore(managers, null); // null = use javax.net.ssl.trustStore system props - int managerCountBeforeAdditionalStores = managers.size(); - - if ( !isOsTrustStoreDisabled(descriptor) ) { - KeyStore platformKeyStore = loadPlatformKeyStore(); - if (platformKeyStore != null) { - addTrustManagerFromKeyStore(managers, platformKeyStore); - } - } else { - log.debug("OS trust store merge disabled"); - } - - KeyStore trustedUrlsKeyStore = TrustedUrlTrustStoreHelper.getTrustedUrlsKeyStore(); - if (trustedUrlsKeyStore != null) { - addTrustManagerFromKeyStore(managers, trustedUrlsKeyStore); - } - - if (managers.size() == managerCountBeforeAdditionalStores) { - return managers; // Nothing new to add; skip installing a composite context - } - - try { - SSLContext ctx = SSLContext.getInstance("TLS"); - ctx.init(null, new TrustManager[]{new CompositeX509TrustManager(managers)}, null); - SSLContext.setDefault(ctx); - log.info("Composite SSL context installed: configured trust store + OS trust store + trusted URLs"); - } catch (GeneralSecurityException e) { - log.warn("Could not install composite SSL context with additional trust stores: " + e.getMessage()); - } - - return managers; - } - - private boolean isOsTrustStoreDisabled(TrustStoreConfigDescriptor descriptor) { - if (EnvHelper.asBoolean(EnvHelper.env("FCLI_DISABLE_OS_TRUSTSTORE"))) { - return true; - } - return descriptor != null && Boolean.FALSE.equals(descriptor.getUseOsTrustStore()); - } - - private void addTrustManagerFromKeyStore(List managers, KeyStore keyStore) { - try { - TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); - tmf.init(keyStore); // null = uses javax.net.ssl.trustStore system props - for (TrustManager tm : tmf.getTrustManagers()) { - if (tm instanceof X509TrustManager x509tm) { - managers.add(x509tm); - } - } - } catch (GeneralSecurityException e) { - log.debug("Could not load trust manager from key store: " + e.getMessage()); - } - } - - private void logAcceptedIssuers(List trustManagers) { - if ( !log.isTraceEnabled() ) { - return; - } - - trustManagers.stream() - .map(X509TrustManager::getAcceptedIssuers) - .filter(Objects::nonNull) - .flatMap(Arrays::stream) - .forEach(cert -> - log.trace("Loaded trusted cert - Subject: {}, Issuer: {}", - cert.getSubjectX500Principal().getName(), - cert.getIssuerX500Principal().getName())); - } - - private KeyStore loadPlatformKeyStore() { - if (PlatformHelper.isWindows()) { - return loadSingleKeyStore("Windows-ROOT"); - } else if (PlatformHelper.isMac()) { - return loadSingleKeyStore("KeychainStore"); - } else { - log.debug("No OS trust store loaded for platform: {}", PlatformHelper.getOSString()); - return null; // Linux has no standard Java-accessible OS trust store - } - } - - private KeyStore loadSingleKeyStore(String type) { - try { - KeyStore ks = KeyStore.getInstance(type); - ks.load(null, null); - log.debug("Loaded OS trust store: {}", type); - return ks; - } catch (Exception | LinkageError e) { - // Provider may be unavailable in GraalVM native images built on a different OS - log.warn("OS trust store unavailable ({}): {}", type, e.getMessage()); - log.debug("OS trust store load failure details", e); - return null; - } - } - - private void initializeTrustStoreFromEnv(String trustStorePropertyKey, String trustStoreTypePropertyKey, - String trustStorePasswordPropertyKey) { - String trustStorePath = System.getenv("FCLI_TRUSTSTORE"); - if (null != trustStorePath && Files.exists(Path.of(trustStorePath))) { - System.setProperty(trustStorePropertyKey, trustStorePath); - - String trustStoreType = "jks"; - if (null != System.getenv("FCLI_TRUSTSTORE_TYPE")) { - trustStoreType = System.getenv("FCLI_TRUSTSTORE_TYPE"); - } else { - String fileName = Paths.get(trustStorePath).getFileName().toString(); - String fileExtension = StringUtils.substringAfterLast(fileName, "."); - if (fileExtension.equals("jks") || fileExtension.equals("p12") || fileExtension.equals("pfx")) { - trustStoreType = fileExtension; - } - } - System.setProperty(trustStoreTypePropertyKey, trustStoreType); - - String trustStorePwd = "changeit"; - if (null != System.getenv("FCLI_TRUSTSTORE_PWD")) { - trustStorePwd = System.getenv("FCLI_TRUSTSTORE_PWD"); - } - System.setProperty(trustStorePasswordPropertyKey, trustStorePwd); - } - } - - private void initializeTrustStoreFromConfig(TrustStoreConfigDescriptor descriptor, String trustStorePropertyKey, - String trustStoreTypePropertyKey, String trustStorePasswordPropertyKey) { - Path absolutePath = Path.of(descriptor.getPath()).toAbsolutePath(); - if (!Files.exists(absolutePath)) { - log.warn("WARN: Trust store cannot be found: " + absolutePath); - } - System.setProperty(trustStorePropertyKey, descriptor.getPath()); - if (StringUtils.isNotBlank(descriptor.getType())) { - System.setProperty(trustStoreTypePropertyKey, descriptor.getType()); - } - if (StringUtils.isNotBlank(descriptor.getPassword())) { - System.setProperty(trustStorePasswordPropertyKey, descriptor.getPassword()); - } + FcliTrustManager.installAsDefault(); + log.debug("Initialized refreshable fcli trust manager"); } private void initializeLocale() { Locale.setDefault(LanguageHelper.getConfiguredLanguageDescriptor().getLocale()); } - - private static final class CompositeX509TrustManager extends X509ExtendedTrustManager { - private static final Logger log = LoggerFactory.getLogger(CompositeX509TrustManager.class); - private final List delegates; - - CompositeX509TrustManager(List delegates) { - this.delegates = List.copyOf(delegates); - log.debug("CompositeX509TrustManager created with {} delegate(s)", delegates.size()); - } - - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { - tryEach("checkClientTrusted", chain, tm -> tm.checkClientTrusted(chain, authType)); - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { - log.debug("checkServerTrusted(chain[{}], authType={})", chain.length, authType); - logChain(chain); - tryEach("checkServerTrusted", chain, tm -> tm.checkServerTrusted(chain, authType)); - } - - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException { - tryEach("checkClientTrusted(socket)", chain, tm -> { - if (tm instanceof X509ExtendedTrustManager ext) { ext.checkClientTrusted(chain, authType, socket); } - else { tm.checkClientTrusted(chain, authType); } - }); - } - - @Override - public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine) throws CertificateException { - tryEach("checkClientTrusted(engine)", chain, tm -> { - if (tm instanceof X509ExtendedTrustManager ext) { ext.checkClientTrusted(chain, authType, engine); } - else { tm.checkClientTrusted(chain, authType); } - }); - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException { - log.debug("checkServerTrusted(chain[{}], authType={}, socket)", chain.length, authType); - logChain(chain); - tryEach("checkServerTrusted(socket)", chain, tm -> { - if (tm instanceof X509ExtendedTrustManager ext) { ext.checkServerTrusted(chain, authType, socket); } - else { tm.checkServerTrusted(chain, authType); } - }); - } - - @Override - public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine) throws CertificateException { - log.debug("checkServerTrusted(chain[{}], authType={}, engine)", chain.length, authType); - logChain(chain); - tryEach("checkServerTrusted(engine)", chain, tm -> { - if (tm instanceof X509ExtendedTrustManager ext) { ext.checkServerTrusted(chain, authType, engine); } - else { tm.checkServerTrusted(chain, authType); } - }); - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return delegates.stream() - .map(X509TrustManager::getAcceptedIssuers) - .filter(Objects::nonNull) - .flatMap(Arrays::stream) - .toArray(X509Certificate[]::new); - } - - private void logChain(X509Certificate[] chain) { - if (log.isDebugEnabled()) { - for (int i = 0; i < chain.length; i++) { - log.debug(" cert[{}]: subject={}, issuer={}", i, - chain[i].getSubjectX500Principal().getName(), - chain[i].getIssuerX500Principal().getName()); - } - } - } - - private void tryEach(String method, X509Certificate[] chain, TrustCheckFactory check) throws CertificateException { - CertificateException last = null; - for (int i = 0; i < delegates.size(); i++) { - X509TrustManager tm = delegates.get(i); - try { - check.check(tm); - log.debug("{}: accepted by delegate[{}] {}", method, i, tm.getClass().getName()); - return; - } catch (CertificateException e) { - log.debug("{}: rejected by delegate[{}] {}: {}", method, i, tm.getClass().getName(), e.getMessage()); - last = e; - } - } - log.debug("{}: all {} delegate(s) rejected the certificate chain", method, delegates.size()); - if (last != null) { throw last; } - throw new CertificateException("No trust manager accepted the certificate chain"); - } - - @FunctionalInterface - private interface TrustCheckFactory { - void check(X509TrustManager tm) throws CertificateException; - } - } } diff --git a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/trust/FcliTrustManager.java b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/trust/FcliTrustManager.java new file mode 100644 index 00000000000..fbc224b210f --- /dev/null +++ b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/trust/FcliTrustManager.java @@ -0,0 +1,525 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.common.http.ssl.trust; + +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.security.GeneralSecurityException; +import java.security.KeyStore; +import java.security.MessageDigest; +import java.security.cert.CertificateException; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509ExtendedTrustManager; +import javax.net.ssl.X509TrustManager; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fortify.cli.common.http.ssl.truststore.helper.TrustStoreConfigDescriptor; +import com.fortify.cli.common.http.ssl.truststore.helper.TrustStoreConfigHelper; +import com.fortify.cli.common.http.ssl.truststore.helper.TrustedUrlCertificateDescriptor; +import com.fortify.cli.common.http.ssl.truststore.helper.TrustedUrlTrustStoreHelper; +import com.fortify.cli.common.util.EnvHelper; +import com.fortify.cli.common.util.PlatformHelper; + +/** + * Process-wide SSL trust manager for fcli. + * + *

{@link #installAsDefault()} is called once during startup so every new TLS + * client uses the same trust manager instance. Long-running processes such as + * RPC and MCP servers can then call {@link #refresh()} or + * {@link #refreshIfChanged()} at runtime to pick up trust-store changes without + * restarting the JVM.

+ * + *

Trust checks always run against an immutable snapshot referenced through an + * {@link AtomicReference}, allowing refresh operations to atomically swap in a + * newly loaded trust configuration without blocking in-flight TLS handshakes.

+ */ +public final class FcliTrustManager extends X509ExtendedTrustManager { + private static final Logger LOG = LoggerFactory.getLogger(FcliTrustManager.class); + private static final String TRUST_STORE_PROPERTY_KEY = "javax.net.ssl.trustStore"; + private static final String TRUST_STORE_TYPE_PROPERTY_KEY = "javax.net.ssl.trustStoreType"; + private static final String TRUST_STORE_PASSWORD_PROPERTY_KEY = "javax.net.ssl.trustStorePassword"; + + private static final FcliTrustManager INSTANCE = new FcliTrustManager(); + private static final AtomicBoolean INSTALLED = new AtomicBoolean(false); + private static final int MAX_CERT_CHANGE_LOG_ENTRIES = 20; + + private final Object refreshLock = new Object(); + private final AtomicReference snapshotRef = new AtomicReference<>(TrustSnapshot.empty()); + private final AtomicLong refreshCounter = new AtomicLong(0); + private final AtomicLong skippedRefreshCounter = new AtomicLong(0); + private final AtomicLong failedRefreshCounter = new AtomicLong(0); + private volatile String trustStateFingerprint = ""; + + private FcliTrustManager() {} + + public static FcliTrustManager getInstance() { + return INSTANCE; + } + + /** + * Installs this trust manager as the JVM default SSL trust manager. + */ + public static void installAsDefault() { + if (INSTALLED.compareAndSet(false, true)) { + try { + SSLContext ctx = SSLContext.getInstance("TLS"); + ctx.init(null, new TrustManager[] {INSTANCE}, null); + SSLContext.setDefault(ctx); + LOG.debug("Installed refreshable fcli SSL trust manager"); + } catch (GeneralSecurityException e) { + INSTALLED.set(false); + LOG.warn("Could not install refreshable fcli SSL trust manager: {}", e.getMessage()); + return; + } + } + refresh(); + } + + /** + * Reloads the current trust configuration and replaces the active snapshot. + */ + public static boolean refresh() { + return INSTANCE.refreshInternal(); + } + + public static boolean refreshIfChanged() { + return INSTANCE.refreshIfChangedInternal(); + } + + private boolean refreshIfChangedInternal() { + var current = trustStateFingerprint; + var latest = computeTrustStateFingerprint(); + if (Objects.equals(current, latest)) { + skippedRefreshCounter.incrementAndGet(); + return true; + } + synchronized (refreshLock) { + if (Objects.equals(trustStateFingerprint, latest)) { + skippedRefreshCounter.incrementAndGet(); + return true; + } + return refreshInternal(latest, "changed"); + } + } + + private boolean refreshInternal() { + synchronized (refreshLock) { + return refreshInternal(computeTrustStateFingerprint(), "forced"); + } + } + + private boolean refreshInternal(String latestFingerprint, String reason) { + var startNanos = System.nanoTime(); + var previousSnapshot = snapshotRef.get(); + var descriptor = TrustStoreConfigHelper.getTrustStoreConfig(); + applyTrustStoreSystemProperties(descriptor); + + List managers = new ArrayList<>(); + addTrustManagerFromKeyStore(managers, null); + + if (!isOsTrustStoreDisabled(descriptor)) { + var platformKeyStore = loadPlatformKeyStore(); + if (platformKeyStore != null) { + addTrustManagerFromKeyStore(managers, platformKeyStore); + } + } else { + LOG.debug("OS trust store merge disabled"); + } + + var trustedUrlsKeyStore = TrustedUrlTrustStoreHelper.getTrustedUrlsKeyStore(); + if (trustedUrlsKeyStore != null) { + addTrustManagerFromKeyStore(managers, trustedUrlsKeyStore); + } + + if (managers.isEmpty()) { + failedRefreshCounter.incrementAndGet(); + LOG.warn("No trust managers available after refresh; keeping previous trust snapshot"); + return false; + } + + var newSnapshot = TrustSnapshot.of(managers); + snapshotRef.set(newSnapshot); + trustStateFingerprint = latestFingerprint; + var refreshId = refreshCounter.incrementAndGet(); + logRefreshSummary(refreshId, reason, startNanos, previousSnapshot, newSnapshot, descriptor); + logChangedCertificates(refreshId, previousSnapshot, newSnapshot); + logAcceptedIssuers(refreshId, newSnapshot.delegates()); + return true; + } + + private void logRefreshSummary(long refreshId, String reason, long startNanos, + TrustSnapshot previousSnapshot, TrustSnapshot newSnapshot, TrustStoreConfigDescriptor descriptor) { + long elapsedMs = (System.nanoTime() - startNanos) / 1_000_000; + int previousDelegates = previousSnapshot.delegates().size(); + int previousCerts = previousSnapshot.acceptedIssuers().length; + int newDelegates = newSnapshot.delegates().size(); + int newCerts = newSnapshot.acceptedIssuers().length; + LOG.debug("Trust refresh#{} reason={} took {}ms delegates {}->{} certs {}->{} skipped={} failed={} path={} useOs={} disableOs={}", + refreshId, + reason, + elapsedMs, + previousDelegates, + newDelegates, + previousCerts, + newCerts, + skippedRefreshCounter.get(), + failedRefreshCounter.get(), + descriptor == null ? "" : Objects.toString(descriptor.getPath(), ""), + descriptor != null && !Boolean.FALSE.equals(descriptor.getUseOsTrustStore()), + EnvHelper.asBoolean(EnvHelper.env("FCLI_DISABLE_OS_TRUSTSTORE"))); + } + + private void logChangedCertificates(long refreshId, TrustSnapshot previousSnapshot, TrustSnapshot newSnapshot) { + if (!LOG.isDebugEnabled()) { + return; + } + var previousCertIds = toCertIds(previousSnapshot.acceptedIssuers()); + var newCertIds = toCertIds(newSnapshot.acceptedIssuers()); + var added = new LinkedHashSet(newCertIds); + added.removeAll(previousCertIds); + var removed = new LinkedHashSet(previousCertIds); + removed.removeAll(newCertIds); + LOG.debug("Trust refresh#{} certificate delta: +{} -{}", refreshId, added.size(), removed.size()); + logCertificateIdSample(refreshId, "added", added); + logCertificateIdSample(refreshId, "removed", removed); + } + + private void logCertificateIdSample(long refreshId, String changeType, Set certIds) { + if (certIds.isEmpty()) { + return; + } + int maxEntries = Math.min(MAX_CERT_CHANGE_LOG_ENTRIES, certIds.size()); + certIds.stream().limit(maxEntries) + .forEach(id -> LOG.debug("Trust refresh#{} {} cert: {}", refreshId, changeType, id)); + if (certIds.size() > maxEntries) { + LOG.debug("Trust refresh#{} {} certs truncated: showing {} of {}", refreshId, changeType, maxEntries, certIds.size()); + } + } + + private Set toCertIds(X509Certificate[] certificates) { + var result = new LinkedHashSet(); + if (certificates == null) { + return result; + } + for (var cert : certificates) { + result.add(certificateId(cert)); + } + return result; + } + + private String certificateId(X509Certificate cert) { + if (cert == null) { + return "null-cert"; + } + return String.format("subject=%s issuer=%s serial=%s", + cert.getSubjectX500Principal().getName(), + cert.getIssuerX500Principal().getName(), + cert.getSerialNumber().toString(16)); + } + + private String computeTrustStateFingerprint() { + var descriptor = TrustStoreConfigHelper.getTrustStoreConfig(); + var descriptorFingerprint = String.join("|", + Objects.toString(descriptor.getPath(), ""), + Objects.toString(descriptor.getType(), ""), + hashSecretForFingerprint(descriptor.getPassword()), + Objects.toString(descriptor.getUseOsTrustStore(), "")); + + var trustedUrlsFingerprint = TrustedUrlTrustStoreHelper.listTrustedUrls() + .sorted(Comparator.comparing(TrustedUrlCertificateDescriptor::getKey)) + .map(d -> String.join("|", + Objects.toString(d.getKey(), ""), + Objects.toString(d.getSha256(), ""), + Objects.toString(d.getNotAfter(), ""))) + .collect(Collectors.joining("::")); + + return String.join("#", + descriptorFingerprint, + trustedUrlsFingerprint, + Objects.toString(EnvHelper.env("FCLI_TRUSTSTORE"), ""), + Objects.toString(EnvHelper.env("FCLI_TRUSTSTORE_TYPE"), ""), + hashSecretForFingerprint(EnvHelper.env("FCLI_TRUSTSTORE_PWD")), + Objects.toString(EnvHelper.env("FCLI_DISABLE_OS_TRUSTSTORE"), ""), + PlatformHelper.getOSString()); + } + + private String hashSecretForFingerprint(String value) { + if (StringUtils.isBlank(value)) { + return ""; + } + try { + var bytes = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + return HexFormat.of().formatHex(bytes); + } catch (GeneralSecurityException e) { + LOG.debug("Unable to hash trust-store secret for fingerprinting: {}", e.getMessage()); + return Integer.toHexString(value.hashCode()); + } + } + + private void applyTrustStoreSystemProperties(TrustStoreConfigDescriptor descriptor) { + System.clearProperty(TRUST_STORE_PROPERTY_KEY); + System.clearProperty(TRUST_STORE_TYPE_PROPERTY_KEY); + System.clearProperty(TRUST_STORE_PASSWORD_PROPERTY_KEY); + + if (descriptor != null && StringUtils.isNotBlank(descriptor.getPath())) { + initializeTrustStoreFromConfig(descriptor); + } else { + initializeTrustStoreFromEnv(); + } + LOG.debug("Trust store file: {}", System.getProperty(TRUST_STORE_PROPERTY_KEY, "NONE")); + } + + private boolean isOsTrustStoreDisabled(TrustStoreConfigDescriptor descriptor) { + if (EnvHelper.asBoolean(EnvHelper.env("FCLI_DISABLE_OS_TRUSTSTORE"))) { + return true; + } + return descriptor != null && Boolean.FALSE.equals(descriptor.getUseOsTrustStore()); + } + + private void addTrustManagerFromKeyStore(List managers, KeyStore keyStore) { + try { + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(keyStore); + for (TrustManager tm : tmf.getTrustManagers()) { + if (tm instanceof X509TrustManager x509tm) { + managers.add(x509tm); + } + } + } catch (GeneralSecurityException e) { + LOG.debug("Could not load trust manager from key store: {}", e.getMessage()); + } + } + + private void logAcceptedIssuers(long refreshId, List trustManagers) { + if (!LOG.isTraceEnabled()) { + return; + } + trustManagers.stream() + .map(X509TrustManager::getAcceptedIssuers) + .filter(Objects::nonNull) + .flatMap(Arrays::stream) + .forEach(cert -> LOG.trace("Trust refresh#{} cert - Subject: {}, Issuer: {}", + refreshId, + cert.getSubjectX500Principal().getName(), + cert.getIssuerX500Principal().getName())); + } + + private KeyStore loadPlatformKeyStore() { + if (PlatformHelper.isWindows()) { + return loadSingleKeyStore("Windows-ROOT"); + } else if (PlatformHelper.isMac()) { + return loadSingleKeyStore("KeychainStore"); + } else { + LOG.debug("No OS trust store loaded for platform: {}", PlatformHelper.getOSString()); + return null; + } + } + + private KeyStore loadSingleKeyStore(String type) { + try { + KeyStore ks = KeyStore.getInstance(type); + ks.load(null, null); + LOG.debug("Loaded OS trust store: {}", type); + return ks; + } catch (Exception | LinkageError e) { + LOG.warn("OS trust store unavailable ({}): {}", type, e.getMessage()); + LOG.debug("OS trust store load failure details", e); + return null; + } + } + + private void initializeTrustStoreFromEnv() { + String trustStorePath = System.getenv("FCLI_TRUSTSTORE"); + if (trustStorePath != null && Files.exists(Path.of(trustStorePath))) { + System.setProperty(TRUST_STORE_PROPERTY_KEY, trustStorePath); + + String trustStoreType = "jks"; + if (System.getenv("FCLI_TRUSTSTORE_TYPE") != null) { + trustStoreType = System.getenv("FCLI_TRUSTSTORE_TYPE"); + } else { + String fileName = Paths.get(trustStorePath).getFileName().toString(); + String fileExtension = StringUtils.substringAfterLast(fileName, "."); + if (fileExtension.equals("jks") || fileExtension.equals("p12") || fileExtension.equals("pfx")) { + trustStoreType = fileExtension; + } + } + System.setProperty(TRUST_STORE_TYPE_PROPERTY_KEY, trustStoreType); + + String trustStorePwd = "changeit"; + if (System.getenv("FCLI_TRUSTSTORE_PWD") != null) { + trustStorePwd = System.getenv("FCLI_TRUSTSTORE_PWD"); + } + System.setProperty(TRUST_STORE_PASSWORD_PROPERTY_KEY, trustStorePwd); + } + } + + private void initializeTrustStoreFromConfig(TrustStoreConfigDescriptor descriptor) { + Path absolutePath = Path.of(descriptor.getPath()).toAbsolutePath(); + if (!Files.exists(absolutePath)) { + LOG.warn("WARN: Trust store cannot be found: {}", absolutePath); + } + System.setProperty(TRUST_STORE_PROPERTY_KEY, descriptor.getPath()); + if (StringUtils.isNotBlank(descriptor.getType())) { + System.setProperty(TRUST_STORE_TYPE_PROPERTY_KEY, descriptor.getType()); + } + if (StringUtils.isNotBlank(descriptor.getPassword())) { + System.setProperty(TRUST_STORE_PASSWORD_PROPERTY_KEY, descriptor.getPassword()); + } + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException { + tryEach("checkClientTrusted", chain, tm -> tm.checkClientTrusted(chain, authType)); + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException { + LOG.debug("checkServerTrusted(chain[{}], authType={})", chainLength(chain), authType); + logChain(chain); + tryEach("checkServerTrusted", chain, tm -> tm.checkServerTrusted(chain, authType)); + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException { + tryEach("checkClientTrusted(socket)", chain, tm -> { + if (tm instanceof X509ExtendedTrustManager ext) { + ext.checkClientTrusted(chain, authType, socket); + } else { + tm.checkClientTrusted(chain, authType); + } + }); + } + + @Override + public void checkClientTrusted(X509Certificate[] chain, String authType, SSLEngine engine) throws CertificateException { + tryEach("checkClientTrusted(engine)", chain, tm -> { + if (tm instanceof X509ExtendedTrustManager ext) { + ext.checkClientTrusted(chain, authType, engine); + } else { + tm.checkClientTrusted(chain, authType); + } + }); + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType, Socket socket) throws CertificateException { + LOG.debug("checkServerTrusted(chain[{}], authType={}, socket)", chainLength(chain), authType); + logChain(chain); + tryEach("checkServerTrusted(socket)", chain, tm -> { + if (tm instanceof X509ExtendedTrustManager ext) { + ext.checkServerTrusted(chain, authType, socket); + } else { + tm.checkServerTrusted(chain, authType); + } + }); + } + + @Override + public void checkServerTrusted(X509Certificate[] chain, String authType, SSLEngine engine) throws CertificateException { + LOG.debug("checkServerTrusted(chain[{}], authType={}, engine)", chainLength(chain), authType); + logChain(chain); + tryEach("checkServerTrusted(engine)", chain, tm -> { + if (tm instanceof X509ExtendedTrustManager ext) { + ext.checkServerTrusted(chain, authType, engine); + } else { + tm.checkServerTrusted(chain, authType); + } + }); + } + + @Override + public X509Certificate[] getAcceptedIssuers() { + return snapshotRef.get().acceptedIssuers(); + } + + private int chainLength(X509Certificate[] chain) { + return chain == null ? 0 : chain.length; + } + + private void logChain(X509Certificate[] chain) { + if (!LOG.isDebugEnabled() || chain == null) { + return; + } + for (int i = 0; i < chain.length; i++) { + LOG.debug(" cert[{}]: subject={}, issuer={}", i, + chain[i].getSubjectX500Principal().getName(), + chain[i].getIssuerX500Principal().getName()); + } + } + + private void tryEach(String method, X509Certificate[] chain, TrustCheckFactory check) throws CertificateException { + var snapshot = snapshotRef.get(); + CertificateException last = null; + var delegates = snapshot.delegates(); + for (int i = 0; i < delegates.size(); i++) { + var tm = delegates.get(i); + try { + check.check(tm); + LOG.debug("{}: accepted by delegate[{}] {}", method, i, tm.getClass().getName()); + return; + } catch (CertificateException e) { + LOG.debug("{}: rejected by delegate[{}] {}: {}", method, i, tm.getClass().getName(), e.getMessage()); + last = e; + } + } + LOG.debug("{}: all {} delegate(s) rejected the certificate chain", method, delegates.size()); + if (last != null) { + throw last; + } + throw new CertificateException("No trust manager accepted the certificate chain"); + } + + private record TrustSnapshot(List delegates, X509Certificate[] acceptedIssuers) { + private static TrustSnapshot empty() { + return new TrustSnapshot(List.of(), new X509Certificate[0]); + } + + private static TrustSnapshot of(List delegates) { + var immutableDelegates = List.copyOf(delegates); + var acceptedIssuers = immutableDelegates.stream() + .map(X509TrustManager::getAcceptedIssuers) + .filter(Objects::nonNull) + .flatMap(Arrays::stream) + .toArray(X509Certificate[]::new); + return new TrustSnapshot(immutableDelegates, acceptedIssuers); + } + } + + @FunctionalInterface + private interface TrustCheckFactory { + void check(X509TrustManager tm) throws CertificateException; + } +} diff --git a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/truststore/helper/TrustStoreConfigHelper.java b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/truststore/helper/TrustStoreConfigHelper.java index 3d84af9b483..4cc5d0ea1ad 100644 --- a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/truststore/helper/TrustStoreConfigHelper.java +++ b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/truststore/helper/TrustStoreConfigHelper.java @@ -14,6 +14,7 @@ import java.nio.file.Path; +import com.fortify.cli.common.http.ssl.trust.FcliTrustManager; import com.fortify.cli.common.util.FcliDataHelper; public final class TrustStoreConfigHelper { @@ -29,11 +30,15 @@ public static final TrustStoreConfigDescriptor getTrustStoreConfig() { public static final TrustStoreConfigDescriptor setTrustStoreConfig(TrustStoreConfigDescriptor descriptor) { Path trustStoreConfigPath = getTrustStoreConfigPath(); FcliDataHelper.saveSecuredFile(trustStoreConfigPath, descriptor, true); + // Refresh trust manager for RPC/MCP servers. + FcliTrustManager.refresh(); return descriptor; } public static final void clearTrustStoreConfig() { FcliDataHelper.deleteFile(getTrustStoreConfigPath(), true); + // Refresh trust manager for RPC/MCP servers. + FcliTrustManager.refresh(); } private static final Path getTrustStoreConfigPath() { diff --git a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/truststore/helper/TrustedUrlTrustStoreHelper.java b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/truststore/helper/TrustedUrlTrustStoreHelper.java index 97fbb255742..cc96e055d4d 100644 --- a/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/truststore/helper/TrustedUrlTrustStoreHelper.java +++ b/fcli-core/fcli-common-core/src/main/java/com/fortify/cli/common/http/ssl/truststore/helper/TrustedUrlTrustStoreHelper.java @@ -45,6 +45,7 @@ import com.fortify.cli.common.exception.FcliSimpleException; import com.fortify.cli.common.exception.FcliTechnicalException; +import com.fortify.cli.common.http.ssl.trust.FcliTrustManager; import com.fortify.cli.common.util.FcliDataHelper; import lombok.AccessLevel; @@ -66,6 +67,8 @@ public static TrustedUrlCertificateDescriptor addTrustedUrl(String sourceUrl) { var rootCertificate = selectRootCertificate(certificateChain); var descriptor = toDescriptor(target, sourceUrl, rootCertificate); FcliDataHelper.saveFile(getDescriptorPath(target.key()), descriptor, true); + // Refresh trust manager for RPC/MCP servers. + FcliTrustManager.refresh(); return descriptor; } @@ -77,6 +80,8 @@ public static TrustedUrlCertificateDescriptor removeTrustedUrl(String sourceUrl) throw new FcliSimpleException("No trusted URL found for "+target.url()); } FcliDataHelper.deleteFile(descriptorPath, true); + // Refresh trust manager for RPC/MCP servers. + FcliTrustManager.refresh(); return existing; } @@ -91,8 +96,11 @@ public static Stream listTrustedUrls() { } public static Stream clearTrustedUrls() { - return listTrustedUrls() - .peek(d->FcliDataHelper.deleteFile(getDescriptorPath(d.getKey()), true)); + var descriptors = listTrustedUrls().toList(); + descriptors.forEach(d->FcliDataHelper.deleteFile(getDescriptorPath(d.getKey()), true)); + // Refresh trust manager for RPC/MCP servers. + FcliTrustManager.refresh(); + return descriptors.stream(); } public static KeyStore getTrustedUrlsKeyStore() { diff --git a/fcli-core/fcli-util/src/main/java/com/fortify/cli/util/rpc_server/helper/RPCServer.java b/fcli-core/fcli-util/src/main/java/com/fortify/cli/util/rpc_server/helper/RPCServer.java index f69f4a46961..323fde38e6b 100644 --- a/fcli-core/fcli-util/src/main/java/com/fortify/cli/util/rpc_server/helper/RPCServer.java +++ b/fcli-core/fcli-util/src/main/java/com/fortify/cli/util/rpc_server/helper/RPCServer.java @@ -32,6 +32,7 @@ import com.fortify.cli.common.cli.util.FcliActionState; import com.fortify.cli.common.cli.util.FcliExecutionContext; import com.fortify.cli.common.cli.util.FcliExecutionContextHolder; +import com.fortify.cli.common.http.ssl.trust.FcliTrustManager; import com.fortify.cli.common.json.JsonHelper; import lombok.extern.slf4j.Slf4j; @@ -225,6 +226,7 @@ private RPCResponse executeMethod(RPCRequest request) { try { JsonNode result; try (var frame = FcliExecutionContextHolder.push(new FcliExecutionContext(registry.getIsolationScope(), new FcliActionState()))) { + FcliTrustManager.refreshIfChanged(); result = handler.execute(request.params()); } return RPCResponse.success(request.id(), result); From 4a0864b4181b4574a522fef76e1d37ffad21a712 Mon Sep 17 00:00:00 2001 From: Ruud Senden <8635138+rsenden@users.noreply.github.com> Date: Wed, 24 Jun 2026 16:58:44 +0200 Subject: [PATCH 09/10] NCD report improvements (#1040) feat: `fcli license ncd-report create`: Add `--end-date` option to allow for generating historical reports feat: `fcli license ncd-report merge`: New command for merging NCD reports feat: `fcli license ncd-report list-contributors`: New command for listing contributors in NCD report feat: `fcli license ncd-report update-contributor-status`: New command for updating contributor status based on manual or AI review --- .../ncd_report/cli/cmd/NcdReportCommands.java | 5 +- .../cli/cmd/NcdReportCreateCommand.java | 14 +- .../cmd/NcdReportListContributorsCommand.java | 161 ++++ .../cli/cmd/NcdReportMergeCommand.java | 732 ++++++++++++++++++ ...dReportUpdateContributorStatusCommand.java | 546 +++++++++++++ .../collector/NcdReportAuthorCollector.java | 3 +- .../collector/NcdReportContext.java | 4 + .../ncd_report/config/NcdReportConfig.java | 25 +- .../config/NcdReportMockSourceConfig.java | 74 ++ .../config/NcdReportSourcesConfig.java | 3 +- .../INcdReportAuthorDescriptor.java | 27 +- .../NcdReportProcessedAuthorDescriptor.java | 22 +- .../ado/NcdReportAdoResultsGenerator.java | 23 +- .../NcdReportGitHubResultsGenerator.java | 25 +- .../NcdReportGitLabResultsGenerator.java | 25 +- .../generator/mock/MockAuthorData.java | 93 +++ .../mock/MockNcdReportAuthorDescriptor.java | 27 + .../mock/MockNcdReportBranchDescriptor.java | 36 + .../mock/MockNcdReportCommitDescriptor.java | 56 ++ .../MockNcdReportRepositoryDescriptor.java | 42 + .../mock/NcdReportMockResultsGenerator.java | 216 ++++++ .../helper/NcdReportContributorHelper.java | 82 ++ .../ncd_report/reader/NcdReportReader.java | 184 +++++ .../validator/NcdReportValidator.java | 131 ++++ .../writer/INcdReportAuthorsWriter.java | 6 +- .../writer/NcdReportAuthorsWriter.java | 40 +- .../NcdReportContributorsCsvSchema.java | 269 +++++++ .../license/i18n/LicenseMessages.properties | 26 +- .../ReportTemplateConfig.yml | 24 - .../cli/ftest/license/NcdReportSpec.groovy | 338 ++++++++ .../resources/runtime/report/mock-authors.csv | 7 + .../runtime/report/mock-authors.json | 26 + .../runtime/report/ncd-report-mock.yml | 19 + .../resources/runtime/report/ncd-report.yml | 17 +- 34 files changed, 3226 insertions(+), 102 deletions(-) create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportListContributorsCommand.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportMergeCommand.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportUpdateContributorStatusCommand.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportMockSourceConfig.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockAuthorData.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportAuthorDescriptor.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportBranchDescriptor.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportCommitDescriptor.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportRepositoryDescriptor.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/NcdReportMockResultsGenerator.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/helper/NcdReportContributorHelper.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/reader/NcdReportReader.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/validator/NcdReportValidator.java create mode 100644 fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/NcdReportContributorsCsvSchema.java delete mode 100644 fcli-other/fcli-functional-test/ReportTemplateConfig.yml create mode 100644 fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/mock-authors.csv create mode 100644 fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/mock-authors.json create mode 100644 fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/ncd-report-mock.yml diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportCommands.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportCommands.java index 383e614bd1e..3c25ba57597 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportCommands.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportCommands.java @@ -20,7 +20,10 @@ name = "ncd-report", subcommands = { NcdReportCreateCommand.class, - NcdReportCreateConfigCommand.class + NcdReportCreateConfigCommand.class, + NcdReportMergeCommand.class, + NcdReportListContributorsCommand.class, + NcdReportUpdateContributorStatusCommand.class } ) public class NcdReportCommands extends AbstractContainerCommand {} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportCreateCommand.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportCreateCommand.java index 411d7f41f04..3fc9168399a 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportCreateCommand.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportCreateCommand.java @@ -13,6 +13,9 @@ package com.fortify.cli.license.ncd_report.cli.cmd; import java.io.File; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneOffset; import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; import com.fortify.cli.common.progress.helper.IProgressWriterI18n; @@ -31,8 +34,10 @@ public final class NcdReportCreateCommand extends AbstractConfigurableReportGenerateCommand { @Getter @Mixin private OutputHelperMixins.CreateWithDetailsOutput outputHelper; @Mixin private UnirestContextMixin unirestContextMixin; - @Option(names = {"-c","--config"}, required = true, defaultValue = "NcdReportConfig.yml") + @Option(names = {"-c","--config"}, defaultValue = "NcdReportConfig.yml") @Getter private File configFile; + @Option(names = {"--end-date"}) + @Getter private LocalDate endDate; @Override protected String getReportTitle() { @@ -44,6 +49,13 @@ protected Class getConfigType() { return NcdReportConfig.class; } + @Override + protected void updateConfig(NcdReportConfig config) { + if ( endDate != null ) { + config.setCommitEndDate(endDate.atTime(LocalTime.MAX).atOffset(ZoneOffset.UTC)); + } + } + @Override protected NcdReportContext createReportContext(NcdReportConfig config, IReportWriter reportWriter, IProgressWriterI18n progressWriter) { return new NcdReportContext(config, reportWriter, progressWriter, unirestContextMixin.getUnirestContext()); diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportListContributorsCommand.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportListContributorsCommand.java new file mode 100644 index 00000000000..40793afd7be --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportListContributorsCommand.java @@ -0,0 +1,161 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.cli.cmd; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.common.output.cli.cmd.AbstractOutputCommand; +import com.fortify.cli.common.output.cli.cmd.IJsonNodeSupplier; +import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; +import com.fortify.cli.license.ncd_report.helper.NcdReportContributorHelper; +import com.fortify.cli.license.ncd_report.reader.NcdReportReader; +import com.fortify.cli.license.ncd_report.writer.NcdReportContributorsCsvSchema; + +import lombok.Getter; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +@Command(name = "list-contributors", aliases = {"lsc"}) +public final class NcdReportListContributorsCommand extends AbstractOutputCommand implements IJsonNodeSupplier { + @Getter @Mixin private OutputHelperMixins.TableWithQuery outputHelper; + + @Option(names = {"-r", "--report"}, required = true) + @Getter private Path reportPath; + + @Override + public JsonNode getJsonNode() { + var result = JsonHelper.getObjectMapper().createArrayNode(); + try ( var reader = new NcdReportReader(reportPath) ) { + var contributors = readContributors(reader); + enrichContributors(contributors); + // Sort contributors by author name with duplicates immediately following their representative, + // and ignored records at the end. This sorting is applied for consistency and readability, + // especially when reading legacy reports that may not be optimally ordered. + var deduped = dedupeByAuthorId(contributors); + var asObjectNodes = deduped.stream() + .map(map -> JsonHelper.getObjectMapper().convertValue(map, ObjectNode.class)) + .toList(); + var sorted = NcdReportContributorsCsvSchema.sortByAuthorNameAndStatus(new java.util.ArrayList<>(asObjectNodes)); + sorted.stream() + .map(this::toOutputRow) + .forEach(result::add); + } + return result; + } + + private List> readContributors(NcdReportReader reader) { + return reader.readContributors(); + } + + private void enrichContributors(List> contributors) { + var representativeByContributingNumber = new java.util.HashMap(); + + // Ensure authorId is present (legacy reports may not include this column). + contributors.forEach(this::ensureAuthorId); + + contributors.stream() + .filter(r -> "contributing".equalsIgnoreCase(getValue(r, NcdReportContributorsCsvSchema.CONTRIBUTION_STATUS))) + .forEach(r -> { + var number = getValue(r, "contributingAuthorNumber"); + if ( StringUtils.isNotBlank(number) ) { + representativeByContributingNumber.put(number, getValue(r, NcdReportContributorsCsvSchema.AUTHOR_ID)); + } + }); + + contributors.stream() + .filter(r -> "duplicate".equalsIgnoreCase(getValue(r, NcdReportContributorsCsvSchema.CONTRIBUTION_STATUS))) + .filter(r -> StringUtils.isBlank(getValue(r, NcdReportContributorsCsvSchema.DUPLICATE_OF))) + .forEach(r -> { + var number = getValue(r, "contributingAuthorNumber"); + var representativeId = representativeByContributingNumber.get(number); + if ( StringUtils.isNotBlank(representativeId) + && !representativeId.equals(getValue(r, NcdReportContributorsCsvSchema.AUTHOR_ID)) ) { + r.put(NcdReportContributorsCsvSchema.DUPLICATE_OF, representativeId); + } + }); + } + + private void ensureAuthorId(Map row) { + if ( StringUtils.isNotBlank(getValue(row, NcdReportContributorsCsvSchema.AUTHOR_ID)) ) { + return; + } + var expressionInput = NcdReportContributorHelper.createExpressionInput( + getValue(row, NcdReportContributorsCsvSchema.AUTHOR_NAME), + getValue(row, NcdReportContributorsCsvSchema.AUTHOR_EMAIL)); + row.put(NcdReportContributorsCsvSchema.AUTHOR_ID, NcdReportContributorHelper.computeAuthorId(expressionInput)); + } + + private List> dedupeByAuthorId(List> contributors) { + var result = new ArrayList>(); + var seenAuthorIds = new HashSet(); + for ( var row : contributors.stream().sorted(compareByAuthorIdPriority()).toList() ) { + var authorId = getValue(row, NcdReportContributorsCsvSchema.AUTHOR_ID); + if ( seenAuthorIds.add(authorId) ) { + result.add(row); + } + } + return result; + } + + private Comparator> compareByAuthorIdPriority() { + return Comparator + ., String>comparing(r -> r.getOrDefault(NcdReportContributorsCsvSchema.AUTHOR_ID, "")) + .thenComparingInt(this::contributionStatusPriority) + .thenComparing(r -> r.getOrDefault(NcdReportContributorsCsvSchema.AUTHOR_NAME, "").toLowerCase()) + .thenComparing(r -> r.getOrDefault(NcdReportContributorsCsvSchema.AUTHOR_EMAIL, "").toLowerCase()); + } + + private int contributionStatusPriority(Map row) { + var status = getValue(row, NcdReportContributorsCsvSchema.CONTRIBUTION_STATUS); + return switch ( status.toLowerCase() ) { + case "contributing" -> 0; + case "duplicate" -> 1; + case "ignored" -> 2; + default -> 3; + }; + } + + private ObjectNode toOutputRow(ObjectNode row) { + return JsonHelper.getObjectMapper().createObjectNode() + .put(NcdReportContributorsCsvSchema.AUTHOR_ID, row.path(NcdReportContributorsCsvSchema.AUTHOR_ID).asText("")) + .put(NcdReportContributorsCsvSchema.AUTHOR_NAME, row.path(NcdReportContributorsCsvSchema.AUTHOR_NAME).asText("")) + .put(NcdReportContributorsCsvSchema.AUTHOR_EMAIL, row.path(NcdReportContributorsCsvSchema.AUTHOR_EMAIL).asText("")) + .put(NcdReportContributorsCsvSchema.CONTRIBUTION_STATUS, row.path(NcdReportContributorsCsvSchema.CONTRIBUTION_STATUS).asText("")) + .put(NcdReportContributorsCsvSchema.DUPLICATE_OF, row.path(NcdReportContributorsCsvSchema.DUPLICATE_OF).asText("")) + .put(NcdReportContributorsCsvSchema.OVERRIDE_STATUS, row.path(NcdReportContributorsCsvSchema.OVERRIDE_STATUS).asText("")) + .put(NcdReportContributorsCsvSchema.OVERRIDE_STATUS_CONFIDENCE, + row.path(NcdReportContributorsCsvSchema.OVERRIDE_STATUS_CONFIDENCE).asText("")) + .put(NcdReportContributorsCsvSchema.OVERRIDE_STATUS_NOTES, + row.path(NcdReportContributorsCsvSchema.OVERRIDE_STATUS_NOTES).asText("")); + } + + private String getValue(Map row, String fieldName) { + return StringUtils.defaultString(row.get(fieldName)); + } + + @Override + public final boolean isSingular() { + return false; + } +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportMergeCommand.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportMergeCommand.java new file mode 100644 index 00000000000..8b35732edfc --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportMergeCommand.java @@ -0,0 +1,732 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.cli.cmd; + +import java.io.BufferedWriter; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.expression.Expression; +import org.springframework.expression.spel.standard.SpelExpressionParser; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fortify.cli.common.exception.FcliSimpleException; +import com.fortify.cli.common.exception.FcliTechnicalException; +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.common.output.cli.mixin.OutputHelperMixins; +import com.fortify.cli.common.output.writer.record.RecordWriterFactory; +import com.fortify.cli.common.report.cli.cmd.AbstractReportGenerateCommand; +import com.fortify.cli.common.report.writer.IReportWriter; +import com.fortify.cli.license.ncd_report.config.NcdReportConfig; +import com.fortify.cli.license.ncd_report.config.NcdReportContributorConfig; +import com.fortify.cli.license.ncd_report.helper.NcdReportContributorHelper; +import com.fortify.cli.license.ncd_report.reader.NcdReportReader; +import com.fortify.cli.license.ncd_report.validator.NcdReportValidator; +import com.fortify.cli.license.ncd_report.writer.NcdReportContributorsCsvSchema; + +import lombok.Getter; +import picocli.CommandLine.Command; +import picocli.CommandLine.Mixin; +import picocli.CommandLine.Option; + +@Command(name = "merge") +public final class NcdReportMergeCommand extends AbstractReportGenerateCommand { + private static final ObjectMapper YAML_MAPPER = createYamlMapper(); + private static final CsvMapper CSV_MAPPER = new CsvMapper(); + private static final List DETAIL_FILE_NAMES = List.of( + "details/repositories.csv", + "details/commits-by-branch.csv", + "details/commits-by-repository.csv", + "details/contributors-by-repository.csv"); + + @Getter @Mixin private OutputHelperMixins.DetailsNoQuery outputHelper; + + @Option(names = {"-r", "--reports"}, required = true, split = ",") + @Getter private List reportPaths; + + @Override + protected String getReportTitle() { + return "Number of Contributing Developers (NCD) Merged Report"; + } + + @Override + protected void generateReport(IReportWriter reportWriter) { + var sourceReports = loadSourceReports(); + var mergedContributorConfig = mergeContributorConfig(sourceReports); + var contributors = mergeContributors(sourceReports, mergedContributorConfig); + synthesizeMissingDuplicateOf(contributors); + writeMergedContributors(reportWriter, contributors); + var detailRowCounts = writeMergedDetails(reportWriter, sourceReports, contributors); + writeMergedConfig(reportWriter, mergedContributorConfig); + copyEmbeddedSources(reportWriter, sourceReports); + updateSummary(reportWriter, contributors, sourceReports, mergedContributorConfig, detailRowCounts); + } + + private List loadSourceReports() { + var result = new ArrayList(); + var checksumErrors = new ArrayList(); + var usedSourceNames = new LinkedHashSet(); + for ( var reportPath : reportPaths ) { + var sourceName = createUniqueSourceName(reportPath, usedSourceNames); + usedSourceNames.add(sourceName); + var sourceRef = "sources/" + sourceName; + try ( var reader = new NcdReportReader(reportPath) ) { + checksumErrors.addAll(NcdReportValidator.validateChecksums(reader)); + var config = reader.readConfig(); + var sourceSummary = reader.readSummary(); + var contributors = readContributors(reader, sourceRef); + var entryNames = reader.listFileEntries(); + var contributorConfig = config.getContributor().orElseGet(NcdReportContributorConfig::new); + result.add(new SourceReport(reportPath.toAbsolutePath(), sourceRef, contributors, entryNames, contributorConfig, config, sourceSummary)); + } + } + if ( !checksumErrors.isEmpty() ) { + throw new FcliSimpleException("Checksum validation failed for source reports:\n\t%s", String.join("\n\t", checksumErrors)); + } + return result; + } + + private List readContributors(NcdReportReader reader, String sourceRef) { + try { + var result = new ArrayList(); + for ( var row : reader.readContributors() ) { + result.add(ContributorRecord.fromRow(row, sourceRef)); + } + return result; + } catch ( Exception e ) { + throw new FcliSimpleException("Error reading contributors.csv from %s:\n\tMessage: %s", reader.getReportPath(), e.getMessage()); + } + } + + private Optional mergeContributorConfig(List sourceReports) { + var ignoreExpressions = collectDistinctExpressions(sourceReports, true); + var duplicateExpressions = collectDistinctExpressions(sourceReports, false); + + if ( ignoreExpressions.isEmpty() && duplicateExpressions.isEmpty() ) { + return Optional.empty(); + } + + var result = new NcdReportContributorConfig(); + result.setIgnoreExpression(combineExpressions(ignoreExpressions)); + result.setDuplicateExpression(combineExpressions(duplicateExpressions)); + return Optional.of(result); + } + + private List collectDistinctExpressions(List sourceReports, boolean ignore) { + var result = new ArrayList(); + for ( var source : sourceReports ) { + var expression = ignore + ? source.contributorConfig().getIgnoreExpression() + : source.contributorConfig().getDuplicateExpression(); + expression + .map(String::trim) + .filter(StringUtils::isNotBlank) + .filter(e -> !result.contains(e)) + .ifPresent(result::add); + } + return result; + } + + private Optional combineExpressions(List expressions) { + if ( expressions.isEmpty() ) { + return Optional.empty(); + } else if ( expressions.size() == 1 ) { + return Optional.of(expressions.get(0)); + } + var combined = expressions.stream() + .map(e -> "(" + e + ")") + .collect(Collectors.joining(" ||\n")); + return Optional.of(combined); + } + + private List mergeContributors(List sourceReports, Optional mergedContributorConfig) { + List allContributors = sourceReports.stream() + .flatMap(s -> s.contributors().stream()) + .collect(Collectors.toCollection(ArrayList::new)); + allContributors = aggregateByAuthorId(allContributors); + + var parser = new SpelExpressionParser(); + var ignoreExpression = mergedContributorConfig + .flatMap(NcdReportContributorConfig::getIgnoreExpression) + .map(parser::parseExpression); + var duplicateExpression = mergedContributorConfig + .flatMap(NcdReportContributorConfig::getDuplicateExpression) + .map(parser::parseExpression); + + int nextAuthorNumber = 1; + var ignoredContributors = new ArrayList(); + var nonIgnoredContributors = new ArrayList(); + for ( var contributor : allContributors ) { + // Preserve source rows that were already marked ignored in the original report. + if ( contributor.isSourceIgnored() || isIgnored(contributor, ignoreExpression) ) { + contributor.authorState("ignored"); + contributor.authorNumber(-1); + contributor.contributionStatus("ignored"); + contributor.contributingAuthorNumber(-1); + ignoredContributors.add(contributor); + } else { + contributor.authorState("processed"); + contributor.authorNumber(nextAuthorNumber++); + nonIgnoredContributors.add(contributor); + } + } + + applyDeduplication(nonIgnoredContributors, duplicateExpression); + + var result = new ArrayList(ignoredContributors.size() + nonIgnoredContributors.size()); + result.addAll(ignoredContributors); + result.addAll(nonIgnoredContributors); + return result; + } + + private List aggregateByAuthorId(List contributors) { + var byAuthorId = new TreeMap(); + for ( var contributor : contributors ) { + byAuthorId.compute(contributor.authorId(), (k, existing) -> { + if ( existing == null ) { + return contributor; + } + existing.addSourceOccurrence(contributor.sourceReport(), contributor.sourceContributionStatus(), + contributor.sourceContributingAuthorNumber()); + return existing; + }); + } + return new ArrayList<>(byAuthorId.values()); + } + + private boolean isIgnored(ContributorRecord contributor, Optional ignoreExpression) { + return ignoreExpression + .map(e -> JsonHelper.evaluateSpelExpression(contributor.expressionInput(), e, Boolean.class)) + .orElse(false); + } + + private void applyDeduplication(List contributors, Optional duplicateExpression) { + int count = contributors.size(); + if ( count == 0 ) { + return; + } + + var parent = new int[count]; + for ( int i = 0; i < count; i++ ) { + parent[i] = i; + } + + if ( duplicateExpression.isPresent() ) { + var expr = duplicateExpression.get(); + for ( int i = 0; i < count; i++ ) { + for ( int j = i + 1; j < count; j++ ) { + if ( isDuplicate(contributors.get(i), contributors.get(j), expr) ) { + union(parent, i, j); + } + } + } + } + + var rootToMinIndex = new TreeMap(); + for ( int i = 0; i < count; i++ ) { + var root = find(parent, i); + var currentIndex = i; + rootToMinIndex.compute(root, (k, v) -> v == null ? currentIndex : Math.min(v, currentIndex)); + } + + var sortedRootEntries = rootToMinIndex.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .collect(Collectors.toList()); + + var rootToContributingNumber = new HashMap(); + int contributingAuthorNumber = 1; + for ( var entry : sortedRootEntries ) { + rootToContributingNumber.put(entry.getKey(), contributingAuthorNumber++); + } + + for ( int i = 0; i < count; i++ ) { + var root = find(parent, i); + var representativeIndex = rootToMinIndex.get(root); + var contributor = contributors.get(i); + contributor.contributingAuthorNumber(rootToContributingNumber.get(root)); + if ( i == representativeIndex ) { + contributor.contributionStatus("contributing"); + } else { + contributor.contributionStatus("duplicate"); + contributor.duplicateOf(contributors.get(representativeIndex).authorId()); + } + } + } + + private void synthesizeMissingDuplicateOf(List contributors) { + // For records marked as duplicate but with missing duplicateOf, synthesize it + // by looking up the representative record (same contributingAuthorNumber with contributing status) + var representativesByNumber = new HashMap(); + + for ( var contributor : contributors ) { + if ( "contributing".equalsIgnoreCase(contributor.contributionStatus()) ) { + representativesByNumber.put(contributor.contributingAuthorNumber(), contributor.authorId()); + } + } + + for ( var contributor : contributors ) { + if ( "duplicate".equalsIgnoreCase(contributor.contributionStatus()) + && StringUtils.isBlank(contributor.duplicateOf) ) { + var representativeId = representativesByNumber.get(contributor.contributingAuthorNumber()); + if ( representativeId != null ) { + contributor.duplicateOf(representativeId); + } + } + } + } + + private boolean isDuplicate(ContributorRecord c1, ContributorRecord c2, Expression expression) { + return JsonHelper.evaluateSpelExpression(createCompareNode(c1, c2), expression, Boolean.class) + || JsonHelper.evaluateSpelExpression(createCompareNode(c2, c1), expression, Boolean.class); + } + + private ObjectNode createCompareNode(ContributorRecord c1, ContributorRecord c2) { + var node = JsonHelper.getObjectMapper().createObjectNode(); + node.set("a1", c1.expressionInput()); + node.set("a2", c2.expressionInput()); + return node; + } + + private void writeMergedContributors(IReportWriter reportWriter, List contributors) { + var writer = reportWriter.recordWriter(RecordWriterFactory.csv, "contributors.csv", false, null); + // Convert to ObjectNode for sorting, then write sorted records + var records = contributors.stream().map(ContributorRecord::toRow).collect(Collectors.toList()); + var sorted = NcdReportContributorsCsvSchema.sortByAuthorNameAndStatus(records); + sorted.forEach(writer::append); + } + + private Map writeMergedDetails(IReportWriter reportWriter, List sourceReports, List mergedContributors) { + var sourceSemanticsByKey = sourceReports.stream() + .flatMap(s -> s.contributors().stream()) + .collect(Collectors.toMap( + c -> sourceAndAuthorIdKey(c.sourceReport(), c.authorId()), + c -> c, + (a, b) -> a, + HashMap::new)); + var mergedSemanticsBySourceAndAuthor = mergedContributors.stream() + .collect(Collectors.toMap( + c -> sourceAndAuthorIdKey(c.sourceReport(), c.authorId()), + c -> c, + (a, b) -> a, + HashMap::new)); + var mergedSemanticsByAuthor = mergedContributors.stream() + .collect(Collectors.toMap( + ContributorRecord::authorId, + c -> c, + this::preferMoreSpecificMergedSemantics, + HashMap::new)); + + var countsByFile = new HashMap(); + for ( var detailFileName : DETAIL_FILE_NAMES ) { + var writer = reportWriter.recordWriter(RecordWriterFactory.csv, detailFileName, false, null); + int fileRowCount = 0; + for ( var sourceReport : sourceReports ) { + try ( var reader = new NcdReportReader(sourceReport.originalPath()) ) { + if ( !sourceReport.entryNames().contains(detailFileName) ) { + continue; + } + for ( var row : readCsvRows(reader, detailFileName) ) { + row.put("sourceReport", sourceReport.sourceRef()); + enrichRowWithSemantics(row, sourceSemanticsByKey, mergedSemanticsBySourceAndAuthor, mergedSemanticsByAuthor); + writer.append(JsonHelper.getObjectMapper().valueToTree(row)); + fileRowCount++; + } + } + } + countsByFile.put(detailFileName, fileRowCount); + } + return countsByFile; + } + + private List> readCsvRows(NcdReportReader reader, String entryName) { + try ( var csvReader = reader.bufferedReader(entryName) ) { + var schema = CsvSchema.emptySchema().withHeader(); + MappingIterator> iterator = CSV_MAPPER + .readerFor(new TypeReference>() {}) + .with(schema) + .readValues(csvReader); + var result = new ArrayList>(); + while ( iterator.hasNext() ) { + result.add(iterator.next()); + } + return result; + } catch ( Exception e ) { + throw new FcliSimpleException("Error reading %s from %s:\n\tMessage: %s", entryName, reader.getReportPath(), e.getMessage()); + } + } + + private void enrichRowWithSemantics( + Map row, + Map sourceSemanticsByKey, + Map mergedSemanticsBySourceAndAuthor, + Map mergedSemanticsByAuthor) + { + var sourceReport = StringUtils.defaultString(row.get("sourceReport")); + var authorId = StringUtils.defaultString(row.get("authorId")); + if ( StringUtils.isBlank(authorId) ) { return; } + + row.put("sourceAuthorId", authorId); + row.put("sourceAuthorState", StringUtils.defaultString(row.get("authorState"))); + + var sourceSemantics = sourceSemanticsByKey.get(sourceAndAuthorIdKey(sourceReport, authorId)); + if ( sourceSemantics != null ) { + row.put("sourceContributionStatus", sourceSemantics.sourceContributionStatus()); + } + + var mergedSemantics = mergedSemanticsBySourceAndAuthor.get(sourceAndAuthorIdKey(sourceReport, authorId)); + if ( mergedSemantics == null ) { + mergedSemantics = mergedSemanticsByAuthor.get(authorId); + } + if ( mergedSemantics != null ) { + row.put("mergedAuthorId", mergedSemantics.authorId()); + row.put("mergedAuthorState", mergedSemantics.authorState()); + row.put("mergedContributionStatus", mergedSemantics.contributionStatus()); + } + } + + private String sourceAndAuthorIdKey(String sourceReport, String authorId) { + return sourceReport + "|" + authorId; + } + + private ContributorRecord preferMoreSpecificMergedSemantics(ContributorRecord c1, ContributorRecord c2) { + var priority1 = mergedStatusPriority(c1.contributionStatus()); + var priority2 = mergedStatusPriority(c2.contributionStatus()); + if ( priority1 != priority2 ) { + return priority1 > priority2 ? c1 : c2; + } + return c1.authorNumber() <= c2.authorNumber() ? c1 : c2; + } + + private int mergedStatusPriority(String status) { + if ( "contributing".equals(status) ) { return 3; } + if ( "duplicate".equals(status) ) { return 2; } + if ( "ignored".equals(status) ) { return 1; } + return 0; + } + + private void writeMergedConfig(IReportWriter reportWriter, Optional contributorConfig) { + var mergedConfig = new NcdReportConfig(); + mergedConfig.setContributor(contributorConfig); + try ( BufferedWriter bw = reportWriter.bufferedWriter("report-config.yaml") ) { + bw.write("# Auto-generated merged NCD report configuration\n"); + bw.write(YAML_MAPPER.writeValueAsString(mergedConfig)); + } catch ( Exception e ) { + throw new FcliSimpleException("Error writing merged report-config.yaml:\n\tMessage: %s", e.getMessage()); + } + } + + private void copyEmbeddedSources(IReportWriter reportWriter, List sourceReports) { + for ( var sourceReport : sourceReports ) { + try ( var reader = new NcdReportReader(sourceReport.originalPath()) ) { + for ( var entryName : sourceReport.entryNames() ) { + var targetEntryName = sourceReport.sourceRef() + "/" + entryName; + reportWriter.copyTextFile(reader.entryPath(entryName), targetEntryName); + } + } + } + } + + private void updateSummary(IReportWriter reportWriter, List contributors, List sources, + Optional mergedContributorConfig, Map detailRowCounts) + { + var summary = reportWriter.summary(); + summary.put("mergedReportCount", sources.size()); + summary.set("mergedSourceReports", JsonHelper.toArrayNode(sources.stream().map(SourceReport::sourceRef).toArray(String[]::new))); + + int total = contributors.size(); + int ignored = (int) contributors.stream().filter(c -> "ignored".equals(c.contributionStatus())).count(); + int duplicate = (int) contributors.stream().filter(c -> "duplicate".equals(c.contributionStatus())).count(); + int contributing = (int) contributors.stream().filter(c -> "contributing".equals(c.contributionStatus())).count(); + int nonIgnored = total - ignored; + + summary.set("authorCount", JsonHelper.getObjectMapper().createObjectNode() + .put("total", total) + .put("contributing", contributing) + .put("ignored", ignored) + .put("nonIgnored", nonIgnored) + .put("duplicate", duplicate)); + summary.set("commitCount", JsonHelper.getObjectMapper().createObjectNode() + .put("analyzed", detailRowCounts.getOrDefault("details/commits-by-branch.csv", 0))); + summary.set("detailRowCount", JsonHelper.getObjectMapper().createObjectNode() + .put("repositories", detailRowCounts.getOrDefault("details/repositories.csv", 0)) + .put("commitsByBranch", detailRowCounts.getOrDefault("details/commits-by-branch.csv", 0)) + .put("commitsByRepository", detailRowCounts.getOrDefault("details/commits-by-repository.csv", 0)) + .put("contributorsByRepository", detailRowCounts.getOrDefault("details/contributors-by-repository.csv", 0))); + + var reportEndDate = sources.stream() + .map(SourceReport::sourceReportEndDate) + .filter(d -> d != null) + .max(LocalDate::compareTo) + .orElse(null); + var reportStartDate = sources.stream() + .map(SourceReport::sourceReportStartDate) + .filter(d -> d != null) + .min(LocalDate::compareTo) + .orElse(null); + if ( reportStartDate != null ) { + summary.put("reportStartDate", reportStartDate.toString()); + } + if ( reportEndDate != null ) { + summary.put("reportEndDate", reportEndDate.toString()); + } + + mergedContributorConfig + .flatMap(NcdReportContributorConfig::getDuplicateExpression) + .ifPresent(e -> summary.put("mergedDuplicateExpression", e)); + } + + private String createUniqueSourceName(Path reportPath, Set usedNames) { + String fileName = reportPath.getFileName().toString(); + String baseName = fileName.replaceFirst("(?i)\\.zip$", ""); + if ( StringUtils.isBlank(baseName) ) { + baseName = "report"; + } + String sanitized = baseName.replaceAll("[^a-zA-Z0-9._-]", "_"); + String candidate = sanitized; + int index = 2; + while ( usedNames.contains(candidate) ) { + candidate = sanitized + "-" + index++; + } + return candidate; + } + + private static int find(int[] parent, int i) { + if ( parent[i] != i ) { + parent[i] = find(parent, parent[i]); + } + return parent[i]; + } + + private static void union(int[] parent, int i, int j) { + int ri = find(parent, i); + int rj = find(parent, j); + if ( ri != rj ) { + parent[rj] = ri; + } + } + + private static ObjectMapper createYamlMapper() { + var mapper = new ObjectMapper(new YAMLFactory()); + mapper.registerModule(new Jdk8Module()); + mapper.registerModule(new JavaTimeModule()); + return mapper; + } + + private static final class ContributorRecord { + private final String authorName; + private final String authorEmail; + private final String sourceReport; + private final Set sourceReports; + private final Set sourceContributionStatuses; + private final int sourceContributingAuthorNumber; + private final ObjectNode expressionInput; + + private String authorId; + private String authorState; + private int authorNumber; + private String contributionStatus; + private int contributingAuthorNumber; + private String duplicateOf; + private String overrideStatus; + private String overrideStatusConfidence; + private String overrideStatusNotes; + + private ContributorRecord(String authorName, String authorEmail, String sourceReport, String sourceContributionStatus, + int sourceContributingAuthorNumber, ObjectNode expressionInput) + { + this.authorName = authorName; + this.authorEmail = authorEmail; + this.sourceReport = sourceReport; + this.sourceReports = new LinkedHashSet<>(); + this.sourceReports.add(sourceReport); + this.sourceContributionStatuses = new LinkedHashSet<>(); + this.sourceContributionStatuses.add(sourceContributionStatus); + this.sourceContributingAuthorNumber = sourceContributingAuthorNumber; + this.expressionInput = expressionInput; + this.authorId = NcdReportContributorHelper.computeAuthorId(expressionInput); + this.authorState = "processed"; + this.authorNumber = -1; + this.contributionStatus = "contributing"; + this.contributingAuthorNumber = -1; + this.duplicateOf = ""; + this.overrideStatus = ""; + this.overrideStatusConfidence = ""; + this.overrideStatusNotes = ""; + } + + static ContributorRecord fromRow(Map row, String sourceReport) { + String authorName = StringUtils.defaultString(row.get("authorName")); + String authorEmail = StringUtils.defaultString(row.get("authorEmail")); + String sourceContributionStatus = StringUtils.defaultString(row.get("contributionStatus")); + int sourceContributingAuthorNumber = parseInt(row.get("contributingAuthorNumber"), -1); + ObjectNode expressionInput = NcdReportContributorHelper.createExpressionInput(authorName, authorEmail); + var record = new ContributorRecord(authorName, authorEmail, sourceReport, sourceContributionStatus, sourceContributingAuthorNumber, expressionInput); + // Preserve any existing override fields from source report + record.duplicateOf = StringUtils.defaultString(row.get("duplicateOf")); + record.overrideStatus = StringUtils.defaultString(row.get("overrideStatus")); + record.overrideStatusConfidence = StringUtils.defaultString(row.get("overrideStatusConfidence")); + record.overrideStatusNotes = StringUtils.defaultString(row.get("overrideStatusNotes")); + return record; + } + + ObjectNode toRow() { + return JsonHelper.getObjectMapper().createObjectNode() + .put("authorId", authorId) + .put("authorName", authorName) + .put("authorEmail", authorEmail) + .put("authorState", authorState) + .put("contributionStatus", contributionStatus) + .put("duplicateOf", duplicateOf) + .put("overrideStatus", overrideStatus) + .put("overrideStatusConfidence", overrideStatusConfidence) + .put("overrideStatusNotes", overrideStatusNotes) + .put("sourceReports", String.join(";", sourceReports)) + .put("sourceContributionStatus", sourceContributionStatus()); + } + + ObjectNode expressionInput() { + return expressionInput; + } + + String authorId() { + return authorId; + } + + String sourceReport() { + return sourceReport; + } + + String sourceContributionStatus() { + return sourceContributionStatuses.size() == 1 + ? sourceContributionStatuses.iterator().next() + : String.join(";", sourceContributionStatuses); + } + + int sourceContributingAuthorNumber() { + return sourceContributingAuthorNumber; + } + + String authorState() { + return authorState; + } + + int authorNumber() { + return authorNumber; + } + + boolean isSourceContributing() { + return sourceContributionStatuses.stream().anyMatch(s -> "contributing".equalsIgnoreCase(s)); + } + + boolean isSourceIgnored() { + return sourceContributionStatuses.stream().anyMatch(s -> "ignored".equalsIgnoreCase(s)); + } + + void addSourceOccurrence(String sourceReport, String sourceContributionStatus, int sourceContributingAuthorNumber) { + this.sourceReports.add(sourceReport); + this.sourceContributionStatuses.add(sourceContributionStatus); + } + + void authorState(String authorState) { + this.authorState = authorState; + } + + void authorNumber(int authorNumber) { + this.authorNumber = authorNumber; + } + + void contributionStatus(String contributionStatus) { + this.contributionStatus = contributionStatus; + } + + String contributionStatus() { + return contributionStatus; + } + + int contributingAuthorNumber() { + return contributingAuthorNumber; + } + + void contributingAuthorNumber(int contributingAuthorNumber) { + this.contributingAuthorNumber = contributingAuthorNumber; + } + + void duplicateOf(String duplicateOf) { + this.duplicateOf = duplicateOf; + } + + private static int parseInt(String value, int defaultValue) { + try { + return Integer.parseInt(StringUtils.defaultIfBlank(value, String.valueOf(defaultValue))); + } catch ( NumberFormatException e ) { + return defaultValue; + } + } + } + + private static record SourceReport( + Path originalPath, + String sourceRef, + List contributors, + List entryNames, + NcdReportContributorConfig contributorConfig, + NcdReportConfig config, + ObjectNode sourceSummary + ) { + SourceReport { + contributors = Collections.unmodifiableList(contributors); + entryNames = Collections.unmodifiableList(entryNames); + } + + LocalDate sourceReportStartDate() { + return parseSummaryDate("reportStartDate"); + } + + LocalDate sourceReportEndDate() { + return parseSummaryDate("reportEndDate"); + } + + private LocalDate parseSummaryDate(String propertyName) { + var value = sourceSummary.path(propertyName).asText(""); + if ( StringUtils.isBlank(value) ) { + return null; + } + try { + return LocalDate.parse(value); + } catch ( Exception e ) { + throw new FcliTechnicalException(String.format( + "Invalid date '%s' for '%s' in source report %s", + value, propertyName, originalPath), e); + } + } + } +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportUpdateContributorStatusCommand.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportUpdateContributorStatusCommand.java new file mode 100644 index 00000000000..191be932eef --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/cli/cmd/NcdReportUpdateContributorStatusCommand.java @@ -0,0 +1,546 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.cli.cmd; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fortify.cli.common.cli.cmd.AbstractRunnableCommand; +import com.fortify.cli.common.exception.FcliSimpleException; +import com.fortify.cli.common.exception.FcliTechnicalException; +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.license.ncd_report.reader.NcdReportReader; +import com.fortify.cli.license.ncd_report.validator.NcdReportValidator; +import com.fortify.cli.license.ncd_report.writer.NcdReportContributorsCsvSchema; + +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +@Command(name = "update-contributor-status", aliases = {"ucs"}) +public final class NcdReportUpdateContributorStatusCommand extends AbstractRunnableCommand { + private static final CsvMapper CSV_MAPPER = new CsvMapper(); + private static final ObjectMapper JSON_MAPPER = JsonHelper.getObjectMapper(); + private static final ObjectMapper YAML_MAPPER = createYamlMapper(); + private static final Set VALID_OVERRIDE_STATUSES = Set.of("contributing", "duplicate", "ignored"); + private static final double DEFAULT_MIN_CONFIDENCE = 0.8; + + @Option(names = {"-r", "--report"}, required = true) + private Path reportPath; + + @Option(names = {"-c", "--contributors"}) + private Path contributorsPath; + + @Option(names = {"--min-confidence"}, defaultValue = "" + DEFAULT_MIN_CONFIDENCE) + private double minConfidence; + + @Override + public Integer call() { + ObjectNode summary = null; + int recordsProcessed = 0; + int updatesApplied = 0; + + try ( var reader = new NcdReportReader(reportPath) ) { + var checksumErrors = NcdReportValidator.validateChecksums(reader); + if ( !checksumErrors.isEmpty() ) { + throw new FcliSimpleException("Report integrity check failed:\n%s", String.join("\n", checksumErrors)); + } + + var updates = readUpdateFile(); + var contributors = readContributors(reader); + var updateResult = applyUpdates(updates, contributors); + recordsProcessed = updateResult.recordsProcessed(); + updatesApplied = updateResult.updatesApplied(); + var warnings = updateResult.warnings(); + synchronizeContributionFields(contributors, warnings); + + if ( !warnings.isEmpty() ) { + System.err.println("Warnings during update:"); + warnings.forEach(w -> System.err.println(" - " + w)); + } + + summary = rewriteContributorsAndSummaryAndChecksums(reader, contributors); + } + + System.out.println(String.format( + "Processed %d update record(s), applied %d update(s) to report %s", + recordsProcessed, + updatesApplied, + reportPath)); + System.out.println("Summary:"); + System.out.print(asYaml(summary)); + + return 0; + } + + // ------------------------------------------------------------------------- + // Reading update input + // ------------------------------------------------------------------------- + + private List> readUpdateFile() { + try { + var content = contributorsPath == null + ? new String(System.in.readAllBytes(), StandardCharsets.UTF_8) + : Files.readString(contributorsPath); + if ( StringUtils.isBlank(content) ) { + return List.of(); + } + return switch ( detectInputFormat(content) ) { + case JSON -> readStructuredUpdates(JSON_MAPPER, content, "JSON"); + case YAML -> readStructuredUpdates(YAML_MAPPER, content, "YAML"); + case CSV -> readCsvUpdates(content); + }; + } catch ( Exception e ) { + throw new FcliSimpleException("Error reading contributor updates from %s:\n\tMessage: %s", getContributorsSource(), e.getMessage()); + } + } + + private InputFormat detectInputFormat(String content) { + var path = contributorsPath == null ? "" : contributorsPath.getFileName().toString().toLowerCase(); + if ( path.endsWith(".json") ) { return InputFormat.JSON; } + if ( path.endsWith(".yaml") || path.endsWith(".yml") ) { return InputFormat.YAML; } + if ( path.endsWith(".csv") ) { return InputFormat.CSV; } + var trimmed = content.stripLeading(); + if ( trimmed.startsWith("{") || trimmed.startsWith("[") ) { return InputFormat.JSON; } + if ( trimmed.startsWith("-") || trimmed.startsWith("---") ) { return InputFormat.YAML; } + var firstLine = trimmed.lines().findFirst().orElse("").trim(); + if ( firstLine.matches("[A-Za-z0-9_-]+\\s*:.*") ) { return InputFormat.YAML; } + return InputFormat.CSV; + } + + private List> readCsvUpdates(String content) throws Exception { + var schema = CsvSchema.emptySchema().withHeader(); + MappingIterator> it = CSV_MAPPER + .readerFor(new TypeReference>() {}) + .with(schema) + .readValues(content); + var result = new ArrayList>(); + while ( it.hasNext() ) { + var row = it.next(); + if ( !row.values().stream().allMatch(StringUtils::isBlank) ) { + result.add(row); + } + } + return result; + } + + private List> readStructuredUpdates(ObjectMapper mapper, String content, String fmt) throws Exception { + var root = mapper.readTree(content); + if ( root == null || root.isNull() ) { return List.of(); } + if ( root.isObject() ) { return List.of(toStringMap(root, fmt)); } + if ( !root.isArray() ) { + throw new FcliSimpleException("%s contributor updates must be an object or array of objects", fmt); + } + var result = new ArrayList>(); + for ( var node : root ) { result.add(toStringMap(node, fmt)); } + return result; + } + + private Map toStringMap(JsonNode node, String fmt) { + if ( !node.isObject() ) { + throw new FcliSimpleException("%s contributor updates must contain only objects", fmt); + } + var result = new LinkedHashMap(); + node.fields().forEachRemaining(e -> result.put(e.getKey(), jsonValueToString(e.getValue()))); + return result; + } + + private String jsonValueToString(JsonNode node) { + if ( node == null || node.isNull() ) { return ""; } + return node.isValueNode() ? node.asText() : node.toString(); + } + + private String getContributorsSource() { + return contributorsPath == null ? "stdin" : contributorsPath.toString(); + } + + // ------------------------------------------------------------------------- + // Applying updates + // ------------------------------------------------------------------------- + + private List> readContributors(NcdReportReader reader) { + return reader.readContributors(); + } + + private UpdateApplicationResult applyUpdates(List> updates, List> contributors) { + var warnings = new ArrayList(); + int updatesApplied = 0; + var byAuthorId = contributors.stream() + .collect(Collectors.groupingBy(c -> c.getOrDefault(NcdReportContributorsCsvSchema.AUTHOR_ID, ""))); + + for ( var update : updates ) { + if ( applyUpdate(update, byAuthorId, warnings) ) { + updatesApplied++; + } + } + return new UpdateApplicationResult(updates.size(), updatesApplied, warnings); + } + + private boolean applyUpdate(Map update, Map>> byAuthorId, List warnings) { + var authorId = StringUtils.defaultString(update.get(NcdReportContributorsCsvSchema.AUTHOR_ID)).trim(); + if ( StringUtils.isBlank(authorId) ) { + warnings.add("Update row missing authorId; skipping"); + return false; + } + var targets = byAuthorId.get(authorId); + if ( targets == null ) { + warnings.add(String.format("authorId %s: not found in report; skipping", authorId)); + return false; + } + if ( !validateUpdateRow(authorId, update, byAuthorId, warnings) ) { + return false; + } + + boolean hasAppliedFields = false; + for ( var contributor : targets ) { + if ( applyFieldsToContributor(authorId, update, contributor, targets.size(), warnings) ) { + hasAppliedFields = true; + } + } + if ( !hasAppliedFields ) { + warnings.add(String.format("authorId %s: no updatable fields found; skipping", authorId)); + } + return hasAppliedFields; + } + + /** + * Cross-field consistency validation for an update row. + * Returns false if the entire row should be skipped. + */ + private boolean validateUpdateRow(String authorId, Map update, + Map>> byAuthorId, List warnings) { + var overrideStatus = StringUtils.defaultString(update.get(NcdReportContributorsCsvSchema.OVERRIDE_STATUS)).trim(); + var duplicateOf = StringUtils.defaultString(update.get(NcdReportContributorsCsvSchema.DUPLICATE_OF)).trim(); + var confidence = StringUtils.defaultString(update.get(NcdReportContributorsCsvSchema.OVERRIDE_STATUS_CONFIDENCE)).trim(); + + // Validate confidence threshold + if ( StringUtils.isNotBlank(confidence) ) { + try { + double val = Double.parseDouble(confidence); + if ( val < minConfidence ) { + warnings.add(String.format( + "authorId %s: confidence %.2f is below minimum %.2f; skipping", + authorId, val, minConfidence)); + return false; + } + } catch ( NumberFormatException e ) { + warnings.add(String.format("authorId %s: invalid confidence value '%s'; skipping", authorId, confidence)); + return false; + } + } + + // Validate overrideStatus value + if ( StringUtils.isNotBlank(overrideStatus) && !VALID_OVERRIDE_STATUSES.contains(overrideStatus) ) { + warnings.add(String.format("authorId %s: unknown overrideStatus '%s'; skipping", authorId, overrideStatus)); + return false; + } + + // duplicate status requires duplicateOf + if ( "duplicate".equals(overrideStatus) && StringUtils.isBlank(duplicateOf) ) { + warnings.add(String.format("authorId %s: overrideStatus 'duplicate' requires a non-blank duplicateOf; skipping", authorId)); + return false; + } + + // non-duplicate status must not set duplicateOf + if ( StringUtils.isNotBlank(duplicateOf) && StringUtils.isNotBlank(overrideStatus) + && !"duplicate".equals(overrideStatus) ) { + warnings.add(String.format( + "authorId %s: duplicateOf set but overrideStatus is '%s' (not 'duplicate'); skipping", + authorId, overrideStatus)); + return false; + } + + // duplicateOf must reference a known authorId + if ( StringUtils.isNotBlank(duplicateOf) ) { + if ( !byAuthorId.containsKey(duplicateOf) ) { + warnings.add(String.format("authorId %s: duplicateOf '%s' references unknown authorId; skipping", authorId, duplicateOf)); + return false; + } + if ( authorId.equals(duplicateOf) ) { + // Self-reference is silently ignored (can arise from lsc roundtrip) + return true; + } + } + + return true; + } + + private boolean applyFieldsToContributor(String authorId, Map update, Map contributor, + int targetCount, List warnings) { + boolean hasAppliedFields = false; + for ( var entry : update.entrySet() ) { + var field = entry.getKey(); + var value = StringUtils.defaultString(entry.getValue()).trim(); + + if ( StringUtils.isBlank(field) || field.equals(NcdReportContributorsCsvSchema.AUTHOR_ID) ) { + continue; + } + if ( NcdReportContributorsCsvSchema.IMMUTABLE_FIELDS.contains(field) ) { + // Only warn on immutable mismatches when authorId uniquely identifies a single row; + // multiple rows share the same authorId when they are aliases of the same person. + if ( targetCount == 1 ) { + warnIfImmutableMismatch(authorId, field, value, contributor, warnings); + } + continue; + } + if ( !NcdReportContributorsCsvSchema.UPDATABLE_FIELDS.contains(field) ) { + warnings.add(String.format("authorId %s: unknown field '%s'; ignoring", authorId, field)); + continue; + } + contributor.put(field, value); + hasAppliedFields = true; + } + return hasAppliedFields; + } + + private void warnIfImmutableMismatch(String authorId, String field, String value, + Map contributor, List warnings) { + if ( StringUtils.isBlank(value) ) { return; } + var existing = StringUtils.defaultString(contributor.get(field)); + if ( !existing.equals(value) ) { + warnings.add(String.format( + "authorId %s: immutable field '%s' in update differs from report value; ignoring", + authorId, field)); + } + } + + private void synchronizeContributionFields(List> contributors, List warnings) { + applyOverrideStatusToContributionStatus(contributors); + recalculateContributingAuthorNumbers(contributors, warnings); + } + + private void applyOverrideStatusToContributionStatus(List> contributors) { + for ( var contributor : contributors ) { + var overrideStatus = StringUtils.defaultString( + contributor.get(NcdReportContributorsCsvSchema.OVERRIDE_STATUS)).trim(); + if ( StringUtils.isBlank(overrideStatus) ) { + continue; + } + contributor.put(NcdReportContributorsCsvSchema.CONTRIBUTION_STATUS, overrideStatus); + if ( !"duplicate".equals(overrideStatus) ) { + contributor.put(NcdReportContributorsCsvSchema.DUPLICATE_OF, ""); + } + } + } + + private void recalculateContributingAuthorNumbers(List> contributors, List warnings) { + var byAuthorId = contributors.stream() + .filter(c -> StringUtils.isNotBlank(c.get(NcdReportContributorsCsvSchema.AUTHOR_ID))) + .collect(Collectors.toMap( + c -> c.get(NcdReportContributorsCsvSchema.AUTHOR_ID), + c -> c, + (left, right) -> left, + LinkedHashMap::new)); + + for ( var contributor : contributors ) { + contributor.put(NcdReportContributorsCsvSchema.CONTRIBUTING_AUTHOR_NUMBER, "-1"); + } + + var contributing = contributors.stream() + .filter(c -> "contributing".equals(normalizeStatus(c))) + .sorted((left, right) -> StringUtils.defaultString(left.get(NcdReportContributorsCsvSchema.AUTHOR_NAME)) + .compareToIgnoreCase(StringUtils.defaultString(right.get(NcdReportContributorsCsvSchema.AUTHOR_NAME)))) + .toList(); + int contributingAuthorNumber = 1; + for ( var contributor : contributing ) { + contributor.put(NcdReportContributorsCsvSchema.CONTRIBUTING_AUTHOR_NUMBER, + String.valueOf(contributingAuthorNumber++)); + } + + for ( var contributor : contributors ) { + if ( !"duplicate".equals(normalizeStatus(contributor)) ) { + continue; + } + var authorId = StringUtils.defaultString(contributor.get(NcdReportContributorsCsvSchema.AUTHOR_ID)); + var duplicateOf = StringUtils.defaultString(contributor.get(NcdReportContributorsCsvSchema.DUPLICATE_OF)).trim(); + if ( StringUtils.isBlank(duplicateOf) ) { + warnings.add(String.format("authorId %s: duplicate status has blank duplicateOf; leaving contributingAuthorNumber as -1", authorId)); + continue; + } + + var representative = resolveContributingRepresentative(duplicateOf, byAuthorId, new HashSet<>()); + if ( representative == null ) { + warnings.add(String.format("authorId %s: duplicateOf '%s' does not resolve to a contributing author; leaving contributingAuthorNumber as -1", + authorId, duplicateOf)); + continue; + } + + contributor.put(NcdReportContributorsCsvSchema.CONTRIBUTING_AUTHOR_NUMBER, + StringUtils.defaultString(representative.get(NcdReportContributorsCsvSchema.CONTRIBUTING_AUTHOR_NUMBER), "-1")); + } + } + + private Map resolveContributingRepresentative(String authorId, + Map> byAuthorId, Set seenAuthorIds) + { + if ( !seenAuthorIds.add(authorId) ) { + return null; + } + var contributor = byAuthorId.get(authorId); + if ( contributor == null ) { + return null; + } + var status = normalizeStatus(contributor); + if ( "contributing".equals(status) ) { + return contributor; + } + if ( "duplicate".equals(status) ) { + var duplicateOf = StringUtils.defaultString(contributor.get(NcdReportContributorsCsvSchema.DUPLICATE_OF)).trim(); + if ( StringUtils.isBlank(duplicateOf) ) { + return null; + } + return resolveContributingRepresentative(duplicateOf, byAuthorId, seenAuthorIds); + } + return null; + } + + private String normalizeStatus(Map contributor) { + return StringUtils.defaultString(contributor.get(NcdReportContributorsCsvSchema.CONTRIBUTION_STATUS)).trim().toLowerCase(); + } + + // ------------------------------------------------------------------------- + // Writing back + // ------------------------------------------------------------------------- + + private ObjectNode rewriteContributorsAndSummaryAndChecksums(NcdReportReader reader, List> contributors) { + var entryPath = reader.entryPath("contributors.csv"); + try { + var sortedContributors = NcdReportContributorsCsvSchema.sortByAuthorNameAndStatus( + contributors.stream() + .map(row -> JSON_MAPPER.convertValue(row, ObjectNode.class)) + .toList()); + var presentColumns = contributors.stream() + .flatMap(map -> map.keySet().stream()) + .collect(Collectors.toSet()); + var csvSchema = NcdReportContributorsCsvSchema.buildSchema(presentColumns); + var outputColumns = NcdReportContributorsCsvSchema.getOutputColumns().stream() + .filter(presentColumns::contains) + .toList(); + var writableContributors = sortedContributors.stream() + .map(row -> JSON_MAPPER.convertValue(row, new TypeReference>() {})) + .map(row -> { + Map writableRow = new java.util.LinkedHashMap<>(); + outputColumns.forEach(column -> writableRow.put(column, row.get(column))); + return writableRow; + }) + .toList(); + var csv = CSV_MAPPER.writer(csvSchema).writeValueAsString(writableContributors); + Files.write(entryPath, csv.getBytes(StandardCharsets.UTF_8)); + updateChecksum(reader, "contributors.csv"); + + var summary = updateSummary(reader, writableContributors); + updateChecksum(reader, "summary.txt"); + return summary; + } catch ( Exception e ) { + throw new FcliTechnicalException(String.format("Error updating contributors.csv in %s", reader.getReportPath()), e); + } + } + + private ObjectNode updateSummary(NcdReportReader reader, List> contributors) { + var summaryPath = reader.entryPath("summary.txt"); + var summary = reader.readSummary().deepCopy(); + + int total = contributors.size(); + int ignored = (int) contributors.stream().filter(c -> "ignored".equals(normalizeStatus(c))).count(); + int duplicate = (int) contributors.stream().filter(c -> "duplicate".equals(normalizeStatus(c))).count(); + int contributing = (int) contributors.stream().filter(c -> "contributing".equals(normalizeStatus(c))).count(); + int nonIgnored = total - ignored; + + summary.set("authorCount", JsonHelper.getObjectMapper().createObjectNode() + .put("total", total) + .put("contributing", contributing) + .put("ignored", ignored) + .put("nonIgnored", nonIgnored) + .put("duplicate", duplicate)); + try { + Files.write(summaryPath, YAML_MAPPER.writeValueAsBytes(summary)); + } catch ( Exception e ) { + throw new FcliTechnicalException(String.format("Error updating summary.txt in %s", reader.getReportPath()), e); + } + return summary; + } + + private String asYaml(ObjectNode summary) { + try { + return YAML_MAPPER.writeValueAsString(summary); + } catch ( Exception e ) { + throw new FcliTechnicalException("Error formatting summary output", e); + } + } + + private void updateChecksum(NcdReportReader reader, String entryName) { + var checksumsPath = reader.entryPath("checksums.sha256"); + try { + var lines = Files.readAllLines(checksumsPath, StandardCharsets.UTF_8); + var updated = new ArrayList(); + var entryChecksum = NcdReportValidator.sha256(reader.entryPath(entryName)); + boolean found = false; + for ( var line : lines ) { + var parts = line.split("\\s+", 2); + if ( parts.length >= 2 ) { + var filename = parts[1].startsWith("*") ? parts[1].substring(1) : parts[1]; + if ( filename.equals(entryName) ) { + updated.add(String.format("%s %s", entryChecksum, entryName)); + found = true; + } else { + updated.add(line); + } + } else { + updated.add(line); + } + } + if ( !found ) { + updated.add(String.format("%s %s", entryChecksum, entryName)); + } + Files.write(checksumsPath, updated, StandardCharsets.UTF_8); + } catch ( Exception e ) { + throw new FcliTechnicalException(String.format("Error updating checksums in %s", reader.getReportPath()), e); + } + } + + // ------------------------------------------------------------------------- + // Utilities + // ------------------------------------------------------------------------- + + private static ObjectMapper createYamlMapper() { + var mapper = new ObjectMapper(new YAMLFactory()); + mapper.registerModule(new Jdk8Module()); + mapper.registerModule(new JavaTimeModule()); + return mapper; + } + + private enum InputFormat { + CSV, JSON, YAML + } + + private record UpdateApplicationResult(int recordsProcessed, int updatesApplied, List warnings) {} +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/collector/NcdReportAuthorCollector.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/collector/NcdReportAuthorCollector.java index 6f0a53adcbd..133d631d7af 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/collector/NcdReportAuthorCollector.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/collector/NcdReportAuthorCollector.java @@ -100,11 +100,12 @@ final void writeResults() { } private final void writeEntry(Entry> e) { + var representativeAuthorId = e.getKey().getAuthorId(); var contributingAuthorNumber = counters.increaseCount(AuthorCounter.contributing); INcdReportAuthorsWriter writer = writers.authorsWriter(); writer.writeContributor(e.getKey(), contributingAuthorNumber); e.getValue().forEach(d->{ - writer.writeDuplicateAuthor(d, contributingAuthorNumber); + writer.writeDuplicateAuthor(d, representativeAuthorId, contributingAuthorNumber); counters.increaseCount(AuthorCounter.duplicate); }); } diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/collector/NcdReportContext.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/collector/NcdReportContext.java index 38e5d964ac0..8f2e0d6ae69 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/collector/NcdReportContext.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/collector/NcdReportContext.java @@ -70,6 +70,10 @@ public INcdReportRepositoryProcessor repositoryProcessor() { @Override @SneakyThrows public void close() { repositoryProcessor.writeResults(); + writers.authorsWriter().close(); + reportWriter.summary() + .put("reportStartDate", reportConfig.getCommitStartDateTime().toLocalDate().toString()) + .put("reportEndDate", reportConfig.getCommitEndDateTime().toLocalDate().toString()); logger().updateSummary(reportWriter.summary()); } } diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportConfig.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportConfig.java index e56de3985c7..f1ac21d3961 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportConfig.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportConfig.java @@ -13,13 +13,13 @@ package com.fortify.cli.license.ncd_report.config; import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.Collection; import java.util.Optional; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.formkiq.graalvm.annotations.Reflectable; import com.fortify.cli.common.report.config.IReportSourceSupplierConfig; -import com.fortify.cli.common.util.DateTimePeriodHelper; -import com.fortify.cli.common.util.DateTimePeriodHelper.Period; import com.fortify.cli.license.ncd_report.collector.NcdReportContext; import lombok.Data; @@ -34,16 +34,29 @@ @Reflectable @NoArgsConstructor @Data public class NcdReportConfig implements IReportSourceSupplierConfig { - private static final DateTimePeriodHelper PERIOD_HELPER = new DateTimePeriodHelper(Period.DAYS); private NcdReportSourcesConfig sources; private Optional contributor; + /** Not a YAML config field — set programmatically from the {@code --end-date} CLI option. */ + @JsonIgnore private OffsetDateTime commitEndDate; @Override public final Collection getSourceConfigs() { - return sources.getSourceConfigs(); + return sources == null ? java.util.List.of() : sources.getSourceConfigs(); } - public final OffsetDateTime getCommitOffsetDateTime() { - return PERIOD_HELPER.getCurrentOffsetDateTimeMinusPeriod("90d"); + /** + * Returns the (inclusive) end of the 90-day reporting window. + * When {@code --end-date} is supplied this is end-of-day on that date; otherwise it is now. + */ + public final OffsetDateTime getCommitEndDateTime() { + return commitEndDate != null ? commitEndDate : OffsetDateTime.now(ZoneOffset.UTC); + } + + /** + * Returns the (inclusive) start of the 90-day reporting window: exactly 90 days before + * {@link #getCommitEndDateTime()}. + */ + public final OffsetDateTime getCommitStartDateTime() { + return getCommitEndDateTime().minusDays(90); } } diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportMockSourceConfig.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportMockSourceConfig.java new file mode 100644 index 00000000000..40a29df0287 --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportMockSourceConfig.java @@ -0,0 +1,74 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.config; + +import java.util.Optional; + +import com.formkiq.graalvm.annotations.Reflectable; +import com.fortify.cli.common.report.generator.IReportResultsGenerator; +import com.fortify.cli.common.rest.unirest.config.IUrlConfig; +import com.fortify.cli.license.ncd_report.collector.NcdReportContext; +import com.fortify.cli.license.ncd_report.generator.mock.NcdReportMockResultsGenerator; + +import kong.unirest.Config; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * Mock SCM source configuration for testing NCD report generation. + * Supports both auto-generated mock commits and reading from external files. + */ +@Reflectable @NoArgsConstructor @AllArgsConstructor +@Data @EqualsAndHashCode(callSuper = true) +public class NcdReportMockSourceConfig extends AbstractNcdReportRepoSelectorConfig implements INcdReportSourceConfig, IUrlConfig { + /** Number of repositories to generate (default 3) */ + private Integer repositoryCount = 3; + + /** Number of authors per repository (default 5) */ + private Integer authorsPerRepository = 5; + + /** Number of commits per author (default 10) */ + private Integer commitsPerAuthor = 10; + + /** Optional path to JSON/YAML/CSV file with custom mock commit data */ + private Optional dataFile = Optional.empty(); + + /** Custom HTTP headers */ + private java.util.List headers = new java.util.ArrayList<>(); + + /** Connection timeout in milliseconds */ + private int connectTimeoutInMillis = Config.DEFAULT_CONNECT_TIMEOUT; + + /** Socket timeout in milliseconds */ + private int socketTimeoutInMillis = Config.DEFAULT_SOCKET_TIMEOUT; + + /** Insecure mode enabled flag */ + private Boolean insecureModeEnabled = false; + + @Override + public String getUrl() { + return "mock://"; + } + + @Override + public Boolean getInsecureModeEnabled() { + return insecureModeEnabled; + } + + @Override + public IReportResultsGenerator generator(NcdReportContext reportContext) { + return new NcdReportMockResultsGenerator(this, reportContext); + } +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportSourcesConfig.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportSourcesConfig.java index bc8cccbcc78..99325bba3f1 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportSourcesConfig.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/config/NcdReportSourcesConfig.java @@ -35,9 +35,10 @@ public class NcdReportSourcesConfig { private Optional github = Optional.empty(); private Optional gitlab = Optional.empty(); private Optional ado = Optional.empty(); + private Optional mock = Optional.empty(); public final List getSourceConfigs() { - return Stream.of(ado, github, gitlab) + return Stream.of(ado, github, gitlab, mock) .filter(Optional::isPresent) .map(Optional::get) .map(INcdReportSourceConfig[].class::cast) diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/descriptor/INcdReportAuthorDescriptor.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/descriptor/INcdReportAuthorDescriptor.java index 1706b1f8647..880b09fca7a 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/descriptor/INcdReportAuthorDescriptor.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/descriptor/INcdReportAuthorDescriptor.java @@ -15,8 +15,8 @@ import org.apache.commons.lang3.StringUtils; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fortify.cli.common.json.JsonHelper; import com.fortify.cli.common.json.JsonNodeHolder; +import com.fortify.cli.license.ncd_report.helper.NcdReportContributorHelper; /** *

Describe a commit author. Implementations must at least @@ -38,27 +38,8 @@ public interface INcdReportAuthorDescriptor { String getEmail(); public default ObjectNode toExpressionInput() { - var name = StringUtils.defaultIfBlank(getName(), ""); - var email = StringUtils.defaultIfBlank(getEmail(), ""); - var lcName = name.toLowerCase(); - var lcEmail = email.toLowerCase(); - var lcEmailDomain = StringUtils.substringAfter(lcEmail, "@"); - var lcEmailName = StringUtils.substringBefore(lcEmail, "@"); - var cleanName = lcName.replaceAll("[^a-z]", ""); - // Remove all special characters, then remove leading digits unless remaining string contains only digits - // TODO Any better way of doing this instead of having to iterate through the string 3 times? - var cleanEmailName = lcEmailName.replaceAll("[^a-z0-9]", ""); - if ( !cleanEmailName.matches("[0-9]+") ) { - cleanEmailName = cleanEmailName.replaceAll("^[0-9]+", ""); - } - return JsonHelper.getObjectMapper().createObjectNode() - .put("name", name) - .put("email", email) - .put("lcName", lcName) - .put("lcEmail", lcEmail) - .put("lcEmailDomain", lcEmailDomain) - .put("lcEmailName", lcEmailName) - .put("cleanName", cleanName) - .put("cleanEmailName", cleanEmailName); + return NcdReportContributorHelper.createExpressionInput( + StringUtils.defaultIfBlank(getName(), ""), + StringUtils.defaultIfBlank(getEmail(), "")); } } diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/descriptor/NcdReportProcessedAuthorDescriptor.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/descriptor/NcdReportProcessedAuthorDescriptor.java index c48fdea43d9..91575d221a7 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/descriptor/NcdReportProcessedAuthorDescriptor.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/descriptor/NcdReportProcessedAuthorDescriptor.java @@ -13,6 +13,7 @@ package com.fortify.cli.license.ncd_report.descriptor; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.license.ncd_report.helper.NcdReportContributorHelper; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -25,10 +26,25 @@ public class NcdReportProcessedAuthorDescriptor { private final ObjectNode expressionInput; public ObjectNode updateReportRecord(ObjectNode objectNode) { - return objectNode.put("authorName", authorDescriptor.getName()) + return objectNode.put("authorId", computeAuthorId()) + .put("authorName", authorDescriptor.getName()) .put("authorEmail", authorDescriptor.getEmail()) - .put("authorState", state.name()) - .put("authorNumber", authorNumber); + .put("authorState", state.name()); + } + + /** + * Computes a stable 16-hex-char identifier for this author derived from the + * normalized lowercase name and email fields in the expression input. + * The value is reproducible across separate runs as long + * as the author's name/email are the same, making it suitable as a stable + * cross-report reference key (e.g. for AI-assisted deduplication annotations). + */ + public String getAuthorId() { + return computeAuthorId(); + } + + private String computeAuthorId() { + return NcdReportContributorHelper.computeAuthorId(expressionInput); } public static enum NcdReportProcessedAuthorState { diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/ado/NcdReportAdoResultsGenerator.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/ado/NcdReportAdoResultsGenerator.java index 8f9dd43324f..563f89a290e 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/ado/NcdReportAdoResultsGenerator.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/ado/NcdReportAdoResultsGenerator.java @@ -112,10 +112,11 @@ private void generateCommitData(AdoRestHelper restHelper, NcdReportAdoRepository } private void generateMostRecentCommitData(AdoRestHelper restHelper, INcdReportRepositoryBranchCommitCollector branchCommitCollector, NcdReportAdoRepositoryDescriptor repoDescriptor, List branchDescriptors) { + String until = reportContext().reportConfig().getCommitEndDateTime().format(DateTimeFormatter.ISO_INSTANT); NcdReportAdoCommitDescriptor mostRecentCommit = null; NcdReportAdoBranchDescriptor mostRecentBranch = null; for ( var branch : branchDescriptors ) { - ObjectNode body = getLatestCommit(restHelper, repoDescriptor, branch); + ObjectNode body = getLatestCommit(restHelper, repoDescriptor, branch, until); ArrayNode commits = (ArrayNode) body.path("value"); if ( commits.size()>0 ) { var commit = JsonHelper.treeToValue(commits.get(0), NcdReportAdoCommitDescriptor.class); @@ -131,13 +132,14 @@ private void generateMostRecentCommitData(AdoRestHelper restHelper, INcdReportRe } private boolean generateCommitDataForBranches(AdoRestHelper restHelper, INcdReportRepositoryBranchCommitCollector branchCommitCollector, NcdReportAdoRepositoryDescriptor repoDescriptor, List branchDescriptors) { - String since = reportContext().reportConfig().getCommitOffsetDateTime().format(DateTimeFormatter.ISO_INSTANT); + String since = reportContext().reportConfig().getCommitStartDateTime().format(DateTimeFormatter.ISO_INSTANT); + String until = reportContext().reportConfig().getCommitEndDateTime().format(DateTimeFormatter.ISO_INSTANT); boolean commitsFound = false; for ( var branchDescriptor : branchDescriptors ) { reportContext().progressWriter().writeI18nProgress("fcli.license.ncd-report.loading.branch-commits", repoDescriptor.getFullName(), branchDescriptor.getName()); List foundFlag = new ArrayList<>(); restHelper.repository(repoDescriptor.getOrganizationName(), repoDescriptor.getProjectName(), repoDescriptor.getId()) - .queryCommits().branchName(branchDescriptor.getName()).fromDate(since).process(commit -> { + .queryCommits().branchName(branchDescriptor.getName()).fromDate(since).toDate(until).process(commit -> { foundFlag.add(true); addCommit(branchCommitCollector, repoDescriptor, branchDescriptor, commit); return Break.FALSE; @@ -165,9 +167,18 @@ private List getBranchDescriptors(AdoRestHelper re return result; } - private ObjectNode getLatestCommit(AdoRestHelper restHelper, NcdReportAdoRepositoryDescriptor repoDescriptor, NcdReportAdoBranchDescriptor branchDescriptor) { - return restHelper.repository(repoDescriptor.getOrganizationName(), repoDescriptor.getProjectName(), repoDescriptor.getId()) - .getLatestCommit(branchDescriptor.getName()); + private ObjectNode getLatestCommit(AdoRestHelper restHelper, NcdReportAdoRepositoryDescriptor repoDescriptor, NcdReportAdoBranchDescriptor branchDescriptor, String until) { + var value = JsonHelper.getObjectMapper().createArrayNode(); + restHelper.repository(repoDescriptor.getOrganizationName(), repoDescriptor.getProjectName(), repoDescriptor.getId()) + .queryCommits() + .branchName(branchDescriptor.getName()) + .toDate(until) + .queryParam("searchCriteria.$top", 1) + .process(commit -> { + value.add(commit); + return Break.TRUE; + }); + return JsonHelper.getObjectMapper().createObjectNode().set("value", value); } private NcdReportAdoRepositoryDescriptor getRepoDescriptor(JsonNode repoNode) { diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/github/NcdReportGitHubResultsGenerator.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/github/NcdReportGitHubResultsGenerator.java index 3c360bc1f98..61f938faa12 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/github/NcdReportGitHubResultsGenerator.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/github/NcdReportGitHubResultsGenerator.java @@ -117,10 +117,12 @@ private void generateCommitData(NcdReportGitHubRepositoryDescriptor repoDescript * method. */ private void generateMostRecentCommitData(INcdReportRepositoryBranchCommitCollector branchCommitCollector, NcdReportGitHubRepositoryDescriptor repoDescriptor, List branchDescriptors) { + String until = reportContext().reportConfig().getCommitEndDateTime() + .format(DateTimeFormatter.ISO_INSTANT); NcdReportGitHubCommitDescriptor mostRecentCommitDescriptor = null; NcdReportGitHubBranchDescriptor mostRecentBranchDescriptor = null; for ( var branchDescriptor : branchDescriptors ) { - var currentCommitResponse = getLatestCommit(repoDescriptor, branchDescriptor); + var currentCommitResponse = getLatestCommit(repoDescriptor, branchDescriptor, until); if ( currentCommitResponse.size()>0 ) { var currentCommitDescriptor = JsonHelper.treeToValue(currentCommitResponse.get(0), NcdReportGitHubCommitDescriptor.class); if ( mostRecentCommitDescriptor==null || currentCommitDescriptor.getDate().isAfter(mostRecentCommitDescriptor.getDate()) ) { @@ -140,14 +142,16 @@ private void generateMostRecentCommitData(INcdReportRepositoryBranchCommitCollec * @return true if any commits were found, false otherwise */ private boolean generateCommitDataForBranches(INcdReportRepositoryBranchCommitCollector branchCommitCollector, NcdReportGitHubRepositoryDescriptor repoDescriptor, List branchDescriptors) { - String since = reportContext().reportConfig().getCommitOffsetDateTime() + String since = reportContext().reportConfig().getCommitStartDateTime() + .format(DateTimeFormatter.ISO_INSTANT); + String until = reportContext().reportConfig().getCommitEndDateTime() .format(DateTimeFormatter.ISO_INSTANT); boolean commitsFound = false; for ( var branchDescriptor : branchDescriptors ) { reportContext().progressWriter().writeI18nProgress("fcli.license.ncd-report.loading.branch-commits", repoDescriptor.getFullName(), branchDescriptor.getName()); List foundFlag = new ArrayList<>(); restHelper.repo(repoDescriptor.getOwnerName(), repoDescriptor.getName()) - .queryCommits().sha(branchDescriptor.getSha()).since(since).process(commit -> { + .queryCommits().sha(branchDescriptor.getSha()).since(since).until(until).process(commit -> { foundFlag.add(true); addCommit(branchCommitCollector, repoDescriptor, branchDescriptor, commit); return Break.FALSE; @@ -185,9 +189,18 @@ private List getBranchDescriptors(NcdReportGitH /** * Get a single commit (most recent) for a branch. */ - private ArrayNode getLatestCommit(NcdReportGitHubRepositoryDescriptor repoDescriptor, NcdReportGitHubBranchDescriptor branchDescriptor) { - return restHelper.repo(repoDescriptor.getOwnerName(), repoDescriptor.getName()) - .getLatestCommit(branchDescriptor.getSha()); + private ArrayNode getLatestCommit(NcdReportGitHubRepositoryDescriptor repoDescriptor, NcdReportGitHubBranchDescriptor branchDescriptor, String until) { + var result = JsonHelper.getObjectMapper().createArrayNode(); + restHelper.repo(repoDescriptor.getOwnerName(), repoDescriptor.getName()) + .queryCommits() + .sha(branchDescriptor.getSha()) + .until(until) + .queryParam("per_page", 1) + .process(commit -> { + result.add(commit); + return Break.TRUE; + }); + return result; } /** diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/gitlab/NcdReportGitLabResultsGenerator.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/gitlab/NcdReportGitLabResultsGenerator.java index 0ca1ca51bb5..100b38d3e2e 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/gitlab/NcdReportGitLabResultsGenerator.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/gitlab/NcdReportGitLabResultsGenerator.java @@ -120,10 +120,12 @@ private void generateCommitData(NcdReportGitLabRepositoryDescriptor repoDescript * method. */ private void generateMostRecentCommitData(INcdReportRepositoryBranchCommitCollector branchCommitCollector, NcdReportGitLabRepositoryDescriptor repoDescriptor, List branchDescriptors) { + String until = reportContext().reportConfig().getCommitEndDateTime() + .format(DateTimeFormatter.ISO_INSTANT); NcdReportGitLabCommitDescriptor mostRecentCommitDescriptor = null; NcdReportGitLabBranchDescriptor mostRecentBranchDescriptor = null; for ( var branchDescriptor : branchDescriptors ) { - var currentCommitResponse = getLatestCommit(repoDescriptor, branchDescriptor); + var currentCommitResponse = getLatestCommit(repoDescriptor, branchDescriptor, until); if ( currentCommitResponse.size()>0 ) { var currentCommitDescriptor = JsonHelper.treeToValue(currentCommitResponse.get(0), NcdReportGitLabCommitDescriptor.class); if ( mostRecentCommitDescriptor==null || currentCommitDescriptor.getDate().isAfter(mostRecentCommitDescriptor.getDate()) ) { @@ -143,14 +145,16 @@ private void generateMostRecentCommitData(INcdReportRepositoryBranchCommitCollec * @return true if any commits were found, false otherwise */ private boolean generateCommitDataForBranches(INcdReportRepositoryBranchCommitCollector branchCommitCollector, NcdReportGitLabRepositoryDescriptor repoDescriptor, List branchDescriptors) { - String since = reportContext().reportConfig().getCommitOffsetDateTime() + String since = reportContext().reportConfig().getCommitStartDateTime() + .format(DateTimeFormatter.ISO_INSTANT); + String until = reportContext().reportConfig().getCommitEndDateTime() .format(DateTimeFormatter.ISO_INSTANT); boolean commitsFound = false; for ( var branchDescriptor : branchDescriptors ) { reportContext().progressWriter().writeI18nProgress("fcli.license.ncd-report.loading.branch-commits", repoDescriptor.getFullName(), branchDescriptor.getName()); List foundFlag = new ArrayList<>(); restHelper.project(repoDescriptor.getId()) - .queryCommits().refName(branchDescriptor.getName()).since(since).process(commit -> { + .queryCommits().refName(branchDescriptor.getName()).since(since).until(until).process(commit -> { foundFlag.add(true); addCommit(branchCommitCollector, repoDescriptor, branchDescriptor, commit); return Break.FALSE; @@ -188,9 +192,18 @@ private List getBranchDescriptors(NcdReportGitL /** * Get a single commit (most recent) for a branch. */ - private ArrayNode getLatestCommit(NcdReportGitLabRepositoryDescriptor repoDescriptor, NcdReportGitLabBranchDescriptor branchDescriptor) { - return restHelper.project(repoDescriptor.getId()) - .getLatestCommit(branchDescriptor.getName()); + private ArrayNode getLatestCommit(NcdReportGitLabRepositoryDescriptor repoDescriptor, NcdReportGitLabBranchDescriptor branchDescriptor, String until) { + var result = JsonHelper.getObjectMapper().createArrayNode(); + restHelper.project(repoDescriptor.getId()) + .queryCommits() + .refName(branchDescriptor.getName()) + .until(until) + .queryParam("per_page", 1) + .process(commit -> { + result.add(commit); + return Break.TRUE; + }); + return result; } /** diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockAuthorData.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockAuthorData.java new file mode 100644 index 00000000000..fb2a12930bf --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockAuthorData.java @@ -0,0 +1,93 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.generator.mock; + +import java.util.Arrays; +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * Mock author dataset with realistic names and built-in duplicate scenarios + * for testing deduplication and AI detection functionality. + */ +@Data @AllArgsConstructor +public class MockAuthorData { + private String name; + private String email; + + /** Predefined realistic authors with duplicates and AI candidates */ + private static final List REALISTIC_AUTHORS = Arrays.asList( + // Real distinct authors + new MockAuthorData("John Smith", "john.smith@example.com"), + new MockAuthorData("Sarah Johnson", "sarah.j@example.com"), + new MockAuthorData("Michael Chen", "m.chen@example.com"), + new MockAuthorData("Emma Williams", "emma.williams@example.com"), + new MockAuthorData("Alex Rodriguez", "alex.r@example.com"), + + // DUPLICATE SCENARIOS: Same person, different name variations + // John Smith via different name formats + new MockAuthorData("J. Smith", "john.smith@example.com"), + new MockAuthorData("John D. Smith", "john.smith@example.com"), + + // Sarah Johnson via nickname + new MockAuthorData("Sara Johnson", "sarah.j@example.com"), + new MockAuthorData("Sarah J.", "sarah.j@example.com"), + + // AI CANDIDATE DUPLICATES: Very similar names, different emails + // Could be same person with different company emails + new MockAuthorData("Michael Chen", "mchen@company.com"), + new MockAuthorData("Mike Chen", "mike.chen@example.com"), + + // Very similar variations (typos or transliteration) + new MockAuthorData("Emma Willams", "emma.w@example.com"), // typo + new MockAuthorData("Emma Williams", "ewilliams@example.com"), // same name, different email + + // Common name variations + new MockAuthorData("Alex Rodriguez", "arodriguez@example.com"), + new MockAuthorData("Alexander Rodriguez", "alex.r@example.com"), + + // Additional distinct authors for volume + new MockAuthorData("Jennifer Park", "j.park@example.com"), + new MockAuthorData("David Brown", "d.brown@example.com"), + new MockAuthorData("Lisa Anderson", "l.anderson@example.com"), + + // Some ignored authors (e.g., bots, test accounts) + new MockAuthorData("MyBot1 [bot]", "bot1@example.com"), + new MockAuthorData("MyBot2 [bot]", "bot2@example.com"), + new MockAuthorData("MyBot3 [bot]", "bot3@example.com"), + new MockAuthorData("MyBot4 [bot]", "bot4@example.com") + ); + + /** + * Get a predefined realistic author by index, cycling through the list. + */ + public static MockAuthorData getRealisticAuthor(int index) { + return REALISTIC_AUTHORS.get(index % REALISTIC_AUTHORS.size()); + } + + /** + * Get total number of realistic authors defined. + */ + public static int getRealisticAuthorCount() { + return REALISTIC_AUTHORS.size(); + } + + /** + * Get all realistic authors. + */ + public static List getAllRealisticAuthors() { + return REALISTIC_AUTHORS; + } +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportAuthorDescriptor.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportAuthorDescriptor.java new file mode 100644 index 00000000000..dc2430d3be7 --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportAuthorDescriptor.java @@ -0,0 +1,27 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.generator.mock; + +import com.fortify.cli.license.ncd_report.descriptor.INcdReportAuthorDescriptor; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * Simple mock implementation of {@link INcdReportAuthorDescriptor}. + */ +@Data @AllArgsConstructor +public class MockNcdReportAuthorDescriptor implements INcdReportAuthorDescriptor { + private String name; + private String email; +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportBranchDescriptor.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportBranchDescriptor.java new file mode 100644 index 00000000000..e122c396f17 --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportBranchDescriptor.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.generator.mock; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.license.ncd_report.descriptor.INcdReportBranchDescriptor; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * Simple mock implementation of {@link INcdReportBranchDescriptor}. + */ +@Data @AllArgsConstructor +public class MockNcdReportBranchDescriptor implements INcdReportBranchDescriptor { + private String name; + private String sha; + + public ObjectNode asJsonNode() { + ObjectNode node = JsonNodeFactory.instance.objectNode(); + node.put("name", name); + node.put("commit", JsonNodeFactory.instance.objectNode().put("sha", sha)); + return node; + } +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportCommitDescriptor.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportCommitDescriptor.java new file mode 100644 index 00000000000..284d9902b15 --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportCommitDescriptor.java @@ -0,0 +1,56 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.generator.mock; + +import java.time.LocalDateTime; +import java.time.OffsetDateTime; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.license.ncd_report.descriptor.INcdReportCommitDescriptor; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * Simple mock implementation of {@link INcdReportCommitDescriptor}. + */ +@Data @AllArgsConstructor +public class MockNcdReportCommitDescriptor implements INcdReportCommitDescriptor { + private String sha; + private OffsetDateTime dateTime; + + @Override + public String getId() { + return sha; + } + + @Override + public LocalDateTime getDate() { + return dateTime.toLocalDateTime(); + } + + @Override + public String getMessage() { + return "Mock commit " + sha; + } + + public ObjectNode asJsonNode() { + ObjectNode node = JsonNodeFactory.instance.objectNode(); + node.put("sha", sha); + node.put("commit", JsonNodeFactory.instance.objectNode() + .put("author", JsonNodeFactory.instance.objectNode() + .put("date", dateTime.toString()))); + return node; + } +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportRepositoryDescriptor.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportRepositoryDescriptor.java new file mode 100644 index 00000000000..c2f4cfd57e1 --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/MockNcdReportRepositoryDescriptor.java @@ -0,0 +1,42 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.generator.mock; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.license.ncd_report.descriptor.INcdReportRepositoryDescriptor; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * Simple mock implementation of {@link INcdReportRepositoryDescriptor}. + */ +@Data @AllArgsConstructor +public class MockNcdReportRepositoryDescriptor implements INcdReportRepositoryDescriptor { + private String fullName; + private String url; + private String visibility; + private boolean fork; + + @Override + public JsonNode asJsonNode() { + ObjectNode node = JsonNodeFactory.instance.objectNode(); + node.put("full_name", fullName); + node.put("html_url", url); + node.put("visibility", visibility); + node.put("fork", fork); + return node; + } +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/NcdReportMockResultsGenerator.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/NcdReportMockResultsGenerator.java new file mode 100644 index 00000000000..aca2ebfee5e --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/generator/mock/NcdReportMockResultsGenerator.java @@ -0,0 +1,216 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.generator.mock; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fortify.cli.common.exception.FcliSimpleException; +import com.fortify.cli.license.ncd_report.collector.INcdReportRepositoryBranchCommitCollector; +import com.fortify.cli.license.ncd_report.collector.NcdReportContext; +import com.fortify.cli.license.ncd_report.config.NcdReportMockSourceConfig; +import com.fortify.cli.license.ncd_report.descriptor.INcdReportRepositoryDescriptor; +import com.fortify.cli.license.ncd_report.descriptor.NcdReportBranchCommitDescriptor; +import com.fortify.cli.license.ncd_report.generator.AbstractNcdReportResultsGenerator; + +/** + * Results generator for mock SCM source, useful for testing NCD report functionality. + * Generates synthetic repositories, branches, and commits with configurable parameters. + * Supports both built-in realistic authors and loading from external JSON/CSV files. + */ +public class NcdReportMockResultsGenerator extends AbstractNcdReportResultsGenerator { + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + private static final ObjectMapper YAML_MAPPER = new ObjectMapper(new YAMLFactory()); + private List authors = new ArrayList<>(); + + public NcdReportMockResultsGenerator(NcdReportMockSourceConfig sourceConfig, NcdReportContext reportContext) { + super(sourceConfig, reportContext); + initializeAuthors(); + } + + /** + * Initialize authors from dataFile if configured, otherwise use built-in realistic data. + */ + private void initializeAuthors() { + if ( sourceConfig().getDataFile().isPresent() ) { + String dataFilePath = sourceConfig().getDataFile().get(); + try { + loadAuthorsFromFile(dataFilePath); + } catch ( Exception e ) { + reportContext().logger().warn("Failed to load mock data from " + dataFilePath + ", falling back to built-in data", e); + authors = new ArrayList<>(MockAuthorData.getAllRealisticAuthors()); + } + } else { + authors = new ArrayList<>(MockAuthorData.getAllRealisticAuthors()); + } + } + + /** + * Load authors from JSON, YAML, or CSV file. + */ + private void loadAuthorsFromFile(String dataFilePath) throws Exception { + var file = new File(dataFilePath); + if ( !file.exists() ) { + throw new FcliSimpleException("Data file not found: " + dataFilePath); + } + + var lcDataFilePath = dataFilePath.toLowerCase(); + if ( lcDataFilePath.endsWith(".json") ) { + loadAuthorsFromStructuredFile(file, JSON_MAPPER, "JSON"); + } else if ( lcDataFilePath.endsWith(".yaml") || lcDataFilePath.endsWith(".yml") ) { + loadAuthorsFromStructuredFile(file, YAML_MAPPER, "YAML"); + } else if ( lcDataFilePath.endsWith(".csv") ) { + loadAuthorsFromCsv(file); + } else { + throw new FcliSimpleException("Unsupported data file format: " + dataFilePath + + ". Supported formats: JSON, YAML, CSV"); + } + } + + /** + * Load authors from JSON or YAML file. Expects array of objects with name and email fields, + * or an object containing an authors array. + */ + private void loadAuthorsFromStructuredFile(File file, ObjectMapper mapper, String formatName) throws IOException { + JsonNode root = mapper.readTree(file); + + if ( root.isArray() ) { + addAuthorsFromArray(root); + } else if ( root.isObject() ) { + JsonNode authorsNode = root.get("authors"); + if ( authorsNode != null && authorsNode.isArray() ) { + addAuthorsFromArray(authorsNode); + } + } + + if ( authors.isEmpty() ) { + throw new FcliSimpleException("No valid authors found in %s file: %s", formatName, file.getPath()); + } + } + + private void addAuthorsFromArray(JsonNode authorsNode) { + for ( JsonNode node : authorsNode ) { + String name = node.has("name") ? node.get("name").asText() : ""; + String email = node.has("email") ? node.get("email").asText() : ""; + if ( !name.isEmpty() && !email.isEmpty() ) { + authors.add(new MockAuthorData(name, email)); + } + } + } + + /** + * Load authors from CSV file. Expects columns: name, email. + */ + private void loadAuthorsFromCsv(File file) throws IOException { + String csvContent = Files.readString(file.toPath(), StandardCharsets.UTF_8); + var schema = CsvSchema.emptySchema().withHeader(); + CsvMapper csvMapper = new CsvMapper(); + + MappingIterator> iterator = csvMapper + .readerFor(new TypeReference>() {}) + .with(schema) + .readValues(csvContent); + + while ( iterator.hasNext() ) { + var row = iterator.next(); + String name = row.get("name"); + String email = row.get("email"); + if ( name != null && !name.trim().isEmpty() && email != null && !email.trim().isEmpty() ) { + authors.add(new MockAuthorData(name.trim(), email.trim())); + } + } + + if ( authors.isEmpty() ) { + throw new FcliSimpleException("No valid authors found in CSV file: " + file.getPath()); + } + } + + @Override + protected void generateResults() { + var reportContext = reportContext(); + var repositoryProcessor = reportContext.repositoryProcessor(); + var endDateTime = reportContext.reportConfig().getCommitEndDateTime(); + + // Generate mock repositories + var repositoryCount = sourceConfig().getRepositoryCount(); + for ( int i = 1; i <= repositoryCount; i++ ) { + final int repoIndex = i; + var repoName = "mock-repo-" + repoIndex; + var repoUrl = "https://mock.example.com/repos/" + repoName; + var repoDescriptor = new MockNcdReportRepositoryDescriptor(repoName, repoUrl, "public", false); + + repositoryProcessor.processRepository( + sourceConfig(), + repoDescriptor, + (repo, branchCollector) -> generateCommitDataForRepository(branchCollector, repoIndex, repoDescriptor, endDateTime) + ); + } + } + + private void generateCommitDataForRepository( + INcdReportRepositoryBranchCommitCollector branchCollector, + int repoIndex, + INcdReportRepositoryDescriptor repositoryDescriptor, + OffsetDateTime reportEndDateTime) { + + var authorsPerRepo = sourceConfig().getAuthorsPerRepository(); + var commitsPerAuthor = sourceConfig().getCommitsPerAuthor(); + var branchDescriptor = new MockNcdReportBranchDescriptor("main", "abc123"); + + // Generate authors and commits using realistic data + int authorIndex = ((repoIndex - 1) * authorsPerRepo); + for ( int a = 0; a < authorsPerRepo; a++ ) { + // Use realistic authors, cycling through available authors + MockAuthorData authorData = authors.get((authorIndex + a) % authors.size()); + var author = new MockNcdReportAuthorDescriptor(authorData.getName(), authorData.getEmail()); + + // Generate commits for this author spread across the report window + some before + for ( int c = 1; c <= commitsPerAuthor; c++ ) { + // Distribute commits: some before the window, most within the window + var daysOffset = (c % 2 == 0) ? -120 + (c / 2) : -30 + (c / 2); + var commitDateTime = reportEndDateTime.plusDays(daysOffset); + + var commitSha = String.format("%040x", (long)repoIndex * 10000 + (long)a * 100 + c); + var commitDescriptor = new MockNcdReportCommitDescriptor(commitSha, commitDateTime); + + // Create the full branch commit descriptor + var branchCommitDescriptor = new NcdReportBranchCommitDescriptor( + repositoryDescriptor, + branchDescriptor, + commitDescriptor, + author + ); + + branchCollector.reportBranchCommit(branchCommitDescriptor); + } + } + } + + @Override + protected String getType() { + return "mock"; + } +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/helper/NcdReportContributorHelper.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/helper/NcdReportContributorHelper.java new file mode 100644 index 00000000000..187794e5c64 --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/helper/NcdReportContributorHelper.java @@ -0,0 +1,82 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.helper; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fortify.cli.common.exception.FcliBugException; +import com.fortify.cli.common.json.JsonHelper; +import com.fortify.cli.license.ncd_report.writer.NcdReportContributorsCsvSchema; + +public final class NcdReportContributorHelper { + public static ObjectNode createExpressionInput(String name, String email) { + var normalizedName = StringUtils.defaultIfBlank(name, ""); + var normalizedEmail = StringUtils.defaultIfBlank(email, ""); + var lcName = normalizedName.toLowerCase(); + var lcEmail = normalizedEmail.toLowerCase(); + var lcEmailDomain = StringUtils.substringAfter(lcEmail, "@"); + var lcEmailName = StringUtils.substringBefore(lcEmail, "@"); + var cleanName = lcName.replaceAll("[^a-z]", ""); + var cleanEmailName = lcEmailName.replaceAll("[^a-z0-9]", ""); + if ( !cleanEmailName.matches("[0-9]+") ) { + cleanEmailName = cleanEmailName.replaceAll("^[0-9]+", ""); + } + return JsonHelper.getObjectMapper().createObjectNode() + .put("name", normalizedName) + .put("email", normalizedEmail) + .put("lcName", lcName) + .put("lcEmail", lcEmail) + .put("lcEmailDomain", lcEmailDomain) + .put("lcEmailName", lcEmailName) + .put(NcdReportContributorsCsvSchema.CLEAN_NAME, cleanName) + .put(NcdReportContributorsCsvSchema.CLEAN_EMAIL_NAME, cleanEmailName); + } + + public static String computeAuthorId(ObjectNode expressionInput) { + var name = expressionInput.path("name").asText(""); + var email = expressionInput.path("email").asText(""); + var input = name.toLowerCase() + ":" + email.toLowerCase(); + try { + var digest = MessageDigest.getInstance("SHA-256"); + var hash = digest.digest(input.getBytes(StandardCharsets.UTF_8)); + return String.format("%032x", new BigInteger(1, hash)).substring(0, 16); + } catch ( NoSuchAlgorithmException e ) { + throw new FcliBugException("SHA-256 not available", e); + } + } + + public static void normalizeContributorRow(Map row) { + var expressionInput = createExpressionInput( + row.get(NcdReportContributorsCsvSchema.AUTHOR_NAME), + row.get(NcdReportContributorsCsvSchema.AUTHOR_EMAIL)); + row.compute(NcdReportContributorsCsvSchema.CLEAN_NAME, + (key, value) -> StringUtils.isBlank(value) + ? expressionInput.path(NcdReportContributorsCsvSchema.CLEAN_NAME).asText("") + : value); + row.compute(NcdReportContributorsCsvSchema.CLEAN_EMAIL_NAME, + (key, value) -> StringUtils.isBlank(value) + ? expressionInput.path(NcdReportContributorsCsvSchema.CLEAN_EMAIL_NAME).asText("") + : value); + row.compute(NcdReportContributorsCsvSchema.AUTHOR_ID, + (key, value) -> StringUtils.isBlank(value) ? computeAuthorId(expressionInput) : value); + } + + private NcdReportContributorHelper() {} +} \ No newline at end of file diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/reader/NcdReportReader.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/reader/NcdReportReader.java new file mode 100644 index 00000000000..102b52edf75 --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/reader/NcdReportReader.java @@ -0,0 +1,184 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.reader; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.MappingIterator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.csv.CsvMapper; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import com.fortify.cli.common.exception.FcliSimpleException; +import com.fortify.cli.common.exception.FcliTechnicalException; +import com.fortify.cli.license.ncd_report.config.NcdReportConfig; +import com.fortify.cli.license.ncd_report.config.NcdReportContributorConfig; +import com.fortify.cli.license.ncd_report.helper.NcdReportContributorHelper; + +import lombok.Getter; + +/** + * Read helper for NCD reports generated by the {@code ncd-report create} + * command. Supports both report directory and report zip input formats. + */ +public final class NcdReportReader implements AutoCloseable { + private static final CsvMapper CSV_MAPPER = new CsvMapper(); + private static final ObjectMapper YAML_MAPPER = createYamlMapper(); + + @Getter private final Path reportPath; + @Getter private final boolean directoryReport; + private final FileSystem fileSystem; + + public NcdReportReader(Path reportPath) { + this.reportPath = reportPath.toAbsolutePath(); + if ( !Files.exists(this.reportPath) ) { + throw new FcliSimpleException("Report path not found: %s", this.reportPath); + } + this.directoryReport = Files.isDirectory(this.reportPath); + this.fileSystem = directoryReport ? null : openZipFileSystem(this.reportPath); + } + + public Path entryPath(String entryName) { + if ( directoryReport ) { + return reportPath.resolve(entryName).normalize(); + } + var normalizedEntry = entryName.replace('\\', '/'); + return fileSystem.getPath(normalizedEntry); + } + + public BufferedReader bufferedReader(String entryName) { + var entryPath = entryPath(entryName); + if ( !Files.exists(entryPath) ) { + throw new FcliSimpleException("Report entry not found: %s in %s", entryName, reportPath); + } + try { + return Files.newBufferedReader(entryPath); + } catch ( IOException e ) { + throw new FcliTechnicalException(String.format("Error opening report entry %s in %s", entryName, reportPath), e); + } + } + + public NcdReportConfig readConfig() { + var configPath = entryPath("report-config.yaml"); + if ( !Files.exists(configPath) ) { + throw new FcliSimpleException("Report entry not found: report-config.yaml in %s", reportPath); + } + try ( var is = Files.newInputStream(configPath) ) { + return YAML_MAPPER.readValue(is, NcdReportConfig.class); + } catch ( Exception e ) { + throw new FcliTechnicalException(String.format("Error reading report config from %s", reportPath), e); + } + } + + public NcdReportContributorConfig readContributorConfig() { + var config = readConfig(); + return config.getContributor().orElseGet(NcdReportContributorConfig::new); + } + + public ObjectNode readSummary() { + var summaryPath = entryPath("summary.txt"); + if ( !Files.exists(summaryPath) ) { + throw new FcliSimpleException("Report entry not found: summary.txt in %s", reportPath); + } + try ( InputStream is = Files.newInputStream(summaryPath) ) { + return YAML_MAPPER.readValue(is, ObjectNode.class); + } catch ( Exception e ) { + throw new FcliTechnicalException(String.format("Error reading summary.txt from %s", reportPath), e); + } + } + + public List> readContributors() { + try ( var csvReader = bufferedReader("contributors.csv") ) { + var schema = CsvSchema.emptySchema().withHeader(); + MappingIterator> iterator = CSV_MAPPER + .readerFor(new TypeReference>() {}) + .with(schema) + .readValues(csvReader); + var result = new ArrayList>(); + while ( iterator.hasNext() ) { + var row = iterator.next(); + NcdReportContributorHelper.normalizeContributorRow(row); + result.add(row); + } + return result; + } catch ( Exception e ) { + throw new FcliSimpleException("Error reading contributors.csv from %s:\n\tMessage: %s", reportPath, e.getMessage()); + } + } + + public List listFileEntries() { + try ( var paths = fileEntryPathStream() ) { + return paths + .filter(Files::isRegularFile) + .map(this::toEntryName) + .sorted() + .collect(Collectors.toList()); + } catch ( IOException e ) { + throw new FcliTechnicalException(String.format("Error listing report entries for %s", reportPath), e); + } + } + + @Override + public void close() { + if ( fileSystem != null ) { + try { + fileSystem.close(); + } catch ( IOException e ) { + throw new FcliTechnicalException(String.format("Error closing report file system for %s", reportPath), e); + } + } + } + + private static FileSystem openZipFileSystem(Path zipPath) { + try { + return FileSystems.newFileSystem(zipPath); + } catch ( Exception e ) { + throw new FcliSimpleException("Invalid report zip file: %s", zipPath); + } + } + + private static ObjectMapper createYamlMapper() { + var mapper = new ObjectMapper(new YAMLFactory()); + mapper.registerModule(new Jdk8Module()); + mapper.registerModule(new JavaTimeModule()); + return mapper; + } + + private Stream fileEntryPathStream() throws IOException { + return directoryReport + ? Files.walk(reportPath) + : Files.walk(fileSystem.getPath("/")); + } + + private String toEntryName(Path filePath) { + if ( directoryReport ) { + return reportPath.relativize(filePath).toString().replace('\\', '/'); + } + return filePath.toString().replace('\\', '/').replaceFirst("^/+", ""); + } +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/validator/NcdReportValidator.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/validator/NcdReportValidator.java new file mode 100644 index 00000000000..fa5d20f1adf --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/validator/NcdReportValidator.java @@ -0,0 +1,131 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.validator; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +import com.fortify.cli.common.exception.FcliBugException; +import com.fortify.cli.common.exception.FcliSimpleException; +import com.fortify.cli.common.exception.FcliTechnicalException; +import com.fortify.cli.license.ncd_report.reader.NcdReportReader; + +/** + * Validates NCD report integrity, including checksum verification. + */ +public final class NcdReportValidator { + private static final Pattern CHECKSUM_LINE_PATTERN = Pattern.compile("^([0-9a-fA-F]{64})\\s+\\*?(.+)$"); + + /** + * Validate checksums in an NCD report, comparing stored checksums against current file content. + * @param reader NcdReportReader for the report to validate + * @return List of validation errors (empty if all valid) + * @throws FcliSimpleException if checksums.sha256 file not found or invalid format + * @throws FcliTechnicalException if checksum computation fails + */ + public static List validateChecksums(NcdReportReader reader) { + var errors = new ArrayList(); + var entryPath = reader.entryPath("checksums.sha256"); + + if ( !Files.exists(entryPath) ) { + throw new FcliSimpleException("Report integrity check failed: checksums.sha256 not found in %s", reader.getReportPath()); + } + + try { + var checksumsByFile = parseChecksumFile(entryPath); + var reportPath = reader.getReportPath(); + + for ( var entry : checksumsByFile.entrySet() ) { + var fileName = entry.getKey(); + var expectedChecksum = entry.getValue(); + var filePath = reader.entryPath(fileName); + + if ( !Files.exists(filePath) ) { + errors.add(String.format("Missing file: %s", fileName)); + } else { + var actualChecksum = sha256(filePath); + if ( !actualChecksum.equals(expectedChecksum) ) { + errors.add(String.format("Checksum mismatch for %s: expected %s, got %s", + fileName, expectedChecksum, actualChecksum)); + } + } + } + } catch ( FcliSimpleException | FcliTechnicalException e ) { + throw e; + } catch ( Exception e ) { + throw new FcliTechnicalException(String.format("Error validating checksums in %s", reader.getReportPath()), e); + } + + return errors; + } + + /** + * Parse checksums.sha256 file and return map of filename to checksum. + * Format: "HEXDIGEST *filename" or "HEXDIGEST filename" + * @param checksumsPath Path to checksums.sha256 file + * @return Map of filename to checksum + * @throws FcliSimpleException if file format is invalid + * @throws FcliTechnicalException if file read fails + */ + private static Map parseChecksumFile(java.nio.file.Path checksumsPath) { + var result = new HashMap(); + try { + var lines = Files.readAllLines(checksumsPath, StandardCharsets.UTF_8); + for ( var line : lines ) { + if ( line.isBlank() ) { + continue; + } + var matcher = CHECKSUM_LINE_PATTERN.matcher(line); + if ( !matcher.matches() ) { + throw new FcliSimpleException("Invalid line in checksums.sha256: %s", line); + } + var checksum = matcher.group(1); + var fileName = matcher.group(2); + result.put(fileName, checksum); + } + return result; + } catch ( FcliSimpleException | FcliTechnicalException e ) { + throw e; + } catch ( Exception e ) { + throw new FcliTechnicalException(String.format("Error reading checksums.sha256 from %s", checksumsPath), e); + } + } + + /** + * Compute SHA-256 checksum of a file. + * @param path Path to file + * @return Uppercase hex digest (64 characters) + * @throws FcliBugException if SHA-256 not available + * @throws FcliTechnicalException if file read fails + */ + public static String sha256(java.nio.file.Path path) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(Files.readAllBytes(path)); + return String.format("%064X", new BigInteger(1, hash)); + } catch ( NoSuchAlgorithmException e ) { + throw new FcliBugException("SHA-256 not available", e); + } catch ( Exception e ) { + throw new FcliTechnicalException(String.format("Error calculating checksum for %s", path), e); + } + } + + private NcdReportValidator() {} +} diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/INcdReportAuthorsWriter.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/INcdReportAuthorsWriter.java index 1bc2d47e86d..84810afa315 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/INcdReportAuthorsWriter.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/INcdReportAuthorsWriter.java @@ -14,8 +14,10 @@ import com.fortify.cli.license.ncd_report.descriptor.NcdReportProcessedAuthorDescriptor; -public interface INcdReportAuthorsWriter { +public interface INcdReportAuthorsWriter extends AutoCloseable { void writeIgnoredAuthor(NcdReportProcessedAuthorDescriptor descriptor); - void writeDuplicateAuthor(NcdReportProcessedAuthorDescriptor descriptor, int contributingAuthorNumber); + void writeDuplicateAuthor(NcdReportProcessedAuthorDescriptor descriptor, String representativeAuthorId, int contributingAuthorNumber); void writeContributor(NcdReportProcessedAuthorDescriptor descriptor, int contributingAuthorNumber); + @Override + void close(); } \ No newline at end of file diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/NcdReportAuthorsWriter.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/NcdReportAuthorsWriter.java index b4d6f2d7906..b551cbcffec 100644 --- a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/NcdReportAuthorsWriter.java +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/NcdReportAuthorsWriter.java @@ -12,6 +12,10 @@ */ package com.fortify.cli.license.ncd_report.writer; +import java.util.ArrayList; +import java.util.List; + +import com.fasterxml.jackson.databind.node.ObjectNode; import com.fortify.cli.common.json.JsonHelper; import com.fortify.cli.common.output.writer.record.IRecordWriter; import com.fortify.cli.common.output.writer.record.RecordWriterFactory; @@ -20,30 +24,42 @@ public final class NcdReportAuthorsWriter implements INcdReportAuthorsWriter { private final IRecordWriter recordWriter; + private final List buffer = new ArrayList<>(); public NcdReportAuthorsWriter(IReportWriter reportWriter) { this.recordWriter = reportWriter.recordWriter(RecordWriterFactory.csv, "contributors.csv", false, null); } - + @Override public void writeIgnoredAuthor(NcdReportProcessedAuthorDescriptor descriptor) { - write(descriptor, "ignored", -1); + write(descriptor, "ignored", ""); } - + @Override - public void writeDuplicateAuthor(NcdReportProcessedAuthorDescriptor descriptor, int contributingAuthorNumber) { - write(descriptor, "duplicate", contributingAuthorNumber); + public void writeDuplicateAuthor(NcdReportProcessedAuthorDescriptor descriptor, String representativeAuthorId, + int contributingAuthorNumber) { + write(descriptor, "duplicate", representativeAuthorId); } - + @Override public void writeContributor(NcdReportProcessedAuthorDescriptor descriptor, int contributingAuthorNumber) { - write(descriptor, "contributing", contributingAuthorNumber); + write(descriptor, "contributing", ""); } - - public void write(NcdReportProcessedAuthorDescriptor descriptor, String status, int contributingAuthorNumber) { - recordWriter.append(descriptor.updateReportRecord( - JsonHelper.getObjectMapper().createObjectNode()) + + private void write(NcdReportProcessedAuthorDescriptor descriptor, String status, String duplicateOf) { + var record = descriptor.updateReportRecord(JsonHelper.getObjectMapper().createObjectNode()) .put("contributionStatus", status) - .put("contributingAuthorNumber", contributingAuthorNumber)); + .put("duplicateOf", duplicateOf); + buffer.add(record); + } + + /** + * Close the writer by sorting all buffered records and writing them to the CSV. + * This method should be called after all records have been buffered. + */ + @Override + public void close() { + var sorted = NcdReportContributorsCsvSchema.sortByAuthorNameAndStatus(buffer); + sorted.forEach(recordWriter::append); } } diff --git a/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/NcdReportContributorsCsvSchema.java b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/NcdReportContributorsCsvSchema.java new file mode 100644 index 00000000000..0b7b2c7daa3 --- /dev/null +++ b/fcli-core/fcli-license/src/main/java/com/fortify/cli/license/ncd_report/writer/NcdReportContributorsCsvSchema.java @@ -0,0 +1,269 @@ +/* + * Copyright 2021-2026 Open Text. + * + * The only warranties for products and services of Open Text + * and its affiliates and licensors ("Open Text") are as may + * be set forth in the express warranty statements accompanying + * such products and services. Nothing herein should be construed + * as constituting an additional warranty. Open Text shall not be + * liable for technical or editorial errors or omissions contained + * herein. The information contained herein is subject to change + * without notice. + */ +package com.fortify.cli.license.ncd_report.writer; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Set; +import java.util.TreeMap; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.dataformat.csv.CsvSchema; + +/** + * Centralized definition of NCD report contributors.csv column schema. + * This ensures consistency across all commands that read or write contributors.csv. + */ +public final class NcdReportContributorsCsvSchema { + // Core author identity fields (immutable, always present) + public static final String AUTHOR_ID = "authorId"; + public static final String AUTHOR_NAME = "authorName"; + public static final String AUTHOR_EMAIL = "authorEmail"; + public static final String CLEAN_NAME = "cleanName"; + public static final String CLEAN_EMAIL_NAME = "cleanEmailName"; + + // Author state fields (immutable, set during create/merge) + public static final String AUTHOR_STATE = "authorState"; + public static final String AUTHOR_NUMBER = "authorNumber"; + + // Contribution status fields (immutable, set during create/merge) + public static final String CONTRIBUTION_STATUS = "contributionStatus"; + public static final String CONTRIBUTING_AUTHOR_NUMBER = "contributingAuthorNumber"; + + // Source report fields (immutable, from merged reports only) + public static final String SOURCE_REPORTS = "sourceReports"; + public static final String SOURCE_CONTRIBUTION_STATUS = "sourceContributionStatus"; + public static final String SOURCE_CONTRIBUTING_AUTHOR_NUMBER = "sourceContributingAuthorNumber"; + public static final String SOURCE_AUTHOR_ID = "sourceAuthorId"; + public static final String SOURCE_AUTHOR_STATE = "sourceAuthorState"; + public static final String SOURCE_AUTHOR_NUMBER = "sourceAuthorNumber"; + + // Merged report fields (immutable, from merged reports only) + public static final String MERGED_AUTHOR_ID = "mergedAuthorId"; + public static final String MERGED_AUTHOR_STATE = "mergedAuthorState"; + public static final String MERGED_AUTHOR_NUMBER = "mergedAuthorNumber"; + public static final String MERGED_CONTRIBUTION_STATUS = "mergedContributionStatus"; + public static final String MERGED_CONTRIBUTING_AUTHOR_NUMBER = "mergedContributingAuthorNumber"; + + // Updatable override fields + public static final String DUPLICATE_OF = "duplicateOf"; + public static final String OVERRIDE_STATUS = "overrideStatus"; + public static final String OVERRIDE_STATUS_CONFIDENCE = "overrideStatusConfidence"; + public static final String OVERRIDE_STATUS_NOTES = "overrideStatusNotes"; + + // All immutable fields (identity, state, and source/merge semantics) + public static final Set IMMUTABLE_FIELDS = Set.of( + AUTHOR_ID, AUTHOR_NAME, AUTHOR_EMAIL, CLEAN_NAME, CLEAN_EMAIL_NAME, + AUTHOR_STATE, AUTHOR_NUMBER, + CONTRIBUTION_STATUS, CONTRIBUTING_AUTHOR_NUMBER, + SOURCE_REPORTS, SOURCE_CONTRIBUTION_STATUS, SOURCE_CONTRIBUTING_AUTHOR_NUMBER, + SOURCE_AUTHOR_ID, SOURCE_AUTHOR_STATE, SOURCE_AUTHOR_NUMBER, + MERGED_AUTHOR_ID, MERGED_AUTHOR_STATE, MERGED_AUTHOR_NUMBER, + MERGED_CONTRIBUTION_STATUS, MERGED_CONTRIBUTING_AUTHOR_NUMBER + ); + + // All updatable fields + public static final Set UPDATABLE_FIELDS = Set.of( + DUPLICATE_OF, + OVERRIDE_STATUS, + OVERRIDE_STATUS_CONFIDENCE, + OVERRIDE_STATUS_NOTES + ); + + // Column order for CSV output + private static final List COLUMN_ORDER = Arrays.asList( + AUTHOR_ID, + AUTHOR_NAME, + AUTHOR_EMAIL, + CLEAN_NAME, + CLEAN_EMAIL_NAME, + AUTHOR_STATE, + CONTRIBUTION_STATUS, + SOURCE_REPORTS, + SOURCE_CONTRIBUTION_STATUS, + SOURCE_AUTHOR_ID, + SOURCE_AUTHOR_STATE, + MERGED_AUTHOR_ID, + MERGED_AUTHOR_STATE, + MERGED_CONTRIBUTION_STATUS, + DUPLICATE_OF, + OVERRIDE_STATUS_CONFIDENCE, + OVERRIDE_STATUS_NOTES, + OVERRIDE_STATUS + ); + + /** + * Build a CsvSchema with all defined columns in the correct order. + * Only includes columns that are actually present in the provided list. + * @param presentColumns Set of column names that should be included in the schema + * @return CsvSchema with header and specified columns + */ + public static CsvSchema buildSchema(Set presentColumns) { + var builder = CsvSchema.builder(); + for ( var column : COLUMN_ORDER ) { + if ( presentColumns.contains(column) ) { + builder.addColumn(column); + } + } + return builder.build().withUseHeader(true); + } + + /** + * Build a CsvSchema with all known columns in the correct order. + * Used when writing a complete contributors.csv with all possible fields. + * @return CsvSchema with header and all columns + */ + public static CsvSchema buildFullSchema() { + var builder = CsvSchema.builder(); + for ( var column : COLUMN_ORDER ) { + builder.addColumn(column); + } + return builder.build().withUseHeader(true); + } + + /** + * Return the list of columns written to contributors.csv output. + */ + public static List getOutputColumns() { + return COLUMN_ORDER; + } + + /** + * Sort contributors for consistent CSV output using a two-phase approach. + * Phase 1: Collect and organize records by status (contributing, duplicate, ignored). + * Phase 2: Build result by iterating contributing records (sorted by name), + * adding each record's duplicates immediately after, then all ignored records at end. + * This sorting is necessary for legacy reports and improves readability. + */ + public static List sortByAuthorNameAndStatus(List contributors) { + var nameComparator = Comparator.comparing( + (ObjectNode r) -> r.path(AUTHOR_NAME).asText("").toLowerCase()); + + var contributing = new ArrayList(); + var duplicates = new ArrayList(); + var ignored = new ArrayList(); + splitByStatus(contributors, contributing, duplicates, ignored); + + var contributingByAuthorId = indexContributingByAuthorId(contributing); + var contributingByNumber = indexContributingByNumber(contributing); + var duplicateByRepId = new TreeMap>(); + var unmatchedDuplicates = new ArrayList(); + groupDuplicates(duplicates, duplicateByRepId, unmatchedDuplicates, contributingByAuthorId, contributingByNumber); + + // Sort each group by author name + contributing.sort(nameComparator); + duplicateByRepId.values().forEach(list -> list.sort(nameComparator)); + unmatchedDuplicates.sort(nameComparator); + ignored.sort(nameComparator); + + // Phase 2: Build result: contributing + their duplicates + unmatched duplicates + ignored + var result = new ArrayList(); + for ( var contribRecord : contributing ) { + result.add(contribRecord); + var repId = contribRecord.path(AUTHOR_ID).asText(""); + if ( duplicateByRepId.containsKey(repId) ) { + result.addAll(duplicateByRepId.get(repId)); + } + } + result.addAll(unmatchedDuplicates); + result.addAll(ignored); + + return result; + } + + private static void splitByStatus(List contributors, List contributing, + List duplicates, List ignored) + { + for ( var record : contributors ) { + var status = record.path(CONTRIBUTION_STATUS).asText("").toLowerCase(); + if ( "contributing".equals(status) ) { + contributing.add(record); + } else if ( "duplicate".equals(status) ) { + duplicates.add(record); + } else if ( "ignored".equals(status) ) { + ignored.add(record); + } + } + } + + private static TreeMap indexContributingByAuthorId(List contributing) { + var result = new TreeMap(); + for ( var record : contributing ) { + var authorId = record.path(AUTHOR_ID).asText("").trim(); + if ( !authorId.isBlank() ) { + result.put(authorId, record); + } + } + return result; + } + + private static TreeMap indexContributingByNumber(List contributing) { + var result = new TreeMap(); + for ( var record : contributing ) { + var number = parsePositiveInt(record.path(CONTRIBUTING_AUTHOR_NUMBER).asText("")); + if ( number > 0 ) { + result.putIfAbsent(number, record); + } + } + return result; + } + + private static void groupDuplicates(List duplicates, TreeMap> duplicateByRepId, + List unmatchedDuplicates, TreeMap contributingByAuthorId, + TreeMap contributingByNumber) + { + for ( var duplicate : duplicates ) { + var representativeAuthorId = resolveRepresentativeAuthorId(duplicate, contributingByAuthorId, contributingByNumber); + if ( representativeAuthorId == null ) { + unmatchedDuplicates.add(duplicate); + continue; + } + duplicate.put(DUPLICATE_OF, representativeAuthorId); + duplicateByRepId.computeIfAbsent(representativeAuthorId, k -> new ArrayList<>()).add(duplicate); + } + } + + private static String resolveRepresentativeAuthorId(ObjectNode duplicate, TreeMap contributingByAuthorId, + TreeMap contributingByNumber) + { + var duplicateOf = duplicate.path(DUPLICATE_OF).asText("").trim(); + if ( !duplicateOf.isBlank() && contributingByAuthorId.containsKey(duplicateOf) ) { + return duplicateOf; + } + + var contributingAuthorNumber = parsePositiveInt(duplicate.path(CONTRIBUTING_AUTHOR_NUMBER).asText("")); + if ( contributingAuthorNumber <= 0 ) { + return null; + } + + var representative = contributingByNumber.get(contributingAuthorNumber); + if ( representative == null ) { + return null; + } + var authorId = representative.path(AUTHOR_ID).asText("").trim(); + return authorId.isBlank() ? null : authorId; + } + + private static int parsePositiveInt(String value) { + try { + var parsed = Integer.parseInt(value.trim()); + return parsed > 0 ? parsed : -1; + } catch ( NumberFormatException e ) { + return -1; + } + } + + private NcdReportContributorsCsvSchema() {} +} diff --git a/fcli-core/fcli-license/src/main/resources/com/fortify/cli/license/i18n/LicenseMessages.properties b/fcli-core/fcli-license/src/main/resources/com/fortify/cli/license/i18n/LicenseMessages.properties index 289ac27ed74..0061d803686 100644 --- a/fcli-core/fcli-license/src/main/resources/com/fortify/cli/license/i18n/LicenseMessages.properties +++ b/fcli-core/fcli-license/src/main/resources/com/fortify/cli/license/i18n/LicenseMessages.properties @@ -42,6 +42,29 @@ fcli.license.ncd-report.create.usage.description.3 = The generated 'checksums.sh fcli.license.ncd-report.create.config = Configuration file; sample can be generated using the 'create-config' command. fcli.license.ncd-report.create.confirm = Confirm delete of existing report output location. fcli.license.ncd-report.create.confirmPrompt = Confirm delete of existing output location %s? +fcli.license.ncd-report.create.end-date = Report end date (yyyy-MM-dd). The 90-day reporting window ends on this date (inclusive). Defaults to today, producing a standard current-period report. +fcli.license.ncd-report.merge.usage.header = Merge multiple NCD reports into a single combined report +fcli.license.ncd-report.merge.usage.description = This command can be used to merge multiple NCD reports into a single combined report with cross-report deduplication.\n\nAs an example, if you have individual teams generate separate NCD reports and a developer moves from one team to another, that developer may be counted as a contributing developer in both team reports, thus counted twice when manually combining the counts. Merging the reports will deduplicate the developer and count them only once in the merged report. +fcli.license.ncd-report.merge.reports = Source report paths (comma-separated or repeated); each entry may be either a report zip file or report directory generated by the `create` or `merge` commands. +fcli.license.ncd-report.merge.confirm = Confirm delete of existing report output location. +fcli.license.ncd-report.merge.confirmPrompt = Confirm delete of existing output location %s? +fcli.license.ncd-report.list-contributors.usage.header = List contributors from an NCD report +fcli.license.ncd-report.list-contributors.usage.description.0 = This command lists contributors from an NCD report for manual identity review, contribution status verification, and contribution status updates. +fcli.license.ncd-report.list-contributors.usage.description.1 = \nTo prepare contribution status updates (manually or AI-assisted), generate output in your preferred output format (JSON, CSV, or YAML) using the `-o/--output` option, then update any of the following fields for the rows for which contribution status needs to be modified: \ + \n- `overrideStatus`: Can be `contributing`, `duplicate`, or `ignored` \ + \n- `overrideStatusConfidence`: Confidence level (0.0-1.0) for the override \ + \n- `overrideStatusNotes`: Additional notes for the override \ + \n- `duplicateOf`: Required if `overrideStatus` is `duplicate` \ + \n\nEach row to be updated must include the original `authorId` field. Other fields may be omitted; any changed contents in those fields will be ignored with a warning. +fcli.license.ncd-report.list-contributors.usage.description.2 = \nFeed the edited output into the `update-contributor-status` command (alias `ucs`) to apply the updates to the report. +fcli.license.ncd-report.list-contributors.report = Report path (zip file or directory). +fcli.license.ncd-report.list-contributors.include = Include contributors with specified status (duplicates, ignored). +fcli.license.ncd-report.list-contributors.output.table.args = authorId,authorName,authorEmail,contributionStatus,duplicateOf +fcli.license.ncd-report.update-contributor-status.usage.header = Update contribution status in an NCD report +fcli.license.ncd-report.update-contributor-status.usage.description = This command takes the edited output of the `list-contributors` command and applies requested contribution status updates to an existing NCD report. Please see the `list-contributors` command for details on how to prepare the updates. +fcli.license.ncd-report.update-contributor-status.report = Report path (zip file or directory). +fcli.license.ncd-report.update-contributor-status.contributors = Contributor updates file in CSV (with header row), JSON (object or array of objects), or YAML (object or array of objects) format. If omitted, updates are read from stdin. Input should use the `list-contributors` output format (typically exported with `-o json|csv|yaml`) and each update row should include `authorId`. +fcli.license.ncd-report.update-contributor-status.min-confidence = Minimum confidence threshold (0.0-1.0); records below this value are warned and skipped (default: 0.8). fcli.license.ncd-report.create-config.usage.header = Generate a sample configuration file for use by the 'generate' command. fcli.license.ncd-report.create-config.config = Name of the sample configuration file to be generated. fcli.license.ncd-report.create-config.confirm = Confirm overwrite of existing configuration file. @@ -62,4 +85,5 @@ report-zip = Write report output to the given zip-file. fcli.license.msp-report.create.output.table.args = reportPath,summary.errorCount fcli.license.msp-report.create-config.output.table.args = path fcli.license.ncd-report.create.output.table.args = reportPath,summary.errorCount -fcli.license.ncd-report.create-config.output.table.args = path \ No newline at end of file +fcli.license.ncd-report.create-config.output.table.args = path +fcli.license.ncd-report.merge.output.table.args = reportPath,summary.authorCount.contributing \ No newline at end of file diff --git a/fcli-other/fcli-functional-test/ReportTemplateConfig.yml b/fcli-other/fcli-functional-test/ReportTemplateConfig.yml deleted file mode 100644 index 590cfd0c5a2..00000000000 --- a/fcli-other/fcli-functional-test/ReportTemplateConfig.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -name: Change-Me # Required: Report template name -description: Change-Me # Optional: Report template description -type: ISSUE # Required: ISSUE, PROJECT or PORTFOLIO -parameters: -- name: Change-Me0 # Required: Report parameter name - description: Change-Me0 # Optional: Report parameter description - identifier: Change-Me0 # Required: ID/name of the report parameter in the BIRT template. - type: SINGLE_SELECT_DEFAULT # Required: BOOLEAN, MULTI_PROJECT, PROJECT_ATTRIBUTE, SINGLE_PROJECT, SINGLE_SELECT_DEFAULT, STRING, or DATE - paramOrder: 0 # Optional: Report parameter ordering value - reportParameterOptions: # Required for parameter type "SINGLE_SELECT_DEFAULT", not allowed for other parameter types - - defaultValue: true # Required: Boolean indicating whether this is the default parameter option - description: Change-Me1 # Optional: Report parameter option description - displayValue: Change-Me1 # Required: Report parameter display value as shown in SSC UI - reportValue: Change-Me1 # Required: The value to be passed to the BIRT template - - defaultValue: false # See above - description: Change-Me2 # See above - displayValue: Change-Me2 # See above - reportValue: Change-Me2 # See above -- name: Change-Me3 # Repeated section above for additional report parameter but different type - paramOrder: 1 - description: Change-Me3 - identifier: Change-Me3 - type: STRING diff --git a/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/license/NcdReportSpec.groovy b/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/license/NcdReportSpec.groovy index 9a9ef923cdc..e22ce8cb886 100644 --- a/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/license/NcdReportSpec.groovy +++ b/fcli-other/fcli-functional-test/src/ftest/groovy/com/fortify/cli/ftest/license/NcdReportSpec.groovy @@ -28,6 +28,53 @@ class NcdReportSpec extends FcliBaseSpec { @Shared @TestResource("runtime/report/ncd-report.yml") String configFile; @Shared @TempDir("ncd-report") String reportOutputDir; @Shared @TempFile("ncd-report.zip") String reportOutputZip; + + private String tempPath(String relativePath) { + return new File(mockReportDir, relativePath).absolutePath + } + + private static String sha256Hex(File file) { + def digest = java.security.MessageDigest.getInstance("SHA-256") + digest.update(java.nio.file.Files.readAllBytes(file.toPath())) + return digest.digest().collect { String.format("%02X", it) }.join("") + } + + private static void updateChecksum(String reportDir, String entryName) { + def checksumsFile = new File("${reportDir}/checksums.sha256") + def entryFile = new File("${reportDir}/${entryName}") + def checksum = sha256Hex(entryFile) + def lines = checksumsFile.readLines() + def updated = [] + def found = false + lines.each { line -> + def parts = line.split(/\s+/, 2) + if ( parts.size() >= 2 ) { + def filename = parts[1].startsWith("*") ? parts[1].substring(1) : parts[1] + if ( filename == entryName ) { + updated << "${checksum} ${entryName}" + found = true + } else { + updated << line + } + } else { + updated << line + } + } + if ( !found ) { + updated << "${checksum} ${entryName}" + } + checksumsFile.text = updated.join("\n") + "\n" + } + + private static void addLegacyNumberColumnsToContributorsCsv(String reportDir) { + def contributorsFile = new File("${reportDir}/contributors.csv") + def lines = contributorsFile.readLines() + def updated = [] + updated << (lines[0] + ",authorNumber,contributingAuthorNumber") + lines.drop(1).each { updated << (it + ",-1,-1") } + contributorsFile.text = updated.join("\n") + "\n" + updateChecksum(reportDir, "contributors.csv") + } def "generate-config"() { def args = "license ncd-report create-config -y -c ${sampleConfigOutputFile} -o yaml" @@ -82,4 +129,295 @@ class NcdReportSpec extends FcliBaseSpec { it.any { it.contains("logCounts:") } } } + + // ===== Mock Source Tests ===== + + @Shared @TestResource("runtime/report/ncd-report-mock.yml") String mockConfigFile; + @Shared @TestResource("runtime/report/mock-authors.json") String mockAuthorsJson; + @Shared @TestResource("runtime/report/mock-authors.csv") String mockAuthorsCsv; + @Shared @TempDir("ncd-report-mock") String mockReportDir; + @Shared @TempFile("ncd-report-mock.zip") String mockReportZip; + + def "mock-generate-dir"() { + def args = "license ncd-report create -y -c ${mockConfigFile} -d ${mockReportDir}" + when: + def result = Fcli.run(args) + then: + new File("${mockReportDir}/summary.txt").exists() + new File("${mockReportDir}/contributors.csv").exists() + new File("${mockReportDir}/checksums.sha256").exists() + def contributorHeader = new File("${mockReportDir}/contributors.csv").readLines().first() + !contributorHeader.contains("authorNumber") + !contributorHeader.contains("contributingAuthorNumber") + verifyAll(result.stdout) { + it.any { it == "reportPath: ${mockReportDir}" } + it.any { it == ' reportType: Number of Contributing Developers (NCD) Report' } + it.any { it.contains("authorCount:") } + it.any { it.contains("commitCount:") } + } + } + + def "mock-generate-with-end-date"() { + def args = "license ncd-report create -y -c ${mockConfigFile} -d ${mockReportDir}-enddate --end-date 2026-06-01" + when: + def result = Fcli.run(args) + then: + new File("${mockReportDir}-enddate/summary.txt").exists() + verifyAll(result.stdout) { + it.any { it == "reportPath: ${mockReportDir}-enddate" } + it.any { it.contains("authorCount:") } + } + } + + def "mock-list-contributors"() { + def reportDir = tempPath("ncd-report-list-contributors") + def createArgs = "license ncd-report create -y -c ${mockConfigFile} -d ${reportDir}" + def listArgs = "license ncd-report list-contributors -r ${reportDir}" + when: + Fcli.run(createArgs) + def result = Fcli.run(listArgs) + then: + result.stdout.size() > 0 + result.stdout.any { it.contains("Author name") || it.contains("Author email") } + } + + def "mock-list-contributors-json"() { + def reportDir = tempPath("ncd-report-list-contributors-json") + def createArgs = "license ncd-report create -y -c ${mockConfigFile} -d ${reportDir}" + def listArgs = "license ncd-report list-contributors -r ${reportDir} -o json" + when: + Fcli.run(createArgs) + def result = Fcli.run(listArgs) + then: + result.stdout.any { it.contains("authorId") } + result.stdout.any { it.contains("authorName") } + result.stdout.any { it.contains("contributionStatus") } + result.stdout.any { it.contains("duplicateOf") } + } + + def "mock-list-contributors-csv"() { + def reportDir = tempPath("ncd-report-list-contributors-csv") + def createArgs = "license ncd-report create -y -c ${mockConfigFile} -d ${reportDir}" + def listArgs = "license ncd-report list-contributors -r ${reportDir} -o csv" + when: + Fcli.run(createArgs) + def result = Fcli.run(listArgs) + then: + result.stdout[0].contains("authorId") + result.stdout[0].contains("authorName") + result.stdout[0].contains("contributionStatus") + result.stdout[0].contains("duplicateOf") + !result.stdout[0].contains("authorNumber") + !result.stdout[0].contains("contributingAuthorNumber") + result.stdout.size() > 2 // Header + at least one author + } + + def "mock-legacy-number-columns-accepted-by-lsc-merge-update"() { + def report1 = tempPath("ncd-report-legacy-source-1") + def report2 = tempPath("ncd-report-legacy-source-2") + def mergedReport = tempPath("ncd-report-legacy-merged") + def listCsv = tempPath("ncd-report-legacy-list.csv") + + when: + Fcli.run("license ncd-report create -y -c ${mockConfigFile} -d ${report1}") + Fcli.run("license ncd-report create -y -c ${mockConfigFile} -d ${report2}") + + // Inject legacy numeric columns into contributors.csv and update checksums, + // simulating reports produced by older fcli versions. + addLegacyNumberColumnsToContributorsCsv(report1) + addLegacyNumberColumnsToContributorsCsv(report2) + + def lscResult = Fcli.run("license ncd-report list-contributors -r ${report1} -o csv --to-file ${listCsv}") + def updateResult = Fcli.run("license ncd-report update-contributor-status -r ${report1} -c ${listCsv}") + def mergeResult = Fcli.run("license ncd-report merge -r ${report1},${report2} -d ${mergedReport} -y") + + def mergedHeader = new File("${mergedReport}/contributors.csv").readLines().first() + then: + new File(listCsv).exists() + lscResult.exitCode == 0 + updateResult.exitCode == 0 + mergeResult.exitCode == 0 + !mergedHeader.contains("authorNumber") + !mergedHeader.contains("contributingAuthorNumber") + } + + def "mock-merge"() { + def config1 = tempPath("ncd-report-mock-1.yml") + def config2 = tempPath("ncd-report-mock-2.yml") + def report1 = tempPath("ncd-report-mock-1") + def report2 = tempPath("ncd-report-mock-2") + def mergedReport = tempPath("ncd-report-merged") + + when: + // Create two separate reports to merge + def createArgs1 = "license ncd-report create -y -c ${mockConfigFile} -d ${report1}" + def result1 = Fcli.run(createArgs1) + + def createArgs2 = "license ncd-report create -y -c ${mockConfigFile} -d ${report2}" + def result2 = Fcli.run(createArgs2) + + // Merge the two reports + def mergeArgs = "license ncd-report merge -r ${report1},${report2} -d ${mergedReport} -y" + def mergeResult = Fcli.run(mergeArgs) + def mergedLines = new File("${mergedReport}/contributors.csv").readLines() + def headerCols = mergedLines.first().split(',', -1) + def statusIndex = headerCols.findIndexOf { it == 'contributionStatus' } + def sourceReportsIndex = headerCols.findIndexOf { it == 'sourceReports' } + def ignoredCount = mergedLines.drop(1).count { row -> + def cols = row.split(',', -1) + statusIndex >= 0 && cols.size() > statusIndex && cols[statusIndex] == 'ignored' + } + then: + new File("${mergedReport}/summary.txt").exists() + new File("${mergedReport}/contributors.csv").exists() + mergeResult.stdout.any { it.contains("mergedReportCount: 2") } + sourceReportsIndex >= 0 + ignoredCount >= 4 + } + + def "mock-update-from-list-output"() { + def report1 = tempPath("ncd-report-update-source") + def tmpListOutput = tempPath("ncd-contributors-list.csv") + + when: + // Create a report for list + def createArgs = "license ncd-report create -y -c ${mockConfigFile} -d ${report1}" + Fcli.run(createArgs) + + // List contributors to CSV + def listArgs = "license ncd-report list-contributors -r ${report1} -o csv --to-file ${tmpListOutput}" + Fcli.run(listArgs) + + // Update the same report with the list output + def updateArgs = "license ncd-report update-contributor-status -r ${report1} -c ${tmpListOutput}" + Fcli.run(updateArgs) + then: + new File(tmpListOutput).exists() + } + + def "mock-list-contributors-realistic-names"() { + def reportDir = tempPath("ncd-report-realistic-names") + def createArgs = "license ncd-report create -y -c ${mockConfigFile} -d ${reportDir}" + def listArgs = "license ncd-report list-contributors -r ${reportDir} -o table" + when: + Fcli.run(createArgs) + def result = Fcli.run(listArgs) + then: + // Should contain realistic author names like "John Smith", "Sarah Johnson", etc. + result.stdout.any { it.contains("Smith") || it.contains("Johnson") || it.contains("Chen") || it.contains("Williams") } + } + + def "mock-detect-duplicates"() { + def duplicateReportDir = tempPath("ncd-report-duplicates") + def tmpListOutput = tempPath("duplicates-list.json") + + when: + // Create report with realistic data (which includes duplicates) + def createArgs = "license ncd-report create -y -c ${mockConfigFile} -d ${duplicateReportDir}" + Fcli.run(createArgs) + + // List all contributors to see duplicates + def listArgs = "license ncd-report list-contributors -r ${duplicateReportDir} -o json --to-file ${tmpListOutput}" + Fcli.run(listArgs) + then: + new File(tmpListOutput).exists() + } + + def "mock-update-ai-duplicates"() { + def reportPath = tempPath("ncd-report-ai-duplicates") + def updateData = tempPath("ai-duplicates.json") + + when: + // Create report + def createArgs = "license ncd-report create -y -c ${mockConfigFile} -d ${reportPath}" + Fcli.run(createArgs) + + def contributorLines = new File("${reportPath}/contributors.csv").readLines().drop(1) + def authorIds = contributorLines.collect { it.split(',', -1)[0] }.findAll { it }.unique() + + // Create update data with AI-detected duplicates + // Format: authorId pairs where AI thinks they're the same person + new File(updateData).text = '''[ + { + "authorId": "''' + authorIds[0] + '''", + "duplicateOf": "''' + authorIds[1] + '''", + "overrideStatusConfidence": "0.95" + }, + { + "authorId": "''' + authorIds[2] + '''", + "duplicateOf": "''' + authorIds[3] + '''", + "overrideStatusConfidence": "0.85" + } +]''' + + // Try to update (may not find exact matches in generated data, but validates the command) + def updateArgs = "license ncd-report update-contributor-status -r ${reportPath} -c ${updateData}" + Fcli.run(updateArgs) + then: + new File(reportPath).exists() + } + + def "mock-datafile-json"() { + def reportDir = tempPath("ncd-report-with-json-data") + def configYaml = tempPath("ncd-report-json-data.yml") + def mockDataFile = mockAuthorsJson + + when: + // Create config that references JSON data file + new File(configYaml).text = """ +|contributor: +| ignoreExpression: > +| lcName matches '.*\\[bot\\]' +| duplicateExpression: > +| a1.cleanName==a2.cleanName || +| a1.cleanEmailName==a2.cleanEmailName || +| a1.cleanName==a2.cleanEmailName +| +|sources: +| mock: +| - repositoryCount: 1 +| authorsPerRepository: 2 +| commitsPerAuthor: 3 +| dataFile: "${mockDataFile}" +""".stripMargin() + + def createArgs = "license ncd-report create -y -c ${configYaml} -d ${reportDir}" + def result = Fcli.run(createArgs) + then: + new File("${reportDir}/summary.txt").exists() + new File("${reportDir}/contributors.csv").exists() + result.stdout.any { it.contains("reportPath") } + } + + def "mock-datafile-csv"() { + def reportDir = tempPath("ncd-report-with-csv-data") + def configYaml = tempPath("ncd-report-csv-data.yml") + def mockDataFile = mockAuthorsCsv + + when: + // Create config that references CSV data file + new File(configYaml).text = """ +|contributor: +| ignoreExpression: > +| lcName matches '.*\\[bot\\]' +| duplicateExpression: > +| a1.cleanName==a2.cleanName || +| a1.cleanEmailName==a2.cleanEmailName || +| a1.cleanName==a2.cleanEmailName +| +|sources: +| mock: +| - repositoryCount: 1 +| authorsPerRepository: 2 +| commitsPerAuthor: 3 +| dataFile: "${mockDataFile}" +""".stripMargin() + + def createArgs = "license ncd-report create -y -c ${configYaml} -d ${reportDir}" + def result = Fcli.run(createArgs) + then: + new File("${reportDir}/summary.txt").exists() + new File("${reportDir}/contributors.csv").exists() + result.stdout.any { it.contains("reportPath") } + } } diff --git a/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/mock-authors.csv b/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/mock-authors.csv new file mode 100644 index 00000000000..2eb4dc6ca05 --- /dev/null +++ b/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/mock-authors.csv @@ -0,0 +1,7 @@ +name,email +"John Smith","john.smith@example.com" +"Sarah Johnson","sarah.j@example.com" +"Michael Chen","m.chen@example.com" +"J. Smith","john.smith@example.com" +"Sarah Johnson","sarah@company.com" +"Michael C.","mchen@example.com" diff --git a/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/mock-authors.json b/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/mock-authors.json new file mode 100644 index 00000000000..d907a6789c8 --- /dev/null +++ b/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/mock-authors.json @@ -0,0 +1,26 @@ +[ + { + "name": "John Smith", + "email": "john.smith@example.com" + }, + { + "name": "Sarah Johnson", + "email": "sarah.j@example.com" + }, + { + "name": "Michael Chen", + "email": "m.chen@example.com" + }, + { + "name": "J. Smith", + "email": "john.smith@example.com" + }, + { + "name": "Sarah Johnson", + "email": "sarah@company.com" + }, + { + "name": "Michael C.", + "email": "mchen@example.com" + } +] diff --git a/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/ncd-report-mock.yml b/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/ncd-report-mock.yml new file mode 100644 index 00000000000..217e422f578 --- /dev/null +++ b/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/ncd-report-mock.yml @@ -0,0 +1,19 @@ +# Configuration file for testing the NCD license reporting functionality with mock data +# This uses the built-in mock source for testing without requiring external API access + +contributor: + ignoreExpression: > + lcName matches '.*\[bot\]' + + duplicateExpression: > + a1.cleanName==a2.cleanName || + a1.cleanEmailName==a2.cleanEmailName || + a1.cleanName==a2.cleanEmailName + +sources: + mock: + - repositoryCount: 3 + # Use enough authors to include built-in bot accounts from MockAuthorData, + # so ignoreExpression behavior is exercised in generated reports. + authorsPerRepository: 8 + commitsPerAuthor: 10 diff --git a/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/ncd-report.yml b/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/ncd-report.yml index 1ea7f5eb545..b3187568715 100644 --- a/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/ncd-report.yml +++ b/fcli-other/fcli-functional-test/src/ftest/resources/runtime/report/ncd-report.yml @@ -15,13 +15,11 @@ sources: # the following (classic) scopes: read:org, read:user, repo tokenExpression: > #env("FCLI_FT_GITHUB_TOKEN") - repositoryIncludeExpression: > - topics.contains("fortify-integration-sample") - organizations: + organizations: - name: fortify - - name: fod-dev + # Limit to a single repository for faster and more stable CI runs repositoryIncludeExpression: > - true + full_name=="fortify/IWA-Java" gitlab: - baseUrl: https://gitlab.com @@ -29,7 +27,8 @@ sources: # and require the following scopes: read_api, read_repository tokenExpression: > #env("FCLI_FT_GITLAB_TOKEN") - repositoryIncludeExpression: > - topics.contains("vulnerable-sample-app") - groups: - - id: Fortify \ No newline at end of file + groups: + - id: Fortify + # Limit to a single repository for faster and more stable CI runs + repositoryIncludeExpression: > + path_with_namespace=="Fortify/example-eightball" \ No newline at end of file From b7c1b8401fdcb14902ec6160c638be04ced67272 Mon Sep 17 00:00:00 2001 From: Ruud Senden <8635138+rsenden@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:00:05 +0200 Subject: [PATCH 10/10] fix: FoD `release-summary` action: Fix exception on releases with no open vulnerabilities --- .../cli/fod/actions/zip/release-summary.yaml | 25 ++----------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/release-summary.yaml b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/release-summary.yaml index 63057d80deb..e49d028f8e5 100644 --- a/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/release-summary.yaml +++ b/fcli-core/fcli-fod/src/main/resources/com/fortify/cli/fod/actions/zip/release-summary.yaml @@ -43,24 +43,6 @@ steps: - var.set: scanType: ${scan.scanType} ossScanDate: ${scan.completedDateTime} -### -# Note: reverted to retrieving OSS counts from the Vulnerabilities API to support prior FoD releases prior to 24.4 -### - - log.progress: Loading Vulnerabilities - - rest.call: - issues: - if: ${ossScanDate!=null} - uri: /api/v3/releases/${r.releaseId}/vulnerabilities?limit=1 - query: - filters: category:Open Source - on.success: - - var.set: - ossTotal: ${issues_raw.totalCount} - ossCritical: ${issues_raw.filters.^[#this.fieldName == 'severity']?.fieldFilterValues?.^[#this.value == "Critical"]?.count?:0} - ossHigh: ${issues_raw.filters.^[#this.fieldName == 'severity']?.fieldFilterValues?.^[#this.value == "High"]?.count?:0} - ossMedium: ${issues_raw.filters.^[#this.fieldName == 'severity']?.fieldFilterValues?.^[#this.value == "Medium"]?.count?:0} - ossLow: ${issues_raw.filters.^[#this.fieldName == 'severity']?.fieldFilterValues?.^[#this.value == "Low"]?.count?:0} - ### - out.write: ${cli.file}: {fmt: summary-md} - if: ${!{'stdout','stderr'}.contains(cli.file)} @@ -84,9 +66,6 @@ formatters: | **Static** | ${(#isBlank(r.staticScanDate)?#fmt('%-16s', 'N/A'):#formatDateTime(dateFmt, r.staticScanDate)) +' | '+#fmt('%8s', r.staticCritical) +' | '+#fmt('%8s', r.staticHigh) +' | '+#fmt('%8s', r.staticMedium) +' | '+#fmt('%8s', r.staticLow) +' |'} | **Dynamic** | ${(#isBlank(r.dynamicScanDate)?#fmt('%-16s', 'N/A'):#formatDateTime(dateFmt, r.dynamicScanDate))+' | '+#fmt('%8s', r.dynamicCritical) +' | '+#fmt('%8s', r.dynamicHigh) +' | '+#fmt('%8s', r.dynamicMedium) +' | '+#fmt('%8s', r.dynamicLow) +' |'} | **Mobile** | ${(#isBlank(r.mobileScanDate)?#fmt('%-16s', 'N/A'):#formatDateTime(dateFmt, r.mobileScanDate)) +' | '+#fmt('%8s', r.mobileCritical) +' | '+#fmt('%8s', r.mobileHigh) +' | '+#fmt('%8s', r.mobileMedium) +' | '+#fmt('%8s', r.mobileLow) +' |'} - | **Open Source** | ${(#isBlank(ossScanDate)?#fmt('%-16s', 'N/A'):#formatDateTime(dateFmt, ossScanDate)) +' | '+#fmt('%8s', (ossCritical!=null?ossCritical:0)) +' | '+#fmt('%8s', (ossHigh!=null?ossHigh:0)) +' | '+#fmt('%8s', (ossMedium!=null?ossMedium:0)) +' | '+#fmt('%8s', (ossLow!=null?ossLow:0)) +' |'} - | **Total** | | ${#fmt('%8s', r.staticCritical+r.dynamicCritical+r.mobileCritical+(ossCritical!=null?ossCritical:0))+' | '+#fmt('%8s', r.staticHigh+r.dynamicHigh+r.mobileHigh+(ossHigh!=null?ossHigh:0))+' | '+#fmt('%8s', r.staticMedium+r.dynamicMedium+r.mobileMedium+(ossMedium!=null?ossMedium:0))+' | '+#fmt('%8s', r.staticLow+r.dynamicLow+r.mobileLow+(ossLow!=null?ossLow:0))+' |'} -# | **Open Source** | ${(#isBlank(ossScanDate)?#fmt('%-16s', 'N/A'):#formatDateTime(dateFmt, ossScanDate)) +' | '+#fmt('%8s', r.openSourceCritical) +' | '+#fmt('%8s', r.openSourceHigh) +' | '+#fmt('%8s', r.openSourceMedium) +' | '+#fmt('%8s', r.openSourceLow) +' |'} -# | **Total** | | ${#fmt('%8s', r.staticCritical+r.dynamicCritical+r.mobileCritical+r.openSourceCritical)+' | '+#fmt('%8s', r.staticHigh+r.dynamicHigh+r.mobileHigh+r.openSourceHigh)+' | '+#fmt('%8s', r.staticMedium+r.dynamicMedium+r.mobileMedium+r.openSourceMedium)+' | '+#fmt('%8s', r.staticLow+r.dynamicLow+r.mobileLow+r.openSourceLow)+' |'} -# Note: reverted to retrieving OSS counts from the Vulnerabilities API to support prior FoD releases prior to 24.4 - uncomment last two lines when this has been done + | **Open Source** | ${(#isBlank(ossScanDate)?#fmt('%-16s', 'N/A'):#formatDateTime(dateFmt, ossScanDate)) +' | '+#fmt('%8s', r.openSourceCritical) +' | '+#fmt('%8s', r.openSourceHigh) +' | '+#fmt('%8s', r.openSourceMedium) +' | '+#fmt('%8s', r.openSourceLow) +' |'} + | **Total** | | ${#fmt('%8s', r.staticCritical+r.dynamicCritical+r.mobileCritical+r.openSourceCritical)+' | '+#fmt('%8s', r.staticHigh+r.dynamicHigh+r.mobileHigh+r.openSourceHigh)+' | '+#fmt('%8s', r.staticMedium+r.dynamicMedium+r.mobileMedium+r.openSourceMedium)+' | '+#fmt('%8s', r.staticLow+r.dynamicLow+r.mobileLow+r.openSourceLow)+' |'} # Note: update ossScanDate when it is available on release object ...