XRAY-141568 - Added graceful error when jf ca breaks due to cvs issue at RT server side

sca/bom/buildinfo/technologies/python/python.go

@@ -295,6 +295,22 @@ func installPipDeps(params technologies.BuildInfoBomGeneratorParams) (setupFileU
 		}
 		setupFileUsed = false
 	}
+	// When CVS hides the pinned version from the simple-index, pip fails
+	// with "No matching distribution found" instead of hitting a 403, so
+	// IsForbiddenOutput never fires. Replace the misleading pip error with
+	// a structured one.
+	if err != nil && params.IsCurationCmd && remoteUrl != "" &&
+		isCvsVersionFilteredOutput(errors.Join(err, reqErr).Error()) {
+		reqFile := params.PipRequirementsFile
+		if reqFile == "" {
+			reqFile = "requirements.txt"
+		}
+		pins, parseErr := parseRequirementsTxtPins(reqFile)
+		if parseErr != nil {
+			log.Debug(fmt.Sprintf("Curation audit: could not list pinned requirements for CVS error message: %s", parseErr.Error()))
+		}
+		err = errors.Join(err, errors.New(formatCvsBlockedRequirementsMessage(reqFile, pins)))
+	}
 	if err != nil || reqErr != nil {
 		if msgToUser := technologies.GetMsgToUserForCurationBlock(params.IsCurationCmd, techutils.Pip, errors.Join(err, reqErr).Error()); msgToUser != "" {
 			err = errors.Join(err, errors.New(msgToUser))

sca/bom/buildinfo/technologies/python/python_cvs_fallback.go

@@ -0,0 +1,68 @@
+package python
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+)
+
+type pinnedRequirement struct {
+	Name    string
+	Version string
+}
+
+var pinnedRequirementRegex = regexp.MustCompile(`^\s*([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?\s*==\s*([^\s;]+)`)
+
+func parseRequirementsTxtPins(reqPath string) ([]pinnedRequirement, error) {
+	data, err := os.ReadFile(reqPath)
+	if err != nil {
+		return nil, fmt.Errorf("reading requirements file %s: %w", reqPath, err)
+	}
+	var pins []pinnedRequirement
+	for _, raw := range strings.Split(string(data), "\n") {
+		// pip's --hash uses `=` not `#`, so the first `#` is always a comment.
+		if i := strings.Index(raw, "#"); i >= 0 {
+			raw = raw[:i]
+		}
+		line := strings.TrimSpace(raw)
+		if line == "" || strings.HasPrefix(line, "-") {
+			continue
+		}
+		m := pinnedRequirementRegex.FindStringSubmatch(line)
+		if m == nil {
+			continue
+		}
+		pins = append(pins, pinnedRequirement{
+			Name:    normalizePyPIName(m[1]),
+			Version: m[2],
+		})
+	}
+	return pins, nil
+}
+
+var pypiNameNormalizeRegex = regexp.MustCompile(`[-_.]+`)
+
+func normalizePyPIName(name string) string {
+	return strings.ToLower(pypiNameNormalizeRegex.ReplaceAllString(name, "-"))
+}
+
+func formatCvsBlockedRequirementsMessage(reqFile string, pins []pinnedRequirement) string {
+	var b strings.Builder
+	b.WriteString("Curation audit cannot complete: Artifactory CVS (Compliant Version Selection) is filtering blocked package versions out of the PyPI simple-index response — including the response served by the curation audit pass-through endpoint that `jf ca` relies on to enumerate dependencies. ")
+	b.WriteString("Pip therefore reports the pinned version as missing instead of letting the curation block be detected.\n\n")
+	if len(pins) > 0 {
+		fmt.Fprintf(&b, "Pinned requirement(s) read from %s:\n", reqFile)
+		for _, p := range pins {
+			fmt.Fprintf(&b, "  - %s==%s\n", p.Name, p.Version)
+		}
+		b.WriteString("\n")
+	}
+	b.WriteString("To unblock the audit, disable CVS on the curated PyPI repository. Curation policies still apply at the file/download endpoint, so packages remain blocked from download — only the audit visibility is restored.\n")
+	return b.String()
+}
+
+func isCvsVersionFilteredOutput(output string) bool {
+	return strings.Contains(output, "No matching distribution found") ||
+		strings.Contains(output, "Could not find a version that satisfies the requirement")
+}

sca/bom/buildinfo/technologies/python/python_cvs_fallback_test.go

@@ -0,0 +1,74 @@
+package python
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseRequirementsTxtPins(t *testing.T) {
+	cases := []struct {
+		name     string
+		contents string
+		want     []pinnedRequirement
+	}{
+		{
+			name: "extracts pins, ignores comments/blank/options/non-pins, normalizes names",
+			contents: "" +
+				"# a comment\n" +
+				"\n" +
+				"-r other.txt\n" +
+				"-e ./local\n" +
+				"langchain>=1.2.15\n" +
+				"requests[security]==2.34.2  # trailing\n" +
+				`flask==3.0.0 ; python_version >= "3.9"` + "\n" +
+				"Langchain_Core==1.4.0\n",
+			want: []pinnedRequirement{
+				{Name: "requests", Version: "2.34.2"},
+				{Name: "flask", Version: "3.0.0"},
+				{Name: "langchain-core", Version: "1.4.0"},
+			},
+		},
+		{
+			name:     "empty file yields no pins",
+			contents: "",
+			want:     nil,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			path := filepath.Join(t.TempDir(), "requirements.txt")
+			require.NoError(t, os.WriteFile(path, []byte(tc.contents), 0o600))
+			got, err := parseRequirementsTxtPins(path)
+			require.NoError(t, err)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}
+
+func TestIsCvsVersionFilteredOutput(t *testing.T) {
+	cases := map[string]bool{
+		"ERROR: No matching distribution found for deepagents==0.5.5":                                true,
+		"ERROR: Could not find a version that satisfies the requirement langchain-core<2.0.0,>=1.3.2": true,
+		"ERROR: 403 Forbidden":                                                                       false,
+	}
+	for output, want := range cases {
+		t.Run(output, func(t *testing.T) {
+			assert.Equal(t, want, isCvsVersionFilteredOutput(output))
+		})
+	}
+}
+
+func TestFormatCvsBlockedRequirementsMessage(t *testing.T) {
+	msg := formatCvsBlockedRequirementsMessage("requirements.txt",
+		[]pinnedRequirement{{Name: "deepagents", Version: "0.5.5"}})
+
+	assert.Contains(t, msg, "deepagents==0.5.5")
+	assert.Contains(t, msg, "requirements.txt")
+	assert.Contains(t, msg, "CVS")
+	assert.Contains(t, strings.ToLower(msg), "disable cvs")
+}