From c80230bc0c7a87d5ace11546144ef0176df3a46a Mon Sep 17 00:00:00 2001 From: Gauri Yadav Date: Fri, 22 May 2026 11:42:56 +0530 Subject: [PATCH] XRAY-141568 - Added graceful error when jf ca breaks due to cvs issue at RT server side --- .../buildinfo/technologies/python/python.go | 16 ++++ .../python/python_cvs_fallback.go | 68 +++++++++++++++++ .../python/python_cvs_fallback_test.go | 74 +++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 sca/bom/buildinfo/technologies/python/python_cvs_fallback.go create mode 100644 sca/bom/buildinfo/technologies/python/python_cvs_fallback_test.go diff --git a/sca/bom/buildinfo/technologies/python/python.go b/sca/bom/buildinfo/technologies/python/python.go index 67192f69b..277021231 100644 --- a/sca/bom/buildinfo/technologies/python/python.go +++ b/sca/bom/buildinfo/technologies/python/python.go @@ -295,6 +295,22 @@ func installPipDeps(params technologies.BuildInfoBomGeneratorParams) (setupFileU } setupFileUsed = false } + // When CVS hides the pinned version from the simple-index, pip fails + // with "No matching distribution found" instead of hitting a 403, so + // IsForbiddenOutput never fires. Replace the misleading pip error with + // a structured one. + if err != nil && params.IsCurationCmd && remoteUrl != "" && + isCvsVersionFilteredOutput(errors.Join(err, reqErr).Error()) { + reqFile := params.PipRequirementsFile + if reqFile == "" { + reqFile = "requirements.txt" + } + pins, parseErr := parseRequirementsTxtPins(reqFile) + if parseErr != nil { + log.Debug(fmt.Sprintf("Curation audit: could not list pinned requirements for CVS error message: %s", parseErr.Error())) + } + err = errors.Join(err, errors.New(formatCvsBlockedRequirementsMessage(reqFile, pins))) + } if err != nil || reqErr != nil { if msgToUser := technologies.GetMsgToUserForCurationBlock(params.IsCurationCmd, techutils.Pip, errors.Join(err, reqErr).Error()); msgToUser != "" { err = errors.Join(err, errors.New(msgToUser)) diff --git a/sca/bom/buildinfo/technologies/python/python_cvs_fallback.go b/sca/bom/buildinfo/technologies/python/python_cvs_fallback.go new file mode 100644 index 000000000..0576f3742 --- /dev/null +++ b/sca/bom/buildinfo/technologies/python/python_cvs_fallback.go @@ -0,0 +1,68 @@ +package python + +import ( + "fmt" + "os" + "regexp" + "strings" +) + +type pinnedRequirement struct { + Name string + Version string +} + +var pinnedRequirementRegex = regexp.MustCompile(`^\s*([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?\s*==\s*([^\s;]+)`) + +func parseRequirementsTxtPins(reqPath string) ([]pinnedRequirement, error) { + data, err := os.ReadFile(reqPath) + if err != nil { + return nil, fmt.Errorf("reading requirements file %s: %w", reqPath, err) + } + var pins []pinnedRequirement + for _, raw := range strings.Split(string(data), "\n") { + // pip's --hash uses `=` not `#`, so the first `#` is always a comment. + if i := strings.Index(raw, "#"); i >= 0 { + raw = raw[:i] + } + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "-") { + continue + } + m := pinnedRequirementRegex.FindStringSubmatch(line) + if m == nil { + continue + } + pins = append(pins, pinnedRequirement{ + Name: normalizePyPIName(m[1]), + Version: m[2], + }) + } + return pins, nil +} + +var pypiNameNormalizeRegex = regexp.MustCompile(`[-_.]+`) + +func normalizePyPIName(name string) string { + return strings.ToLower(pypiNameNormalizeRegex.ReplaceAllString(name, "-")) +} + +func formatCvsBlockedRequirementsMessage(reqFile string, pins []pinnedRequirement) string { + var b strings.Builder + b.WriteString("Curation audit cannot complete: Artifactory CVS (Compliant Version Selection) is filtering blocked package versions out of the PyPI simple-index response — including the response served by the curation audit pass-through endpoint that `jf ca` relies on to enumerate dependencies. ") + b.WriteString("Pip therefore reports the pinned version as missing instead of letting the curation block be detected.\n\n") + if len(pins) > 0 { + fmt.Fprintf(&b, "Pinned requirement(s) read from %s:\n", reqFile) + for _, p := range pins { + fmt.Fprintf(&b, " - %s==%s\n", p.Name, p.Version) + } + b.WriteString("\n") + } + b.WriteString("To unblock the audit, disable CVS on the curated PyPI repository. Curation policies still apply at the file/download endpoint, so packages remain blocked from download — only the audit visibility is restored.\n") + return b.String() +} + +func isCvsVersionFilteredOutput(output string) bool { + return strings.Contains(output, "No matching distribution found") || + strings.Contains(output, "Could not find a version that satisfies the requirement") +} diff --git a/sca/bom/buildinfo/technologies/python/python_cvs_fallback_test.go b/sca/bom/buildinfo/technologies/python/python_cvs_fallback_test.go new file mode 100644 index 000000000..ce10ee8b9 --- /dev/null +++ b/sca/bom/buildinfo/technologies/python/python_cvs_fallback_test.go @@ -0,0 +1,74 @@ +package python + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseRequirementsTxtPins(t *testing.T) { + cases := []struct { + name string + contents string + want []pinnedRequirement + }{ + { + name: "extracts pins, ignores comments/blank/options/non-pins, normalizes names", + contents: "" + + "# a comment\n" + + "\n" + + "-r other.txt\n" + + "-e ./local\n" + + "langchain>=1.2.15\n" + + "requests[security]==2.34.2 # trailing\n" + + `flask==3.0.0 ; python_version >= "3.9"` + "\n" + + "Langchain_Core==1.4.0\n", + want: []pinnedRequirement{ + {Name: "requests", Version: "2.34.2"}, + {Name: "flask", Version: "3.0.0"}, + {Name: "langchain-core", Version: "1.4.0"}, + }, + }, + { + name: "empty file yields no pins", + contents: "", + want: nil, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "requirements.txt") + require.NoError(t, os.WriteFile(path, []byte(tc.contents), 0o600)) + got, err := parseRequirementsTxtPins(path) + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +func TestIsCvsVersionFilteredOutput(t *testing.T) { + cases := map[string]bool{ + "ERROR: No matching distribution found for deepagents==0.5.5": true, + "ERROR: Could not find a version that satisfies the requirement langchain-core<2.0.0,>=1.3.2": true, + "ERROR: 403 Forbidden": false, + } + for output, want := range cases { + t.Run(output, func(t *testing.T) { + assert.Equal(t, want, isCvsVersionFilteredOutput(output)) + }) + } +} + +func TestFormatCvsBlockedRequirementsMessage(t *testing.T) { + msg := formatCvsBlockedRequirementsMessage("requirements.txt", + []pinnedRequirement{{Name: "deepagents", Version: "0.5.5"}}) + + assert.Contains(t, msg, "deepagents==0.5.5") + assert.Contains(t, msg, "requirements.txt") + assert.Contains(t, msg, "CVS") + assert.Contains(t, strings.ToLower(msg), "disable cvs") +}