feat!: finalize PEP639, auto-enable it, remove CLI switch environment --PEP-639 (#928)

## BREAKING Changes
- Handling of PEP 639 changed, as the specification changed during
finalization.
- Handling of PEP 639 is always enabled, as Python Packing specification
2.4 [recognizes
it](https://packaging.python.org/en/latest/specifications/license-expression/).
- CLI parameter `environment --PEP-639` was removed.

## Changed
- License texts from files are always Base64-encoded. In the past, we
did best-effort text decoding; this was dropped for interoperability.
This is considered a non-breaking change, as no data was removed, it was
just transformed according to CycloneDX specification.
- License text gathering was streamlined across PEP639 and PEP621.

## Added
- Finalized implementation of PEP 639.

----

- fixes #843
- fixes #915
- fixes #931

----

TODO/ DONE 
- [x] implement
- [x] add tests

---------

Signed-off-by: Jan Kowalleck <jan.kowalleck@gmail.com>
Signed-off-by: semantic-release <semantic-release@bot.local>
Co-authored-by: semantic-release <semantic-release@bot.local>

Author: jkowalleck

.github/workflows/release.yml

@@ -6,14 +6,15 @@ on:
       release_force:
         # see https://python-semantic-release.readthedocs.io/en/latest/github-action.html#command-line-options
         description: |
-          Force release be one of: [major | minor | patch]
+          Force release be one of: [major | minor | patch | prerelease]
           Leave empty for auto-detect based on commit messages.
         type: choice
         options:
-          - "" # auto - no force
-          - major # force major
-          - minor # force minor
-          - patch # force patch
+          - ""         # auto - no force
+          - major      # force major
+          - minor      # force minor
+          - patch      # force patch
+          - prerelease # force prerelease
         default: ""
         required: false
       prerelease_token:

cyclonedx_py/__init__.py

@@ -17,7 +17,7 @@
 
 # !! version is managed by `semantic_release`
 # do not use typing here, or else `semantic_release` might have issues finding the variable
-__version__ = "6.1.3"  # noqa:Q000
+__version__ = "7.0.1-alpha.2"  # noqa:Q000
 
 # There is no stable/public API.
 # However, you might call the stable CLI instead, like so:

cyclonedx_py/_internal/cli_common.py

@@ -29,9 +29,10 @@
 def add_argument_pyproject(p: 'ArgumentParser') -> 'Action':
     return p.add_argument('--pyproject',
                           metavar='<file>',
-                          help="Path to the root component's `pyproject.toml` file. "
+                          help="Path to the root component's `pyproject.toml` file.\n"
                                'This should point to a file compliant with PEP 621 '
-                               '(storing project metadata).',
+                               '(Storing project metadata in pyproject.toml). '
+                               'Supports PEP 639 (Improving License Clarity with Better Package Metadata). ',
                           dest='pyproject_file',
                           default=None)
 

cyclonedx_py/_internal/environment.py

@@ -27,21 +27,18 @@
 from textwrap import dedent
 from typing import TYPE_CHECKING, Any, Optional
 
+from cyclonedx.factory.license import LicenseFactory
 from cyclonedx.model import Property
-from cyclonedx.model.component import (  # type:ignore[attr-defined]  # ComponentEvidence was moved, but is still importable - ignore/wont-fix for backwards compatibility # noqa:E501
-    Component,
-    ComponentEvidence,
-    ComponentType,
-)
+from cyclonedx.model.component import Component, ComponentType
 from packageurl import PackageURL
 from packaging.requirements import Requirement
 
 from . import BomBuilder, PropertyName, PurlTypePypi
 from .cli_common import add_argument_mc_type, add_argument_pyproject
-from .utils.cdx import find_LicenseExpression, licenses_fixup, make_bom
+from .utils.cdx import licenses_fixup, make_bom
 from .utils.packaging import metadata2extrefs, metadata2licenses, normalize_packagename
 from .utils.pep610 import PackageSourceArchive, PackageSourceVcs, packagesource2extref, packagesource4dist
-from .utils.pep639 import dist2licenses as dist2licenses_pep639
+from .utils.pep639 import dist2licenses_from_files as pep639_dist2licenses_from_files
 from .utils.pyproject import pyproject2component, pyproject2dependencies, pyproject_load
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -114,12 +111,6 @@ def make_argument_parser(**kwargs: Any) -> 'ArgumentParser':
                  • Build an SBOM from uv environment:
                      $ %(prog)s "$(uv python find)"
                """)
-        p.add_argument('--PEP-639',
-                       action='store_true',
-                       dest='pep639',
-                       help='Enable license gathering according to PEP 639 '
-                            '(improving license clarity with better package metadata).\n'
-                            'The behavior may change during the draft development of the PEP.')
         p.add_argument('--gather-license-texts',
                        action='store_true',
                        dest='gather_license_texts',
@@ -140,11 +131,9 @@ def make_argument_parser(**kwargs: Any) -> 'ArgumentParser':
 
     def __init__(self, *,
                  logger: 'Logger',
-                 pep639: bool,
                  gather_license_texts: bool,
                  **__: Any) -> None:
         self._logger = logger
-        self._pep639 = pep639
         self._gather_license_texts = gather_license_texts
 
     def __call__(self, *,  # type:ignore[override]
@@ -156,7 +145,10 @@ def __call__(self, *,  # type:ignore[override]
             rc = None
         else:
             pyproject = pyproject_load(pyproject_file)
-            root_c = pyproject2component(pyproject, ctype=mc_type, fpath=pyproject_file)
+            root_c = pyproject2component(pyproject, ctype=mc_type,
+                                         fpath=pyproject_file,
+                                         gather_license_texts=self._gather_license_texts,
+                                         logger=self._logger)
             root_c.bom_ref.value = 'root-component'
             root_d = tuple(pyproject2dependencies(pyproject))
             rc = (root_c, root_d)
@@ -189,25 +181,17 @@ def __add_components(self, bom: 'Bom',
                 name=dist_name,
                 version=dist_version,
                 description=dist_meta['Summary'] if 'Summary' in dist_meta else None,
-                licenses=licenses_fixup(metadata2licenses(dist_meta)),
                 external_references=metadata2extrefs(dist_meta),
                 # path of dist-package on disc? naaa... a package may have multiple files/folders on disc
             )
-            if self._pep639:
-                pep639_licenses = list(dist2licenses_pep639(dist, self._gather_license_texts, self._logger))
-                pep639_lexp = find_LicenseExpression(pep639_licenses)
-                if pep639_lexp is not None:
-                    component.licenses = (pep639_lexp,)
-                    pep639_licenses.remove(pep639_lexp)
-                if len(pep639_licenses) > 0:
-                    if find_LicenseExpression(component.licenses) is None:
-                        component.licenses.update(pep639_licenses)
-                    else:
-                        # hack for preventing expressions AND named licenses.
-                        # see https://github.com/CycloneDX/cyclonedx-python/issues/826
-                        # see https://github.com/CycloneDX/specification/issues/454
-                        component.evidence = ComponentEvidence(licenses=pep639_licenses)
-                del pep639_lexp, pep639_licenses
+
+            # region licenses
+            component.licenses.update(metadata2licenses(dist_meta, LicenseFactory(),
+                                                        gather_texts=self._gather_license_texts))
+            if self._gather_license_texts:
+                component.licenses.update(pep639_dist2licenses_from_files(dist, logger=self._logger))
+            licenses_fixup(component)
+            # endregion licenses
 
             del dist_meta, dist_name, dist_version
             self.__component_add_extref_and_purl(component, packagesource4dist(dist))

cyclonedx_py/_internal/pipenv.py

@@ -132,7 +132,8 @@ def __call__(self, *,  # type:ignore[override]
             if pyproject_file is None:
                 rc = None
             else:
-                rc = pyproject_file2component(pyproject_file, ctype=mc_type)
+                rc = pyproject_file2component(pyproject_file, ctype=mc_type,
+                                              gather_license_texts=False, logger=self._logger)
                 rc.bom_ref.value = 'root-component'
 
             return self._make_bom(rc,

cyclonedx_py/_internal/py_interop/__init__.py

@@ -0,0 +1,18 @@
+# This file is part of CycloneDX Python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+
+"""this package contains polyfills and alike, so that all pyhton-versions support the same feature set."""

cyclonedx_py/_internal/py_interop/glob.py

@@ -0,0 +1,37 @@
+# This file is part of CycloneDX Python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+
+__all__ = ['glob']
+
+import sys
+from glob import glob as _glob
+
+if sys.version_info >= (3, 10):
+    glob = _glob
+else:
+    from os.path import join, sep
+    from typing import Optional
+
+    def glob(pathname: str, *, root_dir: Optional[str] = None, recursive: bool = False) -> list[str]:
+        if root_dir is not None:
+            pathname = join(root_dir, pathname)
+        files = _glob(pathname, recursive=recursive)
+        if root_dir is not None:
+            if not root_dir.endswith(sep):
+                root_dir += sep
+            files = [f.removeprefix(root_dir) for f in files]
+        return files

cyclonedx_py/_internal/py_interop/packagemetadata.py

@@ -0,0 +1,28 @@
+# This file is part of CycloneDX Python
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+
+__all__ = ['PackageMetadata']
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:  # pragma: nocover
+    import sys
+
+    if sys.version_info >= (3, 10):
+        from importlib.metadata import PackageMetadata
+    else:
+        from email.message import Message as PackageMetadata

cyclonedx_py/_internal/requirements.py

@@ -116,7 +116,8 @@ def __call__(self, *,  # type:ignore[override]
         if pyproject_file is None:
             rc = None
         else:
-            rc = pyproject_file2component(pyproject_file, ctype=mc_type)
+            rc = pyproject_file2component(pyproject_file, ctype=mc_type,
+                                          gather_license_texts=False, logger=self._logger)
             rc.bom_ref.value = 'root-component'
 
         if requirements_file == '-':

cyclonedx_py/_internal/utils/cdx.py

@@ -27,7 +27,11 @@
 from cyclonedx.builder.this import this_component as lib_component
 from cyclonedx.model import ExternalReference, ExternalReferenceType, XsUri
 from cyclonedx.model.bom import Bom
-from cyclonedx.model.component import Component, ComponentType
+from cyclonedx.model.component import (  # type:ignore[attr-defined]  # ComponentEvidence was moved, but is still importable - ignore/wont-fix for backwards compatibility  # noqa:E501
+    Component,
+    ComponentEvidence,
+    ComponentType,
+)
 from cyclonedx.model.license import DisjunctiveLicense, License, LicenseAcknowledgement, LicenseExpression
 
 from ... import __version__ as _THIS_VERSION  # noqa:N812
@@ -95,11 +99,24 @@ def find_LicenseExpression(licenses: Iterable['License']) -> Optional[LicenseExp
     return None
 
 
-def licenses_fixup(licenses: Iterable['License']) -> Iterable['License']:
-    licenses = set(licenses)
-    if (lexp := find_LicenseExpression(licenses)) is not None:
-        return (lexp,)
-    return licenses
+def licenses_fixup(component: 'Component') -> None:
+    """
+    Per CycloneDX spec, there must be EITHER one license expression OR multiple license id/name.
+    If there is an expression, it is used and everything else is moved to evidences, so it is not lost.
+    """
+    # hack for preventing expressions AND named licenses.
+    # see https://github.com/CycloneDX/cyclonedx-python/issues/826
+    # see https://github.com/CycloneDX/specification/issues/454
+    licenses = list(component.licenses)
+    lexp = find_LicenseExpression(licenses)
+    if lexp is None:
+        return
+    component.licenses = (lexp,)
+    licenses.remove(lexp)
+    if len(licenses) > 0:
+        if component.evidence is None:
+            component.evidence = ComponentEvidence()
+        component.evidence.licenses.update(licenses)
 
 
 _MAP_KNOWN_URL_LABELS: dict[str, ExternalReferenceType] = {