Make source provenance generic

Remove harcoded SPDX attributes and make them be generic instead.
Project allowed attributes are configured via the project config,
these supported values a determined by buildstream-sbom's support

Co-authored-by: Jürg Billeter <j@bitron.ch>
Co-authored-by: Joshua Zivkovic <joshuazivkovic@codethink.co.uk>

Author: 3 people

src/buildstream/_project.py

@@ -125,6 +125,8 @@ def __init__(
         self.sandbox: Optional[MappingNode] = None
         self.splits: Optional[MappingNode] = None
 
+        self.source_provenance_fields: Optional[MappingNode] = None  # Source provenance fields and their description
+
         #
         # Private members
         #
@@ -726,6 +728,7 @@ def _validate_toplevel_node(self, node, *, first_pass=False):
                 "sources",
                 "source-caches",
                 "junctions",
+                "source-provenance-fields",
                 "(@)",
                 "(?)",
             ]
@@ -1005,6 +1008,7 @@ def _load_second_pass(self):
                 mount = _HostMount(path, host_path, optional)
 
             self._shell_host_files.append(mount)
+        self.source_provenance_fields = config.get_mapping("source-provenance-fields")
 
     # _load_pass():
     #

src/buildstream/data/projectconfig.yaml

@@ -175,6 +175,12 @@ shell:
   #
   command: [ 'sh', '-i' ]
 
+# Define the set of fields accepted in `provenance` dictionaries of sources.
+#
+source-provenance-fields:
+  homepage: "The project homepage URL"
+  issue-tracker: "The project's issue tracking URL"
+
 # Defaults for bst commands
 #
 defaults:

src/buildstream/element.py

@@ -85,12 +85,12 @@
 from . import utils
 from . import _cachekey
 from . import _site
-from .node import Node
+from .node import Node, MappingNode
 from .plugin import Plugin
 from .sandbox import _SandboxFlags, SandboxCommandError
 from .sandbox._config import SandboxConfig
 from .sandbox._sandboxremote import SandboxRemote
-from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey, _SourceProvenance
+from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey
 from ._artifact import Artifact
 from ._elementproxy import ElementProxy
 from ._elementsources import ElementSources
@@ -102,7 +102,7 @@
 
 if TYPE_CHECKING:
     from typing import Tuple
-    from .node import MappingNode, ScalarNode, SequenceNode
+    from .node import ScalarNode, SequenceNode
     from .types import SourceRef
 
     # pylint: disable=cyclic-import
@@ -2635,19 +2635,33 @@ def __load_sources(self, load_element):
                     del source[Symbol.DIRECTORY]
 
                 # Provenance is optional
-                provenance_node = source.get_mapping(Symbol.PROVENANCE, default=None)
-                provenance = None
+                provenance_node: MappingNode = source.get_mapping(Symbol.PROVENANCE, default=None)
                 if provenance_node:
                     del source[Symbol.PROVENANCE]
-                    provenance = _SourceProvenance.new_from_node(provenance_node)
+                    defined_attrs = (
+                        project._project_conf.get_mapping("source-provenance-fields", None)
+                        or project.source_provenance_fields
+                    )
+                    try:
+                        provenance_node.validate_keys(defined_attrs.keys())
+                    except LoadError as E:
+                        raise LoadError(
+                            "Specified source attribute not defined in project config\n {}".format(E),
+                            LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE,
+                        )
+
+                    # make sure everything is a string
+                    provenance_node = MappingNode.from_dict(
+                        {key: value.as_str() for key, value in provenance_node.items()}
+                    )
 
                 meta_source = MetaSource(
                     self.name,
                     index,
                     self.get_kind(),
                     kind.as_str(),
                     directory,
-                    provenance,
+                    provenance_node,
                     source,
                     load_element.first_pass,
                 )
@@ -3318,9 +3332,11 @@ def __update_cache_keys(self):
             # encode the dependency's weak cache key instead of it's name.
             #
             dependencies = [
-                [e.project_name, e.name, e._get_cache_key(strength=_KeyStrength.WEAK)]
-                if self.BST_STRICT_REBUILD or e in self.__strict_dependencies
-                else [e.project_name, e.name]
+                (
+                    [e.project_name, e.name, e._get_cache_key(strength=_KeyStrength.WEAK)]
+                    if self.BST_STRICT_REBUILD or e in self.__strict_dependencies
+                    else [e.project_name, e.name]
+                )
                 for e in self._dependencies(_Scope.BUILD)
             ]
             self.__weak_cache_key = self._calculate_cache_key(dependencies)

src/buildstream/exceptions.py

@@ -155,3 +155,8 @@ class LoadErrorReason(Enum):
     This warning will be produced when a filename for a target contains invalid
     characters in its name.
     """
+
+    UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE = 29
+    """
+    Thee source provenance attribute specified was not defined in the project config
+    """

src/buildstream/source.py

@@ -38,15 +38,11 @@
   to provide additional source provenance related metadata which will later
   be reported in :class:`.SourceInfo` objects.
 
-  The ``provenance`` dictionary supports the following fields:
+  The ``provenance`` dictionary itself does not have any specific required keys.
 
-  * Homepage
-
-    The ``homepage`` attribute can be used to specify the project homepage URL
-
-  * Issue Tracker
-
-    The ``issue-tracker`` attribute can be used to specify the project's issue tracking URL
+  Any attribute used in the ``provenance`` dictionary of a source must be
+  defined in the project.conf using the ``source-provenance-fields`` dictionary
+  to define the attribute and its significance.
 
   *Since: 2.5*
 
@@ -371,16 +367,26 @@
 
 import os
 from contextlib import contextmanager
-from typing import Iterable, Iterator, Optional, Tuple, Dict, Any, Set, TYPE_CHECKING, Union
+from typing import (
+    Iterable,
+    Iterator,
+    Optional,
+    Tuple,
+    Dict,
+    Any,
+    Set,
+    TYPE_CHECKING,
+    Union,
+)
 from dataclasses import dataclass
 
 from . import _yaml, utils
 from .node import MappingNode
 from .plugin import Plugin
 from .sourcemirror import SourceMirror
-from .types import SourceRef, CoreWarnings, FastEnum, _SourceProvenance
-from ._exceptions import BstError, ImplError, PluginError
-from .exceptions import ErrorDomain
+from .types import SourceRef, CoreWarnings, FastEnum
+from ._exceptions import BstError, ImplError, PluginError, LoadError
+from .exceptions import ErrorDomain, LoadErrorReason
 from ._loader.metasource import MetaSource
 from ._projectrefs import ProjectRefStorage
 from ._cachekey import generate_key
@@ -396,6 +402,8 @@
 
     # pylint: enable=cyclic-import
 
+SourceProvenance = MappingNode
+
 
 class SourceError(BstError):
     """This exception should be raised by :class:`.Source` implementations
@@ -409,9 +417,20 @@ class SourceError(BstError):
     """
 
     def __init__(
-        self, message: str, *, detail: Optional[str] = None, reason: Optional[str] = None, temporary: bool = False
+        self,
+        message: str,
+        *,
+        detail: Optional[str] = None,
+        reason: Optional[str] = None,
+        temporary: bool = False,
     ):
-        super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason, temporary=temporary)
+        super().__init__(
+            message,
+            detail=detail,
+            domain=ErrorDomain.SOURCE,
+            reason=reason,
+            temporary=temporary,
+        )
 
 
 @dataclass
@@ -553,8 +572,7 @@ def __init__(
         self,
         kind: str,
         url: str,
-        homepage: Optional[str],
-        issue_tracker: Optional[str],
+        provenance: Optional[SourceProvenance],
         medium: Union[SourceInfoMedium, str],
         version_type: Union[SourceVersionType, str],
         version: str,
@@ -572,14 +590,9 @@ def __init__(
         The url of the source input
         """
 
-        self.homepage: Optional[str] = homepage
-        """
-        The project homepage URL
-        """
-
-        self.issue_tracker: Optional[str] = issue_tracker
+        self.provenance = provenance
         """
-        The project issue tracking URL
+        The optional YAML node with source provenance attributes
         """
 
         self.medium: Union[SourceInfoMedium, str] = medium
@@ -642,10 +655,14 @@ def serialize(self) -> Dict[str, Union[str, Dict[str, str]]]:
             "url": self.url,
         }
 
-        if self.homepage is not None:
-            version_info["homepage"] = self.homepage
-        if self.issue_tracker is not None:
-            version_info["issue-tracker"] = self.issue_tracker
+        if self.provenance is not None:
+            # need to keep homepage/issue-tracker [also] at the top-level for backward compat
+            if (homepage := self.provenance.get_str("homepage", None)) is not None:
+                version_info["homepage"] = homepage
+            if (issue_tracker := self.provenance.get_str("issue-tracker", None)) is not None:
+                version_info["issue-tracker"] = issue_tracker
+
+            version_info["provenance"] = self.provenance.strip_node_info()
 
         version_info["medium"] = medium_str
         version_info["version-type"] = version_type_str
@@ -824,9 +841,9 @@ def __init__(
         self.__element_kind = meta.element_kind  # The kind of the element owning this source
         self._directory = meta.directory  # Staging relative directory
         self.__variables = variables  # The variables used to resolve the source's config
-        self.__provenance: Optional[
-            _SourceProvenance
-        ] = meta.provenance  # The _SourceProvenance for general user provided SourceInfo
+        self.__provenance: Optional[SourceProvenance] = (
+            meta.provenance
+        )  # The source provenance for general user provided SourceInfo
 
         self.__key = None  # Cache key for source
 
@@ -1393,23 +1410,33 @@ def create_source_info(
 
         *Since: 2.5*
         """
-        homepage = None
-        issue_tracker = None
+        project = self._get_project()
 
+        provenance: SourceProvenance | None
         if provenance_node is not None:
-            provenance: Optional[_SourceProvenance] = _SourceProvenance.new_from_node(provenance_node)
+            # Ensure provenance node keys are valid and values are all strings
+            defined_provenance_fields = (
+                project._project_conf.get_mapping("source-provenance-fields", None) or project.source_provenance_fields
+            )
+
+            try:
+                provenance_node.validate_keys(defined_provenance_fields.keys())
+            except LoadError as E:
+                raise LoadError(
+                    "Specified source attribute not defined in project config\n {}".format(E),
+                    LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE,
+                )
+
+            # Make sure everything is a string
+            provenance = MappingNode.from_dict({key: value.as_str() for key, value in provenance_node.items()})
+
         else:
             provenance = self.__provenance
 
-        if provenance is not None:
-            homepage = provenance.homepage
-            issue_tracker = provenance.issue_tracker
-
         return SourceInfo(
             self.get_kind(),
             url,
-            homepage,
-            issue_tracker,
+            provenance,
             medium,
             version_type,
             version,
@@ -1767,7 +1794,8 @@ def process_value(action, container, path, key, new_value):
                 _yaml.roundtrip_dump(data, filename)
             except OSError as e:
                 raise SourceError(
-                    "{}: Error saving source reference to '{}': {}".format(self, filename, e), reason="save-ref-error"
+                    "{}: Error saving source reference to '{}': {}".format(self, filename, e),
+                    reason="save-ref-error",
                 ) from e
 
         return True

src/buildstream/types.py

@@ -390,42 +390,6 @@ def new_from_node(cls, node: MappingNode) -> "_SourceMirror":
         return cls(name, aliases)
 
 
-# _SourceProvenance()
-#
-# A simple object describing user provided source provenance information
-#
-# Args:
-#    homepage: The project homepage URL
-#    issue_tracker: The project issue reporting URL
-#
-class _SourceProvenance:
-    def __init__(self, homepage: Optional[str], issue_tracker: Optional[str]):
-        self.homepage: Optional[str] = homepage
-        self.issue_tracker: Optional[str] = issue_tracker
-
-    # new_from_node():
-    #
-    # Creates a _SourceProvenance() from a YAML loaded node.
-    #
-    # Args:
-    #    node: The configuration node describing the spec.
-    #
-    # Returns:
-    #    The described _SourceProvenance instance.
-    #
-    # Raises:
-    #    LoadError: If the node is malformed.
-    #
-    @classmethod
-    def new_from_node(cls, node: MappingNode) -> "_SourceProvenance":
-        node.validate_keys(["homepage", "issue-tracker"])
-
-        homepage: Optional[str] = node.get_str("homepage", None)
-        issue_tracker: Optional[str] = node.get_str("issue-tracker", None)
-
-        return cls(homepage, issue_tracker)
-
-
 ########################################
 #           Type aliases               #
 ########################################