Make source provenance generic

Remove harcoded SPDX attributes and make them be generic instead.
Project allowed attributes are configured via the project config,
these supported values a determined by buildstream-sbom's support

Co-authored-by: Jürg Billeter <j@bitron.ch>
Co-authored-by: Joshua Zivkovic <joshuazivkovic@codethink.co.uk>

Author: 2 people

src/buildstream/_project.py

@@ -125,6 +125,8 @@ def __init__(
         self.sandbox: Optional[MappingNode] = None
         self.splits: Optional[MappingNode] = None
 
+        self.source_provenance_fields: Optional[MappingNode] = None  # Source provenance fields and their description
+
         #
         # Private members
         #
@@ -726,6 +728,7 @@ def _validate_toplevel_node(self, node, *, first_pass=False):
                 "sources",
                 "source-caches",
                 "junctions",
+                "source-provenance-fields",
                 "(@)",
                 "(?)",
             ]
@@ -1005,6 +1008,7 @@ def _load_second_pass(self):
                 mount = _HostMount(path, host_path, optional)
 
             self._shell_host_files.append(mount)
+        self.source_provenance_fields = config.get_mapping("source-provenance-fields")
 
     # _load_pass():
     #

src/buildstream/data/projectconfig.yaml

@@ -175,6 +175,12 @@ shell:
   #
   command: [ 'sh', '-i' ]
 
+# Define the set of fields accepted in `provenance` dictionaries of sources.
+#
+source-provenance-fields:
+  homepage: "The project homepage URL"
+  issue-tracker: "The project's issue tracking URL"
+
 # Defaults for bst commands
 #
 defaults:

src/buildstream/element.py

@@ -90,7 +90,7 @@
 from .sandbox import _SandboxFlags, SandboxCommandError
 from .sandbox._config import SandboxConfig
 from .sandbox._sandboxremote import SandboxRemote
-from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey, _SourceProvenance
+from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey
 from ._artifact import Artifact
 from ._elementproxy import ElementProxy
 from ._elementsources import ElementSources
@@ -2635,19 +2635,18 @@ def __load_sources(self, load_element):
                     del source[Symbol.DIRECTORY]
 
                 # Provenance is optional
-                provenance_node = source.get_mapping(Symbol.PROVENANCE, default=None)
-                provenance = None
+                provenance_node: MappingNode = source.get_mapping(Symbol.PROVENANCE, default=None)
                 if provenance_node:
                     del source[Symbol.PROVENANCE]
-                    provenance = _SourceProvenance.new_from_node(provenance_node)
+                    provenance_node.validate_keys(project.source_provenance_fields.keys())
 
                 meta_source = MetaSource(
                     self.name,
                     index,
                     self.get_kind(),
                     kind.as_str(),
                     directory,
-                    provenance,
+                    provenance_node,
                     source,
                     load_element.first_pass,
                 )

src/buildstream/exceptions.py

@@ -155,3 +155,8 @@ class LoadErrorReason(Enum):
     This warning will be produced when a filename for a target contains invalid
     characters in its name.
     """
+
+    UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE = 29
+    """
+    Thee source provenance attribute specified was not defined in the project config
+    """

src/buildstream/source.py

@@ -48,6 +48,8 @@
 
     The ``issue-tracker`` attribute can be used to specify the project's issue tracking URL
 
+TODO: document that fields can be configured in project
+
   *Since: 2.5*
 
 
@@ -370,6 +372,7 @@
 """
 
 import os
+from collections import ChainMap
 from contextlib import contextmanager
 from typing import Iterable, Iterator, Optional, Tuple, Dict, Any, Set, TYPE_CHECKING, Union
 from dataclasses import dataclass
@@ -378,9 +381,9 @@
 from .node import MappingNode
 from .plugin import Plugin
 from .sourcemirror import SourceMirror
-from .types import SourceRef, CoreWarnings, FastEnum, _SourceProvenance
-from ._exceptions import BstError, ImplError, PluginError
-from .exceptions import ErrorDomain
+from .types import SourceRef, CoreWarnings, FastEnum
+from ._exceptions import BstError, ImplError, PluginError, LoadError
+from .exceptions import ErrorDomain, LoadErrorReason
 from ._loader.metasource import MetaSource
 from ._projectrefs import ProjectRefStorage
 from ._cachekey import generate_key
@@ -396,6 +399,7 @@
 
     # pylint: enable=cyclic-import
 
+SourceProvenance = MappingNode
 
 class SourceError(BstError):
     """This exception should be raised by :class:`.Source` implementations
@@ -553,8 +557,7 @@ def __init__(
         self,
         kind: str,
         url: str,
-        homepage: Optional[str],
-        issue_tracker: Optional[str],
+        provenance: Optional[MappingNode],
         medium: Union[SourceInfoMedium, str],
         version_type: Union[SourceVersionType, str],
         version: str,
@@ -572,14 +575,9 @@ def __init__(
         The url of the source input
         """
 
-        self.homepage: Optional[str] = homepage
-        """
-        The project homepage URL
-        """
-
-        self.issue_tracker: Optional[str] = issue_tracker
+        self.provenance = provenance
         """
-        The project issue tracking URL
+        The optional YAML node with source provenance attributes
         """
 
         self.medium: Union[SourceInfoMedium, str] = medium
@@ -642,10 +640,14 @@ def serialize(self) -> Dict[str, Union[str, Dict[str, str]]]:
             "url": self.url,
         }
 
-        if self.homepage is not None:
-            version_info["homepage"] = self.homepage
-        if self.issue_tracker is not None:
-            version_info["issue-tracker"] = self.issue_tracker
+        if self.provenance is not None:
+            # need to keep homepage/issue-tracker [also] at the top-level for backward compat
+            if (homepage := self.provenance.get_str("homepage", None)) is not None:
+                version_info["homepage"] = homepage
+            if (issue_tracker := self.provenance.get_str("issue-tracker", None)) is not None:
+                version_info["issue-tracker"] = issue_tracker
+
+            version_info["provenance"] = self.provenance
 
         version_info["medium"] = medium_str
         version_info["version-type"] = version_type_str
@@ -825,7 +827,7 @@ def __init__(
         self._directory = meta.directory  # Staging relative directory
         self.__variables = variables  # The variables used to resolve the source's config
         self.__provenance: Optional[
-            _SourceProvenance
+            MappingNode
         ] = meta.provenance  # The _SourceProvenance for general user provided SourceInfo
 
         self.__key = None  # Cache key for source
@@ -927,7 +929,16 @@ def set_ref(self, ref, node):
         """
         raise ImplError("Source plugin '{}' does not implement set_ref()".format(self.get_kind()))
 
-    def track(self, *, previous_sources_dir: Optional[str] = None) -> SourceRef:
+    def load_source_provenance(self, node: MappingNode) -> None:
+        raise ImplError("Source plugin '{}' does not implement load_source_provenance()".format(self.get_kind()))
+
+    def get_source_provenance(self) -> SourceProvenance:
+        raise ImplError("Source plugin '{}' does not implement get_source_provenance()".format(self.get_kind()))
+
+    def set_source_provenance(self, source_provenance: SourceProvenance):
+        raise ImplError("Source plugin '{}' does not implement set_source_provenance()".format(self.get_kind()))
+
+    def track(self, *, previous_sources_dir: Optional[str] = None) -> (SourceRef, SourceProvenance):
         """Resolve a new ref from the plugin's track option
 
         Args:
@@ -1393,23 +1404,24 @@ def create_source_info(
 
         *Since: 2.5*
         """
-        homepage = None
-        issue_tracker = None
+        project = self._get_project()
 
         if provenance_node is not None:
-            provenance: Optional[_SourceProvenance] = _SourceProvenance.new_from_node(provenance_node)
+            # TODO: Ensure provenance node keys are valid and values are all strings; same in element if we drop _SourceProvenance
+            try:
+                provenance_node.validate_keys(project.source_provenance_fields.keys())
+            except LoadError as E:
+                raise LoadError("Specified source attribute not defined in project config\n {}".format(E),
+                                LoadErrorReason.UNDEFINED_SOURCE_PROVENANCE_ATTRIBUTE)
+
+            provenance = provenance_node
         else:
             provenance = self.__provenance
 
-        if provenance is not None:
-            homepage = provenance.homepage
-            issue_tracker = provenance.issue_tracker
-
         return SourceInfo(
             self.get_kind(),
             url,
-            homepage,
-            issue_tracker,
+            provenance,
             medium,
             version_type,
             version,
@@ -1778,13 +1790,30 @@ def process_value(action, container, path, key, new_value):
     #   previous_sources_dir (str): directory where previous sources are staged
     #
     def _track(self, previous_sources_dir: Optional[str] = None) -> SourceRef:
+        def verify_provenance_attributes(provenance: SourceProvenance):
+            if type(provenance) is list:
+                # produce a list of unique attrs
+                unique_entries = {attr: "" for single_provenance in provenance for attr in single_provenance.keys()}
+                used_attrs = unique_entries.keys()
+            else:
+                used_attrs = provenance.keys()
+
+            undefined_attributes = list(set(used_attrs) - set(self._get_project().source_provenance_fields.keys()))
+
+            if len(undefined_attributes) > 0:
+                self.warn(f"Required source attributes not defined in project config: {undefined_attributes}")
+
         if self.BST_REQUIRES_PREVIOUS_SOURCES_TRACK:
-            new_ref = self.__do_track(previous_sources_dir=previous_sources_dir)
+            new_ref, source_provenance = self.__do_track(previous_sources_dir=previous_sources_dir)
         else:
-            new_ref = self.__do_track()
+            new_ref, source_provenance = self.__do_track()
 
         current_ref = self.get_ref()  # pylint: disable=assignment-from-no-return
 
+        # in the case of multi-source plugins, set_ref might update the provenance with the new content,
+        # designed to use ref, so grab the old content first
+        current_provenance: list[MappingNode] = self.get_source_provenance()
+
         if new_ref is None:
             # No tracking, keep current ref
             new_ref = current_ref
@@ -1797,6 +1826,18 @@ def _track(self, previous_sources_dir: Optional[str] = None) -> SourceRef:
 
         self._generate_key()
 
+        # need to account for multi-source provenance and process it appropriately
+        if source_provenance is not None:
+            verify_provenance_attributes(source_provenance)
+
+        if current_provenance is not None:
+            current_provenance = list(map(MappingNode.strip_node_info, current_provenance))
+
+        if current_provenance != source_provenance:
+            self.info(f"Updated source provenance content for {self.name}")
+
+            self.set_source_provenance(source_provenance)
+
         return new_ref
 
     # _requires_previous_sources()
@@ -2025,14 +2066,14 @@ def __do_track(self, **kwargs):
         for mirror in reversed(project.get_alias_uris(alias, first_pass=self.__first_pass, tracking=True)):
             new_source = self.__clone_for_uri(mirror)
             try:
-                ref = new_source.track(**kwargs)  # pylint: disable=assignment-from-none
+                ref, provenance = new_source.track(**kwargs)  # pylint: disable=assignment-from-none
             # FIXME: Need to consider temporary vs. permanent failures,
             #        and how this works with retries.
             except BstError as e:
                 last_error = e
                 continue
 
-            return ref
+            return (ref, provenance)
 
         raise last_error
 

src/buildstream/types.py

@@ -390,42 +390,6 @@ def new_from_node(cls, node: MappingNode) -> "_SourceMirror":
         return cls(name, aliases)
 
 
-# _SourceProvenance()
-#
-# A simple object describing user provided source provenance information
-#
-# Args:
-#    homepage: The project homepage URL
-#    issue_tracker: The project issue reporting URL
-#
-class _SourceProvenance:
-    def __init__(self, homepage: Optional[str], issue_tracker: Optional[str]):
-        self.homepage: Optional[str] = homepage
-        self.issue_tracker: Optional[str] = issue_tracker
-
-    # new_from_node():
-    #
-    # Creates a _SourceProvenance() from a YAML loaded node.
-    #
-    # Args:
-    #    node: The configuration node describing the spec.
-    #
-    # Returns:
-    #    The described _SourceProvenance instance.
-    #
-    # Raises:
-    #    LoadError: If the node is malformed.
-    #
-    @classmethod
-    def new_from_node(cls, node: MappingNode) -> "_SourceProvenance":
-        node.validate_keys(["homepage", "issue-tracker"])
-
-        homepage: Optional[str] = node.get_str("homepage", None)
-        issue_tracker: Optional[str] = node.get_str("issue-tracker", None)
-
-        return cls(homepage, issue_tracker)
-
-
 ########################################
 #           Type aliases               #
 ########################################