From ddd5af48ad3ae8f455105b57762ada785d03cf8d Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Sat, 1 Aug 2026 11:25:05 +0200 Subject: [PATCH 01/10] SOLR-17328: Add CycloneDX SBOMs to Solr binary distributions Each binary distribution (full and slim) now ships a CycloneDX 1.6 bom.json describing its actual contents. Two resolvable configurations in :solr:packaging (bomFull, bomSlim) mirror the distribution assembly and are rendered by the CycloneDX Gradle plugin, then a post-processing step adjusts the result: * The metadata declares the "build" lifecycle phase (a CISA Build SBOM) and lists the post-processing and cyclonedx-npm next to the plugin in the tools. * The main component identifies the binary release with a draft "sid" purl (purl-spec issue #516, pkg:sid/apache.org/solr/solr@ with an edition qualifier) and the Solr CPE used by the NVD. * Maven BOM/platform dependencies and the internal :platform project, which are not part of the distribution, are stripped. * Solr project components get valid Maven purls (artifactId instead of the Gradle project name, with the ASF snapshots repository for snapshot builds), a description and the Apache-2.0 license. * The UI artifacts inside the webapp are covered: the npm packages bundled by browserify into the OpenAPI JS client and the Maven artifacts compiled into the wasmJs UI are nested as subassemblies of first-party solr-js-client and solr-ui components, from child SBOMs generated in :solr:webapp:js-client (official cyclonedx-npm tool, runtime dependencies only) and :solr:ui (plugin task over wasmJsRuntimeClasspath). The vendored JavaScript libraries of the AngularJS admin UI are listed from a curated list. * The location of every JAR and JavaScript file in the distribution is recorded as evidence.occurrences by matching SHA-256 hashes against the assembled directories, which also proves the vendored libraries ship unmodified. * Only the SHA-256 hash of each component is kept; the plugin emits eight algorithms per artifact, which only adds bulk. The BOM configurations resolve strictly with the JVM runtime attributes (the packaging project applies jvm-ecosystem for the attribute schema), so BOM-managed and variant-aware dependencies resolve like a runtime classpath instead of silently disappearing. bomFull resolves consistently with bomSlim, matching the distribution layout where server libraries win over module-pulled versions. The child SBOMs degrade gracefully when the UI projects are disabled with -PdisableJsClient / -PdisableUiModule. Assisted-By: Claude Fable 5 --- build.gradle | 1 + .../unreleased/cyclonedx-sboms-SOLR-17328.yml | 12 + gradle/libs.versions.toml | 4 + solr/packaging/build.gradle | 502 +++++++++++++++++- solr/packaging/gradle.lockfile | 331 +++++++++++- solr/server/build.gradle | 35 +- solr/ui/build.gradle.kts | 34 ++ solr/webapp/js-client/build.gradle.kts | 76 ++- 8 files changed, 987 insertions(+), 8 deletions(-) create mode 100644 changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml diff --git a/build.gradle b/build.gradle index 80c97fe1a0f0..e8c107888470 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,7 @@ plugins { alias(libs.plugins.diffplug.spotless) apply false alias(libs.plugins.nodegradle.node) apply false alias(libs.plugins.openapi.generator) apply false + alias(libs.plugins.cyclonedx) apply false alias(libs.plugins.logchange) } diff --git a/changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml b/changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml new file mode 100644 index 000000000000..ed288fec8f89 --- /dev/null +++ b/changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml @@ -0,0 +1,12 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc + +title: > + Ship a CycloneDX SBOM (bom.json) in the root of the full and slim binary distributions, + covering the Java libraries, the Solr artifacts and the UI content of the webapp +type: added +authors: + - name: Piotr P. Karwasz + nick: ppkarwasz +links: + - name: SOLR-17328 + url: https://issues.apache.org/jira/browse/SOLR-17328 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index fe7d4d6d756a..7fae6881d6e9 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -79,6 +79,9 @@ commons-io = "2.22.0" compose = "1.11.1" cuvs-java = "26.06.0" cuvs-lucene = "25.12.0" +cyclonedx = "3.0.2" +# @keep npm tool generating the SBOM of the OpenAPI JS client, installed by :solr:webapp:js-client +cyclonedx-npm = "6.0.0" decompose = "3.5.0" diffplug-spotless = "8.7.0" # @keep Use for dockerfile JRE version @@ -203,6 +206,7 @@ xerial-snappy = "1.1.10.8" [plugins] benmanes-versions = { id = "com.github.ben-manes.versions", version.ref = "benmanes-versions" } compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } +cyclonedx = { id = "org.cyclonedx.bom", version.ref = "cyclonedx" } diffplug-spotless = { id = "com.diffplug.spotless", version.ref = "diffplug-spotless" } jetbrains-compose = { id = "org.jetbrains.compose", version.ref = "compose" } kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index 63c467388ef1..3ac372418772 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -15,8 +15,13 @@ * limitations under the License. */ +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import java.security.MessageDigest import org.apache.tools.ant.filters.ReplaceTokens import org.apache.tools.ant.util.TeeOutputStream +import org.cyclonedx.gradle.CyclonedxDirectTask +import org.cyclonedx.model.Component // This project puts together a "distribution", assembling dependencies from // various other projects. @@ -24,11 +29,424 @@ import org.apache.tools.ant.util.TeeOutputStream plugins { id 'base' id 'distribution' + // Registers the JVM attribute schema, so that the variant-aware resolution of + // the CycloneDX BOM configurations below works like a Java runtime classpath. + id 'jvm-ecosystem' +} + +final APACHE_SNAPSHOTS_QUALIFIER = '&repository_url=https:%2F%2Frepository.apache.org%2Fcontent%2Fgroups%2Fsnapshots%2F' + +// Post-processing of the SBOMs generated by the CycloneDX plugin: +// +// 1. Metadata: +// The "build" lifecycle phase is recorded (this is a "Build SBOM" in CISA's classification) +// and this post-processing step is listed in the tools, next to the CycloneDX plugin. +// 2. Main component: +// The Solr binary archive published on downloads.apache.org is identified by a "sid" purl +// (a draft purl type for software distributed outside package registries, +// see https://github.com/package-url/purl-spec/issues/516) and by the Solr CPE. +// 3. Removed components: +// Maven BOM/platform dependencies (purl qualifier "type=pom") and the internal ":platform" +// project are stripped: they are not part of the binary distribution and the plugin has no +// option to exclude them. +// 4. Solr components: +// The plugin emits invalid purls for Solr projects, built from the Gradle project name and a +// non-standard "project_path" qualifier (e.g. pkg:maven/org.apache.solr/core@11.0.0?project_path=:solr:core). +// They are replaced with the Maven artifactId (base.archivesName) and a "type=jar" qualifier +// (e.g. pkg:maven/org.apache.solr/solr-core@11.0.0?type=jar). +// The project description and the Apache-2.0 license are also added. +// 5. Vendored JavaScript libraries: +// The AngularJS admin UI ships third-party JavaScript files checked into solr/webapp/web/libs. +// Components for them are added from a curated list (version and license read from the file +// headers); the hash matching of step 8 proves that each file still ships unmodified. +// 6. JavaScript client bundle: +// The npm packages bundled by browserify into the OpenAPI JS client +// (server/solr-webapp/webapp/libs/solr/index.js) are nested as subassemblies of a first-party +// "solr-js-client" component, using the SBOM generated by cyclonedx-npm in :solr:webapp:js-client. +// 7. New UI bundle: +// The Maven dependencies compiled into the wasmJs UI (server/solr-webapp/webapp/ui) are nested +// as subassemblies of a first-party "solr-ui" component, using the SBOM generated in :solr:ui, +// together with the npm package bundled by the Kotlin toolchain (see kotlin-js-store/wasm/yarn.lock). +// 8. Archive locations: +// The location of each JAR and JavaScript file within the distribution is recorded as +// "evidence.occurrences": the directories assembled for the distribution are scanned and +// their files are matched to the components by SHA-256 hash. +// 9. Hashes: +// Only the SHA-256 hash of each component is kept: the plugin emits eight algorithms +// per artifact, which only adds bulk. +def postProcessBom = { File bomFile, String edition, Map scanDirs, File jsClientSbomFile, File uiSbomFile -> + def json = new JsonSlurper().parse(bomFile) + + def sha256Of = { File file -> + def digest = MessageDigest.getInstance('SHA-256') + file.eachByte(8192) { buffer, length -> digest.update(buffer, 0, length) } + digest.digest().encodeHex().toString() + } + + // Copies a child SBOM's dependency graph into this one: the child root is replaced + // by the given bundle ref, dropped refs are skipped and entries for refs that + // already exist (the same artifact in both graphs) are merged. + def mergeChildGraph = { List childDeps, String childRootRef, String bundleRef, Set droppedRefs -> + childDeps.each { dep -> + if (dep.ref in droppedRefs) { + return + } + def ref = dep.ref == childRootRef ? bundleRef : dep.ref + def dependsOn = (dep.dependsOn ?: []).findAll { !(it in droppedRefs) } + .collect { it == childRootRef ? bundleRef : it } + def existing = json.dependencies.find { it.ref == ref } + if (existing == null) { + json.dependencies << [ref: ref, dependsOn: dependsOn] + } else { + existing.dependsOn = ((existing.dependsOn ?: []) + dependsOn).unique() + } + } + } + + // 1. Metadata + json.metadata.lifecycles = [[phase: 'build']] + + // Record this post-processing step next to the CycloneDX plugin + if (json.metadata.tools == null) { + json.metadata.tools = [components: []] + } + json.metadata.tools.components << [ + type: 'application', + author: 'The Apache Software Foundation', + name: 'solr-sbom-post-processing', + version: project.version, + description: 'Post-processing of the generated SBOM by the Solr Gradle build (:solr:packaging)', + ] + + // 2. Main component + // Old bom-ref -> new purl, applied to the dependency graph below + def rewrittenRefs = [:] + + Map mainComponent = json.metadata.component + def mainPurl = "pkg:sid/apache.org/solr/solr@${mainComponent.version}?edition=${edition}".toString() + rewrittenRefs[mainComponent.'bom-ref'] = mainPurl + mainComponent.remove('group') + mainComponent.name = 'Apache Solr binary release' + mainComponent.cpe = "cpe:2.3:a:apache:solr:${mainComponent.version}:*:*:*:*:*:*:*".toString() + mainComponent.purl = mainPurl + mainComponent.'bom-ref' = mainPurl + + // Gradle project path encoded in the purls the plugin generates for Solr projects + def projectPathOf = { purl -> + def matcher = purl =~ /[?&]project_path=([^&]+)(&|$)/ + matcher ? URLDecoder.decode(matcher.group(1), 'UTF-8') : null + } + + // 3. Removed components + def removedRefs = json.components.findAll { + it.purl =~ /[?&]type=pom(&|$)/ || projectPathOf(it.purl) == ':platform' + }.collect { it.'bom-ref' } as Set + + // 4. Solr components + Map artifactIdByPath = rootProject.allprojects.collectEntries { + [(it.path): it.base.archivesName.get()] + } + json.components = json.components.collect { Map component -> + def projectPath = projectPathOf(component.purl) + // Skip external components and those about to be removed + if (projectPath == null || component.'bom-ref' in removedRefs) { + return component + } + def artifactId = artifactIdByPath[projectPath] + def repositoryUrlQualifier = component.version.endsWith("-SNAPSHOT") ? APACHE_SNAPSHOTS_QUALIFIER : '' + def purl = "pkg:maven/${component.group}/${artifactId}@${component.version}?type=jar${repositoryUrlQualifier}".toString() + rewrittenRefs[component.'bom-ref'] = purl + component.name = artifactId + component.purl = purl + component.'bom-ref' = purl + component.description = rootProject.project(projectPath).description + component.licenses = [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]] + // Restore the field order the plugin uses for external components + def ordered = [:] + ['type', 'bom-ref', 'group', 'name', 'version', 'description', 'hashes', + 'licenses', 'purl', 'modified', 'properties'].each { key -> + if (component.containsKey(key)) { + ordered[key] = component[key] + } + } + component.forEach { key, value -> + if (!ordered.containsKey(key)) { + ordered[key] = value + } + } + return ordered + } + + // Apply the removals (3) and the ref rewrites (2, 4) to the dependency graph + json.components.removeAll { it.'bom-ref' in removedRefs } + json.dependencies?.removeAll { it.ref in removedRefs } + json.dependencies?.each { dep -> + dep.ref = rewrittenRefs.getOrDefault(dep.ref, dep.ref) + dep.dependsOn?.removeAll { it in removedRefs } + if (dep.dependsOn != null) { + dep.dependsOn = dep.dependsOn.collect { rewrittenRefs.getOrDefault(it, it) } + } + } + + // The UI artifacts of steps 5 to 7 all ship inside the webapp + def webappRef = json.components.find { + it.purl?.startsWith('pkg:maven/org.apache.solr/solr-webapp@') + }?.'bom-ref' ?: mainComponent.'bom-ref' + def webappDependsOn = json.dependencies.find { it.ref == webappRef }.dependsOn + + // 5. Vendored JavaScript libraries + // Entries without a version marker in the file get no version and no purl + def vendoredJsLibs = [ + [file: 'angular.min.js', name: 'angular', version: '1.8.0', license: 'MIT'], + [file: 'angular-chosen.min.js', name: 'angular-chosen-localytics', version: '1.9.2', license: 'MIT'], + [file: 'angular-cookies.min.js', name: 'angular-cookies', version: '1.8.0', license: 'MIT'], + [file: 'angular-resource.min.js', name: 'angular-resource', version: '1.8.0', license: 'MIT'], + [file: 'angular-route.min.js', name: 'angular-route', version: '1.8.0', license: 'MIT'], + [file: 'angular-sanitize.min.js', name: 'angular-sanitize', version: '1.8.0', license: 'MIT'], + [file: 'angular-utf8-base64.min.js', name: 'angular-utf8-base64', license: 'MIT'], + [file: 'chosen.jquery.min.js', name: 'chosen-js', version: '1.8.7', license: 'MIT'], + [file: 'd3.js', name: 'd3', version: '2.8.1', license: 'BSD-3-Clause'], + [file: 'highlight.js', name: 'highlight.js', license: 'BSD-3-Clause'], + [file: 'jquery-3.5.1.min.js', name: 'jquery', version: '3.5.1', license: 'MIT'], + [file: 'jquery-ui.min.js', name: 'jquery-ui', version: '1.12.1', license: 'MIT'], + [file: 'jssha-3.3.1-sha256.min.js', name: 'jssha', version: '3.3.1', license: 'BSD-3-Clause'], + [file: 'jstree.min.js', name: 'jstree', version: '3.3.10', license: 'MIT'], + [file: 'ngtimeago.js', name: 'ngtimeago', license: 'MIT'], + [file: 'ui-grid.min.js', name: 'angular-ui-grid', version: '4.10.0', license: 'MIT'], + ] + def webLibsDir = rootProject.file('solr/webapp/web/libs') + vendoredJsLibs.each { lib -> + def purl = lib.version != null ? "pkg:npm/${lib.name}@${lib.version}".toString() : null + def component = [ + type: 'library', + 'bom-ref': purl ?: "vendored-js:${lib.name}".toString(), + name: lib.name, + ] + if (lib.version != null) { + component.version = lib.version + } + component.hashes = [[alg: 'SHA-256', content: sha256Of(new File(webLibsDir, lib.file))]] + component.licenses = [[license: [id: lib.license]]] + if (purl != null) { + component.purl = purl + } + json.components << component + json.dependencies << [ref: component.'bom-ref', dependsOn: []] + webappDependsOn << component.'bom-ref' + } + + // 6. JavaScript client bundle + if (jsClientSbomFile != null) { + def jsClientBom = new JsonSlurper().parse(jsClientSbomFile) + def bundleRef = "solr-js-client@${project.version}".toString() + json.components << [ + type: 'library', + 'bom-ref': bundleRef, + name: 'solr-js-client', + version: project.version.toString(), + description: 'JavaScript client for the Solr v2 API, generated from its OpenAPI specification', + licenses: [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]], + // The npm packages bundled into the single shipped file by browserify + components: jsClientBom.components, + evidence: [occurrences: [[location: 'server/solr-webapp/webapp/libs/solr/index.js']]], + ] + mergeChildGraph(jsClientBom.dependencies ?: [], jsClientBom.metadata.component.'bom-ref', bundleRef, [] as Set) + webappDependsOn << bundleRef + // Record cyclonedx-npm next to the other tools + jsClientBom.metadata?.tools?.components?.each { tool -> + if (!json.metadata.tools.components.any { it.name == tool.name && it.version == tool.version }) { + json.metadata.tools.components << tool + } + } + } + + // 7. New UI bundle + def uiBundle = null + if (uiSbomFile != null) { + def uiBom = new JsonSlurper().parse(uiSbomFile) + // The ":platform" project and Maven BOMs are on the UI classpath too, see step 3 + def uiDroppedRefs = uiBom.components.findAll { + it.purl =~ /[?&]type=pom(&|$)/ || projectPathOf(it.purl) == ':platform' + }.collect { it.'bom-ref' } as Set + // An artifact present in the Java graph too keeps its top-level component only + def existingRefs = json.components.collect { it.'bom-ref' } as Set + def nested = uiBom.components.findAll { + !(it.'bom-ref' in uiDroppedRefs) && !(it.'bom-ref' in existingRefs) + } + // Bundled by the Kotlin toolchain, see kotlin-js-store/wasm/yarn.lock + nested << [ + type: 'library', + 'bom-ref': 'pkg:npm/%40js-joda/core@3.2.0', + name: '@js-joda/core', + version: '3.2.0', + licenses: [[license: [id: 'BSD-3-Clause']]], + purl: 'pkg:npm/%40js-joda/core@3.2.0', + ] + def bundleRef = "solr-ui@${project.version}".toString() + uiBundle = [ + type: 'library', + 'bom-ref': bundleRef, + name: 'solr-ui', + version: project.version.toString(), + description: 'New Solr admin UI, compiled to WebAssembly', + licenses: [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]], + // The Maven artifacts compiled into the bundle and the npm package above + components: nested, + // Occurrences are filled by the scan of step 8, the bundle file names are content-hashed + ] + json.components << uiBundle + mergeChildGraph(uiBom.dependencies ?: [], uiBom.metadata.component.'bom-ref', bundleRef, uiDroppedRefs) + def uiEntry = json.dependencies.find { it.ref == bundleRef } + uiEntry.dependsOn << 'pkg:npm/%40js-joda/core@3.2.0' + json.dependencies << [ref: 'pkg:npm/%40js-joda/core@3.2.0', dependsOn: []] + webappDependsOn << bundleRef + } + + // 8. Archive locations + // Distribution files indexed by SHA-256 hash; UI bundle files collected on the way + def locationsByHash = [:].withDefault { [] } + def uiLocations = [] + scanDirs.each { prefix, configuration -> + configuration.files.each { root -> + fileTree(root).matching { + include '**/*.jar' + include '**/*.js' + include 'solr-webapp/webapp/ui/**' + }.visit { entry -> + if (!entry.directory) { + def location = "${prefix}/${entry.relativePath}".toString() + locationsByHash[sha256Of(entry.file)] << location + if (entry.relativePath.pathString.startsWith('solr-webapp/webapp/ui/')) { + uiLocations << location + } + } + } + } + } + json.components.each { component -> + def sha256 = component.hashes?.find { it.alg == 'SHA-256' }?.content + if (sha256 != null && locationsByHash.containsKey(sha256)) { + component.evidence = [occurrences: locationsByHash[sha256].sort().collect { [location: it] }] + } + } + if (uiBundle != null) { + uiBundle.evidence = [occurrences: uiLocations.sort().collect { [location: it] }] + } + + // 9. Hashes + // The npm integrity hashes on "externalReferences" are kept, they describe + // the registry tarballs and have no SHA-256 equivalent. + def keepSha256Only + keepSha256Only = { List components -> + components.each { component -> + if (component.hashes != null) { + component.hashes = component.hashes.findAll { it.alg == 'SHA-256' } + if (component.hashes.isEmpty()) { + component.remove('hashes') + } + } + keepSha256Only(component.components ?: []) + } + } + keepSha256Only(json.components) + + bomFile.text = JsonOutput.prettyPrint(JsonOutput.toJson(json)) +} + +tasks.register('cyclonedxFull', CyclonedxDirectTask) { + group = 'Bill of Materials' + description = 'Generates CycloneDX BOM for the full Solr distribution' + + includeConfigs = ['bomFull'] + projectType = Component.Type.APPLICATION + + // The plugin resolves the configuration leniently and without depending on it, + // so the jars of Solr projects must be built first or their hashes are missing. + inputs.files(configurations.bomFull) + .withPropertyName('bomFullArtifacts') + .withNormalizer(ClasspathNormalizer) + + // Distribution directories scanned for the archive location of each artifact + inputs.files(configurations.server, configurations.modules, configurations.crossDcManager) + .withPropertyName('distributionDirs') + .withPathSensitivity(PathSensitivity.RELATIVE) + + // Child SBOMs of the UI bundles; empty when the projects are disabled + inputs.files(configurations.jsClientSbom, configurations.uiSbom) + .withPropertyName('childSboms') + .withPathSensitivity(PathSensitivity.NONE) + + // Sources of the statically listed JavaScript components + inputs.dir(rootProject.file('solr/webapp/web/libs')) + .withPropertyName('vendoredJsLibs') + .withPathSensitivity(PathSensitivity.RELATIVE) + inputs.file(rootProject.file('kotlin-js-store/wasm/yarn.lock')) + .withPropertyName('uiYarnLock') + .withPathSensitivity(PathSensitivity.NONE) + + jsonOutput = cyclonedxDir.get().file("bom-full.json").asFile + + doLast { + postProcessBom(jsonOutput.get().asFile, 'full', [ + 'server': configurations.server, + 'modules': configurations.modules, + 'cross-dc-manager': configurations.crossDcManager, + ], configurations.jsClientSbom.files.find(), configurations.uiSbom.files.find()) + } +} + +tasks.register('cyclonedxSlim', CyclonedxDirectTask) { + group = 'Bill of Materials' + description = 'Generates CycloneDX BOM for the slim Solr distribution' + + includeConfigs = ['bomSlim'] + projectType = Component.Type.APPLICATION + + // The plugin resolves the configuration leniently and without depending on it, + // so the jars of Solr projects must be built first or their hashes are missing. + inputs.files(configurations.bomSlim) + .withPropertyName('bomSlimArtifacts') + .withNormalizer(ClasspathNormalizer) + + // Distribution directories scanned for the archive location of each artifact + inputs.files(configurations.server) + .withPropertyName('distributionDirs') + .withPathSensitivity(PathSensitivity.RELATIVE) + + // Child SBOMs of the UI bundles; empty when the projects are disabled + inputs.files(configurations.jsClientSbom, configurations.uiSbom) + .withPropertyName('childSboms') + .withPathSensitivity(PathSensitivity.NONE) + + // Sources of the statically listed JavaScript components + inputs.dir(rootProject.file('solr/webapp/web/libs')) + .withPropertyName('vendoredJsLibs') + .withPathSensitivity(PathSensitivity.RELATIVE) + inputs.file(rootProject.file('kotlin-js-store/wasm/yarn.lock')) + .withPropertyName('uiYarnLock') + .withPathSensitivity(PathSensitivity.NONE) + + jsonOutput = cyclonedxDir.get().file("bom-slim.json").asFile + + doLast { + postProcessBom(jsonOutput.get().asFile, 'slim', [ + 'server': configurations.server, + ], configurations.jsClientSbom.files.find(), configurations.uiSbom.files.find()) + } +} + +tasks.register('cyclonedx') { + group = 'Bill of Materials' + description = 'Generates CycloneDX BOMs for Solr distributions' + + dependsOn 'cyclonedxFull' + dependsOn 'cyclonedxSlim' } description = 'Solr distribution packaging' ext { + cyclonedxDir = layout.buildDirectory.dir("cyclonedx") distDir = file("$buildDir/solr-${version}") slimDistDir = file("$buildDir/solr-${version}-slim") devDir = file("$buildDir/dev") @@ -49,17 +467,79 @@ configurations { solrSlimTgz solrFullTgzSignature solrSlimTgzSignature + // For the CycloneDX BOM generation + bomSlim { + canBeResolved = true + canBeConsumed = false + } + bomFull { + canBeResolved = true + canBeConsumed = false + extendsFrom bomSlim + } + // Child SBOMs of the UI bundles, merged into the distribution SBOMs by postProcessBom + jsClientSbom { + canBeResolved = true + canBeConsumed = false + } + uiSbom { + canBeResolved = true + canBeConsumed = false + } +} + +// Request the standard JVM runtime variants, like runtimeClasspath does. +// +// The CycloneDX plugin resolves configurations leniently. Without these attributes, +// variant-aware dependencies (e.g. Guava) and platform constraints (e.g. the Jersey BOM) fail to resolve +// and silently disappear. +[configurations.bomSlim, configurations.bomFull].each { conf -> + conf.attributes { + attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY)) + attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, Usage.JAVA_RUNTIME)) + attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, objects.named(LibraryElements, LibraryElements.JAR)) + attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL)) + attribute(TargetJvmEnvironment.TARGET_JVM_ENVIRONMENT_ATTRIBUTE, objects.named(TargetJvmEnvironment, TargetJvmEnvironment.STANDARD_JVM)) + } } +// The distribution keeps the server versions of the libraries shared between the +// server and the modules, so align the full BOM with the versions of the slim one +// (e.g. a module may pull in a newer slf4j-api than the one in server/lib/ext). +configurations.bomFull.shouldResolveConsistentlyWith(configurations.bomSlim) + dependencies { - rootProject.project(":solr:modules").childProjects.values().stream().map {project -> project.path}.each { - module -> modules project(path: module, configuration: "packaging") + rootProject.project(":solr:modules").childProjects.values().stream().map {project -> project.path}.each { module -> + modules project(path: module, configuration: "packaging") + // No "configuration:" here on purpose. + // + // Gradle then selects the variant of the module that matches the attributes declared on the bom configurations above, + // just like it does for a runtime classpath ("variant-aware" resolution). + // Naming a configuration would bypass attribute matching, and imported Maven BOMs (e.g. Jersey's) would no + // longer provide the versions of their managed dependencies. + bomFull project(path: module) } crossDcManager project(path: ":solr:cross-dc-manager", configuration: "packaging") + bomFull project(path: ':solr:cross-dc-manager') example project(path: ":solr:example", configuration: "packaging") server project(path: ":solr:server", configuration: "packaging") + bomSlim project(path: ':solr:server', configuration: 'startJar') + bomSlim project(path: ':solr:server', configuration: 'serverLib') + bomSlim project(path: ':solr:server', configuration: 'libExt') + // Variant-aware dependency instead of server's by-name 'solrCore' + // configuration, see the comment on the modules above. + bomSlim project(path: ':solr:core') + bomSlim project(path: ':solr:server', configuration: 'webapp') + + // Child SBOMs of the UI bundles; empty when the projects are disabled + if (gradle.ext.withJsClient) { + jsClientSbom project(path: ':solr:webapp:js-client', configuration: 'jsClientSbom') + } + if (gradle.ext.withUiModule) { + uiSbom project(path: ':solr:ui', configuration: 'uiSbom') + } docker project(path: ':solr:docker', configuration: 'packaging') @@ -119,6 +599,12 @@ distributions { } }) + // Include CycloneDX BOM + from(cyclonedxDir) { + include 'bom-slim.json' + rename 'bom-slim.json', 'bom.json' + } + // Manually correct posix permissions (matters when packaging on Windows). filesMatching([ "**/*.sh", @@ -127,7 +613,6 @@ distributions { ]) {copy -> copy.permissions { unix("0755") } } - } } full { @@ -140,6 +625,13 @@ distributions { into "modules" }) + // Include CycloneDX BOM + from(cyclonedxDir) { + include 'bom-full.json' + rename 'bom-full.json', 'bom.json' + duplicatesStrategy = DuplicatesStrategy.INCLUDE + } + from(configurations.crossDcManager, { into "cross-dc-manager" filesMatching([ @@ -153,10 +645,12 @@ distributions { } installFullDist { + dependsOn 'cyclonedx' into distDir } installSlimDist { + dependsOn 'cyclonedx' into slimDistDir } @@ -195,10 +689,12 @@ task dev { } fullDistTar { + dependsOn 'cyclonedx' compression = Compression.GZIP } slimDistTar { + dependsOn 'cyclonedx' compression = Compression.GZIP } diff --git a/solr/packaging/gradle.lockfile b/solr/packaging/gradle.lockfile index 67666c69405d..6f7191470d53 100644 --- a/solr/packaging/gradle.lockfile +++ b/solr/packaging/gradle.lockfile @@ -2,4 +2,333 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. # To regenerate this file, run: ./gradlew :solr:packaging:dependencies --write-locks -empty=crossDcManager,docker,docs,example,jarValidation,modules,server,solrFullTgz,solrFullTgzSignature,solrSlimTgz,solrSlimTgzSignature +at.yawk.lz4:lz4-java:1.10.1=bomFull +com.carrotsearch:hppc:0.10.0=bomFull,bomSlim +com.fasterxml.jackson.core:jackson-annotations:2.22=bomFull,bomSlim +com.fasterxml.jackson.core:jackson-core:2.22.0=bomFull,bomSlim +com.fasterxml.jackson.core:jackson-databind:2.22.0=bomFull,bomSlim +com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.22.0=bomFull,bomSlim +com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.22.0=bomFull +com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.22.0=bomFull,bomSlim +com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.22.0=bomFull +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.22.0=bomFull +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22.0=bomFull +com.fasterxml.jackson.module:jackson-module-jakarta-xmlbind-annotations:2.22.0=bomFull,bomSlim +com.fasterxml.jackson.module:jackson-module-scala_2.13:2.22.0=bomFull +com.fasterxml.jackson:jackson-bom:2.22.0=bomFull,bomSlim +com.fasterxml.woodstox:woodstox-core:7.2.1=bomFull,bomSlim +com.github.ben-manes.caffeine:caffeine:3.2.4=bomFull,bomSlim +com.github.luben:zstd-jni:1.5.6-4=bomFull +com.google.android:annotations:4.1.1.4=bomFull +com.google.api-client:google-api-client:2.7.2=bomFull +com.google.api.grpc:gapic-google-cloud-storage-v2:2.69.0=bomFull +com.google.api.grpc:grpc-google-cloud-storage-v2:2.69.0=bomFull +com.google.api.grpc:proto-google-cloud-storage-v2:2.69.0=bomFull +com.google.api.grpc:proto-google-common-protos:2.72.0=bomFull +com.google.api.grpc:proto-google-iam-v1:1.67.0=bomFull +com.google.api:api-common:2.64.0=bomFull +com.google.api:gax-grpc:2.81.0=bomFull +com.google.api:gax-httpjson:2.81.0=bomFull +com.google.api:gax:2.81.0=bomFull +com.google.apis:google-api-services-storage:v1-rev20260204-2.0.0=bomFull +com.google.auth:google-auth-library-credentials:1.48.0=bomFull +com.google.auth:google-auth-library-oauth2-http:1.48.0=bomFull +com.google.auto.value:auto-value-annotations:1.11.1=bomFull +com.google.cloud:google-cloud-bom:0.265.0=bomFull +com.google.cloud:google-cloud-core-grpc:2.71.0=bomFull +com.google.cloud:google-cloud-core-http:2.71.0=bomFull +com.google.cloud:google-cloud-core:2.71.0=bomFull +com.google.cloud:google-cloud-storage:2.69.0=bomFull +com.google.code.gson:gson:2.14.0=bomFull +com.google.errorprone:error_prone_annotations:2.47.0=bomFull,bomSlim +com.google.guava:failureaccess:1.0.3=bomFull,bomSlim +com.google.guava:guava:33.6.0-jre=bomFull,bomSlim +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=bomFull,bomSlim +com.google.http-client:google-http-client-apache-v2:2.1.0=bomFull +com.google.http-client:google-http-client-appengine:2.1.0=bomFull +com.google.http-client:google-http-client-gson:2.1.0=bomFull +com.google.http-client:google-http-client-jackson2:2.1.0=bomFull +com.google.http-client:google-http-client:2.1.0=bomFull +com.google.j2objc:j2objc-annotations:3.1=bomFull,bomSlim +com.google.oauth-client:google-oauth-client:1.39.0=bomFull +com.google.protobuf:protobuf-java-util:4.35.1=bomFull +com.google.protobuf:protobuf-java:4.35.1=bomFull +com.google.re2j:re2j:1.8=bomFull +com.googlecode.json-simple:json-simple:1.1.1=bomFull +com.ibm.icu:icu4j:78.3=bomFull +com.j256.simplemagic:simplemagic:1.17=bomFull,bomSlim +com.jayway.jsonpath:json-path:3.0.0=bomFull,bomSlim +com.knuddels:jtokkit:1.1.0=bomFull +com.lmax:disruptor:4.0.0=bomFull,bomSlim +com.microsoft.onnxruntime:onnxruntime:1.26.0=bomFull +com.nvidia.cuvs.lucene:cuvs-lucene:25.12.0=bomFull +com.nvidia.cuvs:cuvs-java:26.06.0=bomFull +com.squareup.okhttp3:okhttp-jvm:5.4.0=bomFull +com.squareup.okhttp3:okhttp:5.4.0=bomFull +com.squareup.okio:okio-jvm:3.17.0=bomFull +com.squareup.okio:okio:3.17.0=bomFull +com.squareup.retrofit2:converter-jackson:2.9.0=bomFull +com.squareup.retrofit2:retrofit:2.9.0=bomFull +com.tdunning:t-digest:3.3=bomFull,bomSlim +com.thoughtworks.paranamer:paranamer:2.8.3=bomFull +com.typesafe.scala-logging:scala-logging_2.13:3.9.5=bomFull +com.yammer.metrics:metrics-core:2.2.0=bomFull +commons-beanutils:commons-beanutils:1.11.0=bomFull +commons-cli:commons-cli:1.11.0=bomFull,bomSlim +commons-codec:commons-codec:1.22.0=bomFull,bomSlim +commons-collections:commons-collections:3.2.2=bomFull +commons-digester:commons-digester:2.1=bomFull +commons-io:commons-io:2.22.0=bomFull,bomSlim +commons-validator:commons-validator:1.10.1=bomFull +dev.langchain4j:langchain4j-bom:1.17.0=bomFull +dev.langchain4j:langchain4j-cohere:1.17.0-beta27=bomFull +dev.langchain4j:langchain4j-core:1.17.0=bomFull +dev.langchain4j:langchain4j-http-client-jdk:1.17.0=bomFull +dev.langchain4j:langchain4j-http-client:1.17.0=bomFull +dev.langchain4j:langchain4j-hugging-face:1.17.0-beta27=bomFull +dev.langchain4j:langchain4j-mistral-ai:1.17.0=bomFull +dev.langchain4j:langchain4j-open-ai:1.17.0=bomFull +io.dropwizard.metrics:metrics-core:4.2.39=bomFull,bomSlim +io.github.azagniotov:language-detection:12.5.2=bomFull +io.grpc:grpc-alts:1.82.0=bomFull +io.grpc:grpc-api:1.82.0=bomFull +io.grpc:grpc-auth:1.82.0=bomFull +io.grpc:grpc-bom:1.82.0=bomFull +io.grpc:grpc-context:1.82.0=bomFull +io.grpc:grpc-core:1.82.0=bomFull +io.grpc:grpc-googleapis:1.82.0=bomFull +io.grpc:grpc-grpclb:1.82.0=bomFull +io.grpc:grpc-inprocess:1.82.0=bomFull +io.grpc:grpc-netty-shaded:1.82.0=bomFull +io.grpc:grpc-protobuf-lite:1.82.0=bomFull +io.grpc:grpc-protobuf:1.82.0=bomFull +io.grpc:grpc-rls:1.82.0=bomFull +io.grpc:grpc-services:1.82.0=bomFull +io.grpc:grpc-stub:1.82.0=bomFull +io.grpc:grpc-util:1.82.0=bomFull +io.grpc:grpc-xds:1.82.0=bomFull +io.netty:netty-buffer:4.2.15.Final=bomFull,bomSlim +io.netty:netty-codec-base:4.2.15.Final=bomFull,bomSlim +io.netty:netty-common:4.2.15.Final=bomFull,bomSlim +io.netty:netty-handler:4.2.15.Final=bomFull,bomSlim +io.netty:netty-resolver:4.2.15.Final=bomFull,bomSlim +io.netty:netty-tcnative-boringssl-static:2.0.79.Final=bomFull,bomSlim +io.netty:netty-tcnative-classes:2.0.79.Final=bomFull,bomSlim +io.netty:netty-transport-classes-epoll:4.2.15.Final=bomFull,bomSlim +io.netty:netty-transport-native-epoll:4.2.15.Final=bomFull,bomSlim +io.netty:netty-transport-native-unix-common:4.2.15.Final=bomFull,bomSlim +io.netty:netty-transport:4.2.15.Final=bomFull,bomSlim +io.opencensus:opencensus-api:0.31.1=bomFull +io.opencensus:opencensus-contrib-http-util:0.31.1=bomFull +io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37.0-alpha=bomFull +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.22.0-alpha=bomFull,bomSlim +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.22.0=bomFull,bomSlim +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17:2.22.0-alpha=bomFull,bomSlim +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java8:2.22.0-alpha=bomFull,bomSlim +io.opentelemetry.semconv:opentelemetry-semconv:1.37.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-api-incubator:1.56.0-alpha=bomFull,bomSlim +io.opentelemetry:opentelemetry-api:1.56.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-bom:1.56.0=bomFull +io.opentelemetry:opentelemetry-common:1.56.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-context:1.56.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-exporter-common:1.56.0=bomFull +io.opentelemetry:opentelemetry-exporter-otlp-common:1.56.0=bomFull +io.opentelemetry:opentelemetry-exporter-otlp:1.56.0=bomFull +io.opentelemetry:opentelemetry-exporter-prometheus:1.56.0-alpha=bomFull,bomSlim +io.opentelemetry:opentelemetry-exporter-sender-jdk:1.56.0=bomFull +io.opentelemetry:opentelemetry-sdk-common:1.56.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.56.0=bomFull +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.56.0=bomFull +io.opentelemetry:opentelemetry-sdk-logs:1.56.0=bomFull +io.opentelemetry:opentelemetry-sdk-metrics:1.56.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-sdk-trace:1.56.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-sdk:1.56.0=bomFull,bomSlim +io.perfmark:perfmark-api:0.27.0=bomFull +io.prometheus:prometheus-metrics-exposition-formats:1.1.0=bomFull,bomSlim +io.prometheus:prometheus-metrics-model:1.1.0=bomFull,bomSlim +io.sgr:s2-geometry-library-java:1.0.0=bomFull,bomSlim +io.swagger.core.v3:swagger-annotations-jakarta:2.2.52=bomFull,bomSlim +jakarta.activation:jakarta.activation-api:2.1.3=bomFull,bomSlim +jakarta.annotation:jakarta.annotation-api:3.0.0=bomFull,bomSlim +jakarta.inject:jakarta.inject-api:2.0.1=bomFull,bomSlim +jakarta.servlet:jakarta.servlet-api:6.1.0=bomFull,bomSlim +jakarta.validation:jakarta.validation-api:3.1.0=bomFull,bomSlim +jakarta.ws.rs:jakarta.ws.rs-api:4.0.0=bomFull,bomSlim +jakarta.xml.bind:jakarta.xml.bind-api:4.0.2=bomFull,bomSlim +net.sf.jopt-simple:jopt-simple:5.0.4=bomFull +net.sourceforge.argparse4j:argparse4j:0.7.0=bomFull +org.antlr:antlr4-runtime:4.13.2=bomFull,bomSlim +org.apache.calcite.avatica:avatica-core:1.25.0=bomFull +org.apache.calcite.avatica:avatica-metrics:1.25.0=bomFull +org.apache.calcite:calcite-core:1.37.0=bomFull +org.apache.calcite:calcite-linq4j:1.37.0=bomFull +org.apache.commons:commons-exec:1.6.0=bomFull,bomSlim +org.apache.commons:commons-lang3:3.20.0=bomFull,bomSlim +org.apache.commons:commons-math3:3.6.1=bomFull,bomSlim +org.apache.commons:commons-text:1.15.0=bomFull +org.apache.curator:curator-client:5.9.0=bomFull,bomSlim +org.apache.curator:curator-framework:5.9.0=bomFull,bomSlim +org.apache.httpcomponents.client5:httpclient5:5.2.1=bomFull +org.apache.httpcomponents.core5:httpcore5-h2:5.2=bomFull +org.apache.httpcomponents.core5:httpcore5:5.2.3=bomFull +org.apache.httpcomponents:httpclient:4.5.14=bomFull +org.apache.httpcomponents:httpcore:4.4.16=bomFull +org.apache.kafka:kafka-clients:3.9.2=bomFull +org.apache.kafka:kafka-group-coordinator-api:3.9.2=bomFull +org.apache.kafka:kafka-group-coordinator:3.9.2=bomFull +org.apache.kafka:kafka-metadata:3.9.2=bomFull +org.apache.kafka:kafka-raft:3.9.2=bomFull +org.apache.kafka:kafka-server-common:3.9.2=bomFull +org.apache.kafka:kafka-server:3.9.2=bomFull +org.apache.kafka:kafka-storage-api:3.9.2=bomFull +org.apache.kafka:kafka-storage:3.9.2=bomFull +org.apache.kafka:kafka-streams:3.9.2=bomFull +org.apache.kafka:kafka-tools-api:3.9.2=bomFull +org.apache.kafka:kafka-transaction-coordinator:3.9.2=bomFull +org.apache.kafka:kafka_2.13:3.9.2=bomFull +org.apache.logging.log4j:log4j-1.2-api:2.26.0=bomFull,bomSlim +org.apache.logging.log4j:log4j-api:2.26.0=bomFull,bomSlim +org.apache.logging.log4j:log4j-core:2.26.0=bomFull,bomSlim +org.apache.logging.log4j:log4j-layout-template-json:2.26.0=bomFull,bomSlim +org.apache.logging.log4j:log4j-slf4j2-impl:2.26.0=bomFull,bomSlim +org.apache.logging.log4j:log4j-web:2.26.0=bomFull,bomSlim +org.apache.lucene:lucene-analysis-common:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-analysis-icu:10.4.0=bomFull +org.apache.lucene:lucene-analysis-kuromoji:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-analysis-morfologik:10.4.0=bomFull +org.apache.lucene:lucene-analysis-nori:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-analysis-opennlp:10.4.0=bomFull +org.apache.lucene:lucene-analysis-phonetic:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-analysis-smartcn:10.4.0=bomFull +org.apache.lucene:lucene-analysis-stempel:10.4.0=bomFull +org.apache.lucene:lucene-backward-codecs:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-classification:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-codecs:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-core:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-expressions:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-facet:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-grouping:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-highlighter:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-join:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-memory:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-misc:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-queries:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-queryparser:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-sandbox:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-spatial-extras:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-spatial3d:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-suggest:10.4.0=bomFull,bomSlim +org.apache.opennlp:opennlp-dl:2.5.10=bomFull +org.apache.opennlp:opennlp-tools:2.5.10=bomFull +org.apache.tika:tika-core:3.3.1=bomFull +org.apache.zookeeper:zookeeper-jute:3.9.5=bomFull,bomSlim +org.apache.zookeeper:zookeeper:3.9.5=bomFull,bomSlim +org.apiguardian:apiguardian-api:1.1.2=bomFull +org.bitbucket.b_c:jose4j:0.9.6=bomFull +org.carrot2:carrot2-core:4.8.6=bomFull +org.carrot2:morfologik-fsa:2.1.9=bomFull +org.carrot2:morfologik-polish:2.1.9=bomFull +org.carrot2:morfologik-stemming:2.1.9=bomFull +org.checkerframework:checker-qual:4.2.0=bomFull +org.codehaus.janino:commons-compiler:3.1.11=bomFull +org.codehaus.janino:janino:3.1.11=bomFull +org.codehaus.woodstox:stax2-api:4.3.0=bomFull,bomSlim +org.conscrypt:conscrypt-openjdk-uber:2.5.2=bomFull +org.eclipse.jetty.compression:jetty-compression-common:12.1.10=bomFull,bomSlim +org.eclipse.jetty.compression:jetty-compression-gzip:12.1.10=bomFull,bomSlim +org.eclipse.jetty.ee10:jetty-ee10-servlet:12.1.10=bomFull,bomSlim +org.eclipse.jetty.ee10:jetty-ee10-servlets:12.1.10=bomFull,bomSlim +org.eclipse.jetty.ee10:jetty-ee10-webapp:12.1.10=bomFull,bomSlim +org.eclipse.jetty.ee:jetty-ee-webapp:12.1.10=bomFull,bomSlim +org.eclipse.jetty.http2:jetty-http2-client-transport:12.1.10=bomFull,bomSlim +org.eclipse.jetty.http2:jetty-http2-client:12.1.10=bomFull,bomSlim +org.eclipse.jetty.http2:jetty-http2-common:12.1.10=bomFull,bomSlim +org.eclipse.jetty.http2:jetty-http2-hpack:12.1.10=bomFull,bomSlim +org.eclipse.jetty.http2:jetty-http2-server:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-alpn-client:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-alpn-java-client:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-alpn-java-server:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-alpn-server:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-client:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-deploy:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-http:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-io:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-jmx:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-rewrite:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-security:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-server:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-session:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-start:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-util:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-xml:12.1.10=bomFull,bomSlim +org.glassfish.hk2.external:aopalliance-repackaged:4.0.1=bomFull,bomSlim +org.glassfish.hk2:hk2-api:4.0.1=bomFull,bomSlim +org.glassfish.hk2:hk2-locator:4.0.1=bomFull,bomSlim +org.glassfish.hk2:hk2-utils:4.0.1=bomFull,bomSlim +org.glassfish.hk2:osgi-resource-locator:3.0.0=bomFull,bomSlim +org.glassfish.jersey.containers:jersey-container-jetty-http:4.0.2=bomFull,bomSlim +org.glassfish.jersey.core:jersey-client:4.0.2=bomFull,bomSlim +org.glassfish.jersey.core:jersey-common:4.0.2=bomFull,bomSlim +org.glassfish.jersey.core:jersey-server:4.0.2=bomFull,bomSlim +org.glassfish.jersey.ext:jersey-entity-filtering:4.0.2=bomFull,bomSlim +org.glassfish.jersey.inject:jersey-hk2:4.0.2=bomFull,bomSlim +org.glassfish.jersey.media:jersey-media-json-jackson:4.0.2=bomFull,bomSlim +org.glassfish.jersey:jersey-bom:4.0.2=bomFull,bomSlim +org.javassist:javassist:3.30.2-GA=bomFull,bomSlim +org.jetbrains.kotlin:kotlin-stdlib:2.2.21=bomFull +org.jetbrains:annotations:26.1.0=bomFull +org.jspecify:jspecify:1.0.0=bomFull,bomSlim +org.locationtech.jts.io:jts-io-common:1.19.0=bomFull +org.locationtech.jts:jts-core:1.19.0=bomFull +org.locationtech.proj4j:proj4j:1.2.2=bomFull +org.locationtech.spatial4j:spatial4j:0.8=bomFull,bomSlim +org.ow2.asm:asm-commons:9.10.1=bomFull,bomSlim +org.ow2.asm:asm-tree:9.10.1=bomFull,bomSlim +org.ow2.asm:asm:9.10.1=bomFull,bomSlim +org.pcollections:pcollections:4.0.1=bomFull +org.reactivestreams:reactive-streams:1.0.4=bomFull +org.rocksdb:rocksdbjni:7.9.2=bomFull +org.scala-lang.modules:scala-collection-compat_2.13:2.10.0=bomFull +org.scala-lang.modules:scala-java8-compat_2.13:1.0.2=bomFull +org.scala-lang:scala-library:2.13.18=bomFull +org.scala-lang:scala-reflect:2.13.15=bomFull +org.semver4j:semver4j:6.0.0=bomFull,bomSlim +org.slf4j:jcl-over-slf4j:2.0.17=bomFull,bomSlim +org.slf4j:jul-to-slf4j:2.0.17=bomFull,bomSlim +org.slf4j:slf4j-api:2.0.17=bomFull,bomSlim +org.threeten:threetenbp:1.7.3=bomFull +org.xerial.snappy:snappy-java:1.1.10.8=bomFull,bomSlim +software.amazon.awssdk:annotations:2.42.37=bomFull +software.amazon.awssdk:apache-client:2.42.37=bomFull +software.amazon.awssdk:arns:2.42.37=bomFull +software.amazon.awssdk:auth:2.42.37=bomFull +software.amazon.awssdk:aws-core:2.42.37=bomFull +software.amazon.awssdk:aws-query-protocol:2.42.37=bomFull +software.amazon.awssdk:aws-xml-protocol:2.42.37=bomFull +software.amazon.awssdk:bom:2.42.37=bomFull +software.amazon.awssdk:checksums-spi:2.42.37=bomFull +software.amazon.awssdk:checksums:2.42.37=bomFull +software.amazon.awssdk:crt-core:2.42.37=bomFull +software.amazon.awssdk:endpoints-spi:2.42.37=bomFull +software.amazon.awssdk:http-auth-aws-eventstream:2.42.37=bomFull +software.amazon.awssdk:http-auth-aws:2.42.37=bomFull +software.amazon.awssdk:http-auth-spi:2.42.37=bomFull +software.amazon.awssdk:http-auth:2.42.37=bomFull +software.amazon.awssdk:http-client-spi:2.42.37=bomFull +software.amazon.awssdk:identity-spi:2.42.37=bomFull +software.amazon.awssdk:json-utils:2.42.37=bomFull +software.amazon.awssdk:metrics-spi:2.42.37=bomFull +software.amazon.awssdk:profiles:2.42.37=bomFull +software.amazon.awssdk:protocol-core:2.42.37=bomFull +software.amazon.awssdk:regions:2.42.37=bomFull +software.amazon.awssdk:retries-spi:2.42.37=bomFull +software.amazon.awssdk:retries:2.42.37=bomFull +software.amazon.awssdk:s3:2.42.37=bomFull +software.amazon.awssdk:sdk-core:2.42.37=bomFull +software.amazon.awssdk:sts:2.42.37=bomFull +software.amazon.awssdk:third-party-jackson-core:2.42.37=bomFull +software.amazon.awssdk:utils-lite:2.42.37=bomFull +software.amazon.awssdk:utils:2.42.37=bomFull +software.amazon.eventstream:eventstream:1.0.1=bomFull +ua.net.nlp:morfologik-ukrainian-search:4.9.1=bomFull +empty=crossDcManager,docker,docs,example,jarValidation,jsClientSbom,modules,server,solrFullTgz,solrFullTgzSignature,solrSlimTgz,solrSlimTgzSignature,uiSbom diff --git a/solr/server/build.gradle b/solr/server/build.gradle index a22c084d94b0..99120e7ca922 100644 --- a/solr/server/build.gradle +++ b/solr/server/build.gradle @@ -25,14 +25,43 @@ javadoc.enabled(false) compileJava.enabled(false) configurations { - libExt + // === Custom configurations used to assemble the Solr server binary distribution === + + // 1. Jetty Bootstrap JAR + // Output Path: server/start.jar + // Description: Contains the Jetty bootstrap JAR responsible for launching the Solr server. + startJar + + // 2. Server Libraries + // Output Path: server/lib/ + // Description: Contains core libraries required by the Solr server at runtime (mostly Jetty-related JARs). serverLib + + // 3. Extended Server Libraries + // Output Path: server/lib/ext/ + // Description: Includes optional runtime libraries such as logging (SLF4J, Log4j) and metrics (Dropwizard, etc.). + libExt + + // 4. Solr Core JAR + // Output Path: server/solr-webapp/webapp/WEB-INF/lib/ + // Description: Contains the solr-core JAR, which includes the core functionality and indexing logic of Solr. solrCore + + // 5. Solr Web Application Libraries + // Output Path: server/solr-webapp/webapp/WEB-INF/lib/ + // Description: Contains the remaining Solr modules, packaged in exploded WAR format for deployment via Jetty. + webapp + + // === Runtime Configuration === + + // Combines core runtime dependencies for launching the Solr server, + // aggregating required libraries from serverLib, libExt, and solrCore. runtimeClasspath { extendsFrom serverLib, libExt, solrCore } - startJar - webapp + + // Internal configuration used by packaging tasks (e.g., creating distributions). + // This configuration only includes the `packagingDir` folder generated during assembly packaging } diff --git a/solr/ui/build.gradle.kts b/solr/ui/build.gradle.kts index b91ee268c906..cc7e2067fc88 100644 --- a/solr/ui/build.gradle.kts +++ b/solr/ui/build.gradle.kts @@ -15,6 +15,8 @@ * limitations under the License. */ +import org.cyclonedx.gradle.CyclonedxDirectTask +import org.cyclonedx.model.Component import org.jetbrains.compose.desktop.application.dsl.TargetFormat import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpackConfig @@ -239,3 +241,35 @@ artifacts { ) } } + +// CycloneDX SBOM of the Maven dependencies compiled into the wasmJs UI bundle, +// merged into the distribution SBOMs by :solr:packaging. + +val uiSbomFile = layout.buildDirectory.file("cyclonedx/bom-ui.json").get().asFile + +val cyclonedxUi = tasks.register("cyclonedxUi") { + group = "Bill of Materials" + description = "Generates a CycloneDX BOM of the dependencies compiled into the wasmJs UI bundle" + + includeConfigs.set(listOf("wasmJsRuntimeClasspath")) + projectType.set(Component.Type.LIBRARY) + + // The plugin resolves the configuration leniently and without depending on it, + // so the artifacts must be present first or their hashes are missing. + inputs.files(configurations.named("wasmJsRuntimeClasspath")) + .withPropertyName("wasmJsRuntimeArtifacts") + .withNormalizer(ClasspathNormalizer::class) + + jsonOutput.set(uiSbomFile) +} + +val uiSbom = configurations.create("uiSbom") { + isCanBeConsumed = true + isCanBeResolved = false +} + +artifacts { + add("uiSbom", uiSbomFile) { + builtBy(cyclonedxUi) + } +} diff --git a/solr/webapp/js-client/build.gradle.kts b/solr/webapp/js-client/build.gradle.kts index 1ce43c25978b..26a2d9ed00a0 100644 --- a/solr/webapp/js-client/build.gradle.kts +++ b/solr/webapp/js-client/build.gradle.kts @@ -17,6 +17,8 @@ import com.github.gradle.node.npm.task.NpmTask import com.github.gradle.node.npm.task.NpxTask +import groovy.json.JsonOutput +import groovy.json.JsonSlurper // Builds the OpenAPI-generated JS client (from :solr:api) into a single bundled // file, for :solr:webapp to include in the war. This is the only place in the @@ -49,9 +51,29 @@ val syncJSClientSourceCode = tasks.register("syncJSClientSourceCode") { into(jsClientWorkspace) - // Keep the node modules, so that they don't need to be re-downloaded + // Keep the outputs of "npm install", so that they don't need to be regenerated preserve { include("node_modules/**") + include("package-lock.json") + } + + // The OpenAPI generator wrongly declares the @babel/cli build tool as a runtime + // dependency; move it to devDependencies, so that the SBOM of the bundle + // (generated with --omit dev) only lists what browserify actually bundles. + doLast { + val packageJson = File(jsClientWorkspace, "package.json") + @Suppress("UNCHECKED_CAST") + val json = JsonSlurper().parse(packageJson) as MutableMap + + @Suppress("UNCHECKED_CAST") + val dependencies = json["dependencies"] as? MutableMap + dependencies?.remove("@babel/cli")?.let { babelCliVersion -> + @Suppress("UNCHECKED_CAST") + val devDependencies = + json.getOrPut("devDependencies") { mutableMapOf() } as MutableMap + devDependencies["@babel/cli"] = babelCliVersion + } + packageJson.writeText(JsonOutput.prettyPrint(JsonOutput.toJson(json))) } } @@ -64,6 +86,7 @@ val jsClientDownloadDeps = tasks.register("jsClientDownloadDeps") { inputs.dir("$jsClientWorkspace/src") inputs.file("$jsClientWorkspace/package.json") outputs.dir("$jsClientWorkspace/node_modules") + outputs.file("$jsClientWorkspace/package-lock.json") } val jsClientBuild = tasks.register("jsClientBuild") { @@ -117,3 +140,54 @@ artifacts { builtBy(finalizeJsBundleDir) } } + +// CycloneDX SBOM of the bundle, merged into the distribution SBOMs by :solr:packaging + +val jsClientSbomFile = layout.buildDirectory.file("cyclonedx/bom-js-client.json").get().asFile + +val downloadCyclonedxNpm = tasks.register("downloadCyclonedxNpm") { + args.set(listOf("install", "@cyclonedx/cyclonedx-npm@${libs.versions.cyclonedx.npm.get()}")) + + inputs.property("cyclonedx-npm version", libs.versions.cyclonedx.npm.get()) + outputs.dir(project.extra["nodeProjectDir"].toString() + "/node_modules/@cyclonedx/cyclonedx-npm") +} + +val generateJsClientSbom = tasks.register("generateJsClientSbom") { + dependsOn(downloadCyclonedxNpm) + // Needs the package-lock.json and node_modules produced by the install + dependsOn(jsClientDownloadDeps) + + // The full package spec, since the bare "cyclonedx-npm" command name resolves to an + // unrelated npm package. Runs from the node project dir, where downloadCyclonedxNpm + // installed the pinned version, and points at the workspace manifest instead. + command.set("@cyclonedx/cyclonedx-npm@${libs.versions.cyclonedx.npm.get()}") + args.set( + listOf( + // Only the packages bundled into the shipped file, not the build tooling + "--omit", "dev", + // Match the spec version emitted by the CycloneDX Gradle plugin in :solr:packaging + "--spec-version", "1.6", + "--output-reproducible", + "--output-format", "JSON", + "--output-file", jsClientSbomFile.absolutePath, + "$jsClientWorkspace/package.json", + ), + ) + workingDir.set(File(project.extra["nodeProjectDir"].toString())) + + inputs.file("$jsClientWorkspace/package.json") + inputs.file("$jsClientWorkspace/package-lock.json") + inputs.property("cyclonedx-npm version", libs.versions.cyclonedx.npm.get()) + outputs.file(jsClientSbomFile) +} + +val jsClientSbom = configurations.create("jsClientSbom") { + isCanBeConsumed = true + isCanBeResolved = false +} + +artifacts { + add("jsClientSbom", jsClientSbomFile) { + builtBy(generateJsClientSbom) + } +} From 71b86bbc94a2a25fbbf8fcfc643b15c562513bc1 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Wed, 5 Aug 2026 22:06:54 +0200 Subject: [PATCH 02/10] SOLR-17328: Move SBOM generation to gradle/solr/sbom.gradle Review feedback on #4690: the CycloneDX BOM configurations, the two generator tasks and the post-processing filled roughly 480 of the 873 lines of solr/packaging/build.gradle, a file otherwise concerned with assembling the distribution archives. They now live in their own script plugin, applied from the root build next to gradle/solr/packaging.gradle. The move is verbatim except for the CycloneDX types. A script plugin is not compiled against the plugins block classpath of the script that applies it, so CyclonedxDirectTask and Component.Type are looked up by name through the buildscript class loader instead of being imported. What stays in :solr:packaging is the wiring that depends on tasks the distribution plugin only creates later: the bom.json entries in the distribution contents and the dependsOn 'cyclonedx' declarations. Both SBOMs are byte-identical to the ones generated before the move, apart from the per-run serial number and timestamp. Assisted-By: Claude Opus 5 (1M context) --- build.gradle | 1 + gradle/solr/sbom.gradle | 521 ++++++++++++++++++++++++++++++++++++ solr/packaging/build.gradle | 482 +-------------------------------- 3 files changed, 525 insertions(+), 479 deletions(-) create mode 100644 gradle/solr/sbom.gradle diff --git a/build.gradle b/build.gradle index e8c107888470..fcfc382a4116 100644 --- a/build.gradle +++ b/build.gradle @@ -221,6 +221,7 @@ apply from: file('gradle/hacks/turbocharge-jvm-opts.gradle') apply from: file('gradle/hacks/dummy-outputs.gradle') apply from: file('gradle/solr/packaging.gradle') +apply from: file('gradle/solr/sbom.gradle') apply from: file('gradle/solr/solr-forbidden-apis.gradle') apply from: file('gradle/node.gradle') diff --git a/gradle/solr/sbom.gradle b/gradle/solr/sbom.gradle new file mode 100644 index 000000000000..2ca64fad9be5 --- /dev/null +++ b/gradle/solr/sbom.gradle @@ -0,0 +1,521 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import java.security.MessageDigest + +// CycloneDX SBOM generation for the Solr binary distributions. +// +// The ":solr:packaging" project assembles the distributions; this script adds the BOM +// configurations they are computed from, the two generator tasks and the post-processing +// that turns the raw plugin output into the "bom.json" shipped inside each archive. +// +// The child SBOMs merged in below are produced by ":solr:webapp:js-client" (cyclonedx-npm) +// and ":solr:ui" (the CycloneDX Gradle plugin), each in its own build file. + +final APACHE_SNAPSHOTS_QUALIFIER = '&repository_url=https:%2F%2Frepository.apache.org%2Fcontent%2Fgroups%2Fsnapshots%2F' + +configure(project(':solr:packaging')) { + // The CycloneDX plugin is declared in the root plugins block, whose classpath a script + // plugin is not compiled against, so its types are looked up by name instead of imported. + def cyclonedxDirectTaskType = buildscript.classLoader.loadClass('org.cyclonedx.gradle.CyclonedxDirectTask') + def applicationComponentType = buildscript.classLoader.loadClass('org.cyclonedx.model.Component$Type').APPLICATION + + ext { + cyclonedxDir = layout.buildDirectory.dir("cyclonedx") + } + + // For the CycloneDX BOM generation + configurations { + bomSlim { + canBeResolved = true + canBeConsumed = false + } + bomFull { + canBeResolved = true + canBeConsumed = false + extendsFrom bomSlim + } + // Child SBOMs of the UI bundles, merged into the distribution SBOMs by postProcessBom + jsClientSbom { + canBeResolved = true + canBeConsumed = false + } + uiSbom { + canBeResolved = true + canBeConsumed = false + } + } + + // Request the standard JVM runtime variants, like runtimeClasspath does. + // + // The CycloneDX plugin resolves configurations leniently. Without these attributes, + // variant-aware dependencies (e.g. Guava) and platform constraints (e.g. the Jersey BOM) fail to resolve + // and silently disappear. + [configurations.bomSlim, configurations.bomFull].each { conf -> + conf.attributes { + attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY)) + attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, Usage.JAVA_RUNTIME)) + attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, objects.named(LibraryElements, LibraryElements.JAR)) + attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL)) + attribute(TargetJvmEnvironment.TARGET_JVM_ENVIRONMENT_ATTRIBUTE, objects.named(TargetJvmEnvironment, TargetJvmEnvironment.STANDARD_JVM)) + } + } + + // The distribution keeps the server versions of the libraries shared between the + // server and the modules, so align the full BOM with the versions of the slim one + // (e.g. a module may pull in a newer slf4j-api than the one in server/lib/ext). + configurations.bomFull.shouldResolveConsistentlyWith(configurations.bomSlim) + + dependencies { + rootProject.project(":solr:modules").childProjects.values().each { module -> + // No "configuration:" here on purpose. + // + // Gradle then selects the variant of the module that matches the attributes declared on the bom configurations above, + // just like it does for a runtime classpath ("variant-aware" resolution). + // Naming a configuration would bypass attribute matching, and imported Maven BOMs (e.g. Jersey's) would no + // longer provide the versions of their managed dependencies. + bomFull project(path: module.path) + } + + bomFull project(path: ':solr:cross-dc-manager') + + bomSlim project(path: ':solr:server', configuration: 'startJar') + bomSlim project(path: ':solr:server', configuration: 'serverLib') + bomSlim project(path: ':solr:server', configuration: 'libExt') + // Variant-aware dependency instead of server's by-name 'solrCore' + // configuration, see the comment on the modules above. + bomSlim project(path: ':solr:core') + bomSlim project(path: ':solr:server', configuration: 'webapp') + + // Child SBOMs of the UI bundles; empty when the projects are disabled + if (gradle.ext.withJsClient) { + jsClientSbom project(path: ':solr:webapp:js-client', configuration: 'jsClientSbom') + } + if (gradle.ext.withUiModule) { + uiSbom project(path: ':solr:ui', configuration: 'uiSbom') + } + } + + // Post-processing of the SBOMs generated by the CycloneDX plugin: + // + // 1. Metadata: + // The "build" lifecycle phase is recorded (this is a "Build SBOM" in CISA's classification) + // and this post-processing step is listed in the tools, next to the CycloneDX plugin. + // 2. Main component: + // The Solr binary archive published on downloads.apache.org is identified by a "sid" purl + // (a draft purl type for software distributed outside package registries, + // see https://github.com/package-url/purl-spec/issues/516) and by the Solr CPE. + // 3. Removed components: + // Maven BOM/platform dependencies (purl qualifier "type=pom") and the internal ":platform" + // project are stripped: they are not part of the binary distribution and the plugin has no + // option to exclude them. + // 4. Solr components: + // The plugin emits invalid purls for Solr projects, built from the Gradle project name and a + // non-standard "project_path" qualifier (e.g. pkg:maven/org.apache.solr/core@11.0.0?project_path=:solr:core). + // They are replaced with the Maven artifactId (base.archivesName) and a "type=jar" qualifier + // (e.g. pkg:maven/org.apache.solr/solr-core@11.0.0?type=jar). + // The project description and the Apache-2.0 license are also added. + // 5. Vendored JavaScript libraries: + // The AngularJS admin UI ships third-party JavaScript files checked into solr/webapp/web/libs. + // Components for them are added from a curated list (version and license read from the file + // headers); the hash matching of step 8 proves that each file still ships unmodified. + // 6. JavaScript client bundle: + // The npm packages bundled by browserify into the OpenAPI JS client + // (server/solr-webapp/webapp/libs/solr/index.js) are nested as subassemblies of a first-party + // "solr-js-client" component, using the SBOM generated by cyclonedx-npm in :solr:webapp:js-client. + // 7. New UI bundle: + // The Maven dependencies compiled into the wasmJs UI (server/solr-webapp/webapp/ui) are nested + // as subassemblies of a first-party "solr-ui" component, using the SBOM generated in :solr:ui, + // together with the npm package bundled by the Kotlin toolchain (see kotlin-js-store/wasm/yarn.lock). + // 8. Archive locations: + // The location of each JAR and JavaScript file within the distribution is recorded as + // "evidence.occurrences": the directories assembled for the distribution are scanned and + // their files are matched to the components by SHA-256 hash. + // 9. Hashes: + // Only the SHA-256 hash of each component is kept: the plugin emits eight algorithms + // per artifact, which only adds bulk. + def postProcessBom = { File bomFile, String edition, Map scanDirs, File jsClientSbomFile, File uiSbomFile -> + def json = new JsonSlurper().parse(bomFile) + + def sha256Of = { File file -> + def digest = MessageDigest.getInstance('SHA-256') + file.eachByte(8192) { buffer, length -> digest.update(buffer, 0, length) } + digest.digest().encodeHex().toString() + } + + // Copies a child SBOM's dependency graph into this one: the child root is replaced + // by the given bundle ref, dropped refs are skipped and entries for refs that + // already exist (the same artifact in both graphs) are merged. + def mergeChildGraph = { List childDeps, String childRootRef, String bundleRef, Set droppedRefs -> + childDeps.each { dep -> + if (dep.ref in droppedRefs) { + return + } + def ref = dep.ref == childRootRef ? bundleRef : dep.ref + def dependsOn = (dep.dependsOn ?: []).findAll { !(it in droppedRefs) } + .collect { it == childRootRef ? bundleRef : it } + def existing = json.dependencies.find { it.ref == ref } + if (existing == null) { + json.dependencies << [ref: ref, dependsOn: dependsOn] + } else { + existing.dependsOn = ((existing.dependsOn ?: []) + dependsOn).unique() + } + } + } + + // 1. Metadata + json.metadata.lifecycles = [[phase: 'build']] + + // Record this post-processing step next to the CycloneDX plugin + if (json.metadata.tools == null) { + json.metadata.tools = [components: []] + } + json.metadata.tools.components << [ + type: 'application', + author: 'The Apache Software Foundation', + name: 'solr-sbom-post-processing', + version: project.version, + description: 'Post-processing of the generated SBOM by the Solr Gradle build (:solr:packaging)', + ] + + // 2. Main component + // Old bom-ref -> new purl, applied to the dependency graph below + def rewrittenRefs = [:] + + Map mainComponent = json.metadata.component + def mainPurl = "pkg:sid/apache.org/solr/solr@${mainComponent.version}?edition=${edition}".toString() + rewrittenRefs[mainComponent.'bom-ref'] = mainPurl + mainComponent.remove('group') + mainComponent.name = 'Apache Solr binary release' + mainComponent.cpe = "cpe:2.3:a:apache:solr:${mainComponent.version}:*:*:*:*:*:*:*".toString() + mainComponent.purl = mainPurl + mainComponent.'bom-ref' = mainPurl + + // Gradle project path encoded in the purls the plugin generates for Solr projects + def projectPathOf = { purl -> + def matcher = purl =~ /[?&]project_path=([^&]+)(&|$)/ + matcher ? URLDecoder.decode(matcher.group(1), 'UTF-8') : null + } + + // 3. Removed components + def removedRefs = json.components.findAll { + it.purl =~ /[?&]type=pom(&|$)/ || projectPathOf(it.purl) == ':platform' + }.collect { it.'bom-ref' } as Set + + // 4. Solr components + Map artifactIdByPath = rootProject.allprojects.collectEntries { + [(it.path): it.base.archivesName.get()] + } + json.components = json.components.collect { Map component -> + def projectPath = projectPathOf(component.purl) + // Skip external components and those about to be removed + if (projectPath == null || component.'bom-ref' in removedRefs) { + return component + } + def artifactId = artifactIdByPath[projectPath] + def repositoryUrlQualifier = component.version.endsWith("-SNAPSHOT") ? APACHE_SNAPSHOTS_QUALIFIER : '' + def purl = "pkg:maven/${component.group}/${artifactId}@${component.version}?type=jar${repositoryUrlQualifier}".toString() + rewrittenRefs[component.'bom-ref'] = purl + component.name = artifactId + component.purl = purl + component.'bom-ref' = purl + component.description = rootProject.project(projectPath).description + component.licenses = [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]] + // Restore the field order the plugin uses for external components + def ordered = [:] + ['type', 'bom-ref', 'group', 'name', 'version', 'description', 'hashes', + 'licenses', 'purl', 'modified', 'properties'].each { key -> + if (component.containsKey(key)) { + ordered[key] = component[key] + } + } + component.forEach { key, value -> + if (!ordered.containsKey(key)) { + ordered[key] = value + } + } + return ordered + } + + // Apply the removals (3) and the ref rewrites (2, 4) to the dependency graph + json.components.removeAll { it.'bom-ref' in removedRefs } + json.dependencies?.removeAll { it.ref in removedRefs } + json.dependencies?.each { dep -> + dep.ref = rewrittenRefs.getOrDefault(dep.ref, dep.ref) + dep.dependsOn?.removeAll { it in removedRefs } + if (dep.dependsOn != null) { + dep.dependsOn = dep.dependsOn.collect { rewrittenRefs.getOrDefault(it, it) } + } + } + + // The UI artifacts of steps 5 to 7 all ship inside the webapp + def webappRef = json.components.find { + it.purl?.startsWith('pkg:maven/org.apache.solr/solr-webapp@') + }?.'bom-ref' ?: mainComponent.'bom-ref' + def webappDependsOn = json.dependencies.find { it.ref == webappRef }.dependsOn + + // 5. Vendored JavaScript libraries + // Entries without a version marker in the file get no version and no purl + def vendoredJsLibs = [ + [file: 'angular.min.js', name: 'angular', version: '1.8.0', license: 'MIT'], + [file: 'angular-chosen.min.js', name: 'angular-chosen-localytics', version: '1.9.2', license: 'MIT'], + [file: 'angular-cookies.min.js', name: 'angular-cookies', version: '1.8.0', license: 'MIT'], + [file: 'angular-resource.min.js', name: 'angular-resource', version: '1.8.0', license: 'MIT'], + [file: 'angular-route.min.js', name: 'angular-route', version: '1.8.0', license: 'MIT'], + [file: 'angular-sanitize.min.js', name: 'angular-sanitize', version: '1.8.0', license: 'MIT'], + [file: 'angular-utf8-base64.min.js', name: 'angular-utf8-base64', license: 'MIT'], + [file: 'chosen.jquery.min.js', name: 'chosen-js', version: '1.8.7', license: 'MIT'], + [file: 'd3.js', name: 'd3', version: '2.8.1', license: 'BSD-3-Clause'], + [file: 'highlight.js', name: 'highlight.js', license: 'BSD-3-Clause'], + [file: 'jquery-3.5.1.min.js', name: 'jquery', version: '3.5.1', license: 'MIT'], + [file: 'jquery-ui.min.js', name: 'jquery-ui', version: '1.12.1', license: 'MIT'], + [file: 'jssha-3.3.1-sha256.min.js', name: 'jssha', version: '3.3.1', license: 'BSD-3-Clause'], + [file: 'jstree.min.js', name: 'jstree', version: '3.3.10', license: 'MIT'], + [file: 'ngtimeago.js', name: 'ngtimeago', license: 'MIT'], + [file: 'ui-grid.min.js', name: 'angular-ui-grid', version: '4.10.0', license: 'MIT'], + ] + def webLibsDir = rootProject.file('solr/webapp/web/libs') + vendoredJsLibs.each { lib -> + def purl = lib.version != null ? "pkg:npm/${lib.name}@${lib.version}".toString() : null + def component = [ + type: 'library', + 'bom-ref': purl ?: "vendored-js:${lib.name}".toString(), + name: lib.name, + ] + if (lib.version != null) { + component.version = lib.version + } + component.hashes = [[alg: 'SHA-256', content: sha256Of(new File(webLibsDir, lib.file))]] + component.licenses = [[license: [id: lib.license]]] + if (purl != null) { + component.purl = purl + } + json.components << component + json.dependencies << [ref: component.'bom-ref', dependsOn: []] + webappDependsOn << component.'bom-ref' + } + + // 6. JavaScript client bundle + if (jsClientSbomFile != null) { + def jsClientBom = new JsonSlurper().parse(jsClientSbomFile) + def bundleRef = "solr-js-client@${project.version}".toString() + json.components << [ + type: 'library', + 'bom-ref': bundleRef, + name: 'solr-js-client', + version: project.version.toString(), + description: 'JavaScript client for the Solr v2 API, generated from its OpenAPI specification', + licenses: [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]], + // The npm packages bundled into the single shipped file by browserify + components: jsClientBom.components, + evidence: [occurrences: [[location: 'server/solr-webapp/webapp/libs/solr/index.js']]], + ] + mergeChildGraph(jsClientBom.dependencies ?: [], jsClientBom.metadata.component.'bom-ref', bundleRef, [] as Set) + webappDependsOn << bundleRef + // Record cyclonedx-npm next to the other tools + jsClientBom.metadata?.tools?.components?.each { tool -> + if (!json.metadata.tools.components.any { it.name == tool.name && it.version == tool.version }) { + json.metadata.tools.components << tool + } + } + } + + // 7. New UI bundle + def uiBundle = null + if (uiSbomFile != null) { + def uiBom = new JsonSlurper().parse(uiSbomFile) + // The ":platform" project and Maven BOMs are on the UI classpath too, see step 3 + def uiDroppedRefs = uiBom.components.findAll { + it.purl =~ /[?&]type=pom(&|$)/ || projectPathOf(it.purl) == ':platform' + }.collect { it.'bom-ref' } as Set + // An artifact present in the Java graph too keeps its top-level component only + def existingRefs = json.components.collect { it.'bom-ref' } as Set + def nested = uiBom.components.findAll { + !(it.'bom-ref' in uiDroppedRefs) && !(it.'bom-ref' in existingRefs) + } + // Bundled by the Kotlin toolchain, see kotlin-js-store/wasm/yarn.lock + nested << [ + type: 'library', + 'bom-ref': 'pkg:npm/%40js-joda/core@3.2.0', + name: '@js-joda/core', + version: '3.2.0', + licenses: [[license: [id: 'BSD-3-Clause']]], + purl: 'pkg:npm/%40js-joda/core@3.2.0', + ] + def bundleRef = "solr-ui@${project.version}".toString() + uiBundle = [ + type: 'library', + 'bom-ref': bundleRef, + name: 'solr-ui', + version: project.version.toString(), + description: 'New Solr admin UI, compiled to WebAssembly', + licenses: [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]], + // The Maven artifacts compiled into the bundle and the npm package above + components: nested, + // Occurrences are filled by the scan of step 8, the bundle file names are content-hashed + ] + json.components << uiBundle + mergeChildGraph(uiBom.dependencies ?: [], uiBom.metadata.component.'bom-ref', bundleRef, uiDroppedRefs) + def uiEntry = json.dependencies.find { it.ref == bundleRef } + uiEntry.dependsOn << 'pkg:npm/%40js-joda/core@3.2.0' + json.dependencies << [ref: 'pkg:npm/%40js-joda/core@3.2.0', dependsOn: []] + webappDependsOn << bundleRef + } + + // 8. Archive locations + // Distribution files indexed by SHA-256 hash; UI bundle files collected on the way + def locationsByHash = [:].withDefault { [] } + def uiLocations = [] + scanDirs.each { prefix, configuration -> + configuration.files.each { root -> + fileTree(root).matching { + include '**/*.jar' + include '**/*.js' + include 'solr-webapp/webapp/ui/**' + }.visit { entry -> + if (!entry.directory) { + def location = "${prefix}/${entry.relativePath}".toString() + locationsByHash[sha256Of(entry.file)] << location + if (entry.relativePath.pathString.startsWith('solr-webapp/webapp/ui/')) { + uiLocations << location + } + } + } + } + } + json.components.each { component -> + def sha256 = component.hashes?.find { it.alg == 'SHA-256' }?.content + if (sha256 != null && locationsByHash.containsKey(sha256)) { + component.evidence = [occurrences: locationsByHash[sha256].sort().collect { [location: it] }] + } + } + if (uiBundle != null) { + uiBundle.evidence = [occurrences: uiLocations.sort().collect { [location: it] }] + } + + // 9. Hashes + // The npm integrity hashes on "externalReferences" are kept, they describe + // the registry tarballs and have no SHA-256 equivalent. + def keepSha256Only + keepSha256Only = { List components -> + components.each { component -> + if (component.hashes != null) { + component.hashes = component.hashes.findAll { it.alg == 'SHA-256' } + if (component.hashes.isEmpty()) { + component.remove('hashes') + } + } + keepSha256Only(component.components ?: []) + } + } + keepSha256Only(json.components) + + bomFile.text = JsonOutput.prettyPrint(JsonOutput.toJson(json)) + } + + tasks.register('cyclonedxFull', cyclonedxDirectTaskType) { + group = 'Bill of Materials' + description = 'Generates CycloneDX BOM for the full Solr distribution' + + includeConfigs = ['bomFull'] + projectType = applicationComponentType + + // The plugin resolves the configuration leniently and without depending on it, + // so the jars of Solr projects must be built first or their hashes are missing. + inputs.files(configurations.bomFull) + .withPropertyName('bomFullArtifacts') + .withNormalizer(ClasspathNormalizer) + + // Distribution directories scanned for the archive location of each artifact + inputs.files(configurations.server, configurations.modules, configurations.crossDcManager) + .withPropertyName('distributionDirs') + .withPathSensitivity(PathSensitivity.RELATIVE) + + // Child SBOMs of the UI bundles; empty when the projects are disabled + inputs.files(configurations.jsClientSbom, configurations.uiSbom) + .withPropertyName('childSboms') + .withPathSensitivity(PathSensitivity.NONE) + + // Sources of the statically listed JavaScript components + inputs.dir(rootProject.file('solr/webapp/web/libs')) + .withPropertyName('vendoredJsLibs') + .withPathSensitivity(PathSensitivity.RELATIVE) + inputs.file(rootProject.file('kotlin-js-store/wasm/yarn.lock')) + .withPropertyName('uiYarnLock') + .withPathSensitivity(PathSensitivity.NONE) + + jsonOutput = cyclonedxDir.get().file("bom-full.json").asFile + + doLast { + postProcessBom(jsonOutput.get().asFile, 'full', [ + 'server': configurations.server, + 'modules': configurations.modules, + 'cross-dc-manager': configurations.crossDcManager, + ], configurations.jsClientSbom.files.find(), configurations.uiSbom.files.find()) + } + } + + tasks.register('cyclonedxSlim', cyclonedxDirectTaskType) { + group = 'Bill of Materials' + description = 'Generates CycloneDX BOM for the slim Solr distribution' + + includeConfigs = ['bomSlim'] + projectType = applicationComponentType + + // The plugin resolves the configuration leniently and without depending on it, + // so the jars of Solr projects must be built first or their hashes are missing. + inputs.files(configurations.bomSlim) + .withPropertyName('bomSlimArtifacts') + .withNormalizer(ClasspathNormalizer) + + // Distribution directories scanned for the archive location of each artifact + inputs.files(configurations.server) + .withPropertyName('distributionDirs') + .withPathSensitivity(PathSensitivity.RELATIVE) + + // Child SBOMs of the UI bundles; empty when the projects are disabled + inputs.files(configurations.jsClientSbom, configurations.uiSbom) + .withPropertyName('childSboms') + .withPathSensitivity(PathSensitivity.NONE) + + // Sources of the statically listed JavaScript components + inputs.dir(rootProject.file('solr/webapp/web/libs')) + .withPropertyName('vendoredJsLibs') + .withPathSensitivity(PathSensitivity.RELATIVE) + inputs.file(rootProject.file('kotlin-js-store/wasm/yarn.lock')) + .withPropertyName('uiYarnLock') + .withPathSensitivity(PathSensitivity.NONE) + + jsonOutput = cyclonedxDir.get().file("bom-slim.json").asFile + + doLast { + postProcessBom(jsonOutput.get().asFile, 'slim', [ + 'server': configurations.server, + ], configurations.jsClientSbom.files.find(), configurations.uiSbom.files.find()) + } + } + + tasks.register('cyclonedx') { + group = 'Bill of Materials' + description = 'Generates CycloneDX BOMs for Solr distributions' + + dependsOn 'cyclonedxFull' + dependsOn 'cyclonedxSlim' + } +} diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index 3ac372418772..0cd298b46b7f 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -15,13 +15,8 @@ * limitations under the License. */ -import groovy.json.JsonOutput -import groovy.json.JsonSlurper -import java.security.MessageDigest import org.apache.tools.ant.filters.ReplaceTokens import org.apache.tools.ant.util.TeeOutputStream -import org.cyclonedx.gradle.CyclonedxDirectTask -import org.cyclonedx.model.Component // This project puts together a "distribution", assembling dependencies from // various other projects. @@ -29,424 +24,15 @@ import org.cyclonedx.model.Component plugins { id 'base' id 'distribution' - // Registers the JVM attribute schema, so that the variant-aware resolution of - // the CycloneDX BOM configurations below works like a Java runtime classpath. + // Registers the JVM attribute schema, so that the variant-aware resolution of the + // CycloneDX BOM configurations added by gradle/solr/sbom.gradle works like a Java + // runtime classpath. id 'jvm-ecosystem' } -final APACHE_SNAPSHOTS_QUALIFIER = '&repository_url=https:%2F%2Frepository.apache.org%2Fcontent%2Fgroups%2Fsnapshots%2F' - -// Post-processing of the SBOMs generated by the CycloneDX plugin: -// -// 1. Metadata: -// The "build" lifecycle phase is recorded (this is a "Build SBOM" in CISA's classification) -// and this post-processing step is listed in the tools, next to the CycloneDX plugin. -// 2. Main component: -// The Solr binary archive published on downloads.apache.org is identified by a "sid" purl -// (a draft purl type for software distributed outside package registries, -// see https://github.com/package-url/purl-spec/issues/516) and by the Solr CPE. -// 3. Removed components: -// Maven BOM/platform dependencies (purl qualifier "type=pom") and the internal ":platform" -// project are stripped: they are not part of the binary distribution and the plugin has no -// option to exclude them. -// 4. Solr components: -// The plugin emits invalid purls for Solr projects, built from the Gradle project name and a -// non-standard "project_path" qualifier (e.g. pkg:maven/org.apache.solr/core@11.0.0?project_path=:solr:core). -// They are replaced with the Maven artifactId (base.archivesName) and a "type=jar" qualifier -// (e.g. pkg:maven/org.apache.solr/solr-core@11.0.0?type=jar). -// The project description and the Apache-2.0 license are also added. -// 5. Vendored JavaScript libraries: -// The AngularJS admin UI ships third-party JavaScript files checked into solr/webapp/web/libs. -// Components for them are added from a curated list (version and license read from the file -// headers); the hash matching of step 8 proves that each file still ships unmodified. -// 6. JavaScript client bundle: -// The npm packages bundled by browserify into the OpenAPI JS client -// (server/solr-webapp/webapp/libs/solr/index.js) are nested as subassemblies of a first-party -// "solr-js-client" component, using the SBOM generated by cyclonedx-npm in :solr:webapp:js-client. -// 7. New UI bundle: -// The Maven dependencies compiled into the wasmJs UI (server/solr-webapp/webapp/ui) are nested -// as subassemblies of a first-party "solr-ui" component, using the SBOM generated in :solr:ui, -// together with the npm package bundled by the Kotlin toolchain (see kotlin-js-store/wasm/yarn.lock). -// 8. Archive locations: -// The location of each JAR and JavaScript file within the distribution is recorded as -// "evidence.occurrences": the directories assembled for the distribution are scanned and -// their files are matched to the components by SHA-256 hash. -// 9. Hashes: -// Only the SHA-256 hash of each component is kept: the plugin emits eight algorithms -// per artifact, which only adds bulk. -def postProcessBom = { File bomFile, String edition, Map scanDirs, File jsClientSbomFile, File uiSbomFile -> - def json = new JsonSlurper().parse(bomFile) - - def sha256Of = { File file -> - def digest = MessageDigest.getInstance('SHA-256') - file.eachByte(8192) { buffer, length -> digest.update(buffer, 0, length) } - digest.digest().encodeHex().toString() - } - - // Copies a child SBOM's dependency graph into this one: the child root is replaced - // by the given bundle ref, dropped refs are skipped and entries for refs that - // already exist (the same artifact in both graphs) are merged. - def mergeChildGraph = { List childDeps, String childRootRef, String bundleRef, Set droppedRefs -> - childDeps.each { dep -> - if (dep.ref in droppedRefs) { - return - } - def ref = dep.ref == childRootRef ? bundleRef : dep.ref - def dependsOn = (dep.dependsOn ?: []).findAll { !(it in droppedRefs) } - .collect { it == childRootRef ? bundleRef : it } - def existing = json.dependencies.find { it.ref == ref } - if (existing == null) { - json.dependencies << [ref: ref, dependsOn: dependsOn] - } else { - existing.dependsOn = ((existing.dependsOn ?: []) + dependsOn).unique() - } - } - } - - // 1. Metadata - json.metadata.lifecycles = [[phase: 'build']] - - // Record this post-processing step next to the CycloneDX plugin - if (json.metadata.tools == null) { - json.metadata.tools = [components: []] - } - json.metadata.tools.components << [ - type: 'application', - author: 'The Apache Software Foundation', - name: 'solr-sbom-post-processing', - version: project.version, - description: 'Post-processing of the generated SBOM by the Solr Gradle build (:solr:packaging)', - ] - - // 2. Main component - // Old bom-ref -> new purl, applied to the dependency graph below - def rewrittenRefs = [:] - - Map mainComponent = json.metadata.component - def mainPurl = "pkg:sid/apache.org/solr/solr@${mainComponent.version}?edition=${edition}".toString() - rewrittenRefs[mainComponent.'bom-ref'] = mainPurl - mainComponent.remove('group') - mainComponent.name = 'Apache Solr binary release' - mainComponent.cpe = "cpe:2.3:a:apache:solr:${mainComponent.version}:*:*:*:*:*:*:*".toString() - mainComponent.purl = mainPurl - mainComponent.'bom-ref' = mainPurl - - // Gradle project path encoded in the purls the plugin generates for Solr projects - def projectPathOf = { purl -> - def matcher = purl =~ /[?&]project_path=([^&]+)(&|$)/ - matcher ? URLDecoder.decode(matcher.group(1), 'UTF-8') : null - } - - // 3. Removed components - def removedRefs = json.components.findAll { - it.purl =~ /[?&]type=pom(&|$)/ || projectPathOf(it.purl) == ':platform' - }.collect { it.'bom-ref' } as Set - - // 4. Solr components - Map artifactIdByPath = rootProject.allprojects.collectEntries { - [(it.path): it.base.archivesName.get()] - } - json.components = json.components.collect { Map component -> - def projectPath = projectPathOf(component.purl) - // Skip external components and those about to be removed - if (projectPath == null || component.'bom-ref' in removedRefs) { - return component - } - def artifactId = artifactIdByPath[projectPath] - def repositoryUrlQualifier = component.version.endsWith("-SNAPSHOT") ? APACHE_SNAPSHOTS_QUALIFIER : '' - def purl = "pkg:maven/${component.group}/${artifactId}@${component.version}?type=jar${repositoryUrlQualifier}".toString() - rewrittenRefs[component.'bom-ref'] = purl - component.name = artifactId - component.purl = purl - component.'bom-ref' = purl - component.description = rootProject.project(projectPath).description - component.licenses = [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]] - // Restore the field order the plugin uses for external components - def ordered = [:] - ['type', 'bom-ref', 'group', 'name', 'version', 'description', 'hashes', - 'licenses', 'purl', 'modified', 'properties'].each { key -> - if (component.containsKey(key)) { - ordered[key] = component[key] - } - } - component.forEach { key, value -> - if (!ordered.containsKey(key)) { - ordered[key] = value - } - } - return ordered - } - - // Apply the removals (3) and the ref rewrites (2, 4) to the dependency graph - json.components.removeAll { it.'bom-ref' in removedRefs } - json.dependencies?.removeAll { it.ref in removedRefs } - json.dependencies?.each { dep -> - dep.ref = rewrittenRefs.getOrDefault(dep.ref, dep.ref) - dep.dependsOn?.removeAll { it in removedRefs } - if (dep.dependsOn != null) { - dep.dependsOn = dep.dependsOn.collect { rewrittenRefs.getOrDefault(it, it) } - } - } - - // The UI artifacts of steps 5 to 7 all ship inside the webapp - def webappRef = json.components.find { - it.purl?.startsWith('pkg:maven/org.apache.solr/solr-webapp@') - }?.'bom-ref' ?: mainComponent.'bom-ref' - def webappDependsOn = json.dependencies.find { it.ref == webappRef }.dependsOn - - // 5. Vendored JavaScript libraries - // Entries without a version marker in the file get no version and no purl - def vendoredJsLibs = [ - [file: 'angular.min.js', name: 'angular', version: '1.8.0', license: 'MIT'], - [file: 'angular-chosen.min.js', name: 'angular-chosen-localytics', version: '1.9.2', license: 'MIT'], - [file: 'angular-cookies.min.js', name: 'angular-cookies', version: '1.8.0', license: 'MIT'], - [file: 'angular-resource.min.js', name: 'angular-resource', version: '1.8.0', license: 'MIT'], - [file: 'angular-route.min.js', name: 'angular-route', version: '1.8.0', license: 'MIT'], - [file: 'angular-sanitize.min.js', name: 'angular-sanitize', version: '1.8.0', license: 'MIT'], - [file: 'angular-utf8-base64.min.js', name: 'angular-utf8-base64', license: 'MIT'], - [file: 'chosen.jquery.min.js', name: 'chosen-js', version: '1.8.7', license: 'MIT'], - [file: 'd3.js', name: 'd3', version: '2.8.1', license: 'BSD-3-Clause'], - [file: 'highlight.js', name: 'highlight.js', license: 'BSD-3-Clause'], - [file: 'jquery-3.5.1.min.js', name: 'jquery', version: '3.5.1', license: 'MIT'], - [file: 'jquery-ui.min.js', name: 'jquery-ui', version: '1.12.1', license: 'MIT'], - [file: 'jssha-3.3.1-sha256.min.js', name: 'jssha', version: '3.3.1', license: 'BSD-3-Clause'], - [file: 'jstree.min.js', name: 'jstree', version: '3.3.10', license: 'MIT'], - [file: 'ngtimeago.js', name: 'ngtimeago', license: 'MIT'], - [file: 'ui-grid.min.js', name: 'angular-ui-grid', version: '4.10.0', license: 'MIT'], - ] - def webLibsDir = rootProject.file('solr/webapp/web/libs') - vendoredJsLibs.each { lib -> - def purl = lib.version != null ? "pkg:npm/${lib.name}@${lib.version}".toString() : null - def component = [ - type: 'library', - 'bom-ref': purl ?: "vendored-js:${lib.name}".toString(), - name: lib.name, - ] - if (lib.version != null) { - component.version = lib.version - } - component.hashes = [[alg: 'SHA-256', content: sha256Of(new File(webLibsDir, lib.file))]] - component.licenses = [[license: [id: lib.license]]] - if (purl != null) { - component.purl = purl - } - json.components << component - json.dependencies << [ref: component.'bom-ref', dependsOn: []] - webappDependsOn << component.'bom-ref' - } - - // 6. JavaScript client bundle - if (jsClientSbomFile != null) { - def jsClientBom = new JsonSlurper().parse(jsClientSbomFile) - def bundleRef = "solr-js-client@${project.version}".toString() - json.components << [ - type: 'library', - 'bom-ref': bundleRef, - name: 'solr-js-client', - version: project.version.toString(), - description: 'JavaScript client for the Solr v2 API, generated from its OpenAPI specification', - licenses: [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]], - // The npm packages bundled into the single shipped file by browserify - components: jsClientBom.components, - evidence: [occurrences: [[location: 'server/solr-webapp/webapp/libs/solr/index.js']]], - ] - mergeChildGraph(jsClientBom.dependencies ?: [], jsClientBom.metadata.component.'bom-ref', bundleRef, [] as Set) - webappDependsOn << bundleRef - // Record cyclonedx-npm next to the other tools - jsClientBom.metadata?.tools?.components?.each { tool -> - if (!json.metadata.tools.components.any { it.name == tool.name && it.version == tool.version }) { - json.metadata.tools.components << tool - } - } - } - - // 7. New UI bundle - def uiBundle = null - if (uiSbomFile != null) { - def uiBom = new JsonSlurper().parse(uiSbomFile) - // The ":platform" project and Maven BOMs are on the UI classpath too, see step 3 - def uiDroppedRefs = uiBom.components.findAll { - it.purl =~ /[?&]type=pom(&|$)/ || projectPathOf(it.purl) == ':platform' - }.collect { it.'bom-ref' } as Set - // An artifact present in the Java graph too keeps its top-level component only - def existingRefs = json.components.collect { it.'bom-ref' } as Set - def nested = uiBom.components.findAll { - !(it.'bom-ref' in uiDroppedRefs) && !(it.'bom-ref' in existingRefs) - } - // Bundled by the Kotlin toolchain, see kotlin-js-store/wasm/yarn.lock - nested << [ - type: 'library', - 'bom-ref': 'pkg:npm/%40js-joda/core@3.2.0', - name: '@js-joda/core', - version: '3.2.0', - licenses: [[license: [id: 'BSD-3-Clause']]], - purl: 'pkg:npm/%40js-joda/core@3.2.0', - ] - def bundleRef = "solr-ui@${project.version}".toString() - uiBundle = [ - type: 'library', - 'bom-ref': bundleRef, - name: 'solr-ui', - version: project.version.toString(), - description: 'New Solr admin UI, compiled to WebAssembly', - licenses: [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]], - // The Maven artifacts compiled into the bundle and the npm package above - components: nested, - // Occurrences are filled by the scan of step 8, the bundle file names are content-hashed - ] - json.components << uiBundle - mergeChildGraph(uiBom.dependencies ?: [], uiBom.metadata.component.'bom-ref', bundleRef, uiDroppedRefs) - def uiEntry = json.dependencies.find { it.ref == bundleRef } - uiEntry.dependsOn << 'pkg:npm/%40js-joda/core@3.2.0' - json.dependencies << [ref: 'pkg:npm/%40js-joda/core@3.2.0', dependsOn: []] - webappDependsOn << bundleRef - } - - // 8. Archive locations - // Distribution files indexed by SHA-256 hash; UI bundle files collected on the way - def locationsByHash = [:].withDefault { [] } - def uiLocations = [] - scanDirs.each { prefix, configuration -> - configuration.files.each { root -> - fileTree(root).matching { - include '**/*.jar' - include '**/*.js' - include 'solr-webapp/webapp/ui/**' - }.visit { entry -> - if (!entry.directory) { - def location = "${prefix}/${entry.relativePath}".toString() - locationsByHash[sha256Of(entry.file)] << location - if (entry.relativePath.pathString.startsWith('solr-webapp/webapp/ui/')) { - uiLocations << location - } - } - } - } - } - json.components.each { component -> - def sha256 = component.hashes?.find { it.alg == 'SHA-256' }?.content - if (sha256 != null && locationsByHash.containsKey(sha256)) { - component.evidence = [occurrences: locationsByHash[sha256].sort().collect { [location: it] }] - } - } - if (uiBundle != null) { - uiBundle.evidence = [occurrences: uiLocations.sort().collect { [location: it] }] - } - - // 9. Hashes - // The npm integrity hashes on "externalReferences" are kept, they describe - // the registry tarballs and have no SHA-256 equivalent. - def keepSha256Only - keepSha256Only = { List components -> - components.each { component -> - if (component.hashes != null) { - component.hashes = component.hashes.findAll { it.alg == 'SHA-256' } - if (component.hashes.isEmpty()) { - component.remove('hashes') - } - } - keepSha256Only(component.components ?: []) - } - } - keepSha256Only(json.components) - - bomFile.text = JsonOutput.prettyPrint(JsonOutput.toJson(json)) -} - -tasks.register('cyclonedxFull', CyclonedxDirectTask) { - group = 'Bill of Materials' - description = 'Generates CycloneDX BOM for the full Solr distribution' - - includeConfigs = ['bomFull'] - projectType = Component.Type.APPLICATION - - // The plugin resolves the configuration leniently and without depending on it, - // so the jars of Solr projects must be built first or their hashes are missing. - inputs.files(configurations.bomFull) - .withPropertyName('bomFullArtifacts') - .withNormalizer(ClasspathNormalizer) - - // Distribution directories scanned for the archive location of each artifact - inputs.files(configurations.server, configurations.modules, configurations.crossDcManager) - .withPropertyName('distributionDirs') - .withPathSensitivity(PathSensitivity.RELATIVE) - - // Child SBOMs of the UI bundles; empty when the projects are disabled - inputs.files(configurations.jsClientSbom, configurations.uiSbom) - .withPropertyName('childSboms') - .withPathSensitivity(PathSensitivity.NONE) - - // Sources of the statically listed JavaScript components - inputs.dir(rootProject.file('solr/webapp/web/libs')) - .withPropertyName('vendoredJsLibs') - .withPathSensitivity(PathSensitivity.RELATIVE) - inputs.file(rootProject.file('kotlin-js-store/wasm/yarn.lock')) - .withPropertyName('uiYarnLock') - .withPathSensitivity(PathSensitivity.NONE) - - jsonOutput = cyclonedxDir.get().file("bom-full.json").asFile - - doLast { - postProcessBom(jsonOutput.get().asFile, 'full', [ - 'server': configurations.server, - 'modules': configurations.modules, - 'cross-dc-manager': configurations.crossDcManager, - ], configurations.jsClientSbom.files.find(), configurations.uiSbom.files.find()) - } -} - -tasks.register('cyclonedxSlim', CyclonedxDirectTask) { - group = 'Bill of Materials' - description = 'Generates CycloneDX BOM for the slim Solr distribution' - - includeConfigs = ['bomSlim'] - projectType = Component.Type.APPLICATION - - // The plugin resolves the configuration leniently and without depending on it, - // so the jars of Solr projects must be built first or their hashes are missing. - inputs.files(configurations.bomSlim) - .withPropertyName('bomSlimArtifacts') - .withNormalizer(ClasspathNormalizer) - - // Distribution directories scanned for the archive location of each artifact - inputs.files(configurations.server) - .withPropertyName('distributionDirs') - .withPathSensitivity(PathSensitivity.RELATIVE) - - // Child SBOMs of the UI bundles; empty when the projects are disabled - inputs.files(configurations.jsClientSbom, configurations.uiSbom) - .withPropertyName('childSboms') - .withPathSensitivity(PathSensitivity.NONE) - - // Sources of the statically listed JavaScript components - inputs.dir(rootProject.file('solr/webapp/web/libs')) - .withPropertyName('vendoredJsLibs') - .withPathSensitivity(PathSensitivity.RELATIVE) - inputs.file(rootProject.file('kotlin-js-store/wasm/yarn.lock')) - .withPropertyName('uiYarnLock') - .withPathSensitivity(PathSensitivity.NONE) - - jsonOutput = cyclonedxDir.get().file("bom-slim.json").asFile - - doLast { - postProcessBom(jsonOutput.get().asFile, 'slim', [ - 'server': configurations.server, - ], configurations.jsClientSbom.files.find(), configurations.uiSbom.files.find()) - } -} - -tasks.register('cyclonedx') { - group = 'Bill of Materials' - description = 'Generates CycloneDX BOMs for Solr distributions' - - dependsOn 'cyclonedxFull' - dependsOn 'cyclonedxSlim' -} - description = 'Solr distribution packaging' ext { - cyclonedxDir = layout.buildDirectory.dir("cyclonedx") distDir = file("$buildDir/solr-${version}") slimDistDir = file("$buildDir/solr-${version}-slim") devDir = file("$buildDir/dev") @@ -467,79 +53,17 @@ configurations { solrSlimTgz solrFullTgzSignature solrSlimTgzSignature - // For the CycloneDX BOM generation - bomSlim { - canBeResolved = true - canBeConsumed = false - } - bomFull { - canBeResolved = true - canBeConsumed = false - extendsFrom bomSlim - } - // Child SBOMs of the UI bundles, merged into the distribution SBOMs by postProcessBom - jsClientSbom { - canBeResolved = true - canBeConsumed = false - } - uiSbom { - canBeResolved = true - canBeConsumed = false - } -} - -// Request the standard JVM runtime variants, like runtimeClasspath does. -// -// The CycloneDX plugin resolves configurations leniently. Without these attributes, -// variant-aware dependencies (e.g. Guava) and platform constraints (e.g. the Jersey BOM) fail to resolve -// and silently disappear. -[configurations.bomSlim, configurations.bomFull].each { conf -> - conf.attributes { - attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY)) - attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, Usage.JAVA_RUNTIME)) - attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, objects.named(LibraryElements, LibraryElements.JAR)) - attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL)) - attribute(TargetJvmEnvironment.TARGET_JVM_ENVIRONMENT_ATTRIBUTE, objects.named(TargetJvmEnvironment, TargetJvmEnvironment.STANDARD_JVM)) - } } -// The distribution keeps the server versions of the libraries shared between the -// server and the modules, so align the full BOM with the versions of the slim one -// (e.g. a module may pull in a newer slf4j-api than the one in server/lib/ext). -configurations.bomFull.shouldResolveConsistentlyWith(configurations.bomSlim) - dependencies { rootProject.project(":solr:modules").childProjects.values().stream().map {project -> project.path}.each { module -> modules project(path: module, configuration: "packaging") - // No "configuration:" here on purpose. - // - // Gradle then selects the variant of the module that matches the attributes declared on the bom configurations above, - // just like it does for a runtime classpath ("variant-aware" resolution). - // Naming a configuration would bypass attribute matching, and imported Maven BOMs (e.g. Jersey's) would no - // longer provide the versions of their managed dependencies. - bomFull project(path: module) } crossDcManager project(path: ":solr:cross-dc-manager", configuration: "packaging") - bomFull project(path: ':solr:cross-dc-manager') example project(path: ":solr:example", configuration: "packaging") server project(path: ":solr:server", configuration: "packaging") - bomSlim project(path: ':solr:server', configuration: 'startJar') - bomSlim project(path: ':solr:server', configuration: 'serverLib') - bomSlim project(path: ':solr:server', configuration: 'libExt') - // Variant-aware dependency instead of server's by-name 'solrCore' - // configuration, see the comment on the modules above. - bomSlim project(path: ':solr:core') - bomSlim project(path: ':solr:server', configuration: 'webapp') - - // Child SBOMs of the UI bundles; empty when the projects are disabled - if (gradle.ext.withJsClient) { - jsClientSbom project(path: ':solr:webapp:js-client', configuration: 'jsClientSbom') - } - if (gradle.ext.withUiModule) { - uiSbom project(path: ':solr:ui', configuration: 'uiSbom') - } docker project(path: ':solr:docker', configuration: 'packaging') From 49143e66d7e122acbe2f043f777ce57cd51cf13c Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Wed, 5 Aug 2026 22:35:25 +0200 Subject: [PATCH 03/10] SOLR-17328: Update the packaging lock state after merging main The bomFull and bomSlim configurations exist only on this branch, so changes on main that alter what :solr:packaging resolves do not update their entries in solr/packaging/gradle.lockfile. Two such changes had accumulated and broke the build: * SOLR-18187 (#4259) added the anthropic, google-ai-gemini and ollama langchain4j providers to :solr:modules:language-models, which bomFull pulls in transitively. * #4594 bumped the OpenTelemetry and Prometheus stacks. Regenerated with "gradlew resolveAndLockAll collectJarInfos --write-locks"; only the packaging lockfile changed. Assisted-By: Claude Opus 5 (1M context) --- solr/packaging/gradle.lockfile | 53 +++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/solr/packaging/gradle.lockfile b/solr/packaging/gradle.lockfile index 6f7191470d53..72dd1ba6b96f 100644 --- a/solr/packaging/gradle.lockfile +++ b/solr/packaging/gradle.lockfile @@ -80,13 +80,16 @@ commons-collections:commons-collections:3.2.2=bomFull commons-digester:commons-digester:2.1=bomFull commons-io:commons-io:2.22.0=bomFull,bomSlim commons-validator:commons-validator:1.10.1=bomFull +dev.langchain4j:langchain4j-anthropic:1.17.0=bomFull dev.langchain4j:langchain4j-bom:1.17.0=bomFull dev.langchain4j:langchain4j-cohere:1.17.0-beta27=bomFull dev.langchain4j:langchain4j-core:1.17.0=bomFull +dev.langchain4j:langchain4j-google-ai-gemini:1.17.0=bomFull dev.langchain4j:langchain4j-http-client-jdk:1.17.0=bomFull dev.langchain4j:langchain4j-http-client:1.17.0=bomFull dev.langchain4j:langchain4j-hugging-face:1.17.0-beta27=bomFull dev.langchain4j:langchain4j-mistral-ai:1.17.0=bomFull +dev.langchain4j:langchain4j-ollama:1.17.0=bomFull dev.langchain4j:langchain4j-open-ai:1.17.0=bomFull io.dropwizard.metrics:metrics-core:4.2.39=bomFull,bomSlim io.github.azagniotov:language-detection:12.5.2=bomFull @@ -121,31 +124,33 @@ io.netty:netty-transport:4.2.15.Final=bomFull,bomSlim io.opencensus:opencensus-api:0.31.1=bomFull io.opencensus:opencensus-contrib-http-util:0.31.1=bomFull io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37.0-alpha=bomFull -io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.22.0-alpha=bomFull,bomSlim -io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.22.0=bomFull,bomSlim -io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17:2.22.0-alpha=bomFull,bomSlim -io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java8:2.22.0-alpha=bomFull,bomSlim -io.opentelemetry.semconv:opentelemetry-semconv:1.37.0=bomFull,bomSlim -io.opentelemetry:opentelemetry-api-incubator:1.56.0-alpha=bomFull,bomSlim -io.opentelemetry:opentelemetry-api:1.56.0=bomFull,bomSlim -io.opentelemetry:opentelemetry-bom:1.56.0=bomFull -io.opentelemetry:opentelemetry-common:1.56.0=bomFull,bomSlim -io.opentelemetry:opentelemetry-context:1.56.0=bomFull,bomSlim -io.opentelemetry:opentelemetry-exporter-common:1.56.0=bomFull -io.opentelemetry:opentelemetry-exporter-otlp-common:1.56.0=bomFull -io.opentelemetry:opentelemetry-exporter-otlp:1.56.0=bomFull -io.opentelemetry:opentelemetry-exporter-prometheus:1.56.0-alpha=bomFull,bomSlim -io.opentelemetry:opentelemetry-exporter-sender-jdk:1.56.0=bomFull -io.opentelemetry:opentelemetry-sdk-common:1.56.0=bomFull,bomSlim -io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.56.0=bomFull -io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.56.0=bomFull -io.opentelemetry:opentelemetry-sdk-logs:1.56.0=bomFull -io.opentelemetry:opentelemetry-sdk-metrics:1.56.0=bomFull,bomSlim -io.opentelemetry:opentelemetry-sdk-trace:1.56.0=bomFull,bomSlim -io.opentelemetry:opentelemetry-sdk:1.56.0=bomFull,bomSlim +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.27.0-alpha=bomFull,bomSlim +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.27.0=bomFull,bomSlim +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17:2.27.0-alpha=bomFull,bomSlim +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry:2.27.0-alpha=bomFull,bomSlim +io.opentelemetry.semconv:opentelemetry-semconv:1.40.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-api-incubator:1.61.0-alpha=bomFull,bomSlim +io.opentelemetry:opentelemetry-api:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-bom:1.63.0=bomFull +io.opentelemetry:opentelemetry-common:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-context:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-exporter-common:1.63.0=bomFull +io.opentelemetry:opentelemetry-exporter-otlp-common:1.63.0=bomFull +io.opentelemetry:opentelemetry-exporter-otlp:1.63.0=bomFull +io.opentelemetry:opentelemetry-exporter-prometheus:1.63.0-alpha=bomFull,bomSlim +io.opentelemetry:opentelemetry-exporter-sender-jdk:1.63.0=bomFull +io.opentelemetry:opentelemetry-sdk-common:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.63.0=bomFull +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.63.0=bomFull +io.opentelemetry:opentelemetry-sdk-logs:1.63.0=bomFull +io.opentelemetry:opentelemetry-sdk-metrics:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-sdk-trace:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-sdk:1.63.0=bomFull,bomSlim io.perfmark:perfmark-api:0.27.0=bomFull -io.prometheus:prometheus-metrics-exposition-formats:1.1.0=bomFull,bomSlim -io.prometheus:prometheus-metrics-model:1.1.0=bomFull,bomSlim +io.prometheus:prometheus-metrics-config:1.8.0=bomFull,bomSlim +io.prometheus:prometheus-metrics-exposition-formats:1.8.0=bomFull,bomSlim +io.prometheus:prometheus-metrics-exposition-textformats:1.8.0=bomFull,bomSlim +io.prometheus:prometheus-metrics-model:1.8.0=bomFull,bomSlim io.sgr:s2-geometry-library-java:1.0.0=bomFull,bomSlim io.swagger.core.v3:swagger-annotations-jakarta:2.2.52=bomFull,bomSlim jakarta.activation:jakarta.activation-api:2.1.3=bomFull,bomSlim From 53981c1acead986e9ee8200f486c490f1ddaff57 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 6 Aug 2026 00:01:05 +0200 Subject: [PATCH 04/10] SOLR-17328: Describe the vendored JavaScript in a checked manifest Review feedback on #4690: the third-party JavaScript under solr/webapp/web/libs was described by a hard-coded list in the SBOM code, which nothing tied to the directory it described, so a file could be added, bumped or removed and the SBOM would keep reporting the old state. Three of the sixteen files also had no purl at all, leaving them invisible to vulnerability scanners. The components now live in solr/webapp/vendored-libs.json as CycloneDX component objects, merged into the distribution SBOM as they are, and :solr:webapp:validateVendoredLibs fails whenever the manifest and the directory disagree. The generator tasks depend on it, so a stale hash cannot reach a BOM, which also matters for the archive locations of step 8 since those are matched by hash. updateVendoredLibs refreshes the hashes; it cannot invent identity or pedigree, so a new file keeps failing validation until those are filled in by hand. Every component now identifies its upstream package by purl, even where the shipped bytes deviate, so that advisories still match, and records the deviation under "pedigree" with the original in pedigree.ancestors. Comparing each file against upstream established that: * eleven are the pristine upstream artifact with an Apache licence header prepended, and the ancestors carry the verified upstream SHA-256 next to a distribution reference; * ui-grid.min.js is byte-identical to npm angular-ui-grid 4.10.0 and so needs no pedigree at all; * ngtimeago.js is a fork of uttesh/ngtimeago pinned to the untagged commit 0b1e72785a6e, patched by SOLR-7780, LUCENE-6732 and SOLR-13343, which pedigree.commits now records. It is unrelated to the npm package ng-timeago; * jquery-ui.min.js, highlight.js, angular-chosen.min.js and jssha-3.3.1-sha256.min.js are custom or differently built artifacts that match no published file; they keep the upstream purl so that advisories are not missed, with the deviation in pedigree.notes. d3.js and angular-utf8-base64.min.js are served by neither unpkg nor cdnjs under those names; the former is d3.v2.js inside the d3 2.8.1 tarball and the latter exists only in the git repository, hence its pkg:github purl. Both BOMs validate against the CycloneDX 1.6 schema. Apart from the three components that gain a purl, and the pedigree entries, the generated SBOMs are unchanged. Assisted-By: Claude Opus 5 (1M context) --- gradle/solr/sbom.gradle | 57 +-- gradle/validation/precommit.gradle | 1 + solr/webapp/build.gradle | 116 ++++++ solr/webapp/vendored-libs.json | 635 +++++++++++++++++++++++++++++ 4 files changed, 770 insertions(+), 39 deletions(-) create mode 100644 solr/webapp/vendored-libs.json diff --git a/gradle/solr/sbom.gradle b/gradle/solr/sbom.gradle index 2ca64fad9be5..6ed3d027f3c5 100644 --- a/gradle/solr/sbom.gradle +++ b/gradle/solr/sbom.gradle @@ -40,6 +40,10 @@ configure(project(':solr:packaging')) { cyclonedxDir = layout.buildDirectory.dir("cyclonedx") } + // Hand-maintained CycloneDX components for the third-party JavaScript checked into + // solr/webapp/web/libs, merged in by step 5 of the post-processing below. + def vendoredLibsManifest = rootProject.file('solr/webapp/vendored-libs.json') + // For the CycloneDX BOM generation configurations { bomSlim { @@ -133,8 +137,8 @@ configure(project(':solr:packaging')) { // The project description and the Apache-2.0 license are also added. // 5. Vendored JavaScript libraries: // The AngularJS admin UI ships third-party JavaScript files checked into solr/webapp/web/libs. - // Components for them are added from a curated list (version and license read from the file - // headers); the hash matching of step 8 proves that each file still ships unmodified. + // Their components are maintained by hand in solr/webapp/vendored-libs.json: each identifies + // the upstream package by purl, with any deviation of the shipped bytes under "pedigree". // 6. JavaScript client bundle: // The npm packages bundled by browserify into the OpenAPI JS client // (server/solr-webapp/webapp/libs/solr/index.js) are nested as subassemblies of a first-party @@ -271,41 +275,8 @@ configure(project(':solr:packaging')) { def webappDependsOn = json.dependencies.find { it.ref == webappRef }.dependsOn // 5. Vendored JavaScript libraries - // Entries without a version marker in the file get no version and no purl - def vendoredJsLibs = [ - [file: 'angular.min.js', name: 'angular', version: '1.8.0', license: 'MIT'], - [file: 'angular-chosen.min.js', name: 'angular-chosen-localytics', version: '1.9.2', license: 'MIT'], - [file: 'angular-cookies.min.js', name: 'angular-cookies', version: '1.8.0', license: 'MIT'], - [file: 'angular-resource.min.js', name: 'angular-resource', version: '1.8.0', license: 'MIT'], - [file: 'angular-route.min.js', name: 'angular-route', version: '1.8.0', license: 'MIT'], - [file: 'angular-sanitize.min.js', name: 'angular-sanitize', version: '1.8.0', license: 'MIT'], - [file: 'angular-utf8-base64.min.js', name: 'angular-utf8-base64', license: 'MIT'], - [file: 'chosen.jquery.min.js', name: 'chosen-js', version: '1.8.7', license: 'MIT'], - [file: 'd3.js', name: 'd3', version: '2.8.1', license: 'BSD-3-Clause'], - [file: 'highlight.js', name: 'highlight.js', license: 'BSD-3-Clause'], - [file: 'jquery-3.5.1.min.js', name: 'jquery', version: '3.5.1', license: 'MIT'], - [file: 'jquery-ui.min.js', name: 'jquery-ui', version: '1.12.1', license: 'MIT'], - [file: 'jssha-3.3.1-sha256.min.js', name: 'jssha', version: '3.3.1', license: 'BSD-3-Clause'], - [file: 'jstree.min.js', name: 'jstree', version: '3.3.10', license: 'MIT'], - [file: 'ngtimeago.js', name: 'ngtimeago', license: 'MIT'], - [file: 'ui-grid.min.js', name: 'angular-ui-grid', version: '4.10.0', license: 'MIT'], - ] - def webLibsDir = rootProject.file('solr/webapp/web/libs') - vendoredJsLibs.each { lib -> - def purl = lib.version != null ? "pkg:npm/${lib.name}@${lib.version}".toString() : null - def component = [ - type: 'library', - 'bom-ref': purl ?: "vendored-js:${lib.name}".toString(), - name: lib.name, - ] - if (lib.version != null) { - component.version = lib.version - } - component.hashes = [[alg: 'SHA-256', content: sha256Of(new File(webLibsDir, lib.file))]] - component.licenses = [[license: [id: lib.license]]] - if (purl != null) { - component.purl = purl - } + // Ready-made components; validateVendoredLibs guarantees the hashes match the files. + new JsonSlurper().parse(vendoredLibsManifest).each { fileName, component -> json.components << component json.dependencies << [ref: component.'bom-ref', dependsOn: []] webappDependsOn << component.'bom-ref' @@ -452,7 +423,11 @@ configure(project(':solr:packaging')) { .withPropertyName('childSboms') .withPathSensitivity(PathSensitivity.NONE) - // Sources of the statically listed JavaScript components + // The vendored JavaScript components and the files they describe + dependsOn ':solr:webapp:validateVendoredLibs' + inputs.file(vendoredLibsManifest) + .withPropertyName('vendoredLibsManifest') + .withPathSensitivity(PathSensitivity.NONE) inputs.dir(rootProject.file('solr/webapp/web/libs')) .withPropertyName('vendoredJsLibs') .withPathSensitivity(PathSensitivity.RELATIVE) @@ -494,7 +469,11 @@ configure(project(':solr:packaging')) { .withPropertyName('childSboms') .withPathSensitivity(PathSensitivity.NONE) - // Sources of the statically listed JavaScript components + // The vendored JavaScript components and the files they describe + dependsOn ':solr:webapp:validateVendoredLibs' + inputs.file(vendoredLibsManifest) + .withPropertyName('vendoredLibsManifest') + .withPathSensitivity(PathSensitivity.NONE) inputs.dir(rootProject.file('solr/webapp/web/libs')) .withPropertyName('vendoredJsLibs') .withPathSensitivity(PathSensitivity.RELATIVE) diff --git a/gradle/validation/precommit.gradle b/gradle/validation/precommit.gradle index 02d03010016f..0ce9460fa1d3 100644 --- a/gradle/validation/precommit.gradle +++ b/gradle/validation/precommit.gradle @@ -40,6 +40,7 @@ configure(rootProject) { "ecjLint", "validateLogCalls", "validateSourcePatterns", + "validateVendoredLibs", "spotlessCheck" ] } diff --git a/solr/webapp/build.gradle b/solr/webapp/build.gradle index da2ba2cda085..b64d1574cce2 100644 --- a/solr/webapp/build.gradle +++ b/solr/webapp/build.gradle @@ -15,6 +15,10 @@ * limitations under the License. */ +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import java.security.MessageDigest + plugins { id 'java' id 'war' @@ -63,3 +67,115 @@ war { artifacts { war tasks.war } + +// web/libs holds checked-in third-party JavaScript, described for the distribution SBOM by +// vendored-libs.json. These tasks keep the two in sync: validateVendoredLibs fails on any drift, +// updateVendoredLibs refreshes the hashes. The manifest sits outside web/, which goes into the war. +def vendoredLibsManifest = file('vendored-libs.json') +def vendoredLibsDir = file('web/libs') + +def sha256Of = { File f -> + def digest = MessageDigest.getInstance('SHA-256') + f.eachByte(8192) { buffer, length -> digest.update(buffer, 0, length) } + digest.digest().encodeHex().toString() +} + +tasks.register('validateVendoredLibs') { + group = 'Verification' + description = 'Checks that web/libs matches the components declared in vendored-libs.json' + + inputs.file(vendoredLibsManifest).withPropertyName('manifest') + .withPathSensitivity(PathSensitivity.NONE) + inputs.dir(vendoredLibsDir).withPropertyName('vendoredLibs') + .withPathSensitivity(PathSensitivity.RELATIVE) + def marker = layout.buildDirectory.file('vendored-libs-validated.txt') + outputs.file(marker) + + doLast { + def manifest = new JsonSlurper().parse(vendoredLibsManifest) + def onDisk = vendoredLibsDir.listFiles().findAll { it.file }.collectEntries { [(it.name): it] } + def errors = [] + + (onDisk.keySet() - manifest.keySet()).sort().each { + errors << "${it}: present in ${vendoredLibsDir} but absent from ${vendoredLibsManifest.name}." + } + (manifest.keySet() - onDisk.keySet()).sort().each { + errors << "${it}: declared in ${vendoredLibsManifest.name} but no such file in ${vendoredLibsDir}." + } + + manifest.each { fileName, component -> + def file = onDisk[fileName] + if (file == null) { + return + } + ['type', 'bom-ref', 'name', 'purl', 'licenses', 'hashes'].each { field -> + if (!component[field]) { + errors << "${fileName}: component is missing the required '${field}' field." + } + } + def declared = component.hashes?.find { it.alg == 'SHA-256' }?.content + if (declared == null) { + errors << "${fileName}: component declares no SHA-256 hash." + } else { + def actual = sha256Of(file) + if (declared != actual) { + errors << "${fileName}: content changed, SHA-256 is ${actual} but the component declares " + + "${declared}. Review the component (version, licence, pedigree) and run 'gradlew " + + ":solr:webapp:updateVendoredLibs' to refresh the hash." + } + } + } + + if (errors) { + throw new GradleException("Vendored JavaScript does not match ${vendoredLibsManifest}:\n " + + errors.join("\n ")) + } + marker.get().asFile.text = "${manifest.size()} vendored components validated\n" + } +} + +tasks.matching { it.name == 'check' }.configureEach { + it.dependsOn 'validateVendoredLibs' +} + +tasks.register('updateVendoredLibs') { + group = 'Build Dependencies' + description = 'Refreshes the SHA-256 hashes in vendored-libs.json' + + doLast { + // JsonSlurper hands back a LinkedHashMap, so the declaration order survives the round trip. + def manifest = new JsonSlurper().parse(vendoredLibsManifest) + def onDisk = vendoredLibsDir.listFiles().findAll { it.file }.sort { it.name } + + onDisk.each { file -> + def component = manifest[file.name] + if (component == null) { + // A stub only: identity, licence and pedigree cannot be inferred from the file, so + // validateVendoredLibs keeps failing until they are filled in by hand. + logger.lifecycle("${file.name}: new file, added a stub component that must be completed by hand") + manifest[file.name] = [ + type: 'library', + 'bom-ref': null, + name: null, + hashes: [[alg: 'SHA-256', content: sha256Of(file)]], + licenses: null, + purl: null, + ] + } else { + def actual = sha256Of(file) + if (component.hashes?.find { it.alg == 'SHA-256' }?.content != actual) { + logger.lifecycle("${file.name}: refreshed SHA-256") + component.hashes = [[alg: 'SHA-256', content: actual]] + } + } + } + + def removed = manifest.keySet() - onDisk.collect { it.name } as Set + removed.each { + logger.lifecycle("${it}: file is gone, dropping the component") + manifest.remove(it) + } + + vendoredLibsManifest.text = JsonOutput.prettyPrint(JsonOutput.toJson(manifest.sort { it.key })) + "\n" + } +} diff --git a/solr/webapp/vendored-libs.json b/solr/webapp/vendored-libs.json new file mode 100644 index 000000000000..3786bd48e741 --- /dev/null +++ b/solr/webapp/vendored-libs.json @@ -0,0 +1,635 @@ +{ + "angular-chosen.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-chosen-localytics@1.9.2", + "name": "angular-chosen-localytics", + "version": "1.9.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "d132a592badfa97744c42ff5abf533ea5a826d1981bbd0f01d19f5b5d7e0df2b" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-chosen-localytics@1.9.2", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-chosen-localytics", + "version": "1.9.2", + "purl": "pkg:npm/angular-chosen-localytics@1.9.2" + } + ], + "notes": "Differently minified build of 1.9.2; the published dist/angular-chosen.min.js has the same logic but different symbol names, so the shipped bytes are not comparable to it." + } + }, + "angular-cookies.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-cookies@1.8.0", + "name": "angular-cookies", + "version": "1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "395514ba020286029440b1fc14d04b4102cb5d4e0ba7a4117079b802ca0869cb" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-cookies@1.8.0", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-cookies", + "version": "1.8.0", + "purl": "pkg:npm/angular-cookies@1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "eed97b74e2128f3d340325dd9cbfb9b8f70a1a5ade70eccca990d45483aa8700" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/angular-cookies@1.8.0/angular-cookies.min.js" + } + ] + } + ], + "notes": "Apache licence header of 1101 bytes prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "angular-resource.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-resource@1.8.0", + "name": "angular-resource", + "version": "1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "89c2d5cb9a399e7979687172d6875c68d1cd0d2b7a99f96dd83a523614f80c4d" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-resource@1.8.0", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-resource", + "version": "1.8.0", + "purl": "pkg:npm/angular-resource@1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "3a61f95c630f63b39aacf3f8ee66bc13bc9b820d11f749591d1c3d07125ec184" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/angular-resource@1.8.0/angular-resource.min.js" + } + ] + } + ], + "notes": "Apache licence header of 1101 bytes prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "angular-route.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-route@1.8.0", + "name": "angular-route", + "version": "1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "0a9fac6d4809f3408d96c434d36624624ddb3db36419f557d2cd2cf1b949b105" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-route@1.8.0", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-route", + "version": "1.8.0", + "purl": "pkg:npm/angular-route@1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "3422eae4c737ff2d30abfe3df6c30e6b11869d3a30683c5efced151248eb9661" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/angular-route@1.8.0/angular-route.min.js" + } + ] + } + ], + "notes": "Apache licence header of 1101 bytes prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "angular-sanitize.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-sanitize@1.8.0", + "name": "angular-sanitize", + "version": "1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "db5c1a1af40f0c7205a7bc6b592d15cb1e50478665467d7e95c804b20df723d4" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-sanitize@1.8.0", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-sanitize", + "version": "1.8.0", + "purl": "pkg:npm/angular-sanitize@1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "958e6aa9b32f5ef3e86acf16d2413f08baa02f68fbe38baa5d8916282ae1b882" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/angular-sanitize@1.8.0/angular-sanitize.min.js" + } + ] + } + ], + "notes": "Apache licence header of 1101 bytes prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "angular-utf8-base64.min.js": { + "type": "library", + "bom-ref": "pkg:github/stranger82/angular-utf8-base64@v0.0.5", + "name": "angular-utf8-base64", + "version": "0.0.5", + "hashes": [ + { + "alg": "SHA-256", + "content": "b95109c1309c2f7aea93f062d2006ce50eb877b8698d529741f635843dd79b89" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:github/stranger82/angular-utf8-base64@v0.0.5", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-utf8-base64", + "version": "0.0.5", + "purl": "pkg:github/stranger82/angular-utf8-base64@v0.0.5", + "hashes": [ + { + "alg": "SHA-256", + "content": "c8cad92b2f6a528c5e0982aef0ad739d1cd1afb896e8c2ba400a750e2d5d5d57" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://raw.githubusercontent.com/stranger82/angular-utf8-base64/43cea612939acc07e06939f8da97e6f22186d1e2/angular-utf8-base64.min.js" + } + ] + } + ], + "notes": "Apache licence header of 1786 bytes prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "angular.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular@1.8.0", + "name": "angular", + "version": "1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "f5c29b548881dc22e6c2ffa12eae37dc1419b76046ff99a154b02e44f680de29" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular@1.8.0", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular", + "version": "1.8.0", + "purl": "pkg:npm/angular@1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "566f18cb8bc23558701c2cc4f934fe50bcc85629d1aaf5d589f835f2b3e57a9f" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/angular@1.8.0/angular.min.js" + } + ] + } + ], + "notes": "Apache licence header of 1101 bytes prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "chosen.jquery.min.js": { + "type": "library", + "bom-ref": "pkg:npm/chosen-js@1.8.7", + "name": "chosen-js", + "version": "1.8.7", + "hashes": [ + { + "alg": "SHA-256", + "content": "7cd60ffd167c5557fca7b192e6400884642ac3842a495f938c8d412ec36a0f75" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/chosen-js@1.8.7", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "chosen-js", + "version": "1.8.7", + "purl": "pkg:npm/chosen-js@1.8.7", + "hashes": [ + { + "alg": "SHA-256", + "content": "73881513a7e7f8944a311bea8e80e9fad946e256ae74d62b5c8d469dc6df0186" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/chosen-js@1.8.7/chosen.jquery.min.js" + } + ] + } + ], + "notes": "Apache licence header of 1165 bytes prepended and a trailing newline; removing it reproduces the ancestor byte-for-byte." + } + }, + "d3.js": { + "type": "library", + "bom-ref": "pkg:npm/d3@2.8.1", + "name": "d3", + "version": "2.8.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "435b386bb55c7fcd938584ebb3c4a64da3712a043fa267a1ed3da2ac22aa6ef7" + } + ], + "licenses": [ + { + "license": { + "id": "BSD-3-Clause" + } + } + ], + "purl": "pkg:npm/d3@2.8.1", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "d3", + "version": "2.8.1", + "purl": "pkg:npm/d3@2.8.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "d74d3335c81747cb3d2eed71c29a83ef6fe3fa523db15934cdfa5738624a2ea2" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://registry.npmjs.org/d3/-/d3-2.8.1.tgz" + } + ] + } + ], + "notes": "Apache licence header of 1433 bytes prepended; removing it reproduces the ancestor (member d3.v2.js of the tarball) byte-for-byte." + } + }, + "highlight.js": { + "type": "library", + "bom-ref": "pkg:npm/highlight.js", + "name": "highlight.js", + "hashes": [ + { + "alg": "SHA-256", + "content": "b0c3b1298c79a2d28a9a6bdae07f4fc878e7bc46f50e81338f881f3d6a61514f" + } + ], + "licenses": [ + { + "license": { + "id": "BSD-3-Clause" + } + } + ], + "purl": "pkg:npm/highlight.js", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "highlight.js", + "purl": "pkg:npm/highlight.js" + } + ], + "notes": "Custom subset build vendored in 2012 whose version could not be determined: the file carries no version marker and its 'var hljs=new function()' API predates 8.0. Treat every highlight.js advisory as potentially applying." + } + }, + "jquery-3.5.1.min.js": { + "type": "library", + "bom-ref": "pkg:npm/jquery@3.5.1", + "name": "jquery", + "version": "3.5.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "44915672ad9626cb67e995221dd29283ff5620125a10745ac968ddc9c3865c39" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/jquery@3.5.1", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "jquery", + "version": "3.5.1", + "purl": "pkg:npm/jquery@3.5.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "f7f6a5894f1d19ddad6fa392b2ece2c5e578cbf7da4ea805b6885eb6985b6e3d" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/jquery@3.5.1/dist/jquery.min.js" + } + ] + } + ], + "notes": "Apache licence header of 1108 bytes prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "jquery-ui.min.js": { + "type": "library", + "bom-ref": "pkg:npm/jquery-ui@1.12.1", + "name": "jquery-ui", + "version": "1.12.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "4bc1d3b514cad98858c077a3e69f61b862e3808a732b72a4d9da3cd91908037b" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/jquery-ui@1.12.1", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "jquery-ui", + "version": "1.12.1", + "purl": "pkg:npm/jquery-ui@1.12.1" + } + ], + "notes": "Custom subset build of 24 KB against the 253 KB full distribution, so it is not byte-comparable to any published artifact. Any jQuery UI 1.12.1 advisory applies until the affected widget is shown to be absent from this build." + } + }, + "jssha-3.3.1-sha256.min.js": { + "type": "library", + "bom-ref": "pkg:npm/jssha@3.3.1", + "name": "jssha", + "version": "3.3.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "c2e3235b130fe33d129f97c6570272b4b20baf0daab628ba79d6d9d005694921" + } + ], + "licenses": [ + { + "license": { + "id": "BSD-3-Clause" + } + } + ], + "purl": "pkg:npm/jssha@3.3.1", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "jssha", + "version": "3.3.1", + "purl": "pkg:npm/jssha@3.3.1" + } + ], + "notes": "SHA-256-only variant; it matches none of the dist files published in the jssha 3.3.1 npm tarball." + } + }, + "jstree.min.js": { + "type": "library", + "bom-ref": "pkg:npm/jstree@3.3.10", + "name": "jstree", + "version": "3.3.10", + "hashes": [ + { + "alg": "SHA-256", + "content": "612d5f0a58cb09c620d82d39ef060e68453f634fca279f9f3e91feca954f98a5" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/jstree@3.3.10", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "jstree", + "version": "3.3.10", + "purl": "pkg:npm/jstree@3.3.10", + "hashes": [ + { + "alg": "SHA-256", + "content": "26238e200ef64e61a4a47bbff33ce50f1312234806db998b8e93ebefda015a6c" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/jstree@3.3.10/dist/jstree.min.js" + } + ] + } + ], + "notes": "Apache licence header of 1054 bytes prepended and a trailing newline; removing it reproduces the ancestor byte-for-byte." + } + }, + "ngtimeago.js": { + "type": "library", + "bom-ref": "pkg:github/uttesh/ngtimeago@0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "name": "ngtimeago", + "version": "0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "hashes": [ + { + "alg": "SHA-256", + "content": "9ca6b12801166dd2cdb699d18db67c0bd8c3b3c6cec05c2b5ea4e01653877a71" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:github/uttesh/ngtimeago@0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "ngtimeago", + "version": "0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "purl": "pkg:github/uttesh/ngtimeago@0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "hashes": [ + { + "alg": "SHA-256", + "content": "52675691593269327c2f053b7aedda07bbfe9869a610a4447dab6dc79bf96df6" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://raw.githubusercontent.com/uttesh/ngtimeago/0b1e72785a6e0e6edd9389d875db6279b0ea36ba/ngtimeago.js" + }, + { + "type": "vcs", + "url": "https://github.com/uttesh/ngtimeago" + } + ] + } + ], + "commits": [ + { + "uid": "7e8d6d110060498429a5b942789274d7ca217136", + "message": "SOLR-7780: Prevent NaN showing on angular UI dashboard" + }, + { + "uid": "1331a57e3d5d0fd1480fafcbe6c66721eacf331a", + "message": "LUCENE-6732: Remove tabs in JS and XML files" + }, + { + "uid": "9d97ef1027fe8b844fa5c72fbc63ddc3a24a0bc6", + "message": "SOLR-13343: Fix minor web UI spacing issue" + } + ], + "notes": "Apache licence header of 1062 bytes prepended, and the body carries the Solr patches listed in commits. The base commit is untagged and later than the 0.0.2 tag. Not the npm package ng-timeago, which is the unrelated joyingsoft/ng-timeago project." + } + }, + "ui-grid.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-ui-grid@4.10.0", + "name": "angular-ui-grid", + "version": "4.10.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "ec0200dc99b4e7f2576e3a0285ce83b90ed1e93416016595e6078c0afa215a34" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-ui-grid@4.10.0" + } +} From efb4d7f29ac3f168e6b515c0bb0e5a7f09bf308f Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 6 Aug 2026 00:39:30 +0200 Subject: [PATCH 05/10] SOLR-17328: Correct the attribution of the vendored JavaScript Follow-up to 53981c1acea, which described the blocks prepended to the files under solr/webapp/web/libs as Apache licence headers. They are not: each is the upstream project's own licence text carrying the upstream copyright holder, MIT for the AngularJS, jQuery, Chosen, jsTree and ngtimeago files and BSD-3-Clause for d3. The pedigree notes now say which licence text was prepended, without asserting anything about who wrote it. The one file that does carry Apache-2.0 text is angular-utf8-base64.min.js, and that is not the ASF's either: the library encapsulates Vassilis Petroulias's base64.js, released by its own author under Apache-2.0, whose notice the vendoring reproduces. Since the shipped code is therefore part MIT and part Apache-2.0, its licence becomes the SPDX expression "MIT AND Apache-2.0". Attribution moves out of prose and into the fields meant for it. Every component now carries a manufacturer, which is what satisfies the Component Producer element of the 2026 CISA SBOM Minimum Elements; that element replaced the 2021 NTIA Supplier Name and asks for the original project or maintaining organization. Individually authored libraries also list their authors. The copyright field is dropped: it is not a minimum element under either the 2021 or the 2026 guidance, and the attribution it held is now carried by manufacturer and authors. highlight.js records its version as "unknown" rather than omitting it. Component Version is a required element, and the guidance asks the SBOM author to state explicitly that information is unknown instead of leaving it out. The file carries no version marker and its API predates highlight.js 8.0. All producer URLs use https, and each was checked to resolve. Two could not simply change scheme: angular-ui.github.com serves no https, so ui-grid points at its GitHub project, and getharvest.com redirects. Both BOMs still validate against the CycloneDX 1.6 schema. Assisted-By: Claude Opus 5 (1M context) --- solr/webapp/vendored-libs.json | 163 ++++++++++++++++++++++++++++++--- 1 file changed, 148 insertions(+), 15 deletions(-) diff --git a/solr/webapp/vendored-libs.json b/solr/webapp/vendored-libs.json index 3786bd48e741..8a99d148ce1c 100644 --- a/solr/webapp/vendored-libs.json +++ b/solr/webapp/vendored-libs.json @@ -2,6 +2,12 @@ "angular-chosen.min.js": { "type": "library", "bom-ref": "pkg:npm/angular-chosen-localytics@1.9.2", + "manufacturer": { + "name": "Localytics", + "url": [ + "https://www.localytics.com/" + ] + }, "name": "angular-chosen-localytics", "version": "1.9.2", "hashes": [ @@ -33,6 +39,12 @@ "angular-cookies.min.js": { "type": "library", "bom-ref": "pkg:npm/angular-cookies@1.8.0", + "manufacturer": { + "name": "Google, Inc.", + "url": [ + "https://angularjs.org/" + ] + }, "name": "angular-cookies", "version": "1.8.0", "hashes": [ @@ -70,12 +82,18 @@ ] } ], - "notes": "Apache licence header of 1101 bytes prepended; removing it reproduces the ancestor byte-for-byte." + "notes": "MIT licence text, 1101 bytes, prepended; removing it reproduces the ancestor byte-for-byte." } }, "angular-resource.min.js": { "type": "library", "bom-ref": "pkg:npm/angular-resource@1.8.0", + "manufacturer": { + "name": "Google, Inc.", + "url": [ + "https://angularjs.org/" + ] + }, "name": "angular-resource", "version": "1.8.0", "hashes": [ @@ -113,12 +131,18 @@ ] } ], - "notes": "Apache licence header of 1101 bytes prepended; removing it reproduces the ancestor byte-for-byte." + "notes": "MIT licence text, 1101 bytes, prepended; removing it reproduces the ancestor byte-for-byte." } }, "angular-route.min.js": { "type": "library", "bom-ref": "pkg:npm/angular-route@1.8.0", + "manufacturer": { + "name": "Google, Inc.", + "url": [ + "https://angularjs.org/" + ] + }, "name": "angular-route", "version": "1.8.0", "hashes": [ @@ -156,12 +180,18 @@ ] } ], - "notes": "Apache licence header of 1101 bytes prepended; removing it reproduces the ancestor byte-for-byte." + "notes": "MIT licence text, 1101 bytes, prepended; removing it reproduces the ancestor byte-for-byte." } }, "angular-sanitize.min.js": { "type": "library", "bom-ref": "pkg:npm/angular-sanitize@1.8.0", + "manufacturer": { + "name": "Google, Inc.", + "url": [ + "https://angularjs.org/" + ] + }, "name": "angular-sanitize", "version": "1.8.0", "hashes": [ @@ -199,12 +229,26 @@ ] } ], - "notes": "Apache licence header of 1101 bytes prepended; removing it reproduces the ancestor byte-for-byte." + "notes": "MIT licence text, 1101 bytes, prepended; removing it reproduces the ancestor byte-for-byte." } }, "angular-utf8-base64.min.js": { "type": "library", "bom-ref": "pkg:github/stranger82/angular-utf8-base64@v0.0.5", + "manufacturer": { + "name": "Andrey Bezyazychniy", + "url": [ + "https://github.com/stranger82/angular-utf8-base64" + ] + }, + "authors": [ + { + "name": "Andrey Bezyazychniy" + }, + { + "name": "Vassilis Petroulias" + } + ], "name": "angular-utf8-base64", "version": "0.0.5", "hashes": [ @@ -215,9 +259,7 @@ ], "licenses": [ { - "license": { - "id": "MIT" - } + "expression": "MIT AND Apache-2.0" } ], "purl": "pkg:github/stranger82/angular-utf8-base64@v0.0.5", @@ -242,12 +284,18 @@ ] } ], - "notes": "Apache licence header of 1786 bytes prepended; removing it reproduces the ancestor byte-for-byte." + "notes": "Licence texts of 1786 bytes prepended: the MIT terms, and the Apache-2.0 notice covering the base64.js library that this package encapsulates. Removing them reproduces the ancestor byte-for-byte." } }, "angular.min.js": { "type": "library", "bom-ref": "pkg:npm/angular@1.8.0", + "manufacturer": { + "name": "Google, Inc.", + "url": [ + "https://angularjs.org/" + ] + }, "name": "angular", "version": "1.8.0", "hashes": [ @@ -285,12 +333,23 @@ ] } ], - "notes": "Apache licence header of 1101 bytes prepended; removing it reproduces the ancestor byte-for-byte." + "notes": "MIT licence text, 1101 bytes, prepended; removing it reproduces the ancestor byte-for-byte." } }, "chosen.jquery.min.js": { "type": "library", "bom-ref": "pkg:npm/chosen-js@1.8.7", + "manufacturer": { + "name": "Harvest", + "url": [ + "https://www.getharvest.com/" + ] + }, + "authors": [ + { + "name": "Patrick Filler" + } + ], "name": "chosen-js", "version": "1.8.7", "hashes": [ @@ -328,12 +387,23 @@ ] } ], - "notes": "Apache licence header of 1165 bytes prepended and a trailing newline; removing it reproduces the ancestor byte-for-byte." + "notes": "MIT licence text, 1165 bytes, prepended, and a trailing newline appended; removing them reproduces the ancestor byte-for-byte." } }, "d3.js": { "type": "library", "bom-ref": "pkg:npm/d3@2.8.1", + "manufacturer": { + "name": "Michael Bostock", + "url": [ + "https://github.com/d3/d3" + ] + }, + "authors": [ + { + "name": "Michael Bostock" + } + ], "name": "d3", "version": "2.8.1", "hashes": [ @@ -371,13 +441,25 @@ ] } ], - "notes": "Apache licence header of 1433 bytes prepended; removing it reproduces the ancestor (member d3.v2.js of the tarball) byte-for-byte." + "notes": "BSD-3-Clause licence text, 1433 bytes, prepended; removing it reproduces the ancestor (member d3.v2.js of the tarball) byte-for-byte." } }, "highlight.js": { "type": "library", "bom-ref": "pkg:npm/highlight.js", + "manufacturer": { + "name": "Ivan Sagalaev", + "url": [ + "https://github.com/highlightjs/highlight.js" + ] + }, + "authors": [ + { + "name": "Ivan Sagalaev" + } + ], "name": "highlight.js", + "version": "unknown", "hashes": [ { "alg": "SHA-256", @@ -400,12 +482,18 @@ "purl": "pkg:npm/highlight.js" } ], - "notes": "Custom subset build vendored in 2012 whose version could not be determined: the file carries no version marker and its 'var hljs=new function()' API predates 8.0. Treat every highlight.js advisory as potentially applying." + "notes": "Custom subset build vendored in 2012 whose version is unknown to the SBOM author: the file carries no version marker and its 'var hljs=new function()' API predates 8.0. The version is recorded as \"unknown\" rather than omitted. Treat every highlight.js advisory as potentially applying." } }, "jquery-3.5.1.min.js": { "type": "library", "bom-ref": "pkg:npm/jquery@3.5.1", + "manufacturer": { + "name": "jQuery Foundation", + "url": [ + "https://jquery.com/" + ] + }, "name": "jquery", "version": "3.5.1", "hashes": [ @@ -443,12 +531,18 @@ ] } ], - "notes": "Apache licence header of 1108 bytes prepended; removing it reproduces the ancestor byte-for-byte." + "notes": "MIT licence text, 1108 bytes, prepended; removing it reproduces the ancestor byte-for-byte." } }, "jquery-ui.min.js": { "type": "library", "bom-ref": "pkg:npm/jquery-ui@1.12.1", + "manufacturer": { + "name": "jQuery Foundation", + "url": [ + "https://jquery.com/" + ] + }, "name": "jquery-ui", "version": "1.12.1", "hashes": [ @@ -480,6 +574,17 @@ "jssha-3.3.1-sha256.min.js": { "type": "library", "bom-ref": "pkg:npm/jssha@3.3.1", + "manufacturer": { + "name": "Brian Turek", + "url": [ + "https://github.com/Caligatio/jsSHA" + ] + }, + "authors": [ + { + "name": "Brian Turek" + } + ], "name": "jssha", "version": "3.3.1", "hashes": [ @@ -511,6 +616,17 @@ "jstree.min.js": { "type": "library", "bom-ref": "pkg:npm/jstree@3.3.10", + "manufacturer": { + "name": "Ivan Bozhanov", + "url": [ + "https://github.com/vakata/jstree" + ] + }, + "authors": [ + { + "name": "Ivan Bozhanov" + } + ], "name": "jstree", "version": "3.3.10", "hashes": [ @@ -548,12 +664,23 @@ ] } ], - "notes": "Apache licence header of 1054 bytes prepended and a trailing newline; removing it reproduces the ancestor byte-for-byte." + "notes": "MIT licence text, 1054 bytes, prepended, and a trailing newline appended; removing them reproduces the ancestor byte-for-byte." } }, "ngtimeago.js": { "type": "library", "bom-ref": "pkg:github/uttesh/ngtimeago@0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "manufacturer": { + "name": "Uttesh Kumar", + "url": [ + "https://github.com/uttesh/ngtimeago" + ] + }, + "authors": [ + { + "name": "Uttesh Kumar" + } + ], "name": "ngtimeago", "version": "0b1e72785a6e0e6edd9389d875db6279b0ea36ba", "hashes": [ @@ -609,12 +736,18 @@ "message": "SOLR-13343: Fix minor web UI spacing issue" } ], - "notes": "Apache licence header of 1062 bytes prepended, and the body carries the Solr patches listed in commits. The base commit is untagged and later than the 0.0.2 tag. Not the npm package ng-timeago, which is the unrelated joyingsoft/ng-timeago project." + "notes": "MIT licence text, 1062 bytes, prepended, and the body carries the Solr patches listed in commits. The base commit is untagged and later than the 0.0.2 tag. Not the npm package ng-timeago, which is the unrelated joyingsoft/ng-timeago project." } }, "ui-grid.min.js": { "type": "library", "bom-ref": "pkg:npm/angular-ui-grid@4.10.0", + "manufacturer": { + "name": "AngularUI Team", + "url": [ + "https://github.com/angular-ui/ui-grid" + ] + }, "name": "angular-ui-grid", "version": "4.10.0", "hashes": [ From e96844da591274fd1fe95a30888076de2bc5dd81 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 6 Aug 2026 07:16:49 +0200 Subject: [PATCH 06/10] SOLR-17328: Package the module libraries that were actually tested Each module's lib/ directory is assembled from a copyRecursive() of its runtimeLibs configuration. A copied Gradle configuration inherits its source's dependencies and excludes but not its shouldResolveConsistentlyWith alignment and not its lock state, so the alignWithRuntimeClasspath call a few lines above, whose comment promises "the distribution contains the same library versions that were tested", was silently discarded for the copy that gets packaged. The subtraction that follows removes the platform libraries by file, so an unaligned copy that resolved a different version of a library the platform also ships was not recognised as a duplicate and stayed. Five such jars reached the binary distribution: cross-dc-manager/lib/error_prone_annotations-2.49.0.jar modules/cuvs/lib/lucene-backward-codecs-10.2.0.jar modules/gcs-repository/lib/jackson-annotations-2.18.3.jar modules/gcs-repository/lib/jackson-core-2.18.3.jar modules/gcs-repository/lib/jackson-databind-2.18.3.jar They were not the versions the tests ran against, they were absent from the lock state, and they were the only five of the 188 third-party jars under the module lib directories with no solr/licenses entry, so validateJarChecksums never saw them either. The error_prone_annotations one was never a runtime dependency at all: 2.49.0 is locked only for the annotationProcessor and errorprone configurations. Aligning the copy makes both sides of the subtraction resolve to the same file, so all five are now recognised as duplicates of the platform's copies and dropped. Nothing is added: consistent resolution only raises versions of modules already in the graph. Runtime behaviour does not change. SolrResourceLoader appends module lib URLs after the existing ones and the WEB-INF/lib jars sit in the parent loader, so the platform version already won; the five jars were dead weight that only file-level scanners could see. Assisted-By: Claude Opus 5 (1M context) --- gradle/solr/packaging.gradle | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gradle/solr/packaging.gradle b/gradle/solr/packaging.gradle index f8e5e311d83f..4ebc5569fa47 100644 --- a/gradle/solr/packaging.gradle +++ b/gradle/solr/packaging.gradle @@ -91,6 +91,10 @@ configure(allprojects return true } } + // A copied configuration inherits neither the consistent resolution nor the lock state of + // its source, so realign it: without this the module ships whichever versions it resolves + // on its own, which are not the ones that were tested (and not the ones the SBOM lists). + alignWithRuntimeClasspath(externalLibs) return externalLibs - configurations.solrPlatformLibs }, { into "lib" From c8d93636fad8432d9f5c214c5a1f44a0ef83300b Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 6 Aug 2026 07:17:00 +0200 Subject: [PATCH 07/10] SOLR-17328: Fail the build on a JAR the SBOM does not describe The gap fixed in the previous commit was invisible to the build: the distribution shipped five jars that no BOM component described, and nothing noticed. Compare with a file-level scan of the tarball and they show up immediately. Step 8 of the post-processing already indexes every file in the assembled distribution by SHA-256 in order to record evidence.occurrences, so the leftovers of that match are free. Any jar whose hash matches no component now fails the generator task, naming the paths. The check is scoped to jars. The JavaScript client bundle and the wasm UI bundle are single files assembled from many components, so their occurrences are recorded by hand in steps 6 and 7 rather than matched by hash. Verified by reverting the packaging fix and rerunning cyclonedxFull, which fails with exactly those five paths. Assisted-By: Claude Opus 5 (1M context) --- gradle/solr/sbom.gradle | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gradle/solr/sbom.gradle b/gradle/solr/sbom.gradle index 6ed3d027f3c5..6af2bea32cd9 100644 --- a/gradle/solr/sbom.gradle +++ b/gradle/solr/sbom.gradle @@ -151,6 +151,9 @@ configure(project(':solr:packaging')) { // The location of each JAR and JavaScript file within the distribution is recorded as // "evidence.occurrences": the directories assembled for the distribution are scanned and // their files are matched to the components by SHA-256 hash. + // A JAR that matches no component is a build error: the distribution then ships a version + // that the BOM configurations do not resolve, so it is described by neither the SBOM nor + // the license checks. // 9. Hashes: // Only the SHA-256 hash of each component is kept: the plugin emits eight algorithms // per artifact, which only adds bulk. @@ -370,12 +373,24 @@ configure(project(':solr:packaging')) { } } } + def describedHashes = [] as Set json.components.each { component -> def sha256 = component.hashes?.find { it.alg == 'SHA-256' }?.content if (sha256 != null && locationsByHash.containsKey(sha256)) { component.evidence = [occurrences: locationsByHash[sha256].sort().collect { [location: it] }] + describedHashes << sha256 } } + // Only JARs are checked: the JavaScript bundles are single files assembled from many + // components, so their occurrences are recorded by hand in steps 6 and 7 instead. + def undescribedJars = locationsByHash.findAll { hash, locations -> !(hash in describedHashes) } + .values().flatten().findAll { it.endsWith('.jar') }.sort() + if (undescribedJars) { + throw new GradleException("The ${edition} distribution ships JARs that no component of " + + "${bomFile.name} describes:\n " + undescribedJars.join("\n ") + "\n" + + "The packaged versions have drifted from the ones the 'bom${edition.capitalize()}' " + + "configuration resolves, which also means they are not covered by solr/licenses.") + } if (uiBundle != null) { uiBundle.evidence = [occurrences: uiLocations.sort().collect { [location: it] }] } From 0bbf2584848d88954ba8130b804fe7272114a6af Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 6 Aug 2026 09:03:56 +0200 Subject: [PATCH 08/10] SOLR-17328: Ship a single bom.json in the full distribution The full distribution builds on the slim one with "with(distributions.slim.getContents())", which brought in the slim BOM as bom.json, and the full BOM was then added under the same name with DuplicatesStrategy.INCLUDE. The archive therefore carried two bom.json entries, 588 KB and 938 KB, and which one landed on disk was up to the extraction tool: GNU tar overwrites and so happens to leave the correct full BOM, but a tool that stops at the first match would silently give the slim BOM for the full distribution. Lift the shared content into a "commonDistContents" copy spec that holds no BOM, so each distribution adds only its own. Verified to be a no-op otherwise: the full archive listing goes from 2042 to 2041 entries with the duplicate as the only difference, and the slim listing is unchanged down to the file modes and sizes. Assisted-By: Claude Opus 5 (1M context) --- solr/packaging/build.gradle | 110 +++++++++++++++++++----------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index 0cd298b46b7f..f768080ed7a9 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -76,74 +76,79 @@ dependencies { } } +// Everything both distributions contain. Each adds its own CycloneDX BOM as "bom.json" below, so +// this deliberately holds no BOM: if the full distribution inherited the slim one it would end up +// with two "bom.json" entries and it would be up to the extraction tool which one landed on disk. +def commonDistContents = copySpec { + from(rootDir, { + include "LICENSE.txt" + include "NOTICE.txt" + include "CHANGELOG.md" + }) + + from(project(":solr").projectDir, { + include "bin/**" + include "licenses/**" + exclude "licenses/README.committers.txt" + }) + + from(projectDir, { + include "README.txt" + filter(ReplaceTokens, tokens: [ + SOLR_DOC_URL: onlineDocUrl, + SOLR_CHANGES_URL: onlineChangesUrl, + ]) + }) + + from('static/lib', { + into 'lib' + }) + + from(configurations.example, { + into "example" + }) + + from(configurations.server, { + into "server" + }) + + from(configurations.docker, { + into "docker" + filesMatching([ + "scripts/**", + ]) {copy -> + copy.permissions { unix("0755") } + } + }) + + // Manually correct posix permissions (matters when packaging on Windows). + filesMatching([ + "**/*.sh", + "**/bin/solr", + "**/bin/systemd/solr.service", + ]) {copy -> + copy.permissions { unix("0755") } + } +} + distributions { slim { distributionBaseName = 'solr' distributionClassifier = "slim" contents { - - from(rootDir, { - include "LICENSE.txt" - include "NOTICE.txt" - include "CHANGELOG.md" - }) - - from(project(":solr").projectDir, { - include "bin/**" - include "licenses/**" - exclude "licenses/README.committers.txt" - }) - - from(projectDir, { - include "README.txt" - filter(ReplaceTokens, tokens: [ - SOLR_DOC_URL: onlineDocUrl, - SOLR_CHANGES_URL: onlineChangesUrl, - ]) - }) - - from('static/lib', { - into 'lib' - }) - - from(configurations.example, { - into "example" - }) - - from(configurations.server, { - into "server" - }) - - from(configurations.docker, { - into "docker" - filesMatching([ - "scripts/**", - ]) {copy -> - copy.permissions { unix("0755") } - } - }) + with commonDistContents // Include CycloneDX BOM from(cyclonedxDir) { include 'bom-slim.json' rename 'bom-slim.json', 'bom.json' } - - // Manually correct posix permissions (matters when packaging on Windows). - filesMatching([ - "**/*.sh", - "**/bin/solr", - "**/bin/systemd/solr.service", - ]) {copy -> - copy.permissions { unix("0755") } - } } } full { distributionBaseName = 'solr' contents { - // Build on-top of the slim distribution - with(distributions.slim.getContents()) + with commonDistContents from(configurations.modules, { into "modules" @@ -153,7 +158,6 @@ distributions { from(cyclonedxDir) { include 'bom-full.json' rename 'bom-full.json', 'bom.json' - duplicatesStrategy = DuplicatesStrategy.INCLUDE } from(configurations.crossDcManager, { From 89dd0204c0d27a955b62cef02e0c06de5c9bfe68 Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 6 Aug 2026 09:05:21 +0200 Subject: [PATCH 09/10] SOLR-17328: Publish the SBOMs next to the release archives Each distribution already carries its BOM as bom.json in the archive root, which keeps the SBOM with the artifact it describes but means anyone who only wants the SBOM has to download the whole distribution to reach it. Publish the same file next to the archive as well, named after it: solr-.tgz.cdx.json solr--slim.tgz.cdx.json with a checksum and, when signing is enabled, a signature, like every other release artifact. The two files are exposed as separate artifacts of a new "sbom" configuration rather than as a directory, so that computeChecksums and the signing task see each of them. No SBOM is published for the source release: assembleSourceTgz is a raw "git archive" export, so there is no resolved dependency set for a build-time SBOM to describe. smokeTestRelease.py needs a matching change. checkSigs() walked a sorted listing and treated every entry starting with "." as one of its signatures, which the new files break twice over: solr-X.tgz.cdx.json merely starts with solr-X.tgz., and its own .sha512 sorts between the archive and the archive's .sha512, so the archive's signatures are no longer adjacent to it. An entry is now recognised as a signature only when stripping .asc or .sha512 leaves the name of another entry, which drops the dependency on listing order altogether. Verified against a locally assembled release folder, signed with a throwaway key: each sibling is byte-identical to the bom.json inside the corresponding archive, every checksum and signature verifies, checkSigs() passes end to end, and both files validate against the CycloneDX 1.6 schema. Assisted-By: Claude Opus 5 (1M context) --- .../unreleased/cyclonedx-sboms-SOLR-17328.yml | 4 +- dev-docs/gradle-help/publishing.txt | 5 ++ dev-tools/scripts/smokeTestRelease.py | 49 ++++++++++--------- gradle/solr/sbom.gradle | 42 ++++++++++++++++ solr/distribution/build.gradle | 16 ++++++ 5 files changed, 90 insertions(+), 26 deletions(-) diff --git a/changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml b/changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml index ed288fec8f89..8ca333bf5a5b 100644 --- a/changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml +++ b/changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml @@ -1,8 +1,8 @@ # See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc title: > - Ship a CycloneDX SBOM (bom.json) in the root of the full and slim binary distributions, - covering the Java libraries, the Solr artifacts and the UI content of the webapp + Ship a CycloneDX SBOM for the binary distributions, as bom.json in the archive root and as + solr-.tgz.cdx.json next to the archive type: added authors: - name: Piotr P. Karwasz diff --git a/dev-docs/gradle-help/publishing.txt b/dev-docs/gradle-help/publishing.txt index 5b8bc5dd275f..f02d809a2a20 100644 --- a/dev-docs/gradle-help/publishing.txt +++ b/dev-docs/gradle-help/publishing.txt @@ -43,6 +43,11 @@ All distribution artifacts will be placed under: solr/distribution/build/release +Each binary distribution carries its CycloneDX SBOM as "bom.json" in the archive root, and the +same file is published next to the archive as solr-.tgz.cdx.json (and the -slim +equivalent) so that it can be fetched without downloading the distribution. Both get a checksum +and, when signing is enabled, a signature, like every other release artifact. + Artifact signing is optional (but required if you're really making a release). diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index 24a9e395e243..e2804a172ca9 100755 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -237,18 +237,15 @@ def checkAllJARs(topDir, gitRevision, version): def checkSigs(urlString, version, tmpDir, isSigned, keysFile): print(' test basics...') ents = getDirEntries(urlString) - artifact = None changesURL = None openApiURL = None mavenURL = None dockerURL = None - artifactURL = None expectedSigs = [] if isSigned: expectedSigs.append('asc') expectedSigs.extend(['sha512']) - sigs = [] - artifacts = [] + candidates = [] for text, subURL in ents: if text == '.gitrev': @@ -267,31 +264,35 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile): if text not in ('openApi/', 'openApi-%s/' % version): raise RuntimeError('solr: found %s vs expected openApi-%s/' % (text, version)) openApiURL = subURL - elif artifact is None: - artifact = text - artifactURL = subURL + else: + candidates.append((text, subURL)) + + # Split the remaining entries into artifacts and the signatures that belong to them. + # An entry is a signature if its file name is the name of another artifact with a signature extension. + names = {text for text, subURL in candidates} + artifacts = [] + sigs = {} + for text, subURL in candidates: + for ext in expectedSigs: + if text.endswith('.' + ext) and text[:-(len(ext) + 1)] in names: + sigs.setdefault(text[:-(len(ext) + 1)], []).append(ext) + break + else: expected = 'solr-%s' % version - if not artifact.startswith(expected): + if not text.startswith(expected): raise RuntimeError('solr: unknown artifact %s: expected prefix %s' % (text, expected)) - sigs = [] - elif text.startswith(artifact + '.'): - sigs.append(subURL.rsplit(".")[-1:][0]) - else: - if sigs != expectedSigs: - raise RuntimeError('solr: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, sigs)) - artifacts.append((artifact, artifactURL)) - artifact = text - artifactURL = subURL - sigs = [] - - if sigs != []: - artifacts.append((artifact, artifactURL)) - if sigs != expectedSigs: - raise RuntimeError('solr: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, sigs)) + artifacts.append((text, subURL)) + + for artifact, subURL in artifacts: # pylint: disable=redefined-argument-from-local + actualSigs = sorted(sigs.get(artifact, [])) + if actualSigs != expectedSigs: + raise RuntimeError('solr: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, actualSigs)) expected = ['solr-%s-src.tgz' % version, 'solr-%s.tgz' % version, - 'solr-%s-slim.tgz' % version] + 'solr-%s-slim.tgz' % version, + 'solr-%s.tgz.cdx.json' % version, + 'solr-%s-slim.tgz.cdx.json' % version] actual = [x[0] for x in artifacts] expected.sort() diff --git a/gradle/solr/sbom.gradle b/gradle/solr/sbom.gradle index 6af2bea32cd9..0e35ee823782 100644 --- a/gradle/solr/sbom.gradle +++ b/gradle/solr/sbom.gradle @@ -38,6 +38,9 @@ configure(project(':solr:packaging')) { ext { cyclonedxDir = layout.buildDirectory.dir("cyclonedx") + // The same BOMs under the name of the archive each describes, for publishing next to the + // archives in the release folder. See the "cyclonedxReleaseArtifacts" task below. + cyclonedxReleaseDir = layout.buildDirectory.dir("cyclonedx-release") } // Hand-maintained CycloneDX components for the third-party JavaScript checked into @@ -64,6 +67,11 @@ configure(project(':solr:packaging')) { canBeResolved = true canBeConsumed = false } + // The distribution SBOMs as separate release artifacts, consumed by ':solr:distribution' + sbom { + canBeResolved = false + canBeConsumed = true + } } // Request the standard JVM runtime variants, like runtimeClasspath does. @@ -512,4 +520,38 @@ configure(project(':solr:packaging')) { dependsOn 'cyclonedxFull' dependsOn 'cyclonedxSlim' } + + // Each distribution carries its BOM as "bom.json" in the archive root, so that it travels with + // the artifact it describes. The same file is also published next to the archive in the release + // folder, named after it, so that it can be fetched without downloading the whole distribution. + // The archive names are spelled out here as they are for the signatures in build.gradle. + def fullBomFileName = "solr-${version}.tgz.cdx.json".toString() + def slimBomFileName = "solr-${version}-slim.tgz.cdx.json".toString() + + tasks.register('cyclonedxReleaseArtifacts', Sync) { + group = 'Bill of Materials' + description = 'Copies the CycloneDX BOMs to the file names they are published under' + + dependsOn 'cyclonedx' + + from(cyclonedxDir) { + include 'bom-full.json' + rename 'bom-full.json', fullBomFileName + } + from(cyclonedxDir) { + include 'bom-slim.json' + rename 'bom-slim.json', slimBomFileName + } + into cyclonedxReleaseDir + } + + // Published as separate artifacts rather than as the directory, so that ':solr:distribution' + // computes a checksum and a signature for each BOM. + artifacts { + [fullBomFileName, slimBomFileName].each { fileName -> + sbom(cyclonedxReleaseDir.map {it.file(fileName)}) { + builtBy tasks.named('cyclonedxReleaseArtifacts') + } + } + } } diff --git a/solr/distribution/build.gradle b/solr/distribution/build.gradle index b3e2e6e0bee3..ea35dc55964e 100644 --- a/solr/distribution/build.gradle +++ b/solr/distribution/build.gradle @@ -53,12 +53,15 @@ configurations { changesHtml openApiSpecFile docker + sbom } dependencies { changesHtml project(path: ":solr:documentation", configuration: "changesHtml") openApiSpecFile project(path: ":solr:api", configuration: "openapiSpec") docker project(path: ':solr:docker', configuration: 'packagingOfficial') + // The CycloneDX BOMs of the binary distributions, named after the archive each describes + sbom project(path: ":solr:packaging", configuration: "sbom") } def fullDistTarTask = rootProject.getTasksByName("fullDistTar", true)[0] @@ -79,6 +82,9 @@ task computeChecksums(type: buildinfra.checksumClass()) { dependsOn configurations.openApiSpecFile files += configurations.openApiSpecFile + dependsOn configurations.sbom + files += configurations.sbom + outputDir = file("${buildDir}/checksums") } @@ -101,11 +107,20 @@ task signOpenApiSpec(type: Sign) { } } +task signSboms(type: Sign) { + dependsOn configurations.sbom + // These are not artifacts either, see signOpenApiSpec above + doFirst { + sign(*configurations.sbom.files.toArray(new File[0])) + } +} + task signReleaseArchives(type: Sync) { from tasks.signFullBinaryTgz from tasks.signSlimBinaryTgz from tasks.signSourceTgz from tasks.signOpenApiSpec + from tasks.signSboms into "${buildDir}/signatures" } @@ -149,6 +164,7 @@ task assembleRelease(type: Sync) { from tasks.assembleSourceTgz from fullDistTarTask from slimDistTarTask + from configurations.sbom from(tasks.computeChecksums, { exclude {it.file.getName().contains("openapi")} From ce18923706076faad04adf76f9aa73fc06c065fe Mon Sep 17 00:00:00 2001 From: "Piotr P. Karwasz" Date: Thu, 6 Aug 2026 09:05:44 +0200 Subject: [PATCH 10/10] smokeTestRelease.py: Fix the error raised when the OpenAPI spec is missing The closing parenthesis was misplaced, so the tuple was passed to RuntimeError as a second argument and the format string was left with one argument for two placeholders. Rather than reporting the missing file, testOpenApi raised "TypeError: not enough arguments for format string". Assisted-By: Claude Opus 5 (1M context) --- dev-tools/scripts/smokeTestRelease.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index e2804a172ca9..ce942a68fcd3 100755 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -396,7 +396,7 @@ def testOpenApi(version, openApiDirUrl): specFound = True if not specFound: - raise RuntimeError('Did not see %s in %s' % expectedSpecFileName, openApiDirUrl) + raise RuntimeError('Did not see %s in %s' % (expectedSpecFileName, openApiDirUrl)) def testChangelogMd(dir, version):