diff --git a/build.gradle b/build.gradle index 80c97fe1a0f0..fcfc382a4116 100644 --- a/build.gradle +++ b/build.gradle @@ -31,6 +31,7 @@ plugins { alias(libs.plugins.diffplug.spotless) apply false alias(libs.plugins.nodegradle.node) apply false alias(libs.plugins.openapi.generator) apply false + alias(libs.plugins.cyclonedx) apply false alias(libs.plugins.logchange) } @@ -220,6 +221,7 @@ apply from: file('gradle/hacks/turbocharge-jvm-opts.gradle') apply from: file('gradle/hacks/dummy-outputs.gradle') apply from: file('gradle/solr/packaging.gradle') +apply from: file('gradle/solr/sbom.gradle') apply from: file('gradle/solr/solr-forbidden-apis.gradle') apply from: file('gradle/node.gradle') diff --git a/changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml b/changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml new file mode 100644 index 000000000000..8ca333bf5a5b --- /dev/null +++ b/changelog/unreleased/cyclonedx-sboms-SOLR-17328.yml @@ -0,0 +1,12 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc + +title: > + Ship a CycloneDX SBOM for the binary distributions, as bom.json in the archive root and as + solr-.tgz.cdx.json next to the archive +type: added +authors: + - name: Piotr P. Karwasz + nick: ppkarwasz +links: + - name: SOLR-17328 + url: https://issues.apache.org/jira/browse/SOLR-17328 diff --git a/dev-docs/gradle-help/publishing.txt b/dev-docs/gradle-help/publishing.txt index 5b8bc5dd275f..f02d809a2a20 100644 --- a/dev-docs/gradle-help/publishing.txt +++ b/dev-docs/gradle-help/publishing.txt @@ -43,6 +43,11 @@ All distribution artifacts will be placed under: solr/distribution/build/release +Each binary distribution carries its CycloneDX SBOM as "bom.json" in the archive root, and the +same file is published next to the archive as solr-.tgz.cdx.json (and the -slim +equivalent) so that it can be fetched without downloading the distribution. Both get a checksum +and, when signing is enabled, a signature, like every other release artifact. + Artifact signing is optional (but required if you're really making a release). diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index 24a9e395e243..ce942a68fcd3 100755 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -237,18 +237,15 @@ def checkAllJARs(topDir, gitRevision, version): def checkSigs(urlString, version, tmpDir, isSigned, keysFile): print(' test basics...') ents = getDirEntries(urlString) - artifact = None changesURL = None openApiURL = None mavenURL = None dockerURL = None - artifactURL = None expectedSigs = [] if isSigned: expectedSigs.append('asc') expectedSigs.extend(['sha512']) - sigs = [] - artifacts = [] + candidates = [] for text, subURL in ents: if text == '.gitrev': @@ -267,31 +264,35 @@ def checkSigs(urlString, version, tmpDir, isSigned, keysFile): if text not in ('openApi/', 'openApi-%s/' % version): raise RuntimeError('solr: found %s vs expected openApi-%s/' % (text, version)) openApiURL = subURL - elif artifact is None: - artifact = text - artifactURL = subURL + else: + candidates.append((text, subURL)) + + # Split the remaining entries into artifacts and the signatures that belong to them. + # An entry is a signature if its file name is the name of another artifact with a signature extension. + names = {text for text, subURL in candidates} + artifacts = [] + sigs = {} + for text, subURL in candidates: + for ext in expectedSigs: + if text.endswith('.' + ext) and text[:-(len(ext) + 1)] in names: + sigs.setdefault(text[:-(len(ext) + 1)], []).append(ext) + break + else: expected = 'solr-%s' % version - if not artifact.startswith(expected): + if not text.startswith(expected): raise RuntimeError('solr: unknown artifact %s: expected prefix %s' % (text, expected)) - sigs = [] - elif text.startswith(artifact + '.'): - sigs.append(subURL.rsplit(".")[-1:][0]) - else: - if sigs != expectedSigs: - raise RuntimeError('solr: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, sigs)) - artifacts.append((artifact, artifactURL)) - artifact = text - artifactURL = subURL - sigs = [] - - if sigs != []: - artifacts.append((artifact, artifactURL)) - if sigs != expectedSigs: - raise RuntimeError('solr: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, sigs)) + artifacts.append((text, subURL)) + + for artifact, subURL in artifacts: # pylint: disable=redefined-argument-from-local + actualSigs = sorted(sigs.get(artifact, [])) + if actualSigs != expectedSigs: + raise RuntimeError('solr: artifact %s has wrong sigs: expected %s but got %s' % (artifact, expectedSigs, actualSigs)) expected = ['solr-%s-src.tgz' % version, 'solr-%s.tgz' % version, - 'solr-%s-slim.tgz' % version] + 'solr-%s-slim.tgz' % version, + 'solr-%s.tgz.cdx.json' % version, + 'solr-%s-slim.tgz.cdx.json' % version] actual = [x[0] for x in artifacts] expected.sort() @@ -395,7 +396,7 @@ def testOpenApi(version, openApiDirUrl): specFound = True if not specFound: - raise RuntimeError('Did not see %s in %s' % expectedSpecFileName, openApiDirUrl) + raise RuntimeError('Did not see %s in %s' % (expectedSpecFileName, openApiDirUrl)) def testChangelogMd(dir, version): diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e8a77111618c..e2c83b070201 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -79,6 +79,9 @@ commons-io = "2.22.0" compose = "1.11.1" cuvs-java = "26.06.0" cuvs-lucene = "25.12.0" +cyclonedx = "3.0.2" +# @keep npm tool generating the SBOM of the OpenAPI JS client, installed by :solr:webapp:js-client +cyclonedx-npm = "6.0.0" decompose = "3.5.0" diffplug-spotless = "8.7.0" # @keep Use for dockerfile JRE version @@ -203,6 +206,7 @@ xerial-snappy = "1.1.10.8" [plugins] benmanes-versions = { id = "com.github.ben-manes.versions", version.ref = "benmanes-versions" } compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } +cyclonedx = { id = "org.cyclonedx.bom", version.ref = "cyclonedx" } diffplug-spotless = { id = "com.diffplug.spotless", version.ref = "diffplug-spotless" } jetbrains-compose = { id = "org.jetbrains.compose", version.ref = "compose" } kotlin-multiplatform = { id = "org.jetbrains.kotlin.multiplatform", version.ref = "kotlin" } diff --git a/gradle/solr/packaging.gradle b/gradle/solr/packaging.gradle index f8e5e311d83f..4ebc5569fa47 100644 --- a/gradle/solr/packaging.gradle +++ b/gradle/solr/packaging.gradle @@ -91,6 +91,10 @@ configure(allprojects return true } } + // A copied configuration inherits neither the consistent resolution nor the lock state of + // its source, so realign it: without this the module ships whichever versions it resolves + // on its own, which are not the ones that were tested (and not the ones the SBOM lists). + alignWithRuntimeClasspath(externalLibs) return externalLibs - configurations.solrPlatformLibs }, { into "lib" diff --git a/gradle/solr/sbom.gradle b/gradle/solr/sbom.gradle new file mode 100644 index 000000000000..0e35ee823782 --- /dev/null +++ b/gradle/solr/sbom.gradle @@ -0,0 +1,557 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import java.security.MessageDigest + +// CycloneDX SBOM generation for the Solr binary distributions. +// +// The ":solr:packaging" project assembles the distributions; this script adds the BOM +// configurations they are computed from, the two generator tasks and the post-processing +// that turns the raw plugin output into the "bom.json" shipped inside each archive. +// +// The child SBOMs merged in below are produced by ":solr:webapp:js-client" (cyclonedx-npm) +// and ":solr:ui" (the CycloneDX Gradle plugin), each in its own build file. + +final APACHE_SNAPSHOTS_QUALIFIER = '&repository_url=https:%2F%2Frepository.apache.org%2Fcontent%2Fgroups%2Fsnapshots%2F' + +configure(project(':solr:packaging')) { + // The CycloneDX plugin is declared in the root plugins block, whose classpath a script + // plugin is not compiled against, so its types are looked up by name instead of imported. + def cyclonedxDirectTaskType = buildscript.classLoader.loadClass('org.cyclonedx.gradle.CyclonedxDirectTask') + def applicationComponentType = buildscript.classLoader.loadClass('org.cyclonedx.model.Component$Type').APPLICATION + + ext { + cyclonedxDir = layout.buildDirectory.dir("cyclonedx") + // The same BOMs under the name of the archive each describes, for publishing next to the + // archives in the release folder. See the "cyclonedxReleaseArtifacts" task below. + cyclonedxReleaseDir = layout.buildDirectory.dir("cyclonedx-release") + } + + // Hand-maintained CycloneDX components for the third-party JavaScript checked into + // solr/webapp/web/libs, merged in by step 5 of the post-processing below. + def vendoredLibsManifest = rootProject.file('solr/webapp/vendored-libs.json') + + // For the CycloneDX BOM generation + configurations { + bomSlim { + canBeResolved = true + canBeConsumed = false + } + bomFull { + canBeResolved = true + canBeConsumed = false + extendsFrom bomSlim + } + // Child SBOMs of the UI bundles, merged into the distribution SBOMs by postProcessBom + jsClientSbom { + canBeResolved = true + canBeConsumed = false + } + uiSbom { + canBeResolved = true + canBeConsumed = false + } + // The distribution SBOMs as separate release artifacts, consumed by ':solr:distribution' + sbom { + canBeResolved = false + canBeConsumed = true + } + } + + // Request the standard JVM runtime variants, like runtimeClasspath does. + // + // The CycloneDX plugin resolves configurations leniently. Without these attributes, + // variant-aware dependencies (e.g. Guava) and platform constraints (e.g. the Jersey BOM) fail to resolve + // and silently disappear. + [configurations.bomSlim, configurations.bomFull].each { conf -> + conf.attributes { + attribute(Category.CATEGORY_ATTRIBUTE, objects.named(Category, Category.LIBRARY)) + attribute(Usage.USAGE_ATTRIBUTE, objects.named(Usage, Usage.JAVA_RUNTIME)) + attribute(LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE, objects.named(LibraryElements, LibraryElements.JAR)) + attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling, Bundling.EXTERNAL)) + attribute(TargetJvmEnvironment.TARGET_JVM_ENVIRONMENT_ATTRIBUTE, objects.named(TargetJvmEnvironment, TargetJvmEnvironment.STANDARD_JVM)) + } + } + + // The distribution keeps the server versions of the libraries shared between the + // server and the modules, so align the full BOM with the versions of the slim one + // (e.g. a module may pull in a newer slf4j-api than the one in server/lib/ext). + configurations.bomFull.shouldResolveConsistentlyWith(configurations.bomSlim) + + dependencies { + rootProject.project(":solr:modules").childProjects.values().each { module -> + // No "configuration:" here on purpose. + // + // Gradle then selects the variant of the module that matches the attributes declared on the bom configurations above, + // just like it does for a runtime classpath ("variant-aware" resolution). + // Naming a configuration would bypass attribute matching, and imported Maven BOMs (e.g. Jersey's) would no + // longer provide the versions of their managed dependencies. + bomFull project(path: module.path) + } + + bomFull project(path: ':solr:cross-dc-manager') + + bomSlim project(path: ':solr:server', configuration: 'startJar') + bomSlim project(path: ':solr:server', configuration: 'serverLib') + bomSlim project(path: ':solr:server', configuration: 'libExt') + // Variant-aware dependency instead of server's by-name 'solrCore' + // configuration, see the comment on the modules above. + bomSlim project(path: ':solr:core') + bomSlim project(path: ':solr:server', configuration: 'webapp') + + // Child SBOMs of the UI bundles; empty when the projects are disabled + if (gradle.ext.withJsClient) { + jsClientSbom project(path: ':solr:webapp:js-client', configuration: 'jsClientSbom') + } + if (gradle.ext.withUiModule) { + uiSbom project(path: ':solr:ui', configuration: 'uiSbom') + } + } + + // Post-processing of the SBOMs generated by the CycloneDX plugin: + // + // 1. Metadata: + // The "build" lifecycle phase is recorded (this is a "Build SBOM" in CISA's classification) + // and this post-processing step is listed in the tools, next to the CycloneDX plugin. + // 2. Main component: + // The Solr binary archive published on downloads.apache.org is identified by a "sid" purl + // (a draft purl type for software distributed outside package registries, + // see https://github.com/package-url/purl-spec/issues/516) and by the Solr CPE. + // 3. Removed components: + // Maven BOM/platform dependencies (purl qualifier "type=pom") and the internal ":platform" + // project are stripped: they are not part of the binary distribution and the plugin has no + // option to exclude them. + // 4. Solr components: + // The plugin emits invalid purls for Solr projects, built from the Gradle project name and a + // non-standard "project_path" qualifier (e.g. pkg:maven/org.apache.solr/core@11.0.0?project_path=:solr:core). + // They are replaced with the Maven artifactId (base.archivesName) and a "type=jar" qualifier + // (e.g. pkg:maven/org.apache.solr/solr-core@11.0.0?type=jar). + // The project description and the Apache-2.0 license are also added. + // 5. Vendored JavaScript libraries: + // The AngularJS admin UI ships third-party JavaScript files checked into solr/webapp/web/libs. + // Their components are maintained by hand in solr/webapp/vendored-libs.json: each identifies + // the upstream package by purl, with any deviation of the shipped bytes under "pedigree". + // 6. JavaScript client bundle: + // The npm packages bundled by browserify into the OpenAPI JS client + // (server/solr-webapp/webapp/libs/solr/index.js) are nested as subassemblies of a first-party + // "solr-js-client" component, using the SBOM generated by cyclonedx-npm in :solr:webapp:js-client. + // 7. New UI bundle: + // The Maven dependencies compiled into the wasmJs UI (server/solr-webapp/webapp/ui) are nested + // as subassemblies of a first-party "solr-ui" component, using the SBOM generated in :solr:ui, + // together with the npm package bundled by the Kotlin toolchain (see kotlin-js-store/wasm/yarn.lock). + // 8. Archive locations: + // The location of each JAR and JavaScript file within the distribution is recorded as + // "evidence.occurrences": the directories assembled for the distribution are scanned and + // their files are matched to the components by SHA-256 hash. + // A JAR that matches no component is a build error: the distribution then ships a version + // that the BOM configurations do not resolve, so it is described by neither the SBOM nor + // the license checks. + // 9. Hashes: + // Only the SHA-256 hash of each component is kept: the plugin emits eight algorithms + // per artifact, which only adds bulk. + def postProcessBom = { File bomFile, String edition, Map scanDirs, File jsClientSbomFile, File uiSbomFile -> + def json = new JsonSlurper().parse(bomFile) + + def sha256Of = { File file -> + def digest = MessageDigest.getInstance('SHA-256') + file.eachByte(8192) { buffer, length -> digest.update(buffer, 0, length) } + digest.digest().encodeHex().toString() + } + + // Copies a child SBOM's dependency graph into this one: the child root is replaced + // by the given bundle ref, dropped refs are skipped and entries for refs that + // already exist (the same artifact in both graphs) are merged. + def mergeChildGraph = { List childDeps, String childRootRef, String bundleRef, Set droppedRefs -> + childDeps.each { dep -> + if (dep.ref in droppedRefs) { + return + } + def ref = dep.ref == childRootRef ? bundleRef : dep.ref + def dependsOn = (dep.dependsOn ?: []).findAll { !(it in droppedRefs) } + .collect { it == childRootRef ? bundleRef : it } + def existing = json.dependencies.find { it.ref == ref } + if (existing == null) { + json.dependencies << [ref: ref, dependsOn: dependsOn] + } else { + existing.dependsOn = ((existing.dependsOn ?: []) + dependsOn).unique() + } + } + } + + // 1. Metadata + json.metadata.lifecycles = [[phase: 'build']] + + // Record this post-processing step next to the CycloneDX plugin + if (json.metadata.tools == null) { + json.metadata.tools = [components: []] + } + json.metadata.tools.components << [ + type: 'application', + author: 'The Apache Software Foundation', + name: 'solr-sbom-post-processing', + version: project.version, + description: 'Post-processing of the generated SBOM by the Solr Gradle build (:solr:packaging)', + ] + + // 2. Main component + // Old bom-ref -> new purl, applied to the dependency graph below + def rewrittenRefs = [:] + + Map mainComponent = json.metadata.component + def mainPurl = "pkg:sid/apache.org/solr/solr@${mainComponent.version}?edition=${edition}".toString() + rewrittenRefs[mainComponent.'bom-ref'] = mainPurl + mainComponent.remove('group') + mainComponent.name = 'Apache Solr binary release' + mainComponent.cpe = "cpe:2.3:a:apache:solr:${mainComponent.version}:*:*:*:*:*:*:*".toString() + mainComponent.purl = mainPurl + mainComponent.'bom-ref' = mainPurl + + // Gradle project path encoded in the purls the plugin generates for Solr projects + def projectPathOf = { purl -> + def matcher = purl =~ /[?&]project_path=([^&]+)(&|$)/ + matcher ? URLDecoder.decode(matcher.group(1), 'UTF-8') : null + } + + // 3. Removed components + def removedRefs = json.components.findAll { + it.purl =~ /[?&]type=pom(&|$)/ || projectPathOf(it.purl) == ':platform' + }.collect { it.'bom-ref' } as Set + + // 4. Solr components + Map artifactIdByPath = rootProject.allprojects.collectEntries { + [(it.path): it.base.archivesName.get()] + } + json.components = json.components.collect { Map component -> + def projectPath = projectPathOf(component.purl) + // Skip external components and those about to be removed + if (projectPath == null || component.'bom-ref' in removedRefs) { + return component + } + def artifactId = artifactIdByPath[projectPath] + def repositoryUrlQualifier = component.version.endsWith("-SNAPSHOT") ? APACHE_SNAPSHOTS_QUALIFIER : '' + def purl = "pkg:maven/${component.group}/${artifactId}@${component.version}?type=jar${repositoryUrlQualifier}".toString() + rewrittenRefs[component.'bom-ref'] = purl + component.name = artifactId + component.purl = purl + component.'bom-ref' = purl + component.description = rootProject.project(projectPath).description + component.licenses = [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]] + // Restore the field order the plugin uses for external components + def ordered = [:] + ['type', 'bom-ref', 'group', 'name', 'version', 'description', 'hashes', + 'licenses', 'purl', 'modified', 'properties'].each { key -> + if (component.containsKey(key)) { + ordered[key] = component[key] + } + } + component.forEach { key, value -> + if (!ordered.containsKey(key)) { + ordered[key] = value + } + } + return ordered + } + + // Apply the removals (3) and the ref rewrites (2, 4) to the dependency graph + json.components.removeAll { it.'bom-ref' in removedRefs } + json.dependencies?.removeAll { it.ref in removedRefs } + json.dependencies?.each { dep -> + dep.ref = rewrittenRefs.getOrDefault(dep.ref, dep.ref) + dep.dependsOn?.removeAll { it in removedRefs } + if (dep.dependsOn != null) { + dep.dependsOn = dep.dependsOn.collect { rewrittenRefs.getOrDefault(it, it) } + } + } + + // The UI artifacts of steps 5 to 7 all ship inside the webapp + def webappRef = json.components.find { + it.purl?.startsWith('pkg:maven/org.apache.solr/solr-webapp@') + }?.'bom-ref' ?: mainComponent.'bom-ref' + def webappDependsOn = json.dependencies.find { it.ref == webappRef }.dependsOn + + // 5. Vendored JavaScript libraries + // Ready-made components; validateVendoredLibs guarantees the hashes match the files. + new JsonSlurper().parse(vendoredLibsManifest).each { fileName, component -> + json.components << component + json.dependencies << [ref: component.'bom-ref', dependsOn: []] + webappDependsOn << component.'bom-ref' + } + + // 6. JavaScript client bundle + if (jsClientSbomFile != null) { + def jsClientBom = new JsonSlurper().parse(jsClientSbomFile) + def bundleRef = "solr-js-client@${project.version}".toString() + json.components << [ + type: 'library', + 'bom-ref': bundleRef, + name: 'solr-js-client', + version: project.version.toString(), + description: 'JavaScript client for the Solr v2 API, generated from its OpenAPI specification', + licenses: [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]], + // The npm packages bundled into the single shipped file by browserify + components: jsClientBom.components, + evidence: [occurrences: [[location: 'server/solr-webapp/webapp/libs/solr/index.js']]], + ] + mergeChildGraph(jsClientBom.dependencies ?: [], jsClientBom.metadata.component.'bom-ref', bundleRef, [] as Set) + webappDependsOn << bundleRef + // Record cyclonedx-npm next to the other tools + jsClientBom.metadata?.tools?.components?.each { tool -> + if (!json.metadata.tools.components.any { it.name == tool.name && it.version == tool.version }) { + json.metadata.tools.components << tool + } + } + } + + // 7. New UI bundle + def uiBundle = null + if (uiSbomFile != null) { + def uiBom = new JsonSlurper().parse(uiSbomFile) + // The ":platform" project and Maven BOMs are on the UI classpath too, see step 3 + def uiDroppedRefs = uiBom.components.findAll { + it.purl =~ /[?&]type=pom(&|$)/ || projectPathOf(it.purl) == ':platform' + }.collect { it.'bom-ref' } as Set + // An artifact present in the Java graph too keeps its top-level component only + def existingRefs = json.components.collect { it.'bom-ref' } as Set + def nested = uiBom.components.findAll { + !(it.'bom-ref' in uiDroppedRefs) && !(it.'bom-ref' in existingRefs) + } + // Bundled by the Kotlin toolchain, see kotlin-js-store/wasm/yarn.lock + nested << [ + type: 'library', + 'bom-ref': 'pkg:npm/%40js-joda/core@3.2.0', + name: '@js-joda/core', + version: '3.2.0', + licenses: [[license: [id: 'BSD-3-Clause']]], + purl: 'pkg:npm/%40js-joda/core@3.2.0', + ] + def bundleRef = "solr-ui@${project.version}".toString() + uiBundle = [ + type: 'library', + 'bom-ref': bundleRef, + name: 'solr-ui', + version: project.version.toString(), + description: 'New Solr admin UI, compiled to WebAssembly', + licenses: [[license: [id: 'Apache-2.0', url: 'https://www.apache.org/licenses/LICENSE-2.0']]], + // The Maven artifacts compiled into the bundle and the npm package above + components: nested, + // Occurrences are filled by the scan of step 8, the bundle file names are content-hashed + ] + json.components << uiBundle + mergeChildGraph(uiBom.dependencies ?: [], uiBom.metadata.component.'bom-ref', bundleRef, uiDroppedRefs) + def uiEntry = json.dependencies.find { it.ref == bundleRef } + uiEntry.dependsOn << 'pkg:npm/%40js-joda/core@3.2.0' + json.dependencies << [ref: 'pkg:npm/%40js-joda/core@3.2.0', dependsOn: []] + webappDependsOn << bundleRef + } + + // 8. Archive locations + // Distribution files indexed by SHA-256 hash; UI bundle files collected on the way + def locationsByHash = [:].withDefault { [] } + def uiLocations = [] + scanDirs.each { prefix, configuration -> + configuration.files.each { root -> + fileTree(root).matching { + include '**/*.jar' + include '**/*.js' + include 'solr-webapp/webapp/ui/**' + }.visit { entry -> + if (!entry.directory) { + def location = "${prefix}/${entry.relativePath}".toString() + locationsByHash[sha256Of(entry.file)] << location + if (entry.relativePath.pathString.startsWith('solr-webapp/webapp/ui/')) { + uiLocations << location + } + } + } + } + } + def describedHashes = [] as Set + json.components.each { component -> + def sha256 = component.hashes?.find { it.alg == 'SHA-256' }?.content + if (sha256 != null && locationsByHash.containsKey(sha256)) { + component.evidence = [occurrences: locationsByHash[sha256].sort().collect { [location: it] }] + describedHashes << sha256 + } + } + // Only JARs are checked: the JavaScript bundles are single files assembled from many + // components, so their occurrences are recorded by hand in steps 6 and 7 instead. + def undescribedJars = locationsByHash.findAll { hash, locations -> !(hash in describedHashes) } + .values().flatten().findAll { it.endsWith('.jar') }.sort() + if (undescribedJars) { + throw new GradleException("The ${edition} distribution ships JARs that no component of " + + "${bomFile.name} describes:\n " + undescribedJars.join("\n ") + "\n" + + "The packaged versions have drifted from the ones the 'bom${edition.capitalize()}' " + + "configuration resolves, which also means they are not covered by solr/licenses.") + } + if (uiBundle != null) { + uiBundle.evidence = [occurrences: uiLocations.sort().collect { [location: it] }] + } + + // 9. Hashes + // The npm integrity hashes on "externalReferences" are kept, they describe + // the registry tarballs and have no SHA-256 equivalent. + def keepSha256Only + keepSha256Only = { List components -> + components.each { component -> + if (component.hashes != null) { + component.hashes = component.hashes.findAll { it.alg == 'SHA-256' } + if (component.hashes.isEmpty()) { + component.remove('hashes') + } + } + keepSha256Only(component.components ?: []) + } + } + keepSha256Only(json.components) + + bomFile.text = JsonOutput.prettyPrint(JsonOutput.toJson(json)) + } + + tasks.register('cyclonedxFull', cyclonedxDirectTaskType) { + group = 'Bill of Materials' + description = 'Generates CycloneDX BOM for the full Solr distribution' + + includeConfigs = ['bomFull'] + projectType = applicationComponentType + + // The plugin resolves the configuration leniently and without depending on it, + // so the jars of Solr projects must be built first or their hashes are missing. + inputs.files(configurations.bomFull) + .withPropertyName('bomFullArtifacts') + .withNormalizer(ClasspathNormalizer) + + // Distribution directories scanned for the archive location of each artifact + inputs.files(configurations.server, configurations.modules, configurations.crossDcManager) + .withPropertyName('distributionDirs') + .withPathSensitivity(PathSensitivity.RELATIVE) + + // Child SBOMs of the UI bundles; empty when the projects are disabled + inputs.files(configurations.jsClientSbom, configurations.uiSbom) + .withPropertyName('childSboms') + .withPathSensitivity(PathSensitivity.NONE) + + // The vendored JavaScript components and the files they describe + dependsOn ':solr:webapp:validateVendoredLibs' + inputs.file(vendoredLibsManifest) + .withPropertyName('vendoredLibsManifest') + .withPathSensitivity(PathSensitivity.NONE) + inputs.dir(rootProject.file('solr/webapp/web/libs')) + .withPropertyName('vendoredJsLibs') + .withPathSensitivity(PathSensitivity.RELATIVE) + inputs.file(rootProject.file('kotlin-js-store/wasm/yarn.lock')) + .withPropertyName('uiYarnLock') + .withPathSensitivity(PathSensitivity.NONE) + + jsonOutput = cyclonedxDir.get().file("bom-full.json").asFile + + doLast { + postProcessBom(jsonOutput.get().asFile, 'full', [ + 'server': configurations.server, + 'modules': configurations.modules, + 'cross-dc-manager': configurations.crossDcManager, + ], configurations.jsClientSbom.files.find(), configurations.uiSbom.files.find()) + } + } + + tasks.register('cyclonedxSlim', cyclonedxDirectTaskType) { + group = 'Bill of Materials' + description = 'Generates CycloneDX BOM for the slim Solr distribution' + + includeConfigs = ['bomSlim'] + projectType = applicationComponentType + + // The plugin resolves the configuration leniently and without depending on it, + // so the jars of Solr projects must be built first or their hashes are missing. + inputs.files(configurations.bomSlim) + .withPropertyName('bomSlimArtifacts') + .withNormalizer(ClasspathNormalizer) + + // Distribution directories scanned for the archive location of each artifact + inputs.files(configurations.server) + .withPropertyName('distributionDirs') + .withPathSensitivity(PathSensitivity.RELATIVE) + + // Child SBOMs of the UI bundles; empty when the projects are disabled + inputs.files(configurations.jsClientSbom, configurations.uiSbom) + .withPropertyName('childSboms') + .withPathSensitivity(PathSensitivity.NONE) + + // The vendored JavaScript components and the files they describe + dependsOn ':solr:webapp:validateVendoredLibs' + inputs.file(vendoredLibsManifest) + .withPropertyName('vendoredLibsManifest') + .withPathSensitivity(PathSensitivity.NONE) + inputs.dir(rootProject.file('solr/webapp/web/libs')) + .withPropertyName('vendoredJsLibs') + .withPathSensitivity(PathSensitivity.RELATIVE) + inputs.file(rootProject.file('kotlin-js-store/wasm/yarn.lock')) + .withPropertyName('uiYarnLock') + .withPathSensitivity(PathSensitivity.NONE) + + jsonOutput = cyclonedxDir.get().file("bom-slim.json").asFile + + doLast { + postProcessBom(jsonOutput.get().asFile, 'slim', [ + 'server': configurations.server, + ], configurations.jsClientSbom.files.find(), configurations.uiSbom.files.find()) + } + } + + tasks.register('cyclonedx') { + group = 'Bill of Materials' + description = 'Generates CycloneDX BOMs for Solr distributions' + + dependsOn 'cyclonedxFull' + dependsOn 'cyclonedxSlim' + } + + // Each distribution carries its BOM as "bom.json" in the archive root, so that it travels with + // the artifact it describes. The same file is also published next to the archive in the release + // folder, named after it, so that it can be fetched without downloading the whole distribution. + // The archive names are spelled out here as they are for the signatures in build.gradle. + def fullBomFileName = "solr-${version}.tgz.cdx.json".toString() + def slimBomFileName = "solr-${version}-slim.tgz.cdx.json".toString() + + tasks.register('cyclonedxReleaseArtifacts', Sync) { + group = 'Bill of Materials' + description = 'Copies the CycloneDX BOMs to the file names they are published under' + + dependsOn 'cyclonedx' + + from(cyclonedxDir) { + include 'bom-full.json' + rename 'bom-full.json', fullBomFileName + } + from(cyclonedxDir) { + include 'bom-slim.json' + rename 'bom-slim.json', slimBomFileName + } + into cyclonedxReleaseDir + } + + // Published as separate artifacts rather than as the directory, so that ':solr:distribution' + // computes a checksum and a signature for each BOM. + artifacts { + [fullBomFileName, slimBomFileName].each { fileName -> + sbom(cyclonedxReleaseDir.map {it.file(fileName)}) { + builtBy tasks.named('cyclonedxReleaseArtifacts') + } + } + } +} diff --git a/gradle/validation/precommit.gradle b/gradle/validation/precommit.gradle index 02d03010016f..0ce9460fa1d3 100644 --- a/gradle/validation/precommit.gradle +++ b/gradle/validation/precommit.gradle @@ -40,6 +40,7 @@ configure(rootProject) { "ecjLint", "validateLogCalls", "validateSourcePatterns", + "validateVendoredLibs", "spotlessCheck" ] } diff --git a/solr/distribution/build.gradle b/solr/distribution/build.gradle index b3e2e6e0bee3..ea35dc55964e 100644 --- a/solr/distribution/build.gradle +++ b/solr/distribution/build.gradle @@ -53,12 +53,15 @@ configurations { changesHtml openApiSpecFile docker + sbom } dependencies { changesHtml project(path: ":solr:documentation", configuration: "changesHtml") openApiSpecFile project(path: ":solr:api", configuration: "openapiSpec") docker project(path: ':solr:docker', configuration: 'packagingOfficial') + // The CycloneDX BOMs of the binary distributions, named after the archive each describes + sbom project(path: ":solr:packaging", configuration: "sbom") } def fullDistTarTask = rootProject.getTasksByName("fullDistTar", true)[0] @@ -79,6 +82,9 @@ task computeChecksums(type: buildinfra.checksumClass()) { dependsOn configurations.openApiSpecFile files += configurations.openApiSpecFile + dependsOn configurations.sbom + files += configurations.sbom + outputDir = file("${buildDir}/checksums") } @@ -101,11 +107,20 @@ task signOpenApiSpec(type: Sign) { } } +task signSboms(type: Sign) { + dependsOn configurations.sbom + // These are not artifacts either, see signOpenApiSpec above + doFirst { + sign(*configurations.sbom.files.toArray(new File[0])) + } +} + task signReleaseArchives(type: Sync) { from tasks.signFullBinaryTgz from tasks.signSlimBinaryTgz from tasks.signSourceTgz from tasks.signOpenApiSpec + from tasks.signSboms into "${buildDir}/signatures" } @@ -149,6 +164,7 @@ task assembleRelease(type: Sync) { from tasks.assembleSourceTgz from fullDistTarTask from slimDistTarTask + from configurations.sbom from(tasks.computeChecksums, { exclude {it.file.getName().contains("openapi")} diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index 63c467388ef1..f768080ed7a9 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -24,6 +24,10 @@ import org.apache.tools.ant.util.TeeOutputStream plugins { id 'base' id 'distribution' + // Registers the JVM attribute schema, so that the variant-aware resolution of the + // CycloneDX BOM configurations added by gradle/solr/sbom.gradle works like a Java + // runtime classpath. + id 'jvm-ecosystem' } description = 'Solr distribution packaging' @@ -52,8 +56,8 @@ configurations { } dependencies { - rootProject.project(":solr:modules").childProjects.values().stream().map {project -> project.path}.each { - module -> modules project(path: module, configuration: "packaging") + rootProject.project(":solr:modules").childProjects.values().stream().map {project -> project.path}.each { module -> + modules project(path: module, configuration: "packaging") } crossDcManager project(path: ":solr:cross-dc-manager", configuration: "packaging") @@ -72,74 +76,90 @@ dependencies { } } +// Everything both distributions contain. Each adds its own CycloneDX BOM as "bom.json" below, so +// this deliberately holds no BOM: if the full distribution inherited the slim one it would end up +// with two "bom.json" entries and it would be up to the extraction tool which one landed on disk. +def commonDistContents = copySpec { + from(rootDir, { + include "LICENSE.txt" + include "NOTICE.txt" + include "CHANGELOG.md" + }) + + from(project(":solr").projectDir, { + include "bin/**" + include "licenses/**" + exclude "licenses/README.committers.txt" + }) + + from(projectDir, { + include "README.txt" + filter(ReplaceTokens, tokens: [ + SOLR_DOC_URL: onlineDocUrl, + SOLR_CHANGES_URL: onlineChangesUrl, + ]) + }) + + from('static/lib', { + into 'lib' + }) + + from(configurations.example, { + into "example" + }) + + from(configurations.server, { + into "server" + }) + + from(configurations.docker, { + into "docker" + filesMatching([ + "scripts/**", + ]) {copy -> + copy.permissions { unix("0755") } + } + }) + + // Manually correct posix permissions (matters when packaging on Windows). + filesMatching([ + "**/*.sh", + "**/bin/solr", + "**/bin/systemd/solr.service", + ]) {copy -> + copy.permissions { unix("0755") } + } +} + distributions { slim { distributionBaseName = 'solr' distributionClassifier = "slim" contents { + with commonDistContents - from(rootDir, { - include "LICENSE.txt" - include "NOTICE.txt" - include "CHANGELOG.md" - }) - - from(project(":solr").projectDir, { - include "bin/**" - include "licenses/**" - exclude "licenses/README.committers.txt" - }) - - from(projectDir, { - include "README.txt" - filter(ReplaceTokens, tokens: [ - SOLR_DOC_URL: onlineDocUrl, - SOLR_CHANGES_URL: onlineChangesUrl, - ]) - }) - - from('static/lib', { - into 'lib' - }) - - from(configurations.example, { - into "example" - }) - - from(configurations.server, { - into "server" - }) - - from(configurations.docker, { - into "docker" - filesMatching([ - "scripts/**", - ]) {copy -> - copy.permissions { unix("0755") } - } - }) - - // Manually correct posix permissions (matters when packaging on Windows). - filesMatching([ - "**/*.sh", - "**/bin/solr", - "**/bin/systemd/solr.service", - ]) {copy -> - copy.permissions { unix("0755") } + // Include CycloneDX BOM + from(cyclonedxDir) { + include 'bom-slim.json' + rename 'bom-slim.json', 'bom.json' } - } } full { distributionBaseName = 'solr' contents { - // Build on-top of the slim distribution - with(distributions.slim.getContents()) + with commonDistContents from(configurations.modules, { into "modules" }) + // Include CycloneDX BOM + from(cyclonedxDir) { + include 'bom-full.json' + rename 'bom-full.json', 'bom.json' + } + from(configurations.crossDcManager, { into "cross-dc-manager" filesMatching([ @@ -153,10 +173,12 @@ distributions { } installFullDist { + dependsOn 'cyclonedx' into distDir } installSlimDist { + dependsOn 'cyclonedx' into slimDistDir } @@ -195,10 +217,12 @@ task dev { } fullDistTar { + dependsOn 'cyclonedx' compression = Compression.GZIP } slimDistTar { + dependsOn 'cyclonedx' compression = Compression.GZIP } diff --git a/solr/packaging/gradle.lockfile b/solr/packaging/gradle.lockfile index 67666c69405d..72dd1ba6b96f 100644 --- a/solr/packaging/gradle.lockfile +++ b/solr/packaging/gradle.lockfile @@ -2,4 +2,338 @@ # Manual edits can break the build and are not advised. # This file is expected to be part of source control. # To regenerate this file, run: ./gradlew :solr:packaging:dependencies --write-locks -empty=crossDcManager,docker,docs,example,jarValidation,modules,server,solrFullTgz,solrFullTgzSignature,solrSlimTgz,solrSlimTgzSignature +at.yawk.lz4:lz4-java:1.10.1=bomFull +com.carrotsearch:hppc:0.10.0=bomFull,bomSlim +com.fasterxml.jackson.core:jackson-annotations:2.22=bomFull,bomSlim +com.fasterxml.jackson.core:jackson-core:2.22.0=bomFull,bomSlim +com.fasterxml.jackson.core:jackson-databind:2.22.0=bomFull,bomSlim +com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.22.0=bomFull,bomSlim +com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.22.0=bomFull +com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.22.0=bomFull,bomSlim +com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.22.0=bomFull +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.22.0=bomFull +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.22.0=bomFull +com.fasterxml.jackson.module:jackson-module-jakarta-xmlbind-annotations:2.22.0=bomFull,bomSlim +com.fasterxml.jackson.module:jackson-module-scala_2.13:2.22.0=bomFull +com.fasterxml.jackson:jackson-bom:2.22.0=bomFull,bomSlim +com.fasterxml.woodstox:woodstox-core:7.2.1=bomFull,bomSlim +com.github.ben-manes.caffeine:caffeine:3.2.4=bomFull,bomSlim +com.github.luben:zstd-jni:1.5.6-4=bomFull +com.google.android:annotations:4.1.1.4=bomFull +com.google.api-client:google-api-client:2.7.2=bomFull +com.google.api.grpc:gapic-google-cloud-storage-v2:2.69.0=bomFull +com.google.api.grpc:grpc-google-cloud-storage-v2:2.69.0=bomFull +com.google.api.grpc:proto-google-cloud-storage-v2:2.69.0=bomFull +com.google.api.grpc:proto-google-common-protos:2.72.0=bomFull +com.google.api.grpc:proto-google-iam-v1:1.67.0=bomFull +com.google.api:api-common:2.64.0=bomFull +com.google.api:gax-grpc:2.81.0=bomFull +com.google.api:gax-httpjson:2.81.0=bomFull +com.google.api:gax:2.81.0=bomFull +com.google.apis:google-api-services-storage:v1-rev20260204-2.0.0=bomFull +com.google.auth:google-auth-library-credentials:1.48.0=bomFull +com.google.auth:google-auth-library-oauth2-http:1.48.0=bomFull +com.google.auto.value:auto-value-annotations:1.11.1=bomFull +com.google.cloud:google-cloud-bom:0.265.0=bomFull +com.google.cloud:google-cloud-core-grpc:2.71.0=bomFull +com.google.cloud:google-cloud-core-http:2.71.0=bomFull +com.google.cloud:google-cloud-core:2.71.0=bomFull +com.google.cloud:google-cloud-storage:2.69.0=bomFull +com.google.code.gson:gson:2.14.0=bomFull +com.google.errorprone:error_prone_annotations:2.47.0=bomFull,bomSlim +com.google.guava:failureaccess:1.0.3=bomFull,bomSlim +com.google.guava:guava:33.6.0-jre=bomFull,bomSlim +com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava=bomFull,bomSlim +com.google.http-client:google-http-client-apache-v2:2.1.0=bomFull +com.google.http-client:google-http-client-appengine:2.1.0=bomFull +com.google.http-client:google-http-client-gson:2.1.0=bomFull +com.google.http-client:google-http-client-jackson2:2.1.0=bomFull +com.google.http-client:google-http-client:2.1.0=bomFull +com.google.j2objc:j2objc-annotations:3.1=bomFull,bomSlim +com.google.oauth-client:google-oauth-client:1.39.0=bomFull +com.google.protobuf:protobuf-java-util:4.35.1=bomFull +com.google.protobuf:protobuf-java:4.35.1=bomFull +com.google.re2j:re2j:1.8=bomFull +com.googlecode.json-simple:json-simple:1.1.1=bomFull +com.ibm.icu:icu4j:78.3=bomFull +com.j256.simplemagic:simplemagic:1.17=bomFull,bomSlim +com.jayway.jsonpath:json-path:3.0.0=bomFull,bomSlim +com.knuddels:jtokkit:1.1.0=bomFull +com.lmax:disruptor:4.0.0=bomFull,bomSlim +com.microsoft.onnxruntime:onnxruntime:1.26.0=bomFull +com.nvidia.cuvs.lucene:cuvs-lucene:25.12.0=bomFull +com.nvidia.cuvs:cuvs-java:26.06.0=bomFull +com.squareup.okhttp3:okhttp-jvm:5.4.0=bomFull +com.squareup.okhttp3:okhttp:5.4.0=bomFull +com.squareup.okio:okio-jvm:3.17.0=bomFull +com.squareup.okio:okio:3.17.0=bomFull +com.squareup.retrofit2:converter-jackson:2.9.0=bomFull +com.squareup.retrofit2:retrofit:2.9.0=bomFull +com.tdunning:t-digest:3.3=bomFull,bomSlim +com.thoughtworks.paranamer:paranamer:2.8.3=bomFull +com.typesafe.scala-logging:scala-logging_2.13:3.9.5=bomFull +com.yammer.metrics:metrics-core:2.2.0=bomFull +commons-beanutils:commons-beanutils:1.11.0=bomFull +commons-cli:commons-cli:1.11.0=bomFull,bomSlim +commons-codec:commons-codec:1.22.0=bomFull,bomSlim +commons-collections:commons-collections:3.2.2=bomFull +commons-digester:commons-digester:2.1=bomFull +commons-io:commons-io:2.22.0=bomFull,bomSlim +commons-validator:commons-validator:1.10.1=bomFull +dev.langchain4j:langchain4j-anthropic:1.17.0=bomFull +dev.langchain4j:langchain4j-bom:1.17.0=bomFull +dev.langchain4j:langchain4j-cohere:1.17.0-beta27=bomFull +dev.langchain4j:langchain4j-core:1.17.0=bomFull +dev.langchain4j:langchain4j-google-ai-gemini:1.17.0=bomFull +dev.langchain4j:langchain4j-http-client-jdk:1.17.0=bomFull +dev.langchain4j:langchain4j-http-client:1.17.0=bomFull +dev.langchain4j:langchain4j-hugging-face:1.17.0-beta27=bomFull +dev.langchain4j:langchain4j-mistral-ai:1.17.0=bomFull +dev.langchain4j:langchain4j-ollama:1.17.0=bomFull +dev.langchain4j:langchain4j-open-ai:1.17.0=bomFull +io.dropwizard.metrics:metrics-core:4.2.39=bomFull,bomSlim +io.github.azagniotov:language-detection:12.5.2=bomFull +io.grpc:grpc-alts:1.82.0=bomFull +io.grpc:grpc-api:1.82.0=bomFull +io.grpc:grpc-auth:1.82.0=bomFull +io.grpc:grpc-bom:1.82.0=bomFull +io.grpc:grpc-context:1.82.0=bomFull +io.grpc:grpc-core:1.82.0=bomFull +io.grpc:grpc-googleapis:1.82.0=bomFull +io.grpc:grpc-grpclb:1.82.0=bomFull +io.grpc:grpc-inprocess:1.82.0=bomFull +io.grpc:grpc-netty-shaded:1.82.0=bomFull +io.grpc:grpc-protobuf-lite:1.82.0=bomFull +io.grpc:grpc-protobuf:1.82.0=bomFull +io.grpc:grpc-rls:1.82.0=bomFull +io.grpc:grpc-services:1.82.0=bomFull +io.grpc:grpc-stub:1.82.0=bomFull +io.grpc:grpc-util:1.82.0=bomFull +io.grpc:grpc-xds:1.82.0=bomFull +io.netty:netty-buffer:4.2.15.Final=bomFull,bomSlim +io.netty:netty-codec-base:4.2.15.Final=bomFull,bomSlim +io.netty:netty-common:4.2.15.Final=bomFull,bomSlim +io.netty:netty-handler:4.2.15.Final=bomFull,bomSlim +io.netty:netty-resolver:4.2.15.Final=bomFull,bomSlim +io.netty:netty-tcnative-boringssl-static:2.0.79.Final=bomFull,bomSlim +io.netty:netty-tcnative-classes:2.0.79.Final=bomFull,bomSlim +io.netty:netty-transport-classes-epoll:4.2.15.Final=bomFull,bomSlim +io.netty:netty-transport-native-epoll:4.2.15.Final=bomFull,bomSlim +io.netty:netty-transport-native-unix-common:4.2.15.Final=bomFull,bomSlim +io.netty:netty-transport:4.2.15.Final=bomFull,bomSlim +io.opencensus:opencensus-api:0.31.1=bomFull +io.opencensus:opencensus-contrib-http-util:0.31.1=bomFull +io.opentelemetry.contrib:opentelemetry-gcp-resources:1.37.0-alpha=bomFull +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api-incubator:2.27.0-alpha=bomFull,bomSlim +io.opentelemetry.instrumentation:opentelemetry-instrumentation-api:2.27.0=bomFull,bomSlim +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry-java17:2.27.0-alpha=bomFull,bomSlim +io.opentelemetry.instrumentation:opentelemetry-runtime-telemetry:2.27.0-alpha=bomFull,bomSlim +io.opentelemetry.semconv:opentelemetry-semconv:1.40.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-api-incubator:1.61.0-alpha=bomFull,bomSlim +io.opentelemetry:opentelemetry-api:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-bom:1.63.0=bomFull +io.opentelemetry:opentelemetry-common:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-context:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-exporter-common:1.63.0=bomFull +io.opentelemetry:opentelemetry-exporter-otlp-common:1.63.0=bomFull +io.opentelemetry:opentelemetry-exporter-otlp:1.63.0=bomFull +io.opentelemetry:opentelemetry-exporter-prometheus:1.63.0-alpha=bomFull,bomSlim +io.opentelemetry:opentelemetry-exporter-sender-jdk:1.63.0=bomFull +io.opentelemetry:opentelemetry-sdk-common:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure-spi:1.63.0=bomFull +io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.63.0=bomFull +io.opentelemetry:opentelemetry-sdk-logs:1.63.0=bomFull +io.opentelemetry:opentelemetry-sdk-metrics:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-sdk-trace:1.63.0=bomFull,bomSlim +io.opentelemetry:opentelemetry-sdk:1.63.0=bomFull,bomSlim +io.perfmark:perfmark-api:0.27.0=bomFull +io.prometheus:prometheus-metrics-config:1.8.0=bomFull,bomSlim +io.prometheus:prometheus-metrics-exposition-formats:1.8.0=bomFull,bomSlim +io.prometheus:prometheus-metrics-exposition-textformats:1.8.0=bomFull,bomSlim +io.prometheus:prometheus-metrics-model:1.8.0=bomFull,bomSlim +io.sgr:s2-geometry-library-java:1.0.0=bomFull,bomSlim +io.swagger.core.v3:swagger-annotations-jakarta:2.2.52=bomFull,bomSlim +jakarta.activation:jakarta.activation-api:2.1.3=bomFull,bomSlim +jakarta.annotation:jakarta.annotation-api:3.0.0=bomFull,bomSlim +jakarta.inject:jakarta.inject-api:2.0.1=bomFull,bomSlim +jakarta.servlet:jakarta.servlet-api:6.1.0=bomFull,bomSlim +jakarta.validation:jakarta.validation-api:3.1.0=bomFull,bomSlim +jakarta.ws.rs:jakarta.ws.rs-api:4.0.0=bomFull,bomSlim +jakarta.xml.bind:jakarta.xml.bind-api:4.0.2=bomFull,bomSlim +net.sf.jopt-simple:jopt-simple:5.0.4=bomFull +net.sourceforge.argparse4j:argparse4j:0.7.0=bomFull +org.antlr:antlr4-runtime:4.13.2=bomFull,bomSlim +org.apache.calcite.avatica:avatica-core:1.25.0=bomFull +org.apache.calcite.avatica:avatica-metrics:1.25.0=bomFull +org.apache.calcite:calcite-core:1.37.0=bomFull +org.apache.calcite:calcite-linq4j:1.37.0=bomFull +org.apache.commons:commons-exec:1.6.0=bomFull,bomSlim +org.apache.commons:commons-lang3:3.20.0=bomFull,bomSlim +org.apache.commons:commons-math3:3.6.1=bomFull,bomSlim +org.apache.commons:commons-text:1.15.0=bomFull +org.apache.curator:curator-client:5.9.0=bomFull,bomSlim +org.apache.curator:curator-framework:5.9.0=bomFull,bomSlim +org.apache.httpcomponents.client5:httpclient5:5.2.1=bomFull +org.apache.httpcomponents.core5:httpcore5-h2:5.2=bomFull +org.apache.httpcomponents.core5:httpcore5:5.2.3=bomFull +org.apache.httpcomponents:httpclient:4.5.14=bomFull +org.apache.httpcomponents:httpcore:4.4.16=bomFull +org.apache.kafka:kafka-clients:3.9.2=bomFull +org.apache.kafka:kafka-group-coordinator-api:3.9.2=bomFull +org.apache.kafka:kafka-group-coordinator:3.9.2=bomFull +org.apache.kafka:kafka-metadata:3.9.2=bomFull +org.apache.kafka:kafka-raft:3.9.2=bomFull +org.apache.kafka:kafka-server-common:3.9.2=bomFull +org.apache.kafka:kafka-server:3.9.2=bomFull +org.apache.kafka:kafka-storage-api:3.9.2=bomFull +org.apache.kafka:kafka-storage:3.9.2=bomFull +org.apache.kafka:kafka-streams:3.9.2=bomFull +org.apache.kafka:kafka-tools-api:3.9.2=bomFull +org.apache.kafka:kafka-transaction-coordinator:3.9.2=bomFull +org.apache.kafka:kafka_2.13:3.9.2=bomFull +org.apache.logging.log4j:log4j-1.2-api:2.26.0=bomFull,bomSlim +org.apache.logging.log4j:log4j-api:2.26.0=bomFull,bomSlim +org.apache.logging.log4j:log4j-core:2.26.0=bomFull,bomSlim +org.apache.logging.log4j:log4j-layout-template-json:2.26.0=bomFull,bomSlim +org.apache.logging.log4j:log4j-slf4j2-impl:2.26.0=bomFull,bomSlim +org.apache.logging.log4j:log4j-web:2.26.0=bomFull,bomSlim +org.apache.lucene:lucene-analysis-common:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-analysis-icu:10.4.0=bomFull +org.apache.lucene:lucene-analysis-kuromoji:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-analysis-morfologik:10.4.0=bomFull +org.apache.lucene:lucene-analysis-nori:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-analysis-opennlp:10.4.0=bomFull +org.apache.lucene:lucene-analysis-phonetic:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-analysis-smartcn:10.4.0=bomFull +org.apache.lucene:lucene-analysis-stempel:10.4.0=bomFull +org.apache.lucene:lucene-backward-codecs:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-classification:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-codecs:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-core:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-expressions:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-facet:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-grouping:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-highlighter:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-join:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-memory:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-misc:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-queries:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-queryparser:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-sandbox:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-spatial-extras:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-spatial3d:10.4.0=bomFull,bomSlim +org.apache.lucene:lucene-suggest:10.4.0=bomFull,bomSlim +org.apache.opennlp:opennlp-dl:2.5.10=bomFull +org.apache.opennlp:opennlp-tools:2.5.10=bomFull +org.apache.tika:tika-core:3.3.1=bomFull +org.apache.zookeeper:zookeeper-jute:3.9.5=bomFull,bomSlim +org.apache.zookeeper:zookeeper:3.9.5=bomFull,bomSlim +org.apiguardian:apiguardian-api:1.1.2=bomFull +org.bitbucket.b_c:jose4j:0.9.6=bomFull +org.carrot2:carrot2-core:4.8.6=bomFull +org.carrot2:morfologik-fsa:2.1.9=bomFull +org.carrot2:morfologik-polish:2.1.9=bomFull +org.carrot2:morfologik-stemming:2.1.9=bomFull +org.checkerframework:checker-qual:4.2.0=bomFull +org.codehaus.janino:commons-compiler:3.1.11=bomFull +org.codehaus.janino:janino:3.1.11=bomFull +org.codehaus.woodstox:stax2-api:4.3.0=bomFull,bomSlim +org.conscrypt:conscrypt-openjdk-uber:2.5.2=bomFull +org.eclipse.jetty.compression:jetty-compression-common:12.1.10=bomFull,bomSlim +org.eclipse.jetty.compression:jetty-compression-gzip:12.1.10=bomFull,bomSlim +org.eclipse.jetty.ee10:jetty-ee10-servlet:12.1.10=bomFull,bomSlim +org.eclipse.jetty.ee10:jetty-ee10-servlets:12.1.10=bomFull,bomSlim +org.eclipse.jetty.ee10:jetty-ee10-webapp:12.1.10=bomFull,bomSlim +org.eclipse.jetty.ee:jetty-ee-webapp:12.1.10=bomFull,bomSlim +org.eclipse.jetty.http2:jetty-http2-client-transport:12.1.10=bomFull,bomSlim +org.eclipse.jetty.http2:jetty-http2-client:12.1.10=bomFull,bomSlim +org.eclipse.jetty.http2:jetty-http2-common:12.1.10=bomFull,bomSlim +org.eclipse.jetty.http2:jetty-http2-hpack:12.1.10=bomFull,bomSlim +org.eclipse.jetty.http2:jetty-http2-server:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-alpn-client:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-alpn-java-client:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-alpn-java-server:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-alpn-server:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-client:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-deploy:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-http:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-io:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-jmx:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-rewrite:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-security:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-server:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-session:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-start:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-util:12.1.10=bomFull,bomSlim +org.eclipse.jetty:jetty-xml:12.1.10=bomFull,bomSlim +org.glassfish.hk2.external:aopalliance-repackaged:4.0.1=bomFull,bomSlim +org.glassfish.hk2:hk2-api:4.0.1=bomFull,bomSlim +org.glassfish.hk2:hk2-locator:4.0.1=bomFull,bomSlim +org.glassfish.hk2:hk2-utils:4.0.1=bomFull,bomSlim +org.glassfish.hk2:osgi-resource-locator:3.0.0=bomFull,bomSlim +org.glassfish.jersey.containers:jersey-container-jetty-http:4.0.2=bomFull,bomSlim +org.glassfish.jersey.core:jersey-client:4.0.2=bomFull,bomSlim +org.glassfish.jersey.core:jersey-common:4.0.2=bomFull,bomSlim +org.glassfish.jersey.core:jersey-server:4.0.2=bomFull,bomSlim +org.glassfish.jersey.ext:jersey-entity-filtering:4.0.2=bomFull,bomSlim +org.glassfish.jersey.inject:jersey-hk2:4.0.2=bomFull,bomSlim +org.glassfish.jersey.media:jersey-media-json-jackson:4.0.2=bomFull,bomSlim +org.glassfish.jersey:jersey-bom:4.0.2=bomFull,bomSlim +org.javassist:javassist:3.30.2-GA=bomFull,bomSlim +org.jetbrains.kotlin:kotlin-stdlib:2.2.21=bomFull +org.jetbrains:annotations:26.1.0=bomFull +org.jspecify:jspecify:1.0.0=bomFull,bomSlim +org.locationtech.jts.io:jts-io-common:1.19.0=bomFull +org.locationtech.jts:jts-core:1.19.0=bomFull +org.locationtech.proj4j:proj4j:1.2.2=bomFull +org.locationtech.spatial4j:spatial4j:0.8=bomFull,bomSlim +org.ow2.asm:asm-commons:9.10.1=bomFull,bomSlim +org.ow2.asm:asm-tree:9.10.1=bomFull,bomSlim +org.ow2.asm:asm:9.10.1=bomFull,bomSlim +org.pcollections:pcollections:4.0.1=bomFull +org.reactivestreams:reactive-streams:1.0.4=bomFull +org.rocksdb:rocksdbjni:7.9.2=bomFull +org.scala-lang.modules:scala-collection-compat_2.13:2.10.0=bomFull +org.scala-lang.modules:scala-java8-compat_2.13:1.0.2=bomFull +org.scala-lang:scala-library:2.13.18=bomFull +org.scala-lang:scala-reflect:2.13.15=bomFull +org.semver4j:semver4j:6.0.0=bomFull,bomSlim +org.slf4j:jcl-over-slf4j:2.0.17=bomFull,bomSlim +org.slf4j:jul-to-slf4j:2.0.17=bomFull,bomSlim +org.slf4j:slf4j-api:2.0.17=bomFull,bomSlim +org.threeten:threetenbp:1.7.3=bomFull +org.xerial.snappy:snappy-java:1.1.10.8=bomFull,bomSlim +software.amazon.awssdk:annotations:2.42.37=bomFull +software.amazon.awssdk:apache-client:2.42.37=bomFull +software.amazon.awssdk:arns:2.42.37=bomFull +software.amazon.awssdk:auth:2.42.37=bomFull +software.amazon.awssdk:aws-core:2.42.37=bomFull +software.amazon.awssdk:aws-query-protocol:2.42.37=bomFull +software.amazon.awssdk:aws-xml-protocol:2.42.37=bomFull +software.amazon.awssdk:bom:2.42.37=bomFull +software.amazon.awssdk:checksums-spi:2.42.37=bomFull +software.amazon.awssdk:checksums:2.42.37=bomFull +software.amazon.awssdk:crt-core:2.42.37=bomFull +software.amazon.awssdk:endpoints-spi:2.42.37=bomFull +software.amazon.awssdk:http-auth-aws-eventstream:2.42.37=bomFull +software.amazon.awssdk:http-auth-aws:2.42.37=bomFull +software.amazon.awssdk:http-auth-spi:2.42.37=bomFull +software.amazon.awssdk:http-auth:2.42.37=bomFull +software.amazon.awssdk:http-client-spi:2.42.37=bomFull +software.amazon.awssdk:identity-spi:2.42.37=bomFull +software.amazon.awssdk:json-utils:2.42.37=bomFull +software.amazon.awssdk:metrics-spi:2.42.37=bomFull +software.amazon.awssdk:profiles:2.42.37=bomFull +software.amazon.awssdk:protocol-core:2.42.37=bomFull +software.amazon.awssdk:regions:2.42.37=bomFull +software.amazon.awssdk:retries-spi:2.42.37=bomFull +software.amazon.awssdk:retries:2.42.37=bomFull +software.amazon.awssdk:s3:2.42.37=bomFull +software.amazon.awssdk:sdk-core:2.42.37=bomFull +software.amazon.awssdk:sts:2.42.37=bomFull +software.amazon.awssdk:third-party-jackson-core:2.42.37=bomFull +software.amazon.awssdk:utils-lite:2.42.37=bomFull +software.amazon.awssdk:utils:2.42.37=bomFull +software.amazon.eventstream:eventstream:1.0.1=bomFull +ua.net.nlp:morfologik-ukrainian-search:4.9.1=bomFull +empty=crossDcManager,docker,docs,example,jarValidation,jsClientSbom,modules,server,solrFullTgz,solrFullTgzSignature,solrSlimTgz,solrSlimTgzSignature,uiSbom diff --git a/solr/server/build.gradle b/solr/server/build.gradle index a22c084d94b0..99120e7ca922 100644 --- a/solr/server/build.gradle +++ b/solr/server/build.gradle @@ -25,14 +25,43 @@ javadoc.enabled(false) compileJava.enabled(false) configurations { - libExt + // === Custom configurations used to assemble the Solr server binary distribution === + + // 1. Jetty Bootstrap JAR + // Output Path: server/start.jar + // Description: Contains the Jetty bootstrap JAR responsible for launching the Solr server. + startJar + + // 2. Server Libraries + // Output Path: server/lib/ + // Description: Contains core libraries required by the Solr server at runtime (mostly Jetty-related JARs). serverLib + + // 3. Extended Server Libraries + // Output Path: server/lib/ext/ + // Description: Includes optional runtime libraries such as logging (SLF4J, Log4j) and metrics (Dropwizard, etc.). + libExt + + // 4. Solr Core JAR + // Output Path: server/solr-webapp/webapp/WEB-INF/lib/ + // Description: Contains the solr-core JAR, which includes the core functionality and indexing logic of Solr. solrCore + + // 5. Solr Web Application Libraries + // Output Path: server/solr-webapp/webapp/WEB-INF/lib/ + // Description: Contains the remaining Solr modules, packaged in exploded WAR format for deployment via Jetty. + webapp + + // === Runtime Configuration === + + // Combines core runtime dependencies for launching the Solr server, + // aggregating required libraries from serverLib, libExt, and solrCore. runtimeClasspath { extendsFrom serverLib, libExt, solrCore } - startJar - webapp + + // Internal configuration used by packaging tasks (e.g., creating distributions). + // This configuration only includes the `packagingDir` folder generated during assembly packaging } diff --git a/solr/ui/build.gradle.kts b/solr/ui/build.gradle.kts index 0d76bafc4d86..d5c4c73c9608 100644 --- a/solr/ui/build.gradle.kts +++ b/solr/ui/build.gradle.kts @@ -15,6 +15,8 @@ * limitations under the License. */ +import org.cyclonedx.gradle.CyclonedxDirectTask +import org.cyclonedx.model.Component import org.jetbrains.compose.desktop.application.dsl.TargetFormat import org.jetbrains.kotlin.gradle.ExperimentalWasmDsl import org.jetbrains.kotlin.gradle.targets.js.webpack.KotlinWebpackConfig @@ -240,3 +242,35 @@ artifacts { ) } } + +// CycloneDX SBOM of the Maven dependencies compiled into the wasmJs UI bundle, +// merged into the distribution SBOMs by :solr:packaging. + +val uiSbomFile = layout.buildDirectory.file("cyclonedx/bom-ui.json").get().asFile + +val cyclonedxUi = tasks.register("cyclonedxUi") { + group = "Bill of Materials" + description = "Generates a CycloneDX BOM of the dependencies compiled into the wasmJs UI bundle" + + includeConfigs.set(listOf("wasmJsRuntimeClasspath")) + projectType.set(Component.Type.LIBRARY) + + // The plugin resolves the configuration leniently and without depending on it, + // so the artifacts must be present first or their hashes are missing. + inputs.files(configurations.named("wasmJsRuntimeClasspath")) + .withPropertyName("wasmJsRuntimeArtifacts") + .withNormalizer(ClasspathNormalizer::class) + + jsonOutput.set(uiSbomFile) +} + +val uiSbom = configurations.create("uiSbom") { + isCanBeConsumed = true + isCanBeResolved = false +} + +artifacts { + add("uiSbom", uiSbomFile) { + builtBy(cyclonedxUi) + } +} diff --git a/solr/webapp/build.gradle b/solr/webapp/build.gradle index da2ba2cda085..b64d1574cce2 100644 --- a/solr/webapp/build.gradle +++ b/solr/webapp/build.gradle @@ -15,6 +15,10 @@ * limitations under the License. */ +import groovy.json.JsonOutput +import groovy.json.JsonSlurper +import java.security.MessageDigest + plugins { id 'java' id 'war' @@ -63,3 +67,115 @@ war { artifacts { war tasks.war } + +// web/libs holds checked-in third-party JavaScript, described for the distribution SBOM by +// vendored-libs.json. These tasks keep the two in sync: validateVendoredLibs fails on any drift, +// updateVendoredLibs refreshes the hashes. The manifest sits outside web/, which goes into the war. +def vendoredLibsManifest = file('vendored-libs.json') +def vendoredLibsDir = file('web/libs') + +def sha256Of = { File f -> + def digest = MessageDigest.getInstance('SHA-256') + f.eachByte(8192) { buffer, length -> digest.update(buffer, 0, length) } + digest.digest().encodeHex().toString() +} + +tasks.register('validateVendoredLibs') { + group = 'Verification' + description = 'Checks that web/libs matches the components declared in vendored-libs.json' + + inputs.file(vendoredLibsManifest).withPropertyName('manifest') + .withPathSensitivity(PathSensitivity.NONE) + inputs.dir(vendoredLibsDir).withPropertyName('vendoredLibs') + .withPathSensitivity(PathSensitivity.RELATIVE) + def marker = layout.buildDirectory.file('vendored-libs-validated.txt') + outputs.file(marker) + + doLast { + def manifest = new JsonSlurper().parse(vendoredLibsManifest) + def onDisk = vendoredLibsDir.listFiles().findAll { it.file }.collectEntries { [(it.name): it] } + def errors = [] + + (onDisk.keySet() - manifest.keySet()).sort().each { + errors << "${it}: present in ${vendoredLibsDir} but absent from ${vendoredLibsManifest.name}." + } + (manifest.keySet() - onDisk.keySet()).sort().each { + errors << "${it}: declared in ${vendoredLibsManifest.name} but no such file in ${vendoredLibsDir}." + } + + manifest.each { fileName, component -> + def file = onDisk[fileName] + if (file == null) { + return + } + ['type', 'bom-ref', 'name', 'purl', 'licenses', 'hashes'].each { field -> + if (!component[field]) { + errors << "${fileName}: component is missing the required '${field}' field." + } + } + def declared = component.hashes?.find { it.alg == 'SHA-256' }?.content + if (declared == null) { + errors << "${fileName}: component declares no SHA-256 hash." + } else { + def actual = sha256Of(file) + if (declared != actual) { + errors << "${fileName}: content changed, SHA-256 is ${actual} but the component declares " + + "${declared}. Review the component (version, licence, pedigree) and run 'gradlew " + + ":solr:webapp:updateVendoredLibs' to refresh the hash." + } + } + } + + if (errors) { + throw new GradleException("Vendored JavaScript does not match ${vendoredLibsManifest}:\n " + + errors.join("\n ")) + } + marker.get().asFile.text = "${manifest.size()} vendored components validated\n" + } +} + +tasks.matching { it.name == 'check' }.configureEach { + it.dependsOn 'validateVendoredLibs' +} + +tasks.register('updateVendoredLibs') { + group = 'Build Dependencies' + description = 'Refreshes the SHA-256 hashes in vendored-libs.json' + + doLast { + // JsonSlurper hands back a LinkedHashMap, so the declaration order survives the round trip. + def manifest = new JsonSlurper().parse(vendoredLibsManifest) + def onDisk = vendoredLibsDir.listFiles().findAll { it.file }.sort { it.name } + + onDisk.each { file -> + def component = manifest[file.name] + if (component == null) { + // A stub only: identity, licence and pedigree cannot be inferred from the file, so + // validateVendoredLibs keeps failing until they are filled in by hand. + logger.lifecycle("${file.name}: new file, added a stub component that must be completed by hand") + manifest[file.name] = [ + type: 'library', + 'bom-ref': null, + name: null, + hashes: [[alg: 'SHA-256', content: sha256Of(file)]], + licenses: null, + purl: null, + ] + } else { + def actual = sha256Of(file) + if (component.hashes?.find { it.alg == 'SHA-256' }?.content != actual) { + logger.lifecycle("${file.name}: refreshed SHA-256") + component.hashes = [[alg: 'SHA-256', content: actual]] + } + } + } + + def removed = manifest.keySet() - onDisk.collect { it.name } as Set + removed.each { + logger.lifecycle("${it}: file is gone, dropping the component") + manifest.remove(it) + } + + vendoredLibsManifest.text = JsonOutput.prettyPrint(JsonOutput.toJson(manifest.sort { it.key })) + "\n" + } +} diff --git a/solr/webapp/js-client/build.gradle.kts b/solr/webapp/js-client/build.gradle.kts index 1ce43c25978b..26a2d9ed00a0 100644 --- a/solr/webapp/js-client/build.gradle.kts +++ b/solr/webapp/js-client/build.gradle.kts @@ -17,6 +17,8 @@ import com.github.gradle.node.npm.task.NpmTask import com.github.gradle.node.npm.task.NpxTask +import groovy.json.JsonOutput +import groovy.json.JsonSlurper // Builds the OpenAPI-generated JS client (from :solr:api) into a single bundled // file, for :solr:webapp to include in the war. This is the only place in the @@ -49,9 +51,29 @@ val syncJSClientSourceCode = tasks.register("syncJSClientSourceCode") { into(jsClientWorkspace) - // Keep the node modules, so that they don't need to be re-downloaded + // Keep the outputs of "npm install", so that they don't need to be regenerated preserve { include("node_modules/**") + include("package-lock.json") + } + + // The OpenAPI generator wrongly declares the @babel/cli build tool as a runtime + // dependency; move it to devDependencies, so that the SBOM of the bundle + // (generated with --omit dev) only lists what browserify actually bundles. + doLast { + val packageJson = File(jsClientWorkspace, "package.json") + @Suppress("UNCHECKED_CAST") + val json = JsonSlurper().parse(packageJson) as MutableMap + + @Suppress("UNCHECKED_CAST") + val dependencies = json["dependencies"] as? MutableMap + dependencies?.remove("@babel/cli")?.let { babelCliVersion -> + @Suppress("UNCHECKED_CAST") + val devDependencies = + json.getOrPut("devDependencies") { mutableMapOf() } as MutableMap + devDependencies["@babel/cli"] = babelCliVersion + } + packageJson.writeText(JsonOutput.prettyPrint(JsonOutput.toJson(json))) } } @@ -64,6 +86,7 @@ val jsClientDownloadDeps = tasks.register("jsClientDownloadDeps") { inputs.dir("$jsClientWorkspace/src") inputs.file("$jsClientWorkspace/package.json") outputs.dir("$jsClientWorkspace/node_modules") + outputs.file("$jsClientWorkspace/package-lock.json") } val jsClientBuild = tasks.register("jsClientBuild") { @@ -117,3 +140,54 @@ artifacts { builtBy(finalizeJsBundleDir) } } + +// CycloneDX SBOM of the bundle, merged into the distribution SBOMs by :solr:packaging + +val jsClientSbomFile = layout.buildDirectory.file("cyclonedx/bom-js-client.json").get().asFile + +val downloadCyclonedxNpm = tasks.register("downloadCyclonedxNpm") { + args.set(listOf("install", "@cyclonedx/cyclonedx-npm@${libs.versions.cyclonedx.npm.get()}")) + + inputs.property("cyclonedx-npm version", libs.versions.cyclonedx.npm.get()) + outputs.dir(project.extra["nodeProjectDir"].toString() + "/node_modules/@cyclonedx/cyclonedx-npm") +} + +val generateJsClientSbom = tasks.register("generateJsClientSbom") { + dependsOn(downloadCyclonedxNpm) + // Needs the package-lock.json and node_modules produced by the install + dependsOn(jsClientDownloadDeps) + + // The full package spec, since the bare "cyclonedx-npm" command name resolves to an + // unrelated npm package. Runs from the node project dir, where downloadCyclonedxNpm + // installed the pinned version, and points at the workspace manifest instead. + command.set("@cyclonedx/cyclonedx-npm@${libs.versions.cyclonedx.npm.get()}") + args.set( + listOf( + // Only the packages bundled into the shipped file, not the build tooling + "--omit", "dev", + // Match the spec version emitted by the CycloneDX Gradle plugin in :solr:packaging + "--spec-version", "1.6", + "--output-reproducible", + "--output-format", "JSON", + "--output-file", jsClientSbomFile.absolutePath, + "$jsClientWorkspace/package.json", + ), + ) + workingDir.set(File(project.extra["nodeProjectDir"].toString())) + + inputs.file("$jsClientWorkspace/package.json") + inputs.file("$jsClientWorkspace/package-lock.json") + inputs.property("cyclonedx-npm version", libs.versions.cyclonedx.npm.get()) + outputs.file(jsClientSbomFile) +} + +val jsClientSbom = configurations.create("jsClientSbom") { + isCanBeConsumed = true + isCanBeResolved = false +} + +artifacts { + add("jsClientSbom", jsClientSbomFile) { + builtBy(generateJsClientSbom) + } +} diff --git a/solr/webapp/vendored-libs.json b/solr/webapp/vendored-libs.json new file mode 100644 index 000000000000..8a99d148ce1c --- /dev/null +++ b/solr/webapp/vendored-libs.json @@ -0,0 +1,768 @@ +{ + "angular-chosen.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-chosen-localytics@1.9.2", + "manufacturer": { + "name": "Localytics", + "url": [ + "https://www.localytics.com/" + ] + }, + "name": "angular-chosen-localytics", + "version": "1.9.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "d132a592badfa97744c42ff5abf533ea5a826d1981bbd0f01d19f5b5d7e0df2b" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-chosen-localytics@1.9.2", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-chosen-localytics", + "version": "1.9.2", + "purl": "pkg:npm/angular-chosen-localytics@1.9.2" + } + ], + "notes": "Differently minified build of 1.9.2; the published dist/angular-chosen.min.js has the same logic but different symbol names, so the shipped bytes are not comparable to it." + } + }, + "angular-cookies.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-cookies@1.8.0", + "manufacturer": { + "name": "Google, Inc.", + "url": [ + "https://angularjs.org/" + ] + }, + "name": "angular-cookies", + "version": "1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "395514ba020286029440b1fc14d04b4102cb5d4e0ba7a4117079b802ca0869cb" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-cookies@1.8.0", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-cookies", + "version": "1.8.0", + "purl": "pkg:npm/angular-cookies@1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "eed97b74e2128f3d340325dd9cbfb9b8f70a1a5ade70eccca990d45483aa8700" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/angular-cookies@1.8.0/angular-cookies.min.js" + } + ] + } + ], + "notes": "MIT licence text, 1101 bytes, prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "angular-resource.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-resource@1.8.0", + "manufacturer": { + "name": "Google, Inc.", + "url": [ + "https://angularjs.org/" + ] + }, + "name": "angular-resource", + "version": "1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "89c2d5cb9a399e7979687172d6875c68d1cd0d2b7a99f96dd83a523614f80c4d" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-resource@1.8.0", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-resource", + "version": "1.8.0", + "purl": "pkg:npm/angular-resource@1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "3a61f95c630f63b39aacf3f8ee66bc13bc9b820d11f749591d1c3d07125ec184" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/angular-resource@1.8.0/angular-resource.min.js" + } + ] + } + ], + "notes": "MIT licence text, 1101 bytes, prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "angular-route.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-route@1.8.0", + "manufacturer": { + "name": "Google, Inc.", + "url": [ + "https://angularjs.org/" + ] + }, + "name": "angular-route", + "version": "1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "0a9fac6d4809f3408d96c434d36624624ddb3db36419f557d2cd2cf1b949b105" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-route@1.8.0", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-route", + "version": "1.8.0", + "purl": "pkg:npm/angular-route@1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "3422eae4c737ff2d30abfe3df6c30e6b11869d3a30683c5efced151248eb9661" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/angular-route@1.8.0/angular-route.min.js" + } + ] + } + ], + "notes": "MIT licence text, 1101 bytes, prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "angular-sanitize.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-sanitize@1.8.0", + "manufacturer": { + "name": "Google, Inc.", + "url": [ + "https://angularjs.org/" + ] + }, + "name": "angular-sanitize", + "version": "1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "db5c1a1af40f0c7205a7bc6b592d15cb1e50478665467d7e95c804b20df723d4" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-sanitize@1.8.0", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-sanitize", + "version": "1.8.0", + "purl": "pkg:npm/angular-sanitize@1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "958e6aa9b32f5ef3e86acf16d2413f08baa02f68fbe38baa5d8916282ae1b882" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/angular-sanitize@1.8.0/angular-sanitize.min.js" + } + ] + } + ], + "notes": "MIT licence text, 1101 bytes, prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "angular-utf8-base64.min.js": { + "type": "library", + "bom-ref": "pkg:github/stranger82/angular-utf8-base64@v0.0.5", + "manufacturer": { + "name": "Andrey Bezyazychniy", + "url": [ + "https://github.com/stranger82/angular-utf8-base64" + ] + }, + "authors": [ + { + "name": "Andrey Bezyazychniy" + }, + { + "name": "Vassilis Petroulias" + } + ], + "name": "angular-utf8-base64", + "version": "0.0.5", + "hashes": [ + { + "alg": "SHA-256", + "content": "b95109c1309c2f7aea93f062d2006ce50eb877b8698d529741f635843dd79b89" + } + ], + "licenses": [ + { + "expression": "MIT AND Apache-2.0" + } + ], + "purl": "pkg:github/stranger82/angular-utf8-base64@v0.0.5", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular-utf8-base64", + "version": "0.0.5", + "purl": "pkg:github/stranger82/angular-utf8-base64@v0.0.5", + "hashes": [ + { + "alg": "SHA-256", + "content": "c8cad92b2f6a528c5e0982aef0ad739d1cd1afb896e8c2ba400a750e2d5d5d57" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://raw.githubusercontent.com/stranger82/angular-utf8-base64/43cea612939acc07e06939f8da97e6f22186d1e2/angular-utf8-base64.min.js" + } + ] + } + ], + "notes": "Licence texts of 1786 bytes prepended: the MIT terms, and the Apache-2.0 notice covering the base64.js library that this package encapsulates. Removing them reproduces the ancestor byte-for-byte." + } + }, + "angular.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular@1.8.0", + "manufacturer": { + "name": "Google, Inc.", + "url": [ + "https://angularjs.org/" + ] + }, + "name": "angular", + "version": "1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "f5c29b548881dc22e6c2ffa12eae37dc1419b76046ff99a154b02e44f680de29" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular@1.8.0", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "angular", + "version": "1.8.0", + "purl": "pkg:npm/angular@1.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "566f18cb8bc23558701c2cc4f934fe50bcc85629d1aaf5d589f835f2b3e57a9f" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/angular@1.8.0/angular.min.js" + } + ] + } + ], + "notes": "MIT licence text, 1101 bytes, prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "chosen.jquery.min.js": { + "type": "library", + "bom-ref": "pkg:npm/chosen-js@1.8.7", + "manufacturer": { + "name": "Harvest", + "url": [ + "https://www.getharvest.com/" + ] + }, + "authors": [ + { + "name": "Patrick Filler" + } + ], + "name": "chosen-js", + "version": "1.8.7", + "hashes": [ + { + "alg": "SHA-256", + "content": "7cd60ffd167c5557fca7b192e6400884642ac3842a495f938c8d412ec36a0f75" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/chosen-js@1.8.7", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "chosen-js", + "version": "1.8.7", + "purl": "pkg:npm/chosen-js@1.8.7", + "hashes": [ + { + "alg": "SHA-256", + "content": "73881513a7e7f8944a311bea8e80e9fad946e256ae74d62b5c8d469dc6df0186" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/chosen-js@1.8.7/chosen.jquery.min.js" + } + ] + } + ], + "notes": "MIT licence text, 1165 bytes, prepended, and a trailing newline appended; removing them reproduces the ancestor byte-for-byte." + } + }, + "d3.js": { + "type": "library", + "bom-ref": "pkg:npm/d3@2.8.1", + "manufacturer": { + "name": "Michael Bostock", + "url": [ + "https://github.com/d3/d3" + ] + }, + "authors": [ + { + "name": "Michael Bostock" + } + ], + "name": "d3", + "version": "2.8.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "435b386bb55c7fcd938584ebb3c4a64da3712a043fa267a1ed3da2ac22aa6ef7" + } + ], + "licenses": [ + { + "license": { + "id": "BSD-3-Clause" + } + } + ], + "purl": "pkg:npm/d3@2.8.1", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "d3", + "version": "2.8.1", + "purl": "pkg:npm/d3@2.8.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "d74d3335c81747cb3d2eed71c29a83ef6fe3fa523db15934cdfa5738624a2ea2" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://registry.npmjs.org/d3/-/d3-2.8.1.tgz" + } + ] + } + ], + "notes": "BSD-3-Clause licence text, 1433 bytes, prepended; removing it reproduces the ancestor (member d3.v2.js of the tarball) byte-for-byte." + } + }, + "highlight.js": { + "type": "library", + "bom-ref": "pkg:npm/highlight.js", + "manufacturer": { + "name": "Ivan Sagalaev", + "url": [ + "https://github.com/highlightjs/highlight.js" + ] + }, + "authors": [ + { + "name": "Ivan Sagalaev" + } + ], + "name": "highlight.js", + "version": "unknown", + "hashes": [ + { + "alg": "SHA-256", + "content": "b0c3b1298c79a2d28a9a6bdae07f4fc878e7bc46f50e81338f881f3d6a61514f" + } + ], + "licenses": [ + { + "license": { + "id": "BSD-3-Clause" + } + } + ], + "purl": "pkg:npm/highlight.js", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "highlight.js", + "purl": "pkg:npm/highlight.js" + } + ], + "notes": "Custom subset build vendored in 2012 whose version is unknown to the SBOM author: the file carries no version marker and its 'var hljs=new function()' API predates 8.0. The version is recorded as \"unknown\" rather than omitted. Treat every highlight.js advisory as potentially applying." + } + }, + "jquery-3.5.1.min.js": { + "type": "library", + "bom-ref": "pkg:npm/jquery@3.5.1", + "manufacturer": { + "name": "jQuery Foundation", + "url": [ + "https://jquery.com/" + ] + }, + "name": "jquery", + "version": "3.5.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "44915672ad9626cb67e995221dd29283ff5620125a10745ac968ddc9c3865c39" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/jquery@3.5.1", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "jquery", + "version": "3.5.1", + "purl": "pkg:npm/jquery@3.5.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "f7f6a5894f1d19ddad6fa392b2ece2c5e578cbf7da4ea805b6885eb6985b6e3d" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/jquery@3.5.1/dist/jquery.min.js" + } + ] + } + ], + "notes": "MIT licence text, 1108 bytes, prepended; removing it reproduces the ancestor byte-for-byte." + } + }, + "jquery-ui.min.js": { + "type": "library", + "bom-ref": "pkg:npm/jquery-ui@1.12.1", + "manufacturer": { + "name": "jQuery Foundation", + "url": [ + "https://jquery.com/" + ] + }, + "name": "jquery-ui", + "version": "1.12.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "4bc1d3b514cad98858c077a3e69f61b862e3808a732b72a4d9da3cd91908037b" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/jquery-ui@1.12.1", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "jquery-ui", + "version": "1.12.1", + "purl": "pkg:npm/jquery-ui@1.12.1" + } + ], + "notes": "Custom subset build of 24 KB against the 253 KB full distribution, so it is not byte-comparable to any published artifact. Any jQuery UI 1.12.1 advisory applies until the affected widget is shown to be absent from this build." + } + }, + "jssha-3.3.1-sha256.min.js": { + "type": "library", + "bom-ref": "pkg:npm/jssha@3.3.1", + "manufacturer": { + "name": "Brian Turek", + "url": [ + "https://github.com/Caligatio/jsSHA" + ] + }, + "authors": [ + { + "name": "Brian Turek" + } + ], + "name": "jssha", + "version": "3.3.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "c2e3235b130fe33d129f97c6570272b4b20baf0daab628ba79d6d9d005694921" + } + ], + "licenses": [ + { + "license": { + "id": "BSD-3-Clause" + } + } + ], + "purl": "pkg:npm/jssha@3.3.1", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "jssha", + "version": "3.3.1", + "purl": "pkg:npm/jssha@3.3.1" + } + ], + "notes": "SHA-256-only variant; it matches none of the dist files published in the jssha 3.3.1 npm tarball." + } + }, + "jstree.min.js": { + "type": "library", + "bom-ref": "pkg:npm/jstree@3.3.10", + "manufacturer": { + "name": "Ivan Bozhanov", + "url": [ + "https://github.com/vakata/jstree" + ] + }, + "authors": [ + { + "name": "Ivan Bozhanov" + } + ], + "name": "jstree", + "version": "3.3.10", + "hashes": [ + { + "alg": "SHA-256", + "content": "612d5f0a58cb09c620d82d39ef060e68453f634fca279f9f3e91feca954f98a5" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/jstree@3.3.10", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "jstree", + "version": "3.3.10", + "purl": "pkg:npm/jstree@3.3.10", + "hashes": [ + { + "alg": "SHA-256", + "content": "26238e200ef64e61a4a47bbff33ce50f1312234806db998b8e93ebefda015a6c" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://unpkg.com/jstree@3.3.10/dist/jstree.min.js" + } + ] + } + ], + "notes": "MIT licence text, 1054 bytes, prepended, and a trailing newline appended; removing them reproduces the ancestor byte-for-byte." + } + }, + "ngtimeago.js": { + "type": "library", + "bom-ref": "pkg:github/uttesh/ngtimeago@0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "manufacturer": { + "name": "Uttesh Kumar", + "url": [ + "https://github.com/uttesh/ngtimeago" + ] + }, + "authors": [ + { + "name": "Uttesh Kumar" + } + ], + "name": "ngtimeago", + "version": "0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "hashes": [ + { + "alg": "SHA-256", + "content": "9ca6b12801166dd2cdb699d18db67c0bd8c3b3c6cec05c2b5ea4e01653877a71" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:github/uttesh/ngtimeago@0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "pedigree": { + "ancestors": [ + { + "type": "library", + "name": "ngtimeago", + "version": "0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "purl": "pkg:github/uttesh/ngtimeago@0b1e72785a6e0e6edd9389d875db6279b0ea36ba", + "hashes": [ + { + "alg": "SHA-256", + "content": "52675691593269327c2f053b7aedda07bbfe9869a610a4447dab6dc79bf96df6" + } + ], + "externalReferences": [ + { + "type": "distribution", + "url": "https://raw.githubusercontent.com/uttesh/ngtimeago/0b1e72785a6e0e6edd9389d875db6279b0ea36ba/ngtimeago.js" + }, + { + "type": "vcs", + "url": "https://github.com/uttesh/ngtimeago" + } + ] + } + ], + "commits": [ + { + "uid": "7e8d6d110060498429a5b942789274d7ca217136", + "message": "SOLR-7780: Prevent NaN showing on angular UI dashboard" + }, + { + "uid": "1331a57e3d5d0fd1480fafcbe6c66721eacf331a", + "message": "LUCENE-6732: Remove tabs in JS and XML files" + }, + { + "uid": "9d97ef1027fe8b844fa5c72fbc63ddc3a24a0bc6", + "message": "SOLR-13343: Fix minor web UI spacing issue" + } + ], + "notes": "MIT licence text, 1062 bytes, prepended, and the body carries the Solr patches listed in commits. The base commit is untagged and later than the 0.0.2 tag. Not the npm package ng-timeago, which is the unrelated joyingsoft/ng-timeago project." + } + }, + "ui-grid.min.js": { + "type": "library", + "bom-ref": "pkg:npm/angular-ui-grid@4.10.0", + "manufacturer": { + "name": "AngularUI Team", + "url": [ + "https://github.com/angular-ui/ui-grid" + ] + }, + "name": "angular-ui-grid", + "version": "4.10.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "ec0200dc99b4e7f2576e3a0285ce83b90ed1e93416016595e6078c0afa215a34" + } + ], + "licenses": [ + { + "license": { + "id": "MIT" + } + } + ], + "purl": "pkg:npm/angular-ui-grid@4.10.0" + } +}