From 04fc9d78ac1b48cfeb4921f4f5372a889d4d1003 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 16 Jul 2026 08:59:47 +0200 Subject: [PATCH 1/2] feat: Sync supported versions from the Stackable Hub (parked) Regenerates the supported-versions.adoc partials of the product operators from the Stackable Hub API for released docs versions. Nightly keeps the hand-maintained partials since the Hub only knows released data. Responses are cached in the Antora cache dir; the build never fails and never emits warnings when the Hub is unreachable (the production playbook fails builds on warnings). Parked until the Hub exposes the next (unreleased) SDP release: then nightly can be synced from it as well and the operator repos only need to be touched once (delete partials + adopt shared link bar). --- antora-playbook.yml | 1 + lib/hub-supported-versions.js | 111 ++++++++++++++++++++++++++++++++++ local-antora-playbook.yml | 1 + only-dev-antora-playbook.yml | 1 + truly-local-playbook.yml | 1 + 5 files changed, 115 insertions(+) create mode 100644 lib/hub-supported-versions.js diff --git a/antora-playbook.yml b/antora-playbook.yml index 8fc6cefdc..a6981418b 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -29,6 +29,7 @@ antora: extensions: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js + - ./lib/hub-supported-versions.js - ./lib/llms-txt.js content: sources: diff --git a/lib/hub-supported-versions.js b/lib/hub-supported-versions.js new file mode 100644 index 000000000..bf95902e9 --- /dev/null +++ b/lib/hub-supported-versions.js @@ -0,0 +1,111 @@ +// Keeps the "Supported versions" lists on released docs versions in sync with +// the Stackable Hub (whose data comes from the Portal, the source of truth for +// what a release ships). +// +// At build time, the supported-versions.adoc partial of every product operator +// module is regenerated from https://hub.stackable.tech/api/v1/components/ +// - but only for release versions of the docs. The nightly version keeps the +// hand-maintained partial from the operator repo, because the Hub only knows +// released data. +// +// API responses are cached in the Antora cache dir. When the Hub is not +// reachable and no cache exists, the partial is left untouched. This extension +// never fails the build and only logs at info level - the production playbook +// fails builds on warnings, and Hub downtime must never break a docs build. +// +// Useful links: +// Extensions: https://docs.antora.org/antora/latest/extend/extensions/ +// Types of events: https://docs.antora.org/antora/latest/extend/generator-events-reference/ +'use strict' + +const fs = require('fs') +const ospath = require('path') + +const HUB_API = 'https://hub.stackable.tech/api/v1/components' + +// docs module name -> Hub component slug +const MODULE_TO_SLUG = { + airflow: 'airflow', + druid: 'druid', + hbase: 'hbase', + hdfs: 'hdfs', + hive: 'hive', + kafka: 'kafka', + nifi: 'nifi', + opa: 'opa', + opensearch: 'opensearch', + 'spark-k8s': 'spark', + superset: 'superset', + trino: 'trino', + zookeeper: 'zookeeper', +} + +const STATUS_SUFFIX = { + lts: ' (LTS)', + deprecated: ' (deprecated)', + experimental: ' (experimental)', +} + +module.exports.register = function () { + const logger = this.getLogger('hub-supported-versions') + + this.once('contentAggregated', async ({ playbook, contentAggregate }) => { + const cacheDir = ospath.join(playbook.dir || '.', playbook.runtime.cacheDir || './cache', 'hub') + const components = await fetchComponents(cacheDir, logger) + if (!components) return + + let regenerated = 0 + for (const bucket of contentAggregate) { + // Only the 'home' component carries operator docs; nightly (main) keeps + // the partial from the repo since the Hub has no data for unreleased state. + if (bucket.name !== 'home' || bucket.version === 'nightly') continue + for (const file of bucket.files) { + const match = file.path.match(/^modules\/([^/]+)\/partials\/supported-versions\.adoc$/) + if (!match) continue + const slug = MODULE_TO_SLUG[match[1]] + if (!slug) continue + const release = (components[slug] || { releases: [] }).releases + .find((r) => r.release === bucket.version) + if (!release) { + logger.info(`no Hub data for ${slug} in SDP ${bucket.version}, keeping the partial from the repo`) + continue + } + const lines = release.versions.map((v) => `- ${v.version}${STATUS_SUFFIX[v.status] || ''}`) + file.contents = Buffer.from( + `// Regenerated at build time from ${HUB_API}/${slug} (SDP ${bucket.version}).\n` + + `// The Portal is the source of truth for released product versions.\n` + + lines.join('\n') + '\n', 'utf8') + regenerated++ + } + } + logger.info(`regenerated ${regenerated} supported-versions partial(s) from the Hub`) + }) +} + +async function fetchComponents (cacheDir, logger) { + const cacheFile = ospath.join(cacheDir, 'components.json') + try { + const { components: list } = await getJson(`${HUB_API}`) + const components = {} + for (const { slug } of list) { + components[slug] = await getJson(`${HUB_API}/${slug}`) + } + fs.mkdirSync(cacheDir, { recursive: true }) + fs.writeFileSync(cacheFile, JSON.stringify(components)) + return components + } catch (err) { + logger.info(`could not fetch ${HUB_API} (${err.message}), trying cache`) + try { + return JSON.parse(fs.readFileSync(cacheFile, 'utf8')) + } catch { + logger.info('no cached Hub data available, supported-versions partials are used as-is') + return undefined + } + } +} + +async function getJson (url) { + const response = await fetch(url) + if (!response.ok) throw new Error(`${url} returned ${response.status}`) + return await response.json() +} diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index 1469b3fd9..c8550b6fe 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -14,6 +14,7 @@ antora: extensions: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js + - ./lib/hub-supported-versions.js - ./lib/llms-txt.js content: sources: diff --git a/only-dev-antora-playbook.yml b/only-dev-antora-playbook.yml index 14f5e6813..06fccadea 100644 --- a/only-dev-antora-playbook.yml +++ b/only-dev-antora-playbook.yml @@ -14,6 +14,7 @@ antora: extensions: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js + - ./lib/hub-supported-versions.js - ./lib/llms-txt.js content: sources: diff --git a/truly-local-playbook.yml b/truly-local-playbook.yml index 220fd3512..e7e74b1ce 100644 --- a/truly-local-playbook.yml +++ b/truly-local-playbook.yml @@ -14,6 +14,7 @@ antora: extensions: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js + - ./lib/hub-supported-versions.js - ./lib/llms-txt.js content: sources: From 6ac8b8405bcadff105f6d25218f928200411c5b4 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 13 Aug 2026 23:56:26 +0200 Subject: [PATCH 2/2] feat: Sync nightly supported versions from the Hub's upcoming release Unparks the extension. It previously skipped nightly outright, on the grounds that the Hub only knew released data, and was held back until a next SDP release existed so the 16 operator repos would only need visiting once. The Hub already models this properly, which removes the wait. Both /api/v1/releases and /api/v1/components/{slug} return a shipped releases array and a separate upcomingReleases array, and the Hub only exposes an upcoming release once it is deliberately public -- provisional plans stay private. So nightly reads upcomingReleases, and the window between a release shipping and the next one being planned is a normal empty state rather than something to wait out. It renders as an explicit "not been decided yet" line, and an unreleased list is labelled provisional so a half-filled one cannot read as a commitment. Also generates the partial when it is absent instead of only rewriting an existing one. That is what lets the operator repos delete their copies: two places include it, an operator's own index.adoc and this repo's platform-wide operators:supported_versions.adoc, and both have to keep resolving. Since the extension becomes load-bearing once those copies are gone, a missing partial with no Hub data now gets a short unavailable note rather than nothing at all, so a Hub outage still cannot break the build. Moved from contentAggregated to contentClassified, because adding a file needs the content catalog and partials are resolved later, when pages are converted. Verified against a real only-dev build: 13 partials written, the undecided note renders on both the operator page and the platform overview, addFile round-trips retrievably in the catalog, and the released path still produces 26.7 as 3.2.2 / 3.1.6 (deprecated) / 3.0.6 (LTS) / 2.9.3 (deprecated), with an unknown version correctly falling back to the repo copy. Co-Authored-By: Claude --- lib/hub-supported-versions.js | 141 +++++++++++++++++++++++++--------- 1 file changed, 103 insertions(+), 38 deletions(-) diff --git a/lib/hub-supported-versions.js b/lib/hub-supported-versions.js index bf95902e9..895107868 100644 --- a/lib/hub-supported-versions.js +++ b/lib/hub-supported-versions.js @@ -1,17 +1,27 @@ -// Keeps the "Supported versions" lists on released docs versions in sync with -// the Stackable Hub (whose data comes from the Portal, the source of truth for -// what a release ships). +// Keeps the "Supported versions" lists in sync with the Stackable Hub, whose data +// comes from the Portal, the source of truth for what a release ships. // -// At build time, the supported-versions.adoc partial of every product operator -// module is regenerated from https://hub.stackable.tech/api/v1/components/ -// - but only for release versions of the docs. The nightly version keeps the -// hand-maintained partial from the operator repo, because the Hub only knows -// released data. +// At build time the supported-versions.adoc partial of every product operator +// module is generated from https://hub.stackable.tech/api/v1/components/. +// Released docs versions map to that SDP release; nightly maps to the Hub's next +// *public* upcoming release. The Hub keeps provisional plans private and only +// exposes an upcoming release once it is deliberately published, so between a +// release shipping and the next one being planned there is legitimately nothing +// to show - that window renders as "not decided yet" rather than a stale or +// guessed list. // -// API responses are cached in the Antora cache dir. When the Hub is not -// reachable and no cache exists, the partial is left untouched. This extension -// never fails the build and only logs at info level - the production playbook -// fails builds on warnings, and Hub downtime must never break a docs build. +// The partial is created if it does not exist, not merely rewritten. That is what +// lets the operator repos delete their hand-maintained copies: two places include +// it (an operator's own index.adoc and this repo's platform-wide +// operators:supported_versions.adoc), and both must keep resolving. +// +// Because of that, the extension is load-bearing for the build once the repo +// copies are gone, so it must always leave a usable partial behind. API responses +// are cached in the Antora cache dir; if the Hub is unreachable and there is no +// cache, an existing partial is left alone and a missing one gets a short +// "unavailable" note. It never fails the build and only logs at info level - the +// production playbook fails builds on warnings, and Hub downtime must never break +// a docs build. // // Useful links: // Extensions: https://docs.antora.org/antora/latest/extend/extensions/ @@ -22,6 +32,7 @@ const fs = require('fs') const ospath = require('path') const HUB_API = 'https://hub.stackable.tech/api/v1/components' +const PARTIAL = 'supported-versions.adoc' // docs module name -> Hub component slug const MODULE_TO_SLUG = { @@ -44,44 +55,98 @@ const STATUS_SUFFIX = { lts: ' (LTS)', deprecated: ' (deprecated)', experimental: ' (experimental)', + preview: ' (preview)', } module.exports.register = function () { const logger = this.getLogger('hub-supported-versions') - this.once('contentAggregated', async ({ playbook, contentAggregate }) => { + // contentClassified rather than contentAggregated: the content catalog is what + // can add a file, and partials are resolved later, when pages are converted. + this.once('contentClassified', async ({ playbook, contentCatalog }) => { const cacheDir = ospath.join(playbook.dir || '.', playbook.runtime.cacheDir || './cache', 'hub') const components = await fetchComponents(cacheDir, logger) - if (!components) return - - let regenerated = 0 - for (const bucket of contentAggregate) { - // Only the 'home' component carries operator docs; nightly (main) keeps - // the partial from the repo since the Hub has no data for unreleased state. - if (bucket.name !== 'home' || bucket.version === 'nightly') continue - for (const file of bucket.files) { - const match = file.path.match(/^modules\/([^/]+)\/partials\/supported-versions\.adoc$/) - if (!match) continue - const slug = MODULE_TO_SLUG[match[1]] - if (!slug) continue - const release = (components[slug] || { releases: [] }).releases - .find((r) => r.release === bucket.version) - if (!release) { - logger.info(`no Hub data for ${slug} in SDP ${bucket.version}, keeping the partial from the repo`) - continue + + const component = contentCatalog.getComponent('home') + if (!component) return logger.info('no home component, nothing to do') + + let written = 0 + for (const { version } of component.versions) { + for (const [moduleName, slug] of Object.entries(MODULE_TO_SLUG)) { + const existing = contentCatalog.getById({ + component: 'home', version, module: moduleName, family: 'partial', relative: PARTIAL, + }) + // A module we do not carry in this docs version at all: nothing includes + // the partial, so do not invent one. + if (!existing && !contentCatalog.getById({ + component: 'home', version, module: moduleName, family: 'page', relative: 'index.adoc', + })) continue + + const body = renderPartial({ components, slug, version, logger }) + if (!body) continue // no data and a repo copy is present: leave it alone + + if (existing) { + existing.contents = Buffer.from(body, 'utf8') + } else { + contentCatalog.addFile({ + contents: Buffer.from(body, 'utf8'), + src: { component: 'home', version, module: moduleName, family: 'partial', relative: PARTIAL }, + }) } - const lines = release.versions.map((v) => `- ${v.version}${STATUS_SUFFIX[v.status] || ''}`) - file.contents = Buffer.from( - `// Regenerated at build time from ${HUB_API}/${slug} (SDP ${bucket.version}).\n` + - `// The Portal is the source of truth for released product versions.\n` + - lines.join('\n') + '\n', 'utf8') - regenerated++ + written++ } } - logger.info(`regenerated ${regenerated} supported-versions partial(s) from the Hub`) + logger.info(`wrote ${written} supported-versions partial(s) from the Hub`) }) } +// Returns the AsciiDoc body, or undefined to mean "leave whatever is there". +function renderPartial ({ components, slug, version, logger }) { + const header = `// Generated at build time from ${HUB_API}/${slug}.\n` + + '// Do not edit: the Portal is the source of truth. See lib/hub-supported-versions.js.\n' + + if (!components) { + // No Hub data at all. An existing partial is better than anything we can say, + // but a missing one still has to resolve or the include fails the build. + return `${header}// The Stackable Hub was unreachable during this build.\n` + + 'The supported version list is temporarily unavailable.\n' + } + + const component = components[slug] + const target = version === 'nightly' ? nextUpcoming(component) : releaseFor(component, version) + + if (!target || !target.versions || !target.versions.length) { + if (version === 'nightly') { + logger.info(`no public upcoming release for ${slug}, rendering the undecided note on nightly`) + return `${header}// No upcoming SDP release is public yet, so there is nothing to list.\n` + + 'The product versions for the next Stackable Data Platform release have not been decided yet.\n' + } + logger.info(`no Hub data for ${slug} in SDP ${version}, keeping the partial from the repo`) + return undefined + } + + const lines = target.versions.map((v) => `- ${v.version}${STATUS_SUFFIX[v.status] || ''}`) + const provisional = version === 'nightly' + ? `// Provisional: SDP ${target.release || 'next'} has not been released yet.\n` + + `NOTE: These are the planned product versions for the next release, SDP ${target.release || 'next'}. They may still change.\n\n` + : '' + return header + provisional + lines.join('\n') + '\n' +} + +function releaseFor (component, version) { + return (component?.releases || []).find((r) => r.release === version) +} + +// The next public upcoming release: earliest planned date, falling back to the +// order the Hub returned. The Hub only lists upcoming releases it considers +// public, so anything here is safe to show. +function nextUpcoming (component) { + const upcoming = component?.upcomingReleases || [] + if (upcoming.length < 2) return upcoming[0] + return [...upcoming].sort((a, b) => + String(a.plannedReleaseDate || '9999').localeCompare(String(b.plannedReleaseDate || '9999')))[0] +} + async function fetchComponents (cacheDir, logger) { const cacheFile = ospath.join(cacheDir, 'components.json') try { @@ -98,7 +163,7 @@ async function fetchComponents (cacheDir, logger) { try { return JSON.parse(fs.readFileSync(cacheFile, 'utf8')) } catch { - logger.info('no cached Hub data available, supported-versions partials are used as-is') + logger.info('no cached Hub data available') return undefined } }