Automate generated reference doc source updates

.github/workflows/update-generated-reference-docs.yml

@@ -0,0 +1,76 @@
+name: Update generated reference docs
+
+on:
+  schedule:
+    - cron: "17 8 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: update-generated-reference-docs
+  cancel-in-progress: false
+
+jobs:
+  update-generated-reference-docs:
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.EXTERNAL_REPO_TOKEN || github.token }}
+      GH_TOKEN: ${{ secrets.EXTERNAL_REPO_TOKEN || github.token }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+
+      - name: Install direnv
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y direnv
+
+      - name: Trust direnv
+        run: direnv allow .
+
+      - name: Update source pins
+        id: update
+        run: |
+          mkdir -p .internal
+          direnv exec . python3 scripts/update_reference_doc_sources.py \
+            --apply \
+            --summary .internal/reference-doc-update-summary.json
+
+      - name: Regenerate reference docs
+        if: steps.update.outputs.has_updates == 'true'
+        run: direnv exec . npm run generate:all-reference-docs
+
+      - name: Run tests
+        if: steps.update.outputs.has_updates == 'true'
+        run: direnv exec . python3 -m pytest tests
+
+      - name: Check whitespace
+        if: steps.update.outputs.has_updates == 'true'
+        run: git diff --check
+
+      - name: Upload update artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: generated-reference-doc-update
+          if-no-files-found: ignore
+          path: |
+            .internal/reference-doc-update-summary.json
+            .internal/generated/x2mdx/**/*.json
+
+      - name: Create pull request
+        if: steps.update.outputs.has_updates == 'true'
+        uses: peter-evans/create-pull-request@v8
+        with:
+          token: ${{ secrets.DOCS_PR_TOKEN }}
+          branch: generated-reference-docs/update
+          base: main
+          title: Update generated reference docs from upstream releases
+          commit-message: Update generated reference docs from upstream releases
+          body: ${{ steps.update.outputs.pr_body }}

.github/workflows/update-version-dashboard.yml

@@ -0,0 +1,66 @@
+name: Update version dashboard
+
+on:
+  schedule:
+    - cron: "23 */6 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: update-version-dashboard
+  cancel-in-progress: false
+
+jobs:
+  update-version-dashboard:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v5
+
+      - name: Install Nix
+        uses: cachix/install-nix-action@v31
+
+      - name: Install direnv
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y direnv
+
+      - name: Trust direnv
+        run: direnv allow .
+
+      - name: Update version dashboard
+        run: direnv exec . npm run generate:version-compatibility-dashboard
+
+      - name: Detect changes
+        id: changes
+        run: |
+          if [[ -n "$(git status --porcelain)" ]]; then
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+            git status --short
+            git diff --stat
+          else
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Check whitespace
+        if: steps.changes.outputs.has_changes == 'true'
+        run: git diff --check
+
+      - name: Create pull request
+        if: steps.changes.outputs.has_changes == 'true'
+        uses: peter-evans/create-pull-request@v8
+        with:
+          token: ${{ secrets.DOCS_PR_TOKEN || github.token }}
+          branch: version-dashboard/update
+          base: main
+          title: Update version dashboard data
+          commit-message: Update version dashboard data
+          body: |
+            Updates the committed Canton Network version dashboard data from public network, package, and installer sources.
+
+            Validation run by the workflow:
+            - `npm run generate:version-compatibility-dashboard`
+            - `git diff --check`

config/mintlify-openapi/splice-openapi/source-artifacts.json

@@ -4,6 +4,9 @@
   "tag_regex": "^v(?P<version>0\\.[0-9]+\\.[0-9]+)$",
   "min_version": "0.5.10",
   "publish_version": "0.5.18",
+  "versions": [
+    "0.5.18"
+  ],
   "asset_template": "{version}_openapi.tar.gz",
   "nav_dropdown": "API Reference",
   "top_level_group_label": "Splice APIs",

config/x2mdx/grpc-ledger-api-reference/source-artifacts.json

@@ -7,6 +7,16 @@
     "web_url": "https://github.com/DACH-NY/canton"
   },
   "min_version": "3.4.4",
+  "versions": [
+    "3.4.4",
+    "3.4.5",
+    "3.4.6",
+    "3.4.7",
+    "3.4.8",
+    "3.4.9",
+    "3.4.10",
+    "3.4.11"
+  ],
   "metadata_path": "config/x2mdx/protobuf-history/metadata.json",
   "package_prefixes": [
     "com.daml.ledger.api.v2"

config/x2mdx/protobuf-history/source-artifacts.json

@@ -8,5 +8,18 @@
   },
   "min_version": "3.2.0",
   "excluded_versions": ["3.4.1"],
+  "versions": [
+    "3.4.0",
+    "3.4.2",
+    "3.4.3",
+    "3.4.4",
+    "3.4.5",
+    "3.4.6",
+    "3.4.7",
+    "3.4.8",
+    "3.4.9",
+    "3.4.10",
+    "3.4.11"
+  ],
   "metadata_path": "config/x2mdx/protobuf-history/metadata.json"
 }

config/x2mdx/reference-update-policy.json

@@ -0,0 +1,71 @@
+{
+  "surfaces": {
+    "daml-standard-library": {
+      "kind": "git-tag-versions",
+      "config_path": "config/x2mdx/daml-standard-library/source-artifacts.json",
+      "repository": "digital-asset/daml",
+      "remote": "https://github.com/digital-asset/daml.git",
+      "tag_regex": "^v(?P<version>\\d+\\.\\d+\\.\\d+)$",
+      "keep": 3,
+      "publish_latest": true
+    },
+    "json-ledger-api": {
+      "kind": "canton-release-bundle-minors",
+      "config_paths": [
+        "config/x2mdx/ledger-api/source-artifacts.json",
+        "config/x2mdx/ledger-api-asyncapi/source-artifacts.json"
+      ],
+      "repository": "DACH-NY/canton",
+      "remote": "https://github.com/DACH-NY/canton.git",
+      "tag_regex": "^v(?P<canton_version>\\d+\\.\\d+\\.\\d+)$",
+      "keep_minor_count": 2,
+      "validate_release_url": true
+    },
+    "ledger-bindings": {
+      "kind": "maven-javadoc-artifact",
+      "config_path": "config/x2mdx/ledger-bindings/source-artifacts.json",
+      "keep": 4,
+      "validate_javadoc": true
+    },
+    "protobuf-history": {
+      "kind": "canton-tag-versions",
+      "config_path": "config/x2mdx/protobuf-history/source-artifacts.json",
+      "repository": "DACH-NY/canton",
+      "remote": "https://github.com/DACH-NY/canton.git",
+      "tag_regex": "^v(?P<version>\\d+\\.\\d+\\.\\d+)$",
+      "validate_release_url": true
+    },
+    "grpc-ledger-api-reference": {
+      "kind": "canton-tag-versions",
+      "config_path": "config/x2mdx/grpc-ledger-api-reference/source-artifacts.json",
+      "repository": "DACH-NY/canton",
+      "remote": "https://github.com/DACH-NY/canton.git",
+      "tag_regex": "^v(?P<version>\\d+\\.\\d+\\.\\d+)$",
+      "validate_release_url": true
+    },
+    "typescript-bindings": {
+      "kind": "npm-packages",
+      "config_path": "config/x2mdx/typescript-bindings/source-artifacts.json",
+      "keep": 4,
+      "publish_latest": true
+    },
+    "wallet-gateway-openrpc": {
+      "kind": "github-release-versions",
+      "config_path": "config/x2mdx/wallet-gateway-openrpc/source-artifacts.json",
+      "repository": "hyperledger-labs/splice-wallet-kernel",
+      "tag_prefix": "@canton-network/wallet-gateway-remote@",
+      "tag_regex": "^@canton-network/wallet-gateway-remote@(?P<version>\\d+\\.\\d+\\.\\d+)$",
+      "keep": 2,
+      "publish_latest": true
+    },
+    "splice-openapi": {
+      "kind": "github-release-asset-versions",
+      "config_path": "config/mintlify-openapi/splice-openapi/source-artifacts.json",
+      "repository": "digital-asset/decentralized-canton-sync",
+      "tag_regex": "^v(?P<version>0\\.[0-9]+\\.[0-9]+)$",
+      "asset_template": "{version}_openapi.tar.gz",
+      "keep": 1,
+      "publish_latest": true
+    }
+  }
+}

config/x2mdx/wallet-gateway-openrpc/source-artifacts.json

@@ -5,6 +5,10 @@
   "tag_prefix": "@canton-network/wallet-gateway-remote@",
   "min_version": "0.24.0",
   "publish_version": "0.25.0",
+  "versions": [
+    "0.24.0",
+    "0.25.0"
+  ],
   "specs": [
     {
       "spec_id": "dapp-api",

scripts/generate_canton_protobuf_history.py

@@ -124,6 +124,15 @@ def load_excluded_versions(source_config: dict[str, Any]) -> set[str]:
     return set(configured)
 
 
+def configured_versions(source_config: dict[str, Any]) -> set[str] | None:
+    configured = source_config.get("versions")
+    if configured is None:
+        return None
+    if not isinstance(configured, list) or not all(isinstance(item, str) and item for item in configured):
+        raise ValueError("Source config versions must be a list of non-empty strings")
+    return set(configured)
+
+
 def run(args: list[str], *, cwd: Path | None = None, capture: bool = False) -> str:
     kwargs: dict[str, Any] = {
         "cwd": str(cwd) if cwd else None,
@@ -627,7 +636,7 @@ def main() -> int:
     if not isinstance(bundle_proto_dir, str) or not bundle_proto_dir:
         raise ValueError("Source config must define bundle_proto_dir")
 
-    include_versions = set(args.version) if args.version else None
+    include_versions = set(args.version) if args.version else configured_versions(source_config)
     excluded_versions = load_excluded_versions(source_config)
     if include_versions is not None:
         include_versions -= excluded_versions

scripts/generate_grpc_ledger_api_reference.py

@@ -92,6 +92,15 @@ def package_prefixes(source_config: dict[str, Any]) -> tuple[str, ...]:
     return tuple(configured)
 
 
+def configured_versions(source_config: dict[str, Any]) -> set[str] | None:
+    configured = source_config.get("versions")
+    if configured is None:
+        return None
+    if not isinstance(configured, list) or not all(isinstance(item, str) and item for item in configured):
+        raise ValueError("Source config versions must be a list of non-empty strings")
+    return set(configured)
+
+
 def package_matches(package_name: str, *, prefixes: tuple[str, ...]) -> bool:
     return any(package_name.startswith(prefix) for prefix in prefixes)
 
@@ -630,7 +639,7 @@ def main() -> int:
     args = parse_args()
     source_config = load_json(Path(args.source_config).resolve())
     prefixes = package_prefixes(source_config)
-    include_versions = set(args.version) if args.version else None
+    include_versions = set(args.version) if args.version else configured_versions(source_config)
     min_version = args.min_version or source_config.get("min_version") or "0.0.0"
     if not isinstance(min_version, str):
         raise ValueError("min_version must be a string")

scripts/generate_network_component_versions.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import argparse
+import copy
 import html
 import json
 import re
@@ -405,6 +406,31 @@ def build_config(existing_config: dict, snapshot: dict) -> dict:
     }
 
 
+def without_generated_at(config: dict) -> dict:
+    comparable = copy.deepcopy(config)
+    generated = comparable.get("_generated")
+    if isinstance(generated, dict):
+        generated.pop("generatedAt", None)
+    return comparable
+
+
+def preserve_generated_at_if_only_timestamp_changed(existing_config: dict, candidate_config: dict) -> dict:
+    existing_generated = existing_config.get("_generated")
+    if not isinstance(existing_generated, dict):
+        return candidate_config
+    existing_generated_at = existing_generated.get("generatedAt")
+    if not isinstance(existing_generated_at, str) or not existing_generated_at:
+        return candidate_config
+    if without_generated_at(existing_config) != without_generated_at(candidate_config):
+        return candidate_config
+
+    stable_config = copy.deepcopy(candidate_config)
+    candidate_generated = stable_config.setdefault("_generated", {})
+    if isinstance(candidate_generated, dict):
+        candidate_generated["generatedAt"] = existing_generated_at
+    return stable_config
+
+
 def write_json(path: Path, payload: dict) -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     path.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8")
@@ -418,7 +444,10 @@ def main() -> int:
     args = parse_args()
     existing_config = read_existing_config(DEFAULT_REPO_CONFIG_OUTPUT)
     snapshot = collect_snapshot(args.timeout)
-    repo_version_config = build_config(existing_config, snapshot)
+    repo_version_config = preserve_generated_at_if_only_timestamp_changed(
+        existing_config,
+        build_config(existing_config, snapshot),
+    )
 
     if args.dry_run:
         json.dump(repo_version_config, sys.stdout, indent=2)