From a844f19bda39f08ea82f40522c9e16fdb6492a3f Mon Sep 17 00:00:00 2001 From: masonxhuang Date: Sun, 28 Jun 2026 22:24:01 +0800 Subject: [PATCH 1/2] fix(i18n): recover translation CI failures --- .github/actionlint.yaml | 8 + .github/scripts/i18n/plan_full.py | 103 +++++++-- .github/scripts/i18n/summarize_full.py | 17 +- .../scripts/i18n/tests/test_i18n_scripts.py | 131 ++++++++++- .github/workflows/r2-pages.yml | 1 + .github/workflows/translate-all.yml | 208 ++++++++++++++++-- .github/workflows/translate-incremental.yml | 6 + .../translate-locale-finalize-reusable.yml | 146 ++++++++++++ docs/.i18n/translation-ci-temporary-todo.md | 28 +++ 9 files changed, 600 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/translate-locale-finalize-reusable.yml create mode 100644 docs/.i18n/translation-ci-temporary-todo.md diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 517a206b76..795a64957e 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -3,3 +3,11 @@ self-hosted-runner: - crabbox - openclaw - docs + +paths: + .github/workflows/r2-pages.yml: + ignore: + # GitHub added concurrency `queue: max` on 2026-05-07, but actionlint + # 1.7.11 has not learned the key yet. Keep this scoped to R2 Pages. + # https://github.blog/changelog/2026-05-07-github-actions-concurrency-groups-now-allow-larger-queues/ + - 'unexpected key "queue" for "concurrency" section' diff --git a/.github/scripts/i18n/plan_full.py b/.github/scripts/i18n/plan_full.py index 88263a27e7..785ef3e425 100644 --- a/.github/scripts/i18n/plan_full.py +++ b/.github/scripts/i18n/plan_full.py @@ -9,12 +9,16 @@ Parameters: --target-locale: Locale slug/name to rerun, or all. Default: TARGET_LOCALE/all. --batch-size: Maximum locales per follow-up batch. Default: 4. + --docs-root: Docs directory used to size full-translation shards. Default: docs. + --target-docs-per-shard: Desired source documents per shard. Default: 250. + --max-shards: Maximum shards per locale. Default: 4. Outputs: - GITHUB_OUTPUT receives locale_count, canary_locale, canary_locale_slug, - selected_locales, and batch_1 through batch_6 JSON matrices. The step summary - records the selected canary and batch count. Exits non-zero for unknown - locales or oversized batch requests. + GITHUB_OUTPUT receives locale_count, canary locale fields, selected_locales, + shard_total, and batch_1 through batch_6 JSON matrices. Each batch exposes a + shard matrix for translation jobs and a locale matrix for one finalizer per + locale. The step summary records the selected canary and batch count. Exits + non-zero for unknown locales or oversized batch requests. Examples: python .github/scripts/i18n/plan_full.py --target-locale all @@ -25,13 +29,18 @@ import argparse import json +import math import os +import re from dataclasses import dataclass from pathlib import Path MAX_BATCHES = 6 DEFAULT_BATCH_SIZE = 4 +DEFAULT_TARGET_DOCS_PER_SHARD = 250 +DEFAULT_MAX_SHARDS = 4 +LOCALE_DIR_RE = re.compile(r"^[a-z]{2,3}(?:-[A-Za-z0-9]{2,8})?$") LOCALES: tuple[tuple[str, str], ...] = ( ("zh-CN", "zh-cn"), ("zh-TW", "zh-tw"), @@ -64,6 +73,53 @@ class Locale: def matrix_item(self) -> dict[str, str]: return {"locale": self.locale, "locale_slug": self.locale_slug} + def matrix_item_with_shards(self, shard_total: int) -> dict[str, str]: + item = self.matrix_item() + item["shard_total"] = str(shard_total) + return item + + +def is_locale_dir(path: Path) -> bool: + return path.is_dir() and LOCALE_DIR_RE.match(path.name) is not None and (path / ".i18n" / "README.md").exists() + + +def source_doc_count(docs_root: Path) -> int: + locale_dirs = {path.name for path in docs_root.iterdir() if is_locale_dir(path)} + count = 0 + for path in docs_root.rglob("*"): + if not path.is_file() or path.suffix.lower() not in {".md", ".mdx"}: + continue + rel = path.relative_to(docs_root) + parts = rel.parts + if parts and parts[0] in locale_dirs: + continue + rel_posix = rel.as_posix() + if rel_posix.startswith(".i18n/") or rel_posix.startswith(".generated/"): + continue + count += 1 + return count + + +def shard_total_for_doc_count(doc_count: int, target_docs_per_shard: int, max_shards: int) -> int: + if target_docs_per_shard < 1: + raise SystemExit(f"invalid target docs per shard: {target_docs_per_shard}") + if max_shards < 1: + raise SystemExit(f"invalid max shards: {max_shards}") + return max(1, min(max_shards, math.ceil(doc_count / target_docs_per_shard))) + + +def expand_shards(batch: list[Locale], shard_total: int) -> list[dict[str, str]]: + return [ + { + "locale": locale.locale, + "locale_slug": locale.locale_slug, + "shard_index": str(shard_index), + "shard_total": str(shard_total), + } + for locale in batch + for shard_index in range(shard_total) + ] + def all_locales() -> list[Locale]: return [Locale(locale, slug) for locale, slug in LOCALES] @@ -94,11 +150,11 @@ def build_batches(selected: list[Locale], batch_size: int) -> list[list[Locale]] return batches -def matrix_json(locales: list[Locale]) -> str: - return json.dumps({"include": [locale.matrix_item() for locale in locales]}, separators=(",", ":")) +def matrix_json(items: list[dict[str, str]]) -> str: + return json.dumps({"include": items}, separators=(",", ":")) -def append_outputs(selected: list[Locale], batches: list[list[Locale]]) -> None: +def append_outputs(selected: list[Locale], batches: list[list[Locale]], shard_total: int, source_docs: int) -> None: output = os.environ.get("GITHUB_OUTPUT") if not output: return @@ -108,13 +164,16 @@ def append_outputs(selected: list[Locale], batches: list[list[Locale]]) -> None: fh.write(f"canary_locale={canary.locale}\n") fh.write(f"canary_locale_slug={canary.locale_slug}\n") fh.write(f"selected_locales={','.join(locale.locale for locale in selected)}\n") + fh.write(f"source_doc_count={source_docs}\n") + fh.write(f"shard_total={shard_total}\n") for index in range(MAX_BATCHES): batch = batches[index] if index < len(batches) else [] fh.write(f"batch_{index + 1}_count={len(batch)}\n") - fh.write(f"batch_{index + 1}={matrix_json(batch)}\n") + fh.write(f"batch_{index + 1}={matrix_json(expand_shards(batch, shard_total))}\n") + fh.write(f"batch_{index + 1}_locales={matrix_json([locale.matrix_item_with_shards(shard_total) for locale in batch])}\n") -def append_summary(target_locale: str, selected: list[Locale], batches: list[list[Locale]]) -> None: +def append_summary(target_locale: str, selected: list[Locale], batches: list[list[Locale]], shard_total: int, source_docs: int) -> None: summary = os.environ.get("GITHUB_STEP_SUMMARY") if not summary: return @@ -123,19 +182,32 @@ def append_summary(target_locale: str, selected: list[Locale], batches: list[lis fh.write(f"- requested target: `{normalize_target(target_locale)}`\n") fh.write(f"- selected locales: `{', '.join(locale.locale for locale in selected)}`\n") fh.write(f"- canary locale: `{selected[0].locale}`\n") + fh.write(f"- source docs: `{source_docs}`\n") + fh.write(f"- shards per locale: `{shard_total}`\n") for index, batch in enumerate(batches, start=1): fh.write(f"- batch {index}: `{', '.join(locale.locale for locale in batch)}`\n") -def plan_full(target_locale: str, batch_size: int) -> dict[str, object]: +def plan_full( + target_locale: str, + batch_size: int, + docs_root: Path | None = None, + target_docs_per_shard: int = DEFAULT_TARGET_DOCS_PER_SHARD, + max_shards: int = DEFAULT_MAX_SHARDS, +) -> dict[str, object]: selected = select_locales(target_locale) batches = build_batches(selected, batch_size) - append_outputs(selected, batches) - append_summary(target_locale, selected, batches) + source_docs = source_doc_count(docs_root or Path("docs")) + shard_total = shard_total_for_doc_count(source_docs, target_docs_per_shard, max_shards) + append_outputs(selected, batches, shard_total, source_docs) + append_summary(target_locale, selected, batches, shard_total, source_docs) return { "selected": [locale.matrix_item() for locale in selected], "canary": selected[0].matrix_item(), - "batches": [[locale.matrix_item() for locale in batch] for batch in batches], + "source_doc_count": source_docs, + "shard_total": shard_total, + "batches": [expand_shards(batch, shard_total) for batch in batches], + "batch_locales": [[locale.matrix_item_with_shards(shard_total) for locale in batch] for batch in batches], } @@ -153,12 +225,15 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--target-locale", default=os.environ.get("TARGET_LOCALE", "all")) parser.add_argument("--batch-size", default=DEFAULT_BATCH_SIZE, type=int) + parser.add_argument("--docs-root", default="docs", type=Path) + parser.add_argument("--target-docs-per-shard", default=DEFAULT_TARGET_DOCS_PER_SHARD, type=int) + parser.add_argument("--max-shards", default=DEFAULT_MAX_SHARDS, type=int) return parser.parse_args() def main() -> None: args = parse_args() - plan = plan_full(args.target_locale, args.batch_size) + plan = plan_full(args.target_locale, args.batch_size, args.docs_root, args.target_docs_per_shard, args.max_shards) print(json.dumps(plan, indent=2, sort_keys=True)) diff --git a/.github/scripts/i18n/summarize_full.py b/.github/scripts/i18n/summarize_full.py index 845e8f7885..a13c270e69 100644 --- a/.github/scripts/i18n/summarize_full.py +++ b/.github/scripts/i18n/summarize_full.py @@ -73,9 +73,20 @@ def read_artifacts(artifacts_root: Path) -> tuple[dict[str, dict[str, object]], metadata_only_count += 1 if artifact_role == "canary": continue - existing = artifacts.get(locale) - if existing is None or (not existing.get("failed_reason") and failed_reason): - artifacts[locale] = metadata + existing = artifacts.setdefault( + locale, + { + "locale": locale, + "changed_count": 0, + "deleted_count": 0, + "failed_reason": "", + }, + ) + existing["changed_count"] = int(existing.get("changed_count") or 0) + changed_count + existing["deleted_count"] = int(existing.get("deleted_count") or 0) + deleted_count + if failed_reason: + previous = str(existing.get("failed_reason") or "") + existing["failed_reason"] = "; ".join(part for part in [previous, failed_reason] if part) return artifacts, invalid, artifact_count, metadata_only_count diff --git a/.github/scripts/i18n/tests/test_i18n_scripts.py b/.github/scripts/i18n/tests/test_i18n_scripts.py index fb3c4e120b..48c7ffc482 100644 --- a/.github/scripts/i18n/tests/test_i18n_scripts.py +++ b/.github/scripts/i18n/tests/test_i18n_scripts.py @@ -135,7 +135,7 @@ def test_no_generated_docs_are_part_of_this_migration_diff(self) -> None: stdout=subprocess.PIPE, ).stdout.splitlines() changed_paths = changed + untracked - allowed_docs_paths = {"docs/.i18n/translation-workflow.md"} + allowed_docs_paths = {"docs/.i18n/translation-workflow.md", "docs/.i18n/translation-ci-temporary-todo.md"} generated_docs = [ path for path in changed_paths @@ -192,10 +192,17 @@ def test_full_workflow_gates_batches_after_canary(self) -> None: self.assertIn(f"translate-batch-{index}:", text) self.assertIn("needs.translate-canary.result == 'success'", text) self.assertIn("inputs.canary_only != true", text) + for index in range(1, 6): + self.assertIn(f"needs.finalize-batch-{index}.result == 'success'", text) self.assertIn("artifact_role: canary", text) self.assertIn("canary_source_path: channels/line.md", text) self.assertIn("canary_live_path: channels/line", text) self.assertIn("canary_expected_h1: LINE", text) + self.assertIn("batch_1_locales:", text) + self.assertIn("shard_index: ${{ matrix.shard_index }}", text) + self.assertIn("shard_total: ${{ matrix.shard_total }}", text) + self.assertIn("commit_locale: false", text) + self.assertIn("translate-locale-finalize-reusable.yml", text) self.assertRegex(text, r"translate-canary:[\s\S]*?artifact_role: canary[\s\S]*?commit_locale: true") self.assertIn("inputs.commit_locale || inputs.artifact_role == 'canary'", reusable) self.assertIn("inputs.artifact_role == 'canary' || steps.apply.outputs.changed_count != '0'", reusable) @@ -224,12 +231,20 @@ def test_full_workflow_gates_batches_after_canary(self) -> None: self.assertIn('echo "I18N_SCRIPT_DIR=${I18N_SCRIPT_DIR}" >> "$GITHUB_ENV"', finalize_reusable) self.assertIn("ref: ${{ github.workflow_sha }}", finalize_reusable) self.assertIn('python "${I18N_SCRIPT_DIR}/dispatch_r2_pages.py"', finalize_reusable) + locale_finalize_reusable = (REPO_ROOT / ".github/workflows/translate-locale-finalize-reusable.yml").read_text(encoding="utf-8") + self.assertIn("Download locale shard artifacts", locale_finalize_reusable) + self.assertIn('python "${I18N_SCRIPT_DIR}/apply_artifacts.py"', locale_finalize_reusable) + self.assertIn('python "${I18N_SCRIPT_DIR}/commit_locale_artifact.py"', locale_finalize_reusable) + self.assertIn('python "${I18N_SCRIPT_DIR}/dispatch_r2_pages.py"', locale_finalize_reusable) self.assertIn("provider-preflight:", text) self.assertIn("Translate Full completed with failed or cancelled work", text) r2_pages = (REPO_ROOT / ".github/workflows/r2-pages.yml").read_text(encoding="utf-8") + actionlint_config = (REPO_ROOT / ".github/actionlint.yaml").read_text(encoding="utf-8") self.assertIn("- locale", r2_pages) self.assertIn("- page", r2_pages) - self.assertRegex(r2_pages, r"group: r2-pages\s+cancel-in-progress: false") + self.assertRegex(r2_pages, r"group: r2-pages\s+queue: max\s+cancel-in-progress: false") + self.assertIn(".github/workflows/r2-pages.yml:", actionlint_config) + self.assertIn('unexpected key "queue" for "concurrency" section', actionlint_config) self.assertIn("run-name: R2 Pages", r2_pages) self.assertIn("request_id:", r2_pages) self.assertIn("Fail stale scoped translation deploy", r2_pages) @@ -251,19 +266,41 @@ def test_prepare_path_selection_matches_incremental_rules(self) -> None: self.assertFalse(prepare.incremental_should_translate_paths(["docs/.i18n/glossary.fr.json"])) self.assertTrue(prepare.incremental_should_translate_paths(["docs/.i18n/glossary.fr.json", "docs/guide/setup.mdx"])) + def test_incremental_workflow_schedules_all_expected_finalizer_locales(self) -> None: + text = (REPO_ROOT / ".github/workflows/translate-incremental.yml").read_text(encoding="utf-8") + + for locale, slug in apply_artifacts.parse_expected(apply_artifacts.DEFAULT_EXPECTED_LOCALES).items(): + self.assertIn(f"locale: {slug}", text) + self.assertIn(f"locale_slug: {locale}", text) + self.assertIn(f'!docs/{slug}/**', text) + def test_full_plan_all_uses_canary_and_small_batches(self) -> None: - result = plan_full.plan_full("all", 4) + result = plan_full.plan_full("all", 4, FIXTURES / "pending-docs" / "docs") self.assertEqual("zh-CN", result["canary"]["locale"]) self.assertEqual(5, len(result["batches"])) - self.assertLessEqual(max(len(batch) for batch in result["batches"]), 4) + self.assertEqual(1, result["shard_total"]) + self.assertLessEqual(max(len(batch) for batch in result["batch_locales"]), 4) self.assertEqual(20, sum(len(batch) for batch in result["batches"])) + def test_full_plan_shards_large_batches_without_increasing_locale_batch_size(self) -> None: + result = plan_full.plan_full("ru", 4, FIXTURES / "pending-docs" / "docs", target_docs_per_shard=1, max_shards=4) + + self.assertEqual(2, result["shard_total"]) + self.assertEqual( + [ + {"locale": "ru", "locale_slug": "ru", "shard_index": "0", "shard_total": "2"}, + {"locale": "ru", "locale_slug": "ru", "shard_index": "1", "shard_total": "2"}, + ], + result["batches"][0], + ) + self.assertEqual([[{"locale": "ru", "locale_slug": "ru", "shard_total": "2"}]], result["batch_locales"]) + def test_full_plan_manual_single_locale_only_selects_target(self) -> None: - result = plan_full.plan_full("fr", 3) + result = plan_full.plan_full("fr", 3, FIXTURES / "pending-docs" / "docs") self.assertEqual({"locale": "fr", "locale_slug": "fr"}, result["canary"]) - self.assertEqual([[{"locale": "fr", "locale_slug": "fr"}]], result["batches"]) + self.assertEqual([[{"locale": "fr", "locale_slug": "fr", "shard_index": "0", "shard_total": "1"}]], result["batches"]) with self.assertRaises(SystemExit): - plan_full.plan_full("xx", 3) + plan_full.plan_full("xx", 3, FIXTURES / "pending-docs" / "docs") def test_provider_preflight_classifies_key_model_and_quota_failures(self) -> None: self.assertEqual((False, "invalid_key", "OpenAI rejected the translation API key"), provider_preflight.classify_response(401, "{}")) @@ -1004,6 +1041,33 @@ def test_full_summary_ignores_canary_as_locale_success_and_reports_missing(self) self.assertEqual([], summary.successful) self.assertEqual(["fr: no artifact"], summary.skipped) + def test_full_summary_aggregates_locale_shard_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + artifacts = Path(tmp) + for index, changed_count in enumerate([2, 3]): + self._write_artifact( + artifacts, + f"fr-s{index}of2", + metadata={ + "artifact_role": "locale", + "failed_reason": "", + "locale": "fr", + "locale_slug": "fr", + "mode": "full", + "shard_index": index, + "shard_total": 2, + "source_sha": "source-a", + "changed_count": changed_count, + "deleted_count": 1, + }, + ) + + summary = summarize_full.summarize_full(["fr"], artifacts, "success", "success") + + self.assertEqual(["fr: changed=5 deleted=2"], summary.successful) + self.assertEqual([], summary.failed) + self.assertEqual([], summary.skipped) + def test_apply_artifacts_applies_normal_fixture(self) -> None: with tempfile.TemporaryDirectory() as tmp: repo = self._repo_with_source(tmp) @@ -1046,6 +1110,59 @@ def test_apply_artifacts_applies_normal_fixture(self) -> None: self.assertTrue((repo / "docs/fr/index.md").exists()) self.assertIn("Index FR", (repo / "docs/fr/index.md").read_text(encoding="utf-8")) + def test_apply_artifacts_applies_all_locale_shards_together(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + repo = self._repo_with_source(tmp) + (repo / "docs/guide").mkdir() + (repo / "docs/guide/setup.md").write_text("# Setup\n", encoding="utf-8") + run_git(repo, "add", ".") + run_git(repo, "commit", "-m", "add source") + artifacts = repo / ".openclaw-sync/i18n-artifacts" + self._write_artifact( + artifacts, + "fr-s0of2", + metadata={ + "failed_reason": "", + "locale": "fr", + "locale_slug": "fr", + "mode": "full", + "shard_index": 0, + "shard_total": 2, + "source_sha": "source-a", + }, + changed=["docs/fr/index.md"], + payload={"docs/fr/index.md": "# Index FR\n"}, + ) + self._write_artifact( + artifacts, + "fr-s1of2", + metadata={ + "failed_reason": "", + "locale": "fr", + "locale_slug": "fr", + "mode": "full", + "shard_index": 1, + "shard_total": 2, + "source_sha": "source-a", + }, + changed=["docs/fr/guide/setup.md"], + payload={"docs/fr/guide/setup.md": "# Setup FR\n"}, + ) + + with chdir(repo): + result = apply_artifacts.apply_artifacts( + source_sha="source-a", + mode="full", + shard_total=2, + expected_locales="fr=fr", + artifacts_root=artifacts, + skip_checkout_main=True, + ) + + self.assertEqual(0, result["incomplete_count"]) + self.assertIn("Index FR", (repo / "docs/fr/index.md").read_text(encoding="utf-8")) + self.assertIn("Setup FR", (repo / "docs/fr/guide/setup.md").read_text(encoding="utf-8")) + def test_apply_artifacts_reports_missing_metadata_fixture(self) -> None: with tempfile.TemporaryDirectory() as tmp: repo = self._repo_with_source(tmp) diff --git a/.github/workflows/r2-pages.yml b/.github/workflows/r2-pages.yml index caf3affed6..e28ff5baa3 100644 --- a/.github/workflows/r2-pages.yml +++ b/.github/workflows/r2-pages.yml @@ -58,6 +58,7 @@ permissions: concurrency: group: r2-pages + queue: max cancel-in-progress: false jobs: diff --git a/.github/workflows/translate-all.yml b/.github/workflows/translate-all.yml index 6beba6733c..cfaa29cb14 100644 --- a/.github/workflows/translate-all.yml +++ b/.github/workflows/translate-all.yml @@ -94,18 +94,26 @@ jobs: canary_locale: ${{ steps.plan.outputs.canary_locale }} canary_locale_slug: ${{ steps.plan.outputs.canary_locale_slug }} selected_locales: ${{ steps.plan.outputs.selected_locales }} + source_doc_count: ${{ steps.plan.outputs.source_doc_count }} + shard_total: ${{ steps.plan.outputs.shard_total }} batch_1_count: ${{ steps.plan.outputs.batch_1_count }} batch_1: ${{ steps.plan.outputs.batch_1 }} + batch_1_locales: ${{ steps.plan.outputs.batch_1_locales }} batch_2_count: ${{ steps.plan.outputs.batch_2_count }} batch_2: ${{ steps.plan.outputs.batch_2 }} + batch_2_locales: ${{ steps.plan.outputs.batch_2_locales }} batch_3_count: ${{ steps.plan.outputs.batch_3_count }} batch_3: ${{ steps.plan.outputs.batch_3 }} + batch_3_locales: ${{ steps.plan.outputs.batch_3_locales }} batch_4_count: ${{ steps.plan.outputs.batch_4_count }} batch_4: ${{ steps.plan.outputs.batch_4 }} + batch_4_locales: ${{ steps.plan.outputs.batch_4_locales }} batch_5_count: ${{ steps.plan.outputs.batch_5_count }} batch_5: ${{ steps.plan.outputs.batch_5 }} + batch_5_locales: ${{ steps.plan.outputs.batch_5_locales }} batch_6_count: ${{ steps.plan.outputs.batch_6_count }} batch_6: ${{ steps.plan.outputs.batch_6 }} + batch_6_locales: ${{ steps.plan.outputs.batch_6_locales }} steps: - name: Check out uses: actions/checkout@v6 @@ -185,14 +193,35 @@ jobs: publish_ref: ${{ needs.prepare.outputs.publish_ref }} source_sha: ${{ needs.prepare.outputs.source_sha }} mode: ${{ needs.prepare.outputs.mode }} - shard_index: "0" - shard_total: "1" + shard_index: ${{ matrix.shard_index }} + shard_total: ${{ matrix.shard_total }} worker_parallel: "3" thinking_effort: "medium" pending_limit: "0" artifact_prefix: i18n artifact_role: locale - commit_locale: true + commit_locale: false + secrets: inherit + + finalize-batch-1: + name: Finalize full batch 1 + needs: + - prepare + - plan + - translate-batch-1 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-batch-1.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_1_count != '0' + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJSON(needs.plan.outputs.batch_1_locales) }} + uses: ./.github/workflows/translate-locale-finalize-reusable.yml + with: + locale: ${{ matrix.locale }} + locale_slug: ${{ matrix.locale_slug }} + source_sha: ${{ needs.prepare.outputs.source_sha }} + mode: ${{ needs.prepare.outputs.mode }} + shard_total: ${{ matrix.shard_total }} + artifact_prefix: i18n secrets: inherit translate-batch-2: @@ -202,7 +231,8 @@ jobs: - plan - translate-canary - translate-batch-1 - if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-canary.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_2_count != '0' + - finalize-batch-1 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-canary.result == 'success' && needs.finalize-batch-1.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_2_count != '0' strategy: fail-fast: false max-parallel: 3 @@ -214,14 +244,35 @@ jobs: publish_ref: ${{ needs.prepare.outputs.publish_ref }} source_sha: ${{ needs.prepare.outputs.source_sha }} mode: ${{ needs.prepare.outputs.mode }} - shard_index: "0" - shard_total: "1" + shard_index: ${{ matrix.shard_index }} + shard_total: ${{ matrix.shard_total }} worker_parallel: "3" thinking_effort: "medium" pending_limit: "0" artifact_prefix: i18n artifact_role: locale - commit_locale: true + commit_locale: false + secrets: inherit + + finalize-batch-2: + name: Finalize full batch 2 + needs: + - prepare + - plan + - translate-batch-2 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-batch-2.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_2_count != '0' + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJSON(needs.plan.outputs.batch_2_locales) }} + uses: ./.github/workflows/translate-locale-finalize-reusable.yml + with: + locale: ${{ matrix.locale }} + locale_slug: ${{ matrix.locale_slug }} + source_sha: ${{ needs.prepare.outputs.source_sha }} + mode: ${{ needs.prepare.outputs.mode }} + shard_total: ${{ matrix.shard_total }} + artifact_prefix: i18n secrets: inherit translate-batch-3: @@ -231,7 +282,8 @@ jobs: - plan - translate-canary - translate-batch-2 - if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-canary.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_3_count != '0' + - finalize-batch-2 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-canary.result == 'success' && needs.finalize-batch-2.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_3_count != '0' strategy: fail-fast: false max-parallel: 3 @@ -243,14 +295,35 @@ jobs: publish_ref: ${{ needs.prepare.outputs.publish_ref }} source_sha: ${{ needs.prepare.outputs.source_sha }} mode: ${{ needs.prepare.outputs.mode }} - shard_index: "0" - shard_total: "1" + shard_index: ${{ matrix.shard_index }} + shard_total: ${{ matrix.shard_total }} worker_parallel: "3" thinking_effort: "medium" pending_limit: "0" artifact_prefix: i18n artifact_role: locale - commit_locale: true + commit_locale: false + secrets: inherit + + finalize-batch-3: + name: Finalize full batch 3 + needs: + - prepare + - plan + - translate-batch-3 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-batch-3.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_3_count != '0' + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJSON(needs.plan.outputs.batch_3_locales) }} + uses: ./.github/workflows/translate-locale-finalize-reusable.yml + with: + locale: ${{ matrix.locale }} + locale_slug: ${{ matrix.locale_slug }} + source_sha: ${{ needs.prepare.outputs.source_sha }} + mode: ${{ needs.prepare.outputs.mode }} + shard_total: ${{ matrix.shard_total }} + artifact_prefix: i18n secrets: inherit translate-batch-4: @@ -260,7 +333,8 @@ jobs: - plan - translate-canary - translate-batch-3 - if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-canary.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_4_count != '0' + - finalize-batch-3 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-canary.result == 'success' && needs.finalize-batch-3.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_4_count != '0' strategy: fail-fast: false max-parallel: 3 @@ -272,14 +346,35 @@ jobs: publish_ref: ${{ needs.prepare.outputs.publish_ref }} source_sha: ${{ needs.prepare.outputs.source_sha }} mode: ${{ needs.prepare.outputs.mode }} - shard_index: "0" - shard_total: "1" + shard_index: ${{ matrix.shard_index }} + shard_total: ${{ matrix.shard_total }} worker_parallel: "3" thinking_effort: "medium" pending_limit: "0" artifact_prefix: i18n artifact_role: locale - commit_locale: true + commit_locale: false + secrets: inherit + + finalize-batch-4: + name: Finalize full batch 4 + needs: + - prepare + - plan + - translate-batch-4 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-batch-4.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_4_count != '0' + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJSON(needs.plan.outputs.batch_4_locales) }} + uses: ./.github/workflows/translate-locale-finalize-reusable.yml + with: + locale: ${{ matrix.locale }} + locale_slug: ${{ matrix.locale_slug }} + source_sha: ${{ needs.prepare.outputs.source_sha }} + mode: ${{ needs.prepare.outputs.mode }} + shard_total: ${{ matrix.shard_total }} + artifact_prefix: i18n secrets: inherit translate-batch-5: @@ -289,7 +384,8 @@ jobs: - plan - translate-canary - translate-batch-4 - if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-canary.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_5_count != '0' + - finalize-batch-4 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-canary.result == 'success' && needs.finalize-batch-4.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_5_count != '0' strategy: fail-fast: false max-parallel: 3 @@ -301,14 +397,35 @@ jobs: publish_ref: ${{ needs.prepare.outputs.publish_ref }} source_sha: ${{ needs.prepare.outputs.source_sha }} mode: ${{ needs.prepare.outputs.mode }} - shard_index: "0" - shard_total: "1" + shard_index: ${{ matrix.shard_index }} + shard_total: ${{ matrix.shard_total }} worker_parallel: "3" thinking_effort: "medium" pending_limit: "0" artifact_prefix: i18n artifact_role: locale - commit_locale: true + commit_locale: false + secrets: inherit + + finalize-batch-5: + name: Finalize full batch 5 + needs: + - prepare + - plan + - translate-batch-5 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-batch-5.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_5_count != '0' + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJSON(needs.plan.outputs.batch_5_locales) }} + uses: ./.github/workflows/translate-locale-finalize-reusable.yml + with: + locale: ${{ matrix.locale }} + locale_slug: ${{ matrix.locale_slug }} + source_sha: ${{ needs.prepare.outputs.source_sha }} + mode: ${{ needs.prepare.outputs.mode }} + shard_total: ${{ matrix.shard_total }} + artifact_prefix: i18n secrets: inherit translate-batch-6: @@ -318,7 +435,8 @@ jobs: - plan - translate-canary - translate-batch-5 - if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-canary.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_6_count != '0' + - finalize-batch-5 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-canary.result == 'success' && needs.finalize-batch-5.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_6_count != '0' strategy: fail-fast: false max-parallel: 3 @@ -330,14 +448,35 @@ jobs: publish_ref: ${{ needs.prepare.outputs.publish_ref }} source_sha: ${{ needs.prepare.outputs.source_sha }} mode: ${{ needs.prepare.outputs.mode }} - shard_index: "0" - shard_total: "1" + shard_index: ${{ matrix.shard_index }} + shard_total: ${{ matrix.shard_total }} worker_parallel: "3" thinking_effort: "medium" pending_limit: "0" artifact_prefix: i18n artifact_role: locale - commit_locale: true + commit_locale: false + secrets: inherit + + finalize-batch-6: + name: Finalize full batch 6 + needs: + - prepare + - plan + - translate-batch-6 + if: always() && needs.prepare.outputs.should_translate == 'true' && needs.translate-batch-6.result == 'success' && inputs.canary_only != true && needs.plan.outputs.batch_6_count != '0' + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJSON(needs.plan.outputs.batch_6_locales) }} + uses: ./.github/workflows/translate-locale-finalize-reusable.yml + with: + locale: ${{ matrix.locale }} + locale_slug: ${{ matrix.locale_slug }} + source_sha: ${{ needs.prepare.outputs.source_sha }} + mode: ${{ needs.prepare.outputs.mode }} + shard_total: ${{ matrix.shard_total }} + artifact_prefix: i18n secrets: inherit full-status: @@ -353,6 +492,12 @@ jobs: - translate-batch-4 - translate-batch-5 - translate-batch-6 + - finalize-batch-1 + - finalize-batch-2 + - finalize-batch-3 + - finalize-batch-4 + - finalize-batch-5 + - finalize-batch-6 if: always() && needs.prepare.outputs.should_translate == 'true' runs-on: ubuntu-latest steps: @@ -379,6 +524,12 @@ jobs: BATCH_4_RESULT: ${{ needs.translate-batch-4.result }} BATCH_5_RESULT: ${{ needs.translate-batch-5.result }} BATCH_6_RESULT: ${{ needs.translate-batch-6.result }} + FINALIZE_1_RESULT: ${{ needs.finalize-batch-1.result }} + FINALIZE_2_RESULT: ${{ needs.finalize-batch-2.result }} + FINALIZE_3_RESULT: ${{ needs.finalize-batch-3.result }} + FINALIZE_4_RESULT: ${{ needs.finalize-batch-4.result }} + FINALIZE_5_RESULT: ${{ needs.finalize-batch-5.result }} + FINALIZE_6_RESULT: ${{ needs.finalize-batch-6.result }} run: | set -euo pipefail @@ -396,6 +547,12 @@ jobs: echo "- batch 4: \`${BATCH_4_RESULT}\`" echo "- batch 5: \`${BATCH_5_RESULT}\`" echo "- batch 6: \`${BATCH_6_RESULT}\`" + echo "- finalize batch 1: \`${FINALIZE_1_RESULT}\`" + echo "- finalize batch 2: \`${FINALIZE_2_RESULT}\`" + echo "- finalize batch 3: \`${FINALIZE_3_RESULT}\`" + echo "- finalize batch 4: \`${FINALIZE_4_RESULT}\`" + echo "- finalize batch 5: \`${FINALIZE_5_RESULT}\`" + echo "- finalize batch 6: \`${FINALIZE_6_RESULT}\`" } >> "$GITHUB_STEP_SUMMARY" python .github/scripts/i18n/summarize_full.py \ @@ -403,7 +560,10 @@ jobs: --provider-result "${PROVIDER_RESULT}" \ --canary-result "${CANARY_RESULT}" - for result in "${PROVIDER_RESULT}" "${CANARY_RESULT}" "${BATCH_1_RESULT}" "${BATCH_2_RESULT}" "${BATCH_3_RESULT}" "${BATCH_4_RESULT}" "${BATCH_5_RESULT}" "${BATCH_6_RESULT}"; do + for result in \ + "${PROVIDER_RESULT}" "${CANARY_RESULT}" \ + "${BATCH_1_RESULT}" "${BATCH_2_RESULT}" "${BATCH_3_RESULT}" "${BATCH_4_RESULT}" "${BATCH_5_RESULT}" "${BATCH_6_RESULT}" \ + "${FINALIZE_1_RESULT}" "${FINALIZE_2_RESULT}" "${FINALIZE_3_RESULT}" "${FINALIZE_4_RESULT}" "${FINALIZE_5_RESULT}" "${FINALIZE_6_RESULT}"; do case "${result}" in success|skipped) ;; diff --git a/.github/workflows/translate-incremental.yml b/.github/workflows/translate-incremental.yml index 0cd6de0474..d392cb946e 100644 --- a/.github/workflows/translate-incremental.yml +++ b/.github/workflows/translate-incremental.yml @@ -11,6 +11,7 @@ on: - "!docs/es/**" - "!docs/fa/**" - "!docs/fr/**" + - "!docs/hi/**" - "!docs/id/**" - "!docs/it/**" - "!docs/ja-JP/**" @@ -18,6 +19,7 @@ on: - "!docs/nl/**" - "!docs/pl/**" - "!docs/pt-BR/**" + - "!docs/ru/**" - "!docs/th/**" - "!docs/tr/**" - "!docs/uk/**" @@ -101,6 +103,8 @@ jobs: locale_slug: de - locale: fr locale_slug: fr + - locale: hi + locale_slug: hi - locale: ar locale_slug: ar - locale: it @@ -111,6 +115,8 @@ jobs: locale_slug: nl - locale: fa locale_slug: fa + - locale: ru + locale_slug: ru - locale: tr locale_slug: tr - locale: uk diff --git a/.github/workflows/translate-locale-finalize-reusable.yml b/.github/workflows/translate-locale-finalize-reusable.yml new file mode 100644 index 0000000000..472f426256 --- /dev/null +++ b/.github/workflows/translate-locale-finalize-reusable.yml @@ -0,0 +1,146 @@ +name: Translate Locale Finalize Reusable + +on: + workflow_call: + inputs: + locale: + required: true + type: string + locale_slug: + required: true + type: string + source_sha: + required: true + type: string + mode: + required: true + type: string + shard_total: + required: true + type: string + artifact_prefix: + required: true + type: string + +jobs: + finalize-locale: + name: Finalize ${{ inputs.locale }} locale artifacts + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + actions: write + contents: write + steps: + # Finalizers apply artifacts to latest main, but deploy/commit control + # logic must come from this workflow ref so branch canaries test the fix. + - name: Check out workflow scripts + uses: actions/checkout@v6 + with: + ref: ${{ github.workflow_sha }} + path: .openclaw-sync/workflow-ref + sparse-checkout: .github/scripts/i18n + persist-credentials: false + + - name: Stage workflow scripts + run: | + set -euo pipefail + I18N_SCRIPT_DIR="${RUNNER_TEMP}/openclaw-i18n-scripts" + echo "I18N_SCRIPT_DIR=${I18N_SCRIPT_DIR}" >> "$GITHUB_ENV" + mkdir -p "${I18N_SCRIPT_DIR}" + cp -R .openclaw-sync/workflow-ref/.github/scripts/i18n/. "${I18N_SCRIPT_DIR}/" + + - name: Check out latest main + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Download locale shard artifacts + uses: actions/download-artifact@v8 + with: + pattern: ${{ inputs.artifact_prefix }}-${{ inputs.locale_slug }}-s*of${{ inputs.shard_total }}-${{ inputs.source_sha }} + path: .openclaw-sync/i18n-artifacts + + - name: Apply locale shard artifacts + id: apply + env: + SOURCE_SHA: ${{ inputs.source_sha }} + MODE: ${{ inputs.mode }} + SHARD_TOTAL: ${{ inputs.shard_total }} + EXPECTED_LOCALES: ${{ inputs.locale_slug }}=${{ inputs.locale }} + run: | + set -euo pipefail + + python "${I18N_SCRIPT_DIR}/apply_artifacts.py" \ + --source-sha "${SOURCE_SHA}" \ + --mode "${MODE}" \ + --shard-total "${SHARD_TOTAL}" \ + --expected-locales "${EXPECTED_LOCALES}" + + - name: Set up Node for locale validation + if: steps.apply.outputs.changed_count != '0' + uses: actions/setup-node@v6 + with: + node-version: 24 + cache: npm + + - name: Install docs dependencies + if: steps.apply.outputs.changed_count != '0' + run: npm ci + + - name: Install validation system dependencies + if: steps.apply.outputs.changed_count != '0' + run: sudo apt-get update && sudo apt-get install -y librsvg2-bin + + - name: Install Playwright browser + if: steps.apply.outputs.changed_count != '0' + run: npx playwright install --with-deps chromium + + - name: Check docs before locale finalization + if: steps.apply.outputs.changed_count != '0' + run: npm run docs:check + + - name: Commit locale refresh + id: locale_commit + if: steps.apply.outputs.changed_count != '0' + env: + ARTIFACT_ROLE: locale + BASE_SOURCE_SHA: ${{ steps.apply.outputs.base_source_sha }} + LOCALE: ${{ inputs.locale }} + LOCALE_SLUG: ${{ inputs.locale_slug }} + SHARD_INDEX: "0" + SHARD_TOTAL: ${{ inputs.shard_total }} + run: | + set -euo pipefail + + python "${I18N_SCRIPT_DIR}/commit_locale_artifact.py" + + - name: Fail uncommitted locale refresh + if: steps.apply.outputs.changed_count != '0' && steps.locale_commit.outputs.committed != 'true' + run: | + { + echo "Locale shard artifacts applied changes but did not commit them." + echo "Failing so the translation status does not report an unpublished refresh as successful." + } >&2 + exit 1 + + - name: Dispatch locale docs deploy + if: steps.locale_commit.outputs.committed == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + + python "${I18N_SCRIPT_DIR}/dispatch_r2_pages.py" \ + --ref "${{ github.ref_name }}" \ + --artifact-scope locale \ + --locale "${{ inputs.locale }}" \ + --no-force-upload + + - name: Fail incomplete locale artifact + if: steps.apply.outputs.incomplete_count != '0' + run: | + { + echo "Locale finalizer found missing or failed shard artifact data." + cat .openclaw-sync/i18n-incomplete-locales.txt + } >&2 + exit 1 diff --git a/docs/.i18n/translation-ci-temporary-todo.md b/docs/.i18n/translation-ci-temporary-todo.md new file mode 100644 index 0000000000..a02c9b44ca --- /dev/null +++ b/docs/.i18n/translation-ci-temporary-todo.md @@ -0,0 +1,28 @@ +# Translation CI Temporary TODO + +Temporary operator note for the current translation CI failures. Remove this file after the workflow fixes land and the affected locales have been recovered. + +## Issues To Fix + +1. Add `queue: max` to the `r2-pages.yml` concurrency group and update tests/documentation to match GitHub's queue semantics. `cancel-in-progress: false` protects the running upload but does not preserve every pending upload; without `queue: max`, rapid locale-scoped dispatches can cancel older pending R2 Pages runs before any upload job starts. +2. Align the incremental locale matrix with the finalizer's expected locale list. The finalizer currently expects `hi` and `ru` artifacts, but `translate-incremental.yml` does not schedule those locales, so an otherwise successful aggregate incremental run can still fail with incomplete locales after committing and publishing successful artifacts. +3. Split large full-translation locales into multiple shards. A `Translate ru shard 0/1` job that runs for about two hours and ends with `Error: The operation was canceled` is a translation job timeout, not an R2 publish queue cancellation. Full runs with `pending_limit: "0"` can produce hundreds of documents for one locale; a single shard with `worker_parallel: "3"` may not finish within the reusable workflow's 120-minute timeout. + - Use a conservative first-pass shard policy: `shard_total = ceil(source_doc_count / 250)`, capped to `1..4`. Source docs are an upper bound for full pending docs, so this avoids under-sharding large locales without adding locale-specific planning state. The observed `ru` case had `696` pending docs, so it should run as `3` shards. + - Keep overall translation concurrency bounded while adding shards: preserve full batch `max-parallel: 3` and `worker_parallel: "3"` unless a separate budget review changes them. Sharding should reduce per-job duration, not increase peak active workers. + - Prefer a locale-level finalizer for sharded full translation. Shard jobs should upload artifacts only; one finalizer for that locale should download every shard artifact, apply them together, run one docs check, push one locale commit, and dispatch one locale-scoped R2 publish. Do not let each shard independently commit and publish the same locale. + +## Post-Fix Operator Steps + +After the workflow fixes land on `main`, publish and recover in this order: + +1. Manually run `R2 Pages` on `main` with `artifact_scope=full` and `force_upload=true` to publish any translation commits that already reached `main`. This only publishes committed docs; it does not recover translations that exist only in old workflow artifacts. +2. Rerun failed or missing locales as targeted `Translate Full` runs instead of rerunning all locales. Use `target_locale=` for each incomplete locale, such as `hi` or `ru`, after the matrix/finalizer locale list has been corrected. +3. Do not rely on rerunning only an old finalizer/commit job after changing workflow code. Reusable workflows execute from the workflow ref of that old run, and finalizers can only apply artifacts that were actually produced and are still available. When in doubt, rerun the affected locale workflow so translation, validation, commit, and R2 publish use the fixed control plane. +4. Watch the dispatched `R2 Pages` runs after the fix: pending R2 uploads should queue instead of being cancelled with a higher-priority waiting-request annotation. A locale job that runs close to two hours and cancels during `docs-i18n` output should be treated as a translation timeout and handled by sharding or a timeout-budget change. + +## Acceptance Checklist + +1. Trigger targeted recovery for every known incomplete language after the fix lands. At minimum, rerun `Translate Full` for `hi` and `ru`; add any other failed or cancelled locale from the latest full/incremental summaries. +2. Confirm each targeted locale produces all expected shard artifacts, the locale-level finalizer commits the translated pages and translation memory, and the commit reaches `main`. +3. Confirm each recovered locale dispatches and completes a locale-scoped `R2 Pages` publish. If a publish is missed but the commit is already on `main`, run a manual full `R2 Pages` publish to push the committed output. +4. Confirm the latest full or incremental workflow summary no longer lists the recovered languages as missing, failed, cancelled, or incomplete. From ad03a84f89c61b92a3e06d77601dec9b2929d0b0 Mon Sep 17 00:00:00 2001 From: Vincent Koc <25068+vincentkoc@users.noreply.github.com> Date: Sun, 28 Jun 2026 10:44:18 -0700 Subject: [PATCH 2/2] fix(i18n): tolerate locales without translation memory --- .github/scripts/i18n/commit_locale_artifact.py | 14 +++++++++++--- .github/scripts/i18n/tests/test_i18n_scripts.py | 11 +++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/.github/scripts/i18n/commit_locale_artifact.py b/.github/scripts/i18n/commit_locale_artifact.py index 7d9a5e348d..6be3470f95 100644 --- a/.github/scripts/i18n/commit_locale_artifact.py +++ b/.github/scripts/i18n/commit_locale_artifact.py @@ -82,12 +82,20 @@ def ensure_base_current(base_source_sha: str, locale: str) -> bool: def has_locale_changes(locale: str) -> bool: result = run( - ["git", "status", "--porcelain", "--untracked-files=all", "--", f"docs/{locale}", f"docs/.i18n/{locale}.tm.jsonl"], + ["git", "status", "--porcelain", "--untracked-files=all", "--", *locale_pathspecs(locale)], check=True, ) return bool(result.stdout.strip()) +def locale_pathspecs(locale: str) -> list[str]: + tm_path = f"docs/.i18n/{locale}.tm.jsonl" + paths = [f"docs/{locale}"] + if Path(tm_path).exists() or run(["git", "ls-files", "--error-unmatch", tm_path], check=False).returncode == 0: + paths.append(tm_path) + return paths + + def pending_allowed(locale: str, locale_slug: str, shard_index: str, shard_total: str) -> set[str]: pending_file = Path(".openclaw-sync") / f"docs-i18n-{locale_slug}-s{shard_index}of{shard_total}.txt" allowed = {f"docs/.i18n/{locale}.tm.jsonl"} @@ -122,7 +130,7 @@ def artifact_allowed(locale: str, artifact_dir: str) -> set[str]: def enforce_canary_scope(locale: str, allowed: set[str]) -> None: - status = git_stdout(["status", "--porcelain", "--untracked-files=all", "--", f"docs/{locale}", f"docs/.i18n/{locale}.tm.jsonl"]) + status = git_stdout(["status", "--porcelain", "--untracked-files=all", "--", *locale_pathspecs(locale)]) changed = {line[3:] for line in status.splitlines() if line.strip()} bad = sorted(path for path in changed if path not in allowed) if bad: @@ -153,7 +161,7 @@ def commit_locale( git_stdout(["config", "user.name", "openclaw-docs-i18n[bot]"]) git_stdout(["config", "user.email", "openclaw-docs-i18n[bot]@users.noreply.github.com"]) - git_stdout(["add", f"docs/{locale}", f"docs/.i18n/{locale}.tm.jsonl"]) + git_stdout(["add", *locale_pathspecs(locale)]) git_stdout(["commit", "-m", f"chore(i18n): refresh {locale} translations"]) for attempt in range(1, attempts + 1): diff --git a/.github/scripts/i18n/tests/test_i18n_scripts.py b/.github/scripts/i18n/tests/test_i18n_scripts.py index 48c7ffc482..7ec09702cb 100644 --- a/.github/scripts/i18n/tests/test_i18n_scripts.py +++ b/.github/scripts/i18n/tests/test_i18n_scripts.py @@ -620,6 +620,17 @@ def test_canary_commit_scope_allows_only_sampled_page_and_tm(self) -> None: allowed = commit_locale_artifact.artifact_allowed("fr", str(artifact)) commit_locale_artifact.enforce_canary_scope("fr", allowed) + def test_locale_pathspecs_allow_new_locale_without_tm(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + repo = Path(tmp) + init_repo(repo) + (repo / "docs/hi").mkdir(parents=True) + (repo / "docs/hi/index.md").write_text("# Hindi\n", encoding="utf-8") + + with chdir(repo): + self.assertEqual(["docs/hi"], commit_locale_artifact.locale_pathspecs("hi")) + self.assertTrue(commit_locale_artifact.has_locale_changes("hi")) + def test_canary_commit_scope_rejects_unrelated_locale_deletes_not_in_artifact(self) -> None: with tempfile.TemporaryDirectory() as tmp: repo = Path(tmp)