diff --git a/.github/workflows/add-submodules.yml b/.github/workflows/add-submodules.yml index 3e1f285..ba81a94 100644 --- a/.github/workflows/add-submodules.yml +++ b/.github/workflows/add-submodules.yml @@ -40,113 +40,12 @@ jobs: SUBMODULES_ORG: ${{ vars.SUBMODULES_ORG }} run: | set -euo pipefail - # shellcheck source=assets/env.sh source "$GITHUB_WORKSPACE/.github/workflows/assets/env.sh" # shellcheck source=assets/lib.sh source "$GITHUB_WORKSPACE/.github/workflows/assets/lib.sh" # shellcheck source=assets/add_submodules.sh source "$GITHUB_WORKSPACE/.github/workflows/assets/add_submodules.sh" - - init_translation_state - init_add_submodule_summary_buckets - - begin_phase "$PHASE_SETUP" "Validate inputs and prepare workspace" - validate_secrets - parse_and_validate_lang_codes - echo "Lang codes: ${lang_codes_arr[*]}" >&2 - - WORK_DIR=$(mktemp -d) - trap 'rm -rf "$WORK_DIR"' EXIT - BOOST_WORK="$WORK_DIR/boost" - TRANS_DIR="$WORK_DIR/translations" - mkdir -p "$BOOST_WORK" - - # Configure git credential helper so all github.com pushes are authenticated - # without embedding tokens in remote URLs. - gh auth setup-git - - # libs_ref and boost_org are read by add_submodules.sh (sourced). - # shellcheck disable=SC2153,SC2034 - libs_ref="${LIBS_REF:?}" - # shellcheck disable=SC2153,SC2034 - boost_org="${BOOST_ORG:?}" - end_phase - - # ── Main ───────────────────────────────────────────────────────────── - - begin_phase "$PHASE_ENSURE_BRANCHES" "Ensure local branches in translations repo" - rc=0 - ensure_translations_cloned "$ORG" "$TRANSLATIONS_REPO" "$TRANS_DIR" || rc=$? - if [[ $rc -eq 0 ]]; then - for lang_code in "${lang_codes_arr[@]}"; do - ensure_local_branch_in_translations "$TRANS_DIR" "$lang_code" || rc=$? - [[ $rc -ne 0 ]] && break - done - fi - end_phase - [[ $rc -ne 0 ]] && exit $rc - - begin_phase "$PHASE_PROCESS_SUBMODULES" "Process submodules" - if [[ -n "${SUBMODULES:-}" ]]; then - mapfile -t submodule_names < <(parse_list "$SUBMODULES") - echo "Using ${#submodule_names[@]} submodules from input." >&2 - else - # Fetch .gitmodules from boostorg/boost at LIBS_REF to discover all libs/ submodules. - echo "Fetching .gitmodules from boostorg/boost at $LIBS_REF..." >&2 - gitmodules_content=$(gh api \ - "repos/boostorg/boost/contents/.gitmodules?ref=$LIBS_REF" \ - -H "Accept: application/vnd.github.v3.raw" 2>/dev/null) || { - phase_err "Failed to fetch .gitmodules" - end_phase - exit 1 - } - mapfile -t submodule_names < <( - echo "$gitmodules_content" \ - | grep '^\s*path\s*=' \ - | sed 's/.*=\s*//' \ - | { grep '^libs/' || true; } \ - | sed 's|^libs/||' - ) - echo "Found ${#submodule_names[@]} libs submodules." >&2 - fi - - [[ ${#submodule_names[@]} -eq 0 ]] && { - echo "No submodules to process, nothing to do." >&2 - end_phase - exit 0 - } - - total=${#submodule_names[@]} - submodule_fatal=0 - for i in "${!submodule_names[@]}"; do - sub="${submodule_names[$i]}" - echo "[$(( i + 1 ))/$total] $sub ..." >&2 - if add_one_submodule "$sub"; then - record_submodule_update "$sub" || true - else - rc=$? - if [[ $rc -eq 2 ]]; then - record_submodule_fatal "$sub" - submodule_fatal=$((submodule_fatal + 1)) - fi - fi - done - - # Buckets filled by add_one_submodule. - print_submodule_processing_summary - [[ $submodule_fatal -gt 0 ]] && \ - phase_err "$submodule_fatal submodule(s) failed with errors." - end_phase - - begin_phase "$PHASE_FINALIZE_TRANSLATIONS" "Finalize translations repo" - finalize_rc=0 - finalize_translations_repo "$TRANS_DIR" "$LIBS_REF" "${lang_codes_arr[@]}" || finalize_rc=$? - end_phase - - exit_rc=0 - [[ $submodule_fatal -gt 0 ]] && exit_rc=1 - [[ $finalize_rc -ne 0 ]] && exit_rc=$finalize_rc - [[ $exit_rc -ne 0 ]] && exit $exit_rc - - echo "Done." >&2 + # shellcheck source=assets/submodule_ops.sh + source "$GITHUB_WORKSPACE/.github/workflows/assets/submodule_ops.sh" + add_submodules_main diff --git a/.github/workflows/assets/submodule_ops.sh b/.github/workflows/assets/submodule_ops.sh new file mode 100644 index 0000000..5a57396 --- /dev/null +++ b/.github/workflows/assets/submodule_ops.sh @@ -0,0 +1,168 @@ +# shellcheck shell=bash +# Shared orchestration for add-submodules and start-translation workflows. +# Source after env.sh and lib.sh; add-submodules also sources add_submodules.sh. +# +# Globals set/consumed: +# WORK_DIR, BOOST_WORK, TRANS_DIR, ORG_WORK (optional) +# submodule_names, submodule_fatal, libs_ref, boost_org +# lang_codes_arr, SUBMODULES, LIBS_REF (env) +# shellcheck disable=SC2034,SC2154 + +# Create temp workspace dirs. Pass "with_org_work" to also set ORG_WORK. +init_translation_work_dirs() { + local with_org_work="${1:-}" + WORK_DIR=$(mktemp -d) + trap 'rm -rf "$WORK_DIR"' EXIT + BOOST_WORK="$WORK_DIR/boost" + TRANS_DIR="$WORK_DIR/translations" + mkdir -p "$BOOST_WORK" "$TRANS_DIR" + if [[ "$with_org_work" == "with_org_work" ]]; then + ORG_WORK="$WORK_DIR/$MODULE_ORG" + mkdir -p "$ORG_WORK" + fi +} + +# Emit libs/ submodule basenames (one per line) from raw .gitmodules content. +libs_submodule_names_from_gitmodules_content() { + local content="$1" + echo "$content" \ + | grep '^\s*path\s*=' \ + | sed 's/.*=\s*//' \ + | { grep '^libs/' || true; } \ + | sed 's|^libs/||' +} + +# Emit libs/ submodule basenames from a .gitmodules file path. +libs_submodule_names_from_gitmodules_file() { + local gitmodules_file="$1" + git config -f "$gitmodules_file" --get-regexp 'submodule\..*\.path' 2>/dev/null \ + | awk '{print $2}' \ + | { grep '^libs/' || true; } \ + | sed 's|^libs/||' +} + +# Fetch boostorg/boost .gitmodules at ref; print raw content. Return 1 on failure. +fetch_boost_gitmodules_at_ref() { + local ref="$1" + gh api \ + "repos/boostorg/boost/contents/.gitmodules?ref=$ref" \ + -H "Accept: application/vnd.github.v3.raw" 2>/dev/null +} + +# Populate global submodule_names from SUBMODULES env or boost .gitmodules at LIBS_REF. +# Return 1 when auto-discovery fetch fails. +resolve_add_submodules_names() { + # shellcheck disable=SC2153 + local libs_ref_for_fetch="${libs_ref:-$LIBS_REF}" + if [[ -n "${SUBMODULES:-}" ]]; then + mapfile -t submodule_names < <(parse_list "$SUBMODULES") + echo "Using ${#submodule_names[@]} submodules from input." >&2 + return 0 + fi + echo "Fetching .gitmodules from boostorg/boost at ${libs_ref_for_fetch}..." >&2 + local gitmodules_content + gitmodules_content=$(fetch_boost_gitmodules_at_ref "$libs_ref_for_fetch") || { + phase_err "Failed to fetch .gitmodules" + return 1 + } + mapfile -t submodule_names < <(libs_submodule_names_from_gitmodules_content "$gitmodules_content") + echo "Found ${#submodule_names[@]} libs submodules." >&2 +} + +# Clone translations repo and ensure local-{lang} branches exist. +ensure_all_translation_lang_branches() { + local rc=0 + ensure_translations_cloned "$ORG" "$TRANSLATIONS_REPO" "$TRANS_DIR" || rc=$? + if [[ $rc -eq 0 ]]; then + local lang_code + for lang_code in "${lang_codes_arr[@]}"; do + ensure_local_branch_in_translations "$TRANS_DIR" "$lang_code" || rc=$? + [[ $rc -ne 0 ]] && break + done + fi + return "$rc" +} + +# Run $1 (function name) for each remaining submodule name; update UPDATES / SUBMODULE_FATAL. +process_submodule_list() { + local processor="$1" + shift + local -a names=("$@") + local total=${#names[@]} i sub rc + submodule_fatal=0 + for i in "${!names[@]}"; do + sub="${names[$i]}" + echo "[$(( i + 1 ))/$total] $sub ..." >&2 + if "$processor" "$sub"; then + record_submodule_update "$sub" || true + else + rc=$? + if [[ $rc -eq 2 ]]; then + record_submodule_fatal "$sub" + submodule_fatal=$((submodule_fatal + 1)) + fi + fi + done + print_submodule_processing_summary + if [[ $submodule_fatal -gt 0 ]]; then + phase_err "$submodule_fatal submodule(s) failed with errors." + fi + return 0 +} + +# Combine submodule_fatal count with finalize_rc; return combined exit code. +combine_batch_and_finalize_rc() { + local finalize_rc="${1:-0}" + local exit_rc=0 + [[ "${submodule_fatal:-0}" -gt 0 ]] && exit_rc=1 + [[ "$finalize_rc" -ne 0 ]] && exit_rc=$finalize_rc + return "$exit_rc" +} + +# add-submodules.yml entry point (sources env.sh, lib.sh, add_submodules.sh before calling). +add_submodules_main() { + local rc finalize_rc exit_rc + + init_translation_state + init_add_submodule_summary_buckets + + begin_phase "$PHASE_SETUP" "Validate inputs and prepare workspace" + validate_secrets + parse_and_validate_lang_codes + echo "Lang codes: ${lang_codes_arr[*]}" >&2 + + init_translation_work_dirs + gh auth setup-git + libs_ref="${LIBS_REF:?}" + boost_org="${BOOST_ORG:?}" + end_phase + + begin_phase "$PHASE_ENSURE_BRANCHES" "Ensure local branches in translations repo" + rc=0 + ensure_all_translation_lang_branches || rc=$? + end_phase + [[ $rc -ne 0 ]] && exit $rc + + begin_phase "$PHASE_PROCESS_SUBMODULES" "Process submodules" + resolve_add_submodules_names || { + end_phase + exit 1 + } + + [[ ${#submodule_names[@]} -eq 0 ]] && { + echo "No submodules to process, nothing to do." >&2 + end_phase + exit 0 + } + + process_submodule_list add_one_submodule "${submodule_names[@]}" + end_phase + + begin_phase "$PHASE_FINALIZE_TRANSLATIONS" "Finalize translations repo" + finalize_rc=0 + finalize_translations_repo "$TRANS_DIR" "$LIBS_REF" "${lang_codes_arr[@]}" || finalize_rc=$? + end_phase + + combine_batch_and_finalize_rc "$finalize_rc" || exit $? + echo "Done." >&2 +} diff --git a/.github/workflows/start-translation.yml b/.github/workflows/start-translation.yml index da18355..f21b860 100644 --- a/.github/workflows/start-translation.yml +++ b/.github/workflows/start-translation.yml @@ -105,6 +105,8 @@ jobs: source "$GITHUB_WORKSPACE/.github/workflows/assets/lib.sh" # shellcheck source=assets/translation.sh source "$GITHUB_WORKSPACE/.github/workflows/assets/translation.sh" + # shellcheck source=assets/submodule_ops.sh + source "$GITHUB_WORKSPACE/.github/workflows/assets/submodule_ops.sh" init_translation_state init_submodule_summary_buckets @@ -112,12 +114,7 @@ jobs: begin_phase "$PHASE_SETUP" "Prepare workspace" validate_secrets - WORK_DIR=$(mktemp -d) - trap 'rm -rf "$WORK_DIR"' EXIT - BOOST_WORK="$WORK_DIR/boost" - TRANS_DIR="$WORK_DIR/translations" - mkdir -p "$BOOST_WORK" - + init_translation_work_dirs with_org_work gh auth setup-git # shellcheck disable=SC2034 @@ -126,19 +123,13 @@ jobs: # shellcheck disable=SC2034 libs_ref="${LIBS_REF:?}" - ORG_WORK="$WORK_DIR/$MODULE_ORG" - mkdir -p "$ORG_WORK" - [[ ! -f .gitmodules ]] && { phase_err ".gitmodules not found (run from translations repo with master checked out)" end_phase exit 1 } - mapfile -t submodule_names < <( - git config -f .gitmodules --get-regexp 'submodule\..*\.path' 2>/dev/null \ - | awk '{print $2}' | { grep '^libs/' || true; } | sed 's|^libs/||' - ) + mapfile -t submodule_names < <(libs_submodule_names_from_gitmodules_file ".gitmodules") [[ ${#submodule_names[@]} -eq 0 ]] && { echo "No libs/ submodules in .gitmodules, nothing to sync." >&2 @@ -158,26 +149,7 @@ jobs: exit $rc fi - total=${#submodule_names[@]} - submodule_fatal=0 - for i in "${!submodule_names[@]}"; do - sub="${submodule_names[$i]}" - echo "[$(( i + 1 ))/$total] $sub ..." >&2 - if sync_one_submodule "$sub"; then - record_submodule_update "$sub" || true - else - rc=$? - if [[ $rc -eq 2 ]]; then - record_submodule_fatal "$sub" - submodule_fatal=$((submodule_fatal + 1)) - fi - fi - done - - # Buckets filled by sync_one_submodule. - print_submodule_processing_summary - [[ $submodule_fatal -gt 0 ]] && \ - phase_err "$submodule_fatal submodule(s) failed with errors." + process_submodule_list sync_one_submodule "${submodule_names[@]}" end_phase begin_phase "$PHASE_FINALIZE_TRANSLATIONS" "Finalize translations master" @@ -185,17 +157,15 @@ jobs: finalize_translations_master "$TRANS_DIR" "$LIBS_REF" || rc=$? end_phase - exit_rc=0 - [[ $submodule_fatal -gt 0 ]] && exit_rc=1 - [[ $rc -ne 0 ]] && exit_rc=$rc - if [[ $exit_rc -eq 0 ]]; then + combine_batch_and_finalize_rc "$rc" || exit_rc=$? + if [[ ${exit_rc:-0} -eq 0 ]]; then updated_json=$(submodule_names_to_json "${UPDATES[@]}") echo "updated_submodules=$updated_json" >> "$GITHUB_OUTPUT" echo "Submodules for start-local: $updated_json" >&2 + echo "Done." >&2 + else + exit "$exit_rc" fi - [[ $exit_rc -ne 0 ]] && exit $exit_rc - - echo "Done." >&2 start-local: # always() still evaluates outputs when sync-mirrors fails; updated_submodules is only set on full success. @@ -243,6 +213,8 @@ jobs: source "$GITHUB_WORKSPACE/.github/workflows/assets/lib.sh" # shellcheck source=assets/translation.sh source "$GITHUB_WORKSPACE/.github/workflows/assets/translation.sh" + # shellcheck source=assets/submodule_ops.sh + source "$GITHUB_WORKSPACE/.github/workflows/assets/submodule_ops.sh" init_translation_state init_submodule_summary_buckets @@ -250,12 +222,7 @@ jobs: begin_phase "$PHASE_SETUP" "Prepare workspace" validate_secrets weblate - WORK_DIR=$(mktemp -d) - trap 'rm -rf "$WORK_DIR"' EXIT - BOOST_WORK="$WORK_DIR/boost" - TRANS_DIR="$WORK_DIR/translations" - mkdir -p "$BOOST_WORK" - + init_translation_work_dirs with_org_work gh auth setup-git # shellcheck disable=SC2034 @@ -263,9 +230,6 @@ jobs: # shellcheck disable=SC2034,SC2153 libs_ref="${LIBS_REF:?}" - ORG_WORK="$WORK_DIR/$MODULE_ORG" - mkdir -p "$ORG_WORK" - echo "Lang code: $LANG_CODE" >&2 # Read by sync_one_submodule in translation.sh (sourced above). @@ -295,26 +259,7 @@ jobs: begin_phase "$PHASE_PROCESS_SUBMODULES" "Process local branches" echo "Processing local branches for ${#submodule_names[@]} submodules." >&2 - total=${#submodule_names[@]} - submodule_fatal=0 - for i in "${!submodule_names[@]}"; do - sub="${submodule_names[$i]}" - echo "[$(( i + 1 ))/$total] $sub ..." >&2 - if sync_one_submodule "$sub"; then - record_submodule_update "$sub" || true - else - rc=$? - if [[ $rc -eq 2 ]]; then - record_submodule_fatal "$sub" - submodule_fatal=$((submodule_fatal + 1)) - fi - fi - done - - # Buckets filled by sync_one_submodule. - print_submodule_processing_summary - [[ $submodule_fatal -gt 0 ]] && \ - phase_err "$submodule_fatal submodule(s) failed with errors." + process_submodule_list sync_one_submodule "${submodule_names[@]}" end_phase begin_phase "$PHASE_FINALIZE_TRANSLATIONS" "Finalize translations local branch" diff --git a/scripts/lint.sh b/scripts/lint.sh index 2d808e1..ce2853a 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -171,6 +171,7 @@ echo "lint: actionlint ${ACTIONLINT_VERSION} ($("$ACTIONLINT_BIN" -version | hea .github/workflows/assets/lib.sh \ .github/workflows/assets/translation.sh \ .github/workflows/assets/add_submodules.sh \ + .github/workflows/assets/submodule_ops.sh \ scripts/*.sh \ tests/helpers/*.bash diff --git a/tests/helpers/common.bash b/tests/helpers/common.bash index 20a21cd..dad96c2 100644 --- a/tests/helpers/common.bash +++ b/tests/helpers/common.bash @@ -32,6 +32,15 @@ load_add_submodules() { source "$ASSETS_DIR/add_submodules.sh" } +load_submodule_ops() { + load_lib + export GITHUB_WORKSPACE="$REPO_ROOT" + # shellcheck source=/dev/null + source "$ASSETS_DIR/add_submodules.sh" + # shellcheck source=/dev/null + source "$ASSETS_DIR/submodule_ops.sh" +} + # Run a function and capture its exit code (works under set -e in callers). run_fn() { local errexit_was_on=0 diff --git a/tests/test_submodule_ops.bats b/tests/test_submodule_ops.bats new file mode 100644 index 0000000..6865bd5 --- /dev/null +++ b/tests/test_submodule_ops.bats @@ -0,0 +1,139 @@ +#!/usr/bin/env bats + +setup() { + # shellcheck source=tests/helpers/common.bash + source "$BATS_TEST_DIRNAME/helpers/common.bash" + load_submodule_ops + init_translation_state + init_submodule_summary_buckets +} + +@test "libs_submodule_names_from_gitmodules_content: extracts libs basenames only" { + local content output + content='[submodule "libs/algorithm"] + path = libs/algorithm +[submodule "tools/quickbook"] + path = tools/quickbook +[submodule "libs/system"] + path = libs/system +' + output=$(libs_submodule_names_from_gitmodules_content "$content") + [ "$output" = $'algorithm\nsystem' ] +} + +@test "libs_submodule_names_from_gitmodules_file: reads paths from file" { + local gitmodules + gitmodules="$(mktemp)" + cat >"$gitmodules" <<'EOF' +[submodule "libs/json"] + path = libs/json +[submodule "other"] + path = other/path +EOF + run libs_submodule_names_from_gitmodules_file "$gitmodules" + rm -f "$gitmodules" + [ "$status" -eq 0 ] + [ "$output" = "json" ] +} + +@test "resolve_add_submodules_names: uses SUBMODULES when set" { + SUBMODULES="algorithm, system" + LIBS_REF="develop" + resolve_add_submodules_names + [ "${#submodule_names[@]}" -eq 2 ] + [ "${submodule_names[0]}" = "algorithm" ] + [ "${submodule_names[1]}" = "system" ] +} + +@test "resolve_add_submodules_names: auto-discovers via fetch_boost_gitmodules_at_ref" { + unset SUBMODULES + libs_ref="boost-1.90.0" + fetch_boost_gitmodules_at_ref() { + echo '[submodule "libs/unordered"] + path = libs/unordered +' + } + resolve_add_submodules_names + [ "${#submodule_names[@]}" -eq 1 ] + [ "${submodule_names[0]}" = "unordered" ] +} + +@test "resolve_add_submodules_names: returns 1 when fetch fails" { + unset SUBMODULES + libs_ref="develop" + fetch_boost_gitmodules_at_ref() { return 1; } + set +e + resolve_add_submodules_names + status=$? + set -e + [ "$status" -eq 1 ] +} + +@test "process_submodule_list: records updates and fatals by exit code" { + stub_processor() { + case "$1" in + ok) return 0 ;; + skip) return 1 ;; + fail) return 2 ;; + esac + } + + set +e + process_submodule_list stub_processor ok skip fail + status=$? + set -e + [ "$status" -eq 0 ] + [ "${#UPDATES[@]}" -eq 1 ] + [ "${UPDATES[0]}" = "ok" ] + [ "${#SUBMODULE_FATAL[@]}" -eq 1 ] + [ "${SUBMODULE_FATAL[0]}" = "fail" ] + [ "$submodule_fatal" -eq 1 ] +} + +@test "process_submodule_list: returns 0 when no fatal failures" { + stub_processor() { return 0; } + process_submodule_list stub_processor ok ok +} + +@test "combine_batch_and_finalize_rc: zero when no failures" { + submodule_fatal=0 + run combine_batch_and_finalize_rc 0 + [ "$status" -eq 0 ] +} + +@test "combine_batch_and_finalize_rc: 1 when submodule_fatal > 0" { + submodule_fatal=2 + run combine_batch_and_finalize_rc 0 + [ "$status" -eq 1 ] +} + +@test "combine_batch_and_finalize_rc: finalize rc wins when non-zero" { + submodule_fatal=0 + run combine_batch_and_finalize_rc 3 + [ "$status" -eq 3 ] +} + +@test "combine_batch_and_finalize_rc: finalize rc wins over batch fatal" { + submodule_fatal=2 + run combine_batch_and_finalize_rc 3 + [ "$status" -eq 3 ] +} + +@test "combine_batch_and_finalize_rc: submodule_fatal sets exit 1 even if finalize is 0" { + submodule_fatal=1 + run combine_batch_and_finalize_rc 0 + [ "$status" -eq 1 ] +} + +@test "init_translation_work_dirs: creates BOOST_WORK and optional ORG_WORK" { + run bash -c ' + set -euo pipefail + # shellcheck source=tests/helpers/common.bash + source "$1/helpers/common.bash" + load_submodule_ops + init_translation_work_dirs with_org_work + [[ -d "$BOOST_WORK" && -d "$TRANS_DIR" && -d "$ORG_WORK" ]] + [[ "$ORG_WORK" == "$WORK_DIR/$MODULE_ORG" ]] + ' _ "$BATS_TEST_DIRNAME" + [ "$status" -eq 0 ] +}