From 1ef41e6a9024b931c5f42fde20f61cf0166e518e Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Sun, 9 Aug 2026 09:18:26 -0400 Subject: [PATCH] ci: handle large repo mirroring via GitLab import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add size-aware mirroring logic that routes large empty projects through GitLab’s server-side import API before pushing, to avoid GitLab.com’s 5 GiB push limit. The workflow now tracks project emptiness, starts authenticated imports for private sources, waits for import completion with clear status/permission/time-out handling, and then proceeds with mirror push. It also removes GitHub-only `refs/pull/*` refs from the local mirror before push. --- .../workflows/__mirror-github-to-gitlab.yml | 266 +++++++++++++++++- 1 file changed, 258 insertions(+), 8 deletions(-) diff --git a/.github/workflows/__mirror-github-to-gitlab.yml b/.github/workflows/__mirror-github-to-gitlab.yml index 324775b3..d04befbd 100644 --- a/.github/workflows/__mirror-github-to-gitlab.yml +++ b/.github/workflows/__mirror-github-to-gitlab.yml @@ -56,6 +56,7 @@ jobs: const repositoryData = repos.map((repo) => ({ cloneUrl: repo.clone_url, name: repo.name, + sizeKiB: repo.size, targetVisibility: repo.visibility === 'public' ? 'public' : 'private', ...gitlabTarget(repo), })); @@ -149,6 +150,57 @@ jobs: curl "${curl_args[@]}" "${url}" } + wait_for_gitlab_import() { + local project_id="$1" + local source_name="$2" + local deadline="$((SECONDS + 10800))" + local import_error + local import_status + local status + + while ((SECONDS < deadline)); do + if ! status="$(gitlab_request GET "${GITLAB_API_URL}/projects/${project_id}/import")"; then + echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab import." + return 1 + fi + if [[ "${status}" != "200" ]]; then + if [[ "${status}" == "403" ]]; then + echo "::error title=Mirror failed: ${source_name}::GITLAB_TOKEN needs Import API Read access." + return 1 + fi + echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab import (HTTP ${status})." + return 1 + fi + + import_status="$(jq -er '.import_status' "${response_file}")" + case "${import_status}" in + finished) + echo "GitLab server-side import finished." + return 0 + ;; + scheduled | started) + echo "GitLab server-side import status: ${import_status}." + sleep 30 + ;; + failed) + import_error="$(jq -r '.import_error // "No import error detail was provided."' "${response_file}")" + import_error="${import_error//$'\r'/ }" + import_error="${import_error//$'\n'/ }" + echo "GitLab import failed: ${import_error}" + echo "::error title=Mirror failed: ${source_name}::GitLab server-side import failed." + return 1 + ;; + *) + echo "::error title=Mirror failed: ${source_name}::Unexpected GitLab import status: ${import_status}." + return 1 + ;; + esac + done + + echo "::error title=Mirror failed: ${source_name}::GitLab server-side import timed out after three hours." + return 1 + } + encoded_group="$(jq -rn --arg value "${GITLAB_GROUP}" '$value | @uri')" if ! status="$(gitlab_request GET "${GITLAB_API_URL}/groups/${encoded_group}")"; then echo "::error::Unable to query the GitLab group." @@ -162,54 +214,99 @@ jobs: github_auth="$(printf 'x-access-token:%s' "${GITHUB_TOKEN}" | base64 --wrap=0)" echo "::add-mask::${github_auth}" + github_token_uri="$(jq -rn --arg value "${GITHUB_TOKEN}" '$value | @uri')" + echo "::add-mask::${github_token_uri}" + + # GitLab.com limits each Git push to 5 GiB. Seed large empty projects through GitLab's import API, + # using a lower threshold because GitHub's reported repository size is not the resulting pack size. + gitlab_import_threshold_kib="$((4 * 1024 * 1024))" repository_count="$(jq -er 'length' "${repository_file}")" + successful_repository_count=0 + failed_repository_names=() + failed_repository_reasons=() for ((index = 0; index < repository_count; index++)); do repository="$(jq -ec ".[${index}]" "${repository_file}")" repository_number="$((index + 1))" source_name="$(jq -er '.name' <<< "${repository}")" source_clone_url="$(jq -er '.cloneUrl' <<< "${repository}")" + source_size_kib="$(jq -er '.sizeKiB' <<< "${repository}")" target_name="$(jq -er '.targetName' <<< "${repository}")" target_path="$(jq -er '.targetPath' <<< "${repository}")" target_visibility="$(jq -er '.targetVisibility' <<< "${repository}")" description="Mirror of ${source_clone_url}" project_path="${GITLAB_GROUP}/${target_path}" encoded_project_path="$(jq -rn --arg value "${project_path}" '$value | @uri')" + repository_log="${temp_root}/repository-${index}.log" + + set +e + ( + set -euo pipefail + trap 'echo "::endgroup::"' EXIT + + echo "::group::[${repository_number}/${repository_count}] ${source_name} -> ${project_path}" + echo "Visibility: ${target_visibility}" + echo "Checking for the GitLab project..." - echo "::group::[${repository_number}/${repository_count}] ${source_name} -> ${project_path}" - echo "Visibility: ${target_visibility}" - echo "Checking for the GitLab project..." + project_created=false + gitlab_import_started=false if ! status="$(gitlab_request GET "${GITLAB_API_URL}/projects/${encoded_project_path}")"; then echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab project." exit 1 fi - if [[ "${status}" == "404" ]]; then - echo "Project does not exist; creating ${project_path}..." + if [[ "${status}" == "404" ]] || [[ "${status}" == "301" ]]; then + if [[ "${status}" == "301" ]]; then + echo "The exact GitLab path redirects elsewhere; reclaiming ${project_path}." + else + echo "Project does not exist; creating ${project_path}..." + fi + + create_import_url="" + if ((source_size_kib >= gitlab_import_threshold_kib)); then + create_import_url="${source_clone_url}" + if [[ "${target_visibility}" == "private" ]]; then + create_import_url="https://x-access-token:${github_token_uri}@${source_clone_url#https://}" + echo "::add-mask::${create_import_url}" + fi + echo "The large repository will be imported as part of project creation." + fi + payload="$( jq -nc \ + --arg description "${description}" \ + --arg import_url "${create_import_url}" \ --arg name "${target_name}" \ --arg path "${target_path}" \ --argjson namespace_id "${group_id}" \ --arg visibility "${target_visibility}" \ '{ + description: $description, name: $name, path: $path, namespace_id: $namespace_id, visibility: $visibility, initialize_with_readme: false - }' + } + if $import_url == "" then {} else {import_url: $import_url} end' )" if ! status="$(gitlab_request POST "${GITLAB_API_URL}/projects" "${payload}")"; then echo "::error title=Mirror failed: ${source_name}::Unable to create the GitLab project." exit 1 fi if [[ "${status}" != "201" ]]; then + create_error="$(jq -r '.message // "No response message was provided."' "${response_file}")" + create_error="${create_error//$'\r'/ }" + create_error="${create_error//$'\n'/ }" + echo "GitLab project creation response: ${create_error}" error_message="Unable to create the GitLab project (HTTP ${status})." echo "::error title=Mirror failed: ${source_name}::${error_message}" exit 1 fi + project_created=true + if [[ -n "${create_import_url}" ]]; then + gitlab_import_started=true + fi elif [[ "${status}" != "200" ]]; then echo "::error title=Mirror failed: ${source_name}::Unable to query the GitLab project (HTTP ${status})." exit 1 @@ -218,6 +315,13 @@ jobs: fi project_id="$(jq -er '.id' "${response_file}")" + if [[ "${project_created}" == "true" ]]; then + project_empty=true + else + project_empty="$( + jq -r '(.empty_repo == true) or (.default_branch == null)' "${response_file}" + )" + fi # Make a private source private before changing metadata or pushing any Git data. if [[ "${target_visibility}" == "private" ]]; then @@ -267,6 +371,75 @@ jobs: target_clone_url="$(jq -er '.ssh_url_to_repo' "${response_file}")" + gitlab_incremental_seed=false + if [[ "${project_empty}" == "true" ]] && \ + ((source_size_kib >= gitlab_import_threshold_kib)) && \ + [[ "${gitlab_import_started}" != "true" ]]; then + echo "Empty large project detected (${source_size_kib} KiB reported by GitHub)." + echo "Scheduling a GitLab server-side Git import to avoid the 5 GiB push limit..." + if [[ "${target_visibility}" == "private" ]]; then + import_payload="$( + jq -nc \ + --arg import_url "${source_clone_url}" \ + --arg import_url_user "x-access-token" \ + --arg import_url_password "${GITHUB_TOKEN}" \ + '{ + import_url: $import_url, + import_url_user: $import_url_user, + import_url_password: $import_url_password + }' + )" + else + import_payload="$( + jq -nc \ + --arg import_url "${source_clone_url}" \ + '{import_url: $import_url}' + )" + fi + + if ! status="$( + gitlab_request POST "${GITLAB_API_URL}/projects/${project_id}/import/git" "${import_payload}" + )"; then + echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import." + exit 1 + fi + if [[ "${status}" == "403" ]]; then + echo "::error title=Mirror failed: ${source_name}::GITLAB_TOKEN needs Import API Create access." + exit 1 + fi + + import_response_message="$(jq -r '.message // "No response message was provided."' "${response_file}")" + import_response_message="${import_response_message//$'\r'/ }" + import_response_message="${import_response_message//$'\n'/ }" + if [[ "${status}" == "409" ]]; then + case "${import_response_message}" in + "Import already in progress") + echo "A GitLab import is already in progress; continuing to monitor it." + gitlab_import_started=true + ;; + "Project already has a repository") + echo "GitLab has repository storage from an earlier push; using incremental seeding." + gitlab_incremental_seed=true + ;; + *) + echo "GitLab import conflict: ${import_response_message}" + echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import." + exit 1 + ;; + esac + elif [[ "${status}" != "201" ]] && [[ "${status}" != "202" ]]; then + echo "GitLab import response: ${import_response_message}" + echo "::error title=Mirror failed: ${source_name}::Unable to schedule the GitLab import" \ + "(HTTP ${status})." + exit 1 + else + gitlab_import_started=true + fi + elif [[ "${project_empty}" != "true" ]] && \ + ((source_size_kib >= gitlab_import_threshold_kib)); then + echo "Large project is already populated; using an incremental Git push." + fi + mirror_dir="${temp_root}/repository.git" echo "Cloning ${source_clone_url} as a mirror..." if ! git \ @@ -275,6 +448,39 @@ jobs: echo "::error title=Mirror failed: ${source_name}::Unable to clone the GitHub repository." exit 1 fi + + mapfile -t pull_refs < <( + git -C "${mirror_dir}" for-each-ref --format='%(refname)' refs/pull/ + ) + if ((${#pull_refs[@]} > 0)); then + echo "Removing ${#pull_refs[@]} GitHub-only pull request refs from the mirror..." + printf 'delete %s\n' "${pull_refs[@]}" | git -C "${mirror_dir}" update-ref --stdin + fi + + if [[ "${gitlab_import_started}" == "true" ]]; then + echo "Waiting for the GitLab server-side import to finish..." + if ! wait_for_gitlab_import "${project_id}" "${source_name}"; then + exit 1 + fi + fi + + if [[ "${gitlab_incremental_seed}" == "true" ]]; then + mapfile -t seed_refs < <( + git -C "${mirror_dir}" for-each-ref --format='%(refname)' refs/heads/ refs/tags/ + ) + echo "Seeding ${#seed_refs[@]} branches and tags in separate pushes..." + for seed_ref in "${seed_refs[@]}"; do + echo "Seeding ${seed_ref}..." + if ! git \ + -C "${mirror_dir}" \ + -c core.sshCommand="${gitlab_ssh_command}" \ + push --progress "${target_clone_url}" "+${seed_ref}:${seed_ref}"; then + echo "::error title=Mirror failed: ${source_name}::Unable to seed ${seed_ref} on GitLab." + exit 1 + fi + done + fi + echo "Pushing the mirror to ${target_clone_url}..." if ! git \ -C "${mirror_dir}" \ @@ -285,7 +491,51 @@ jobs: fi rm -rf "${mirror_dir}" echo "Completed mirror for ${source_name}." - echo "::endgroup::" + ) 2>&1 | tee "${repository_log}" + repository_status="${PIPESTATUS[0]}" + set -e + + rm -rf "${temp_root}/repository.git" + if ((repository_status == 0)); then + successful_repository_count="$((successful_repository_count + 1))" + else + error_line="$(grep -E '^::error(::| )' "${repository_log}" | tail -n 1 || true)" + if [[ -n "${error_line}" ]]; then + failure_reason="${error_line##*::}" + else + failure_reason="Repository mirroring failed; inspect its log group." + fi + failure_reason="${failure_reason//$'\r'/ }" + failure_reason="${failure_reason//$'\n'/ }" + failed_repository_names+=("${source_name}") + failed_repository_reasons+=("${failure_reason}") + echo "Recorded mirror failure for ${source_name}; continuing with the next repository." + fi + rm -f "${repository_log}" done - echo "Mirrored ${repository_count} repositories to GitLab." + failed_repository_count="${#failed_repository_names[@]}" + { + echo "## GitHub to GitLab mirror summary" + echo + echo "- Successful: ${successful_repository_count}" + echo "- Failed: ${failed_repository_count}" + if ((failed_repository_count > 0)); then + echo + echo "| Repository | Failure |" + echo "| --- | --- |" + for ((failure_index = 0; failure_index < failed_repository_count; failure_index++)); do + summary_name="${failed_repository_names[$failure_index]//|/\\|}" + summary_reason="${failed_repository_reasons[$failure_index]//|/\\|}" + printf '| %s | %s |\n' "\`${summary_name}\`" "${summary_reason}" + done + fi + } >> "${GITHUB_STEP_SUMMARY}" + + if ((failed_repository_count > 0)); then + echo "::error::Failed to mirror ${failed_repository_count} of ${repository_count} repositories." \ + "See the step summary." + exit 1 + fi + + echo "Mirrored all ${repository_count} repositories to GitLab."