Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 125 additions & 12 deletions .github/scripts/download-build-artifacts.sh
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,99 @@ fi
artifact_present() {
local run_id="$1"
local artifact="$2"
gh api "repos/$GH_REPO/actions/runs/$run_id/artifacts" \
--paginate \
--jq '.artifacts[].name' |
grep -Fxq "$artifact"

local artifact_names
artifact_names="$(
gh api "repos/$GH_REPO/actions/runs/$run_id/artifacts?per_page=100" \
--paginate \
--jq '.artifacts[].name'
)" || {
echo "failed to list artifacts for $workflow run $run_id" >&2
exit 1
}
printf '%s\n' "$artifact_names" |
grep -Fxq -- "$artifact"
}

merge_checksum_manifest() {
local existing="$1"
local incoming="$2"
python3 - "$existing" "$incoming" <<'PY'
from __future__ import annotations
import sys
import tempfile
from pathlib import Path
existing = Path(sys.argv[1])
incoming = Path(sys.argv[2])
entries: dict[str, str] = {}
def read_manifest(path: Path) -> None:
with path.open("r", encoding="utf-8") as handle:
for line_number, line in enumerate(handle, 1):
stripped = line.strip()
if not stripped:
continue
parts = stripped.split(None, 1)
if len(parts) != 2:
raise SystemExit(f"{path}: invalid checksum line {line_number}: {line.rstrip()}")
digest, raw_name = parts[0], parts[1].strip()
if len(digest) != 64 or any(char not in "0123456789abcdef" for char in digest):
raise SystemExit(f"{path}: invalid checksum digest on line {line_number}: {digest}")
name = raw_name.removeprefix("./")
if not name or "/" in name:
raise SystemExit(f"{path}: invalid checksum asset name on line {line_number}: {raw_name}")
previous = entries.get(name)
if previous is not None and previous != digest:
raise SystemExit(
f"{path}: conflicting checksum for {name}: {previous} vs {digest}"
)
entries[name] = digest
read_manifest(existing)
read_manifest(incoming)
with tempfile.NamedTemporaryFile(
"w",
encoding="utf-8",
newline="\n",
dir=str(existing.parent),
delete=False,
) as handle:
temp_path = Path(handle.name)
for name in sorted(entries):
handle.write(f"{entries[name]} ./{name}\n")
temp_path.replace(existing)
PY
}

merge_downloaded_artifact() {
local artifact="$1"
local source_dir="$2"

local source
while IFS= read -r source; do
[[ -n "$source" ]] || continue
local relative_path="${source#"$source_dir"/}"
local target="$destination/$relative_path"
mkdir -p "$(dirname "$target")"
if [[ -e "$target" ]]; then
if [[ -f "$target" ]] && cmp -s "$source" "$target"; then
continue
fi
if [[ -f "$target" && -f "$source" && "$(basename "$target")" == *-release-assets.sha256 ]]; then
if ! merge_checksum_manifest "$target" "$source"; then
return 1
fi
continue
fi
echo "artifact $artifact would overwrite $relative_path with different bytes" >&2
return 1
fi
cp -p "$source" "$target"
done < <(find "$source_dir" -type f -print | sort)
}

required_job_success() {
Expand All @@ -53,15 +142,27 @@ required_job_success() {
return 0
fi

local jobs_file
jobs_file="$(mktemp)"
if ! gh run view "$run_id" --repo "$GH_REPO" --json jobs > "$jobs_file"; then
rm -f "$jobs_file"
return 1
fi

local conclusion
conclusion="$(
GH_RUN_JSON="$(gh run view "$run_id" --json jobs)" REQUIRED_JOB="$required_job" bun -e '
const data = JSON.parse(process.env.GH_RUN_JSON ?? "{}");
const required = process.env.REQUIRED_JOB ?? "";
if ! conclusion="$(
bun -e '
const fs = require("node:fs");
const data = JSON.parse(fs.readFileSync(Bun.argv[1], "utf8"));
const required = Bun.argv[2] ?? "";
const job = (data.jobs ?? []).find((candidate) => candidate?.name === required);
console.log(job?.conclusion ?? "");
'
)" || return 1
' "$jobs_file" "$required_job"
)"; then
rm -f "$jobs_file"
return 1
fi
rm -f "$jobs_file"
[[ "$conclusion" == "success" ]]
}

Expand Down Expand Up @@ -97,13 +198,15 @@ else
done < <(
if [[ -n "$required_job" ]]; then
gh run list \
--repo "$GH_REPO" \
--workflow "$workflow" \
--commit "$sha" \
--limit 20 \
--json databaseId,status,conclusion,event,createdAt \
--jq '.[].databaseId'
else
gh run list \
--repo "$GH_REPO" \
--workflow "$workflow" \
--commit "$sha" \
--limit 20 \
Expand All @@ -121,7 +224,17 @@ fi
mkdir -p "$destination"
for artifact in "${artifacts[@]}"; do
echo "Downloading $workflow artifact $artifact from run $run_id"
gh run download "$run_id" \
artifact_dir="$(mktemp -d)"
if ! gh run download "$run_id" \
--repo "$GH_REPO" \
--name "$artifact" \
--dir "$destination"
--dir "$artifact_dir"; then
rm -rf "$artifact_dir"
exit 1
fi
if ! merge_downloaded_artifact "$artifact" "$artifact_dir"; then
rm -rf "$artifact_dir"
exit 1
fi
rm -rf "$artifact_dir"
done
45 changes: 33 additions & 12 deletions .github/scripts/require-workflow-success.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,35 +46,55 @@ emit_run_id() {
}

required_artifacts_present() {
run_id="$1"
local run_id="$1"
if [[ "${#required_artifacts[@]}" -eq 0 ]]; then
return 0
fi

artifacts="$(gh api "repos/$GH_REPO/actions/runs/$run_id/artifacts" \
--paginate \
--jq '.artifacts[].name')" || return 1
local artifacts
artifacts="$(
gh api "repos/$GH_REPO/actions/runs/$run_id/artifacts?per_page=100" \
--paginate \
--jq '.artifacts[].name'
)" || {
echo "failed to list artifacts for $workflow run $run_id" >&2
return 1
}
local expected
for expected in "${required_artifacts[@]}"; do
if ! printf '%s\n' "$artifacts" | grep -Fxq "$expected"; then
if ! printf '%s\n' "$artifacts" | grep -Fxq -- "$expected"; then
return 1
fi
done
}

required_job_success() {
run_id="$1"
local run_id="$1"
if [[ -z "$required_job" ]]; then
return 0
fi

conclusion="$(
GH_RUN_JSON="$(gh run view "$run_id" --json jobs)" REQUIRED_JOB="$required_job" bun -e '
const data = JSON.parse(process.env.GH_RUN_JSON ?? "{}");
const required = process.env.REQUIRED_JOB ?? "";
local jobs_file
jobs_file="$(mktemp)"
if ! gh run view "$run_id" --repo "$GH_REPO" --json jobs > "$jobs_file"; then
rm -f "$jobs_file"
return 1
fi

local conclusion
if ! conclusion="$(
bun -e '
const fs = require("node:fs");
const data = JSON.parse(fs.readFileSync(Bun.argv[1], "utf8"));
const required = Bun.argv[2] ?? "";
const job = (data.jobs ?? []).find((candidate) => candidate?.name === required);
console.log(job?.conclusion ?? "");
'
)" || return 1
' "$jobs_file" "$required_job"
)"; then
rm -f "$jobs_file"
return 1
fi
rm -f "$jobs_file"
[[ "$conclusion" == "success" ]]
}

Expand All @@ -90,6 +110,7 @@ fi
deadline=$((SECONDS + timeout))
while true; do
runs="$(gh run list \
--repo "$GH_REPO" \
--workflow "$workflow" \
--commit "$sha" \
--limit 10 \
Expand Down
60 changes: 32 additions & 28 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -280,34 +280,6 @@ jobs:
NODE
pnpm --version

- name: Create release-please target branch
if: ${{ inputs.operation == 'publish' && steps.release_head.outputs.uses_temporary_target_branch == 'true' }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh auth setup-git
git push origin "${RELEASE_HEAD_SHA}:refs/heads/${{ steps.release_head.outputs.target_branch }}" --force

- name: Create release-please GitHub releases
id: release_please
if: ${{ inputs.operation == 'publish' }}
uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071
with:
token: ${{ secrets.GITHUB_TOKEN }}
target-branch: ${{ steps.release_head.outputs.target_branch }}
config-file: release-please-config.json
manifest-file: .release-please-manifest.json
skip-github-pull-request: true

- name: Remove release-please target branch
if: ${{ always() && inputs.operation == 'publish' && steps.release_head.outputs.uses_temporary_target_branch == 'true' }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh auth setup-git
git push origin ":refs/heads/${{ steps.release_head.outputs.target_branch }}"

- name: Plan product releases
id: release_plan
run: |
Expand Down Expand Up @@ -477,6 +449,10 @@ jobs:
with:
bun-version: ${{ env.BUN_VERSION }}

- name: Install TypeScript release tooling
if: ${{ steps.release_plan.outputs.has_release_changes == 'true' && steps.release_plan.outputs.product_oliphaunt_js == 'true' }}
run: pnpm install --frozen-lockfile

- name: Download native helper release assets
if: ${{ steps.release_plan.outputs.has_release_changes == 'true' && (steps.release_plan.outputs.product_oliphaunt_broker == 'true' || steps.release_plan.outputs.product_oliphaunt_node_direct == 'true') }}
env:
Expand Down Expand Up @@ -537,6 +513,34 @@ jobs:
PRODUCTS_JSON: ${{ steps.release_plan.outputs.products_json }}
run: tools/release/release.py publish-dry-run --products-json "${PRODUCTS_JSON}" --head-ref "$RELEASE_HEAD_SHA"

- name: Create release-please target branch
if: ${{ inputs.operation == 'publish' && steps.release_plan.outputs.has_release_changes == 'true' && steps.release_head.outputs.uses_temporary_target_branch == 'true' }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh auth setup-git
git push origin "${RELEASE_HEAD_SHA}:refs/heads/${{ steps.release_head.outputs.target_branch }}" --force

- name: Create release-please GitHub releases
id: release_please
if: ${{ inputs.operation == 'publish' && steps.release_plan.outputs.has_release_changes == 'true' }}
uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071
with:
token: ${{ secrets.GITHUB_TOKEN }}
target-branch: ${{ steps.release_head.outputs.target_branch }}
config-file: release-please-config.json
manifest-file: .release-please-manifest.json
skip-github-pull-request: true

- name: Remove release-please target branch
if: ${{ always() && inputs.operation == 'publish' && steps.release_head.outputs.uses_temporary_target_branch == 'true' }}
continue-on-error: true
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh auth setup-git
git push origin ":refs/heads/${{ steps.release_head.outputs.target_branch }}"

- name: Publish liboliphaunt GitHub release assets
if: ${{ steps.release_plan.outputs.has_release_changes == 'true' && inputs.operation == 'publish' && steps.release_plan.outputs.product_liboliphaunt_native == 'true' }}
env:
Expand Down
6 changes: 5 additions & 1 deletion tools/policy/check-crate-size.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,11 @@ for crate_file in $crate_files; do
continue
fi

message="crate size warning: $crate_file is ${size_mib}MiB > ${limit_mib}MiB"
label="warning"
if [ "$mode" = "--enforce" ]; then
label="error"
fi
message="crate size $label: $crate_file is ${size_mib}MiB > ${limit_mib}MiB"
echo "$message" >&2
failed=1
done
Expand Down
Loading
Loading