diff --git a/tools/release-tools/01-check-env.sh b/tools/release-tools/01-check-env.sh index c6e47048e916f2..431b7cfd10c8de 100755 --- a/tools/release-tools/01-check-env.sh +++ b/tools/release-tools/01-check-env.sh @@ -46,7 +46,7 @@ problems=0 echo "== Apache Doris ${TAG} - signing environment check ==" # 1. required tools -for t in git gpg svn sha512sum curl gzip; do +for t in git gpg svn svnmucc sha512sum curl gzip; do if command -v "$t" >/dev/null 2>&1; then ok "tool: $t"; else err "missing tool: $t"; problems=$((problems+1)); fi done diff --git a/tools/release-tools/04-release-complete.sh b/tools/release-tools/04-release-complete.sh new file mode 100755 index 00000000000000..84691d55a4c6e6 --- /dev/null +++ b/tools/release-tools/04-release-complete.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Step 04 - publish the passed RC source artifact to the Apache release SVN +# and generate the [ANNOUNCE] email draft. +# +# The release SVN commit is public and requires PMC permission. This script +# pauses for confirmation before changing SVN, and it never sends email. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=release.env +source "${HERE}/release.env" + +ok() { echo "[ OK ] $*"; } +warn() { echo "[WARN] $*"; } +die() { echo "[FAIL] $*" >&2; exit 1; } +confirm() { local a; read -r -p "$1 [y/N] " a; [[ "$a" == y || "$a" == Y ]]; } + +mail_only=0 +usage() { + cat </dev/null 2>&1 || die "missing tool: $1" +} + +write_announce_email() { + local rn subject body_file eml_file + + rn="${ANNOUNCE_RELEASE_NOTES_URL:-${RELEASE_NOTES_URL:-}}" + if [[ -z "$rn" ]]; then + read -r -p "Release Notes URL for announce email: " rn + fi + [[ -n "$rn" ]] || die "release notes url required" + + mkdir -p "$WORK_DIR" + subject="[ANNOUNCE] Apache Doris ${VERSION} release" + body_file="$WORK_DIR/announce-email.txt" + eml_file="$WORK_DIR/announce-email.eml" + + read -r -d '' BODY < "$body_file" + { + echo "To: ${DEV_LIST}" + echo "Subject: ${subject}" + echo "Content-Type: text/plain; charset=UTF-8" + echo + printf '%s\n' "$BODY" + } > "$eml_file" + + ok "subject: ${subject}" + ok "body: ${body_file}" + ok "eml: ${eml_file} (open in your apache.org mail client)" + echo "----------------------------------------------------------------" + cat "$body_file" + echo "----------------------------------------------------------------" + echo "Review, then SEND MANUALLY from your @apache.org address to ${DEV_LIST}." + echo "(Not auto-sent by design - it's a public ASF list.)" +} + +publish_to_release_svn() { + local src_tar src_asc src_sha512 dst_tar dst_asc dst_sha512 checksum_dir src_tar_file final_sha512_file release_parent_missing + local -a svnmucc_ops + + require_tool svn + require_tool svnmucc + require_tool gpg + require_tool sha512sum + + src_tar="${DEV_SVN_DIR}/${PKG_BASE}.tar.gz" + src_asc="${src_tar}.asc" + src_sha512="${src_tar}.sha512" + dst_tar="${RELEASE_PKG_BASE}.tar.gz" + dst_asc="${dst_tar}.asc" + dst_sha512="${dst_tar}.sha512" + + echo "== Apache Doris ${TAG} - complete release ==" + echo "Source dev SVN folder: ${DEV_SVN_DIR}/" + echo "Target release SVN folder: ${RELEASE_SVN_DIR}/" + echo + echo "Source files:" + echo " ${PKG_BASE}.tar.gz" + echo " ${PKG_BASE}.tar.gz.asc" + echo " ${PKG_BASE}.tar.gz.sha512" + echo "Target files:" + echo " ${dst_tar}" + echo " ${dst_asc}" + echo " ${dst_sha512}" + echo + warn "Only PMC members can write to the release SVN directory." + + svn info "${svn_auth[@]}" "$src_tar" >/dev/null || die "source artifact not found: $src_tar" + svn info "${svn_auth[@]}" "$src_asc" >/dev/null || die "source signature not found: $src_asc" + svn info "${svn_auth[@]}" "$src_sha512" >/dev/null || die "source checksum not found: $src_sha512" + if svn info "${svn_auth[@]}" "$RELEASE_SVN_DIR" >/dev/null 2>&1; then + die "release SVN folder already exists: $RELEASE_SVN_DIR (use --mail-only to regenerate the email)" + fi + release_parent_missing=0 + if svn info "${svn_auth[@]}" "$RELEASE_SVN_PARENT_DIR" >/dev/null 2>&1; then + ok "release SVN parent exists: ${RELEASE_SVN_PARENT_DIR}" + else + release_parent_missing=1 + warn "release SVN parent will be created: ${RELEASE_SVN_PARENT_DIR}" + fi + + checksum_dir="$(mktemp -d)" + trap 'rm -rf "$checksum_dir"' EXIT + + src_tar_file="${PKG_BASE}.tar.gz" + final_sha512_file="${checksum_dir}/${dst_sha512}" + svn cat "${svn_auth[@]}" "$src_tar" > "${checksum_dir}/${src_tar_file}" + svn cat "${svn_auth[@]}" "$src_sha512" > "${checksum_dir}/${src_tar_file}.sha512" + svn cat "${svn_auth[@]}" "$src_asc" > "${checksum_dir}/${src_tar_file}.asc" + ( + cd "$checksum_dir" + sha512sum --check "${src_tar_file}.sha512" + gpg --verify "${src_tar_file}.asc" "$src_tar_file" + cp "$src_tar_file" "$dst_tar" + sha512sum "$dst_tar" > "$dst_sha512" + sha512sum --check "$dst_sha512" + ) + ok "source RC checksum and signature verified: ${src_tar_file}" + ok "final sha512 ok: ${dst_sha512}" + + echo "--- svnmucc operations ---" + svnmucc_ops=() + if [[ "$release_parent_missing" -eq 1 ]]; then + echo "mkdir ${RELEASE_SVN_PARENT_DIR}" + svnmucc_ops+=(mkdir "$RELEASE_SVN_PARENT_DIR") + fi + echo "mkdir ${RELEASE_SVN_DIR}" + svnmucc_ops+=(mkdir "$RELEASE_SVN_DIR") + echo "mv ${src_tar}" + echo " -> ${RELEASE_SVN_DIR}/${dst_tar}" + svnmucc_ops+=(mv "$src_tar" "${RELEASE_SVN_DIR}/${dst_tar}") + echo "mv ${src_asc}" + echo " -> ${RELEASE_SVN_DIR}/${dst_asc}" + svnmucc_ops+=(mv "$src_asc" "${RELEASE_SVN_DIR}/${dst_asc}") + echo "put ${final_sha512_file}" + echo " -> ${RELEASE_SVN_DIR}/${dst_sha512}" + svnmucc_ops+=(put "$final_sha512_file" "${RELEASE_SVN_DIR}/${dst_sha512}") + echo "rm ${DEV_SVN_DIR}" + svnmucc_ops+=(rm "$DEV_SVN_DIR") + echo + echo "Will commit the URL operations above in one SVN revision." + confirm "FINAL confirm - publish release SVN and remove dev RC now?" || { warn "stopping before SVN commit."; exit 0; } + + if ! svnmucc "${svnmucc_auth[@]}" -m "Release Doris ${VERSION}" "${svnmucc_ops[@]}"; then + die "svnmucc release publish failed" + fi + rm -rf "$checksum_dir" + trap - EXIT + ok "committed release artifacts: ${RELEASE_SVN_DIR}/" + ok "removed dev RC folder: ${DEV_SVN_DIR}/" +} + +if [[ "$mail_only" -eq 0 ]]; then + publish_to_release_svn +else + ok "mail-only mode: skipping SVN publish" +fi + +write_announce_email diff --git a/tools/release-tools/README.md b/tools/release-tools/README.md index 20e34504d96ad5..d82509c54ff993 100644 --- a/tools/release-tools/README.md +++ b/tools/release-tools/README.md @@ -19,81 +19,267 @@ under the License. # Doris release helper scripts -Helper scripts for an Apache Doris Release Manager (RM) to cut a **source** release candidate: -package and sign the source tarball, upload it to the Apache dev SVN, and draft the `[VOTE]` email. +This directory contains helper scripts for an Apache Doris Release Manager (RM) +to create and complete a source release candidate (RC). The scripts package and +sign the source tarball, upload the source artifacts to the Apache dev SVN, +draft the `[VOTE]` email, publish the passed RC to the Apache release SVN, and +draft the `[ANNOUNCE]` email. + +These scripts are not a full release automation system. They assume the release +branch, release notes, cherry-picks, build validation, and release tag have +already been prepared. + +## If you are an AI agent + +When a user says something like "read this file and tell me how to start +releasing Doris 4.0.7", give the RM a concrete checklist. Do not just summarize +the scripts. + +For `4.0.7`, explain this sequence: + +1. Confirm the RC number. If the RM does not provide one, ask whether this is + `rc01` or a later candidate such as `rc02`. The tag is + `${VERSION}-${RC}`, for example `4.0.7-rc01`. +2. Confirm that the tag already exists locally and on the Apache Doris GitHub + remote. These scripts verify the tag, but they do not create it. +3. Tell the RM to edit `release.env` before running any script. +4. Tell the RM to export ASF SVN credentials in the shell. +5. Run `./01-check-env.sh` until it reports `environment looks READY`. +6. Run `./02-package-sign-upload.sh` only after the RM has checked the target + dev SVN URL and is ready to publish the RC source artifacts. +7. Run `./03-vote-mail.sh`, review the generated vote email, then send it + manually from the RM's `@apache.org` address. +8. After the vote passes and the `[RESULT]` email has been sent manually, run + `./04-release-complete.sh` to publish the source release artifacts and draft + the announce email. + +Also make these boundaries clear: + +- The vote artifacts in Apache dist SVN are source-only. +- `BIN_FILES` is optional. If set, step 02 signs convenience binary tarballs + locally, but the scripts do not upload those binaries. +- The scripts never send public emails. The RM must review and send the vote, + result, and announce emails manually. +- Step 04 writes to the Apache release SVN and requires PMC permission. + +## Quick start for a new RC + +Run all commands from this directory: + +```bash +cd tools/release-tools +``` + +Before running any script, edit `release.env` for the target version and RC. +For example, to prepare Doris `4.0.7-rc01`: + +```bash +VERSION="4.0.7" +RC="rc01" +TAG="${VERSION}-${RC}" +GIT_REMOTE="upstream-apache" + +APACHE_ID="" +APACHE_EMAIL="@apache.org" +SIGNER_NAME="" +SIGNING_KEY="" + +RELEASE_NOTES_URL="" +ANNOUNCE_RELEASE_NOTES_URL="" +``` + +If you need to advertise convenience binaries in the vote email, set +`BIN_FILES` to absolute paths for those prebuilt tarballs. Leave `BIN_FILES=()` +for the source-only flow. + +Export ASF credentials in the same shell. The scripts use these credentials for +SVN upload and KEYS publishing: + +```bash +export ASF_USERNAME= +export ASF_PASSWORD='' +``` + +Then run the release flow: + +```bash +./01-check-env.sh +./02-package-sign-upload.sh +./03-vote-mail.sh +``` + +Review the generated vote email in `WORK_DIR`, then send it manually to +`dev@doris.apache.org` from your `@apache.org` address. + +After the vote passes: + +1. Tally the votes and send the `[RESULT]` email manually. +2. Run `./04-release-complete.sh`. +3. Check that the new source artifacts are visible on `downloads.apache.org`. +4. Update the Doris download page and release metadata as needed, then check + that the public download page points at the new release. +5. Wait for the download and CDN propagation window. +6. Review the generated announce email in `WORK_DIR`. +7. Only after those checks pass, send the `[ANNOUNCE]` email manually. + +Use this command only when the release SVN step has already been done and you +only need to regenerate the announce email: + +```bash +./04-release-complete.sh --mail-only +``` ## Prerequisites + Have these ready before you run anything: -- The release **tag is already created and pushed** to `apache/doris` — these scripts only verify it - (branch prep, patch merges and tag creation are out of scope). -- A local clone of `apache/doris` with that tag fetched. -- These tools on `PATH`: `git`, `gpg`, `svn`, `sha512sum`, `curl`, `gzip`. -- A GPG signing key — or let step 01 import an existing one, or generate and publish a new one. -- ASF credentials exported in your shell (used for the SVN upload and the KEYS publish): - ```bash - export ASF_USERNAME= - export ASF_PASSWORD='' - ``` -- `release.env` filled in for this release — see [Configuration](#configuration) for the fields. - -## Steps -Run from this directory, in order: -1. **Check the signing environment** — `./01-check-env.sh`. Re-run until it ends with - `environment looks READY`. -2. **Package, sign & upload** — `./02-package-sign-upload.sh`. Builds and signs the source tarball - and uploads it to the dev SVN. It pauses twice for confirmation; nothing public happens before - the final confirm. -3. **Draft the vote email** — `./03-vote-mail.sh`. Prompts for the Release Notes URL and prints the - draft; review it and send it yourself from your `@apache.org` address. - ---- - -Everything below is reference detail. - -## Files -- `release.env` — all configuration (version, paths, signing key, SVN URLs, email). **Edit this first.** -- `01-check-env.sh` — check / prepare the GPG signing environment and ASF credentials. -- `02-package-sign-upload.sh` — `git archive` the tag, GPG-sign, sha512, sign any prebuilt binaries - locally, then upload the source artifacts to the dev SVN. -- `03-vote-mail.sh` — generate the `[VOTE]` email draft. - -## Configuration -The scripts are reusable across releases — they hold no version; edit `release.env` each time. -Set at least: -- `VERSION` / `RC` — e.g. `4.0.6` and `rc02`; `TAG` is derived as `${VERSION}-${RC}`. -- `GIT_REMOTE` — the git remote pointing at `github.com/apache/doris`. -- `APACHE_ID` / `APACHE_EMAIL` / `SIGNER_NAME` — your committer id, `@apache.org` email, and the - display name used to sign the vote email. -- `SIGNING_KEY` — fingerprint of the key to sign with (leave empty to auto-detect a single local secret key). -- `BIN_FILES` — optional absolute paths to prebuilt binary tarballs to sign locally (see below). Leave - the list empty (the default) to skip binary signing and run the source-only flow. When set, step 03 - advertises each binary in the vote email under `BIN_DOWNLOAD_BASE`; when empty that section is omitted. - -`REPO_DIR` defaults to the enclosing checkout (`${ROOT}/../../`) since these scripts live inside -`apache/doris`; override it only if you run them against a different clone. - -The SVN URLs, dev mailing list and verify-guide link rarely change; the defaults target the official -Doris dist repos. The **vote SVN artifacts are source-only**: `apache-doris--src.tar.gz` + `.asc` -+ `.sha512`. All script output (source tarball, signatures, SVN checkouts, email draft) goes to -`WORK_DIR` (`/`, i.e. `tools/release-tools/`, by default). - -## What each step does -- **01** verifies the required tools, your `GPG_TTY` / `gpg.conf`, resolves (or helps you create) a - signing key, checks it is present in the live published `KEYS`, runs a test sign + verify, and - confirms your ASF credentials. It is read-mostly: every state-changing action (edit `gpg.conf`, - import / generate a key, publish `KEYS`) prompts before acting. -- **02** checks the local tag matches `GIT_REMOTE`, builds the source tarball from the tag with - `git archive`, signs it and writes the `.sha512`. If `BIN_FILES` is non-empty it then GPG-signs and - sha512s each prebuilt binary tarball, writing the `.asc` / `.sha512` sidecars **next to** each binary - (these are NOT uploaded — you upload the binaries and their sidecars yourself). Finally it uploads - the three **source** files to `dev/doris//` on the Apache dist SVN. **Eyeball the target URL** - at the confirm prompt before committing — nothing public happens until the final confirm. -- **03** writes `vote-email.txt` and `vote-email.eml` into `WORK_DIR` and prints the draft. Review it - and send it yourself from your `@apache.org` address. - -## Not automated (on purpose) -- Sending the vote email — it goes to a public ASF list, so you review and send it manually. -- Uploading binary packages — step 02 can *sign* the tarballs listed in `BIN_FILES`, but binaries are - not part of the ASF source vote, so you upload them (with their `.asc`/`.sha512`) manually. -- Post-vote steps — tallying the result, the result email, and moving the artifacts from `dev/` to - `release/` once the vote passes. + +- The release tag has already been created and pushed to `apache/doris`. + Branch preparation, patch merges, validation, and tag creation are out of + scope for these scripts. +- The local Doris checkout has fetched that tag. +- `release.env` is filled in for this exact release. +- These tools are on `PATH`: `git`, `gpg`, `svn`, `svnmucc`, `sha512sum`, + `curl`, and `gzip`. +- A GPG signing key is available locally, or the RM is ready to let + `01-check-env.sh` import an existing key or generate and publish a new key. +- ASF credentials are exported as `ASF_USERNAME` and `ASF_PASSWORD`. +- The RM can write to the required Apache SVN locations. Step 04 requires PMC + permission for the release SVN commit. + +## Configuration reference + +All scripts source `release.env`. Edit it once per RC. + +Set these fields first: + +- `VERSION`: Final release version, for example `4.0.7`. +- `RC`: Candidate number, for example `rc01` or `rc02`. +- `TAG`: Derived as `${VERSION}-${RC}`. Do not set it to the final version + without the RC suffix. +- `GIT_REMOTE`: Git remote that points at `github.com/apache/doris`. +- `APACHE_ID`: Apache committer id. +- `APACHE_EMAIL`: Apache email address. +- `SIGNER_NAME`: Display name used in generated emails. +- `SIGNING_KEY`: Fingerprint passed to `gpg -u`. Leave it empty only when this + machine has exactly one usable local secret key. +- `RELEASE_NOTES_URL`: Link used in the vote email. If empty, step 03 prompts. +- `ANNOUNCE_RELEASE_NOTES_URL`: Link used in the announce email. If empty, step + 04 reuses `RELEASE_NOTES_URL` or prompts. +- `BIN_FILES`: Optional absolute paths to prebuilt convenience binary tarballs. + +The default `REPO_DIR` is the enclosing Doris checkout because these scripts +live under `tools/release-tools`. Override it only when you deliberately run the +scripts against another clone. + +The default SVN URLs target the official Doris dist repositories: + +- Dev RC source artifacts: + `https://dist.apache.org/repos/dist/dev/doris/-/` +- Final source release artifacts: + `https://dist.apache.org/repos/dist/release/doris///` + +Step 02 uploads source artifacts with the RC suffix: + +- `apache-doris---src.tar.gz` +- `apache-doris---src.tar.gz.asc` +- `apache-doris---src.tar.gz.sha512` + +Step 04 publishes final source artifacts without the RC suffix: + +- `apache-doris--src.tar.gz` +- `apache-doris--src.tar.gz.asc` +- `apache-doris--src.tar.gz.sha512` + +All generated files go to `WORK_DIR`, which defaults to +`tools/release-tools/-`. + +## Script details + +### 01-check-env.sh + +Use this script to prepare and verify the signing environment. It checks +required tools, `GPG_TTY`, GPG SHA512 digest preferences, the signing key, live +published `KEYS`, test signing, and ASF credentials. + +This script is mostly read-only. It prompts before every state-changing action, +including editing `gpg.conf`, importing or generating a key, and publishing the +key to the Doris KEYS files. + +Expected success line: + +```text +environment looks READY for - +``` + +### 02-package-sign-upload.sh + +Use this script to build, sign, checksum, and upload the RC source artifacts. + +It verifies that the local tag exists, the tag exists on `GIT_REMOTE`, and both +tags point to the same commit. It then creates the source tarball with +`git archive`, writes `.asc` and `.sha512` sidecars, verifies both, and uploads +the three source files to the Apache dev SVN. + +If `BIN_FILES` is non-empty, this script also signs and checksums those binary +tarballs in place. It does not upload binaries. + +This script pauses twice before touching the public dev SVN. Check the printed +target URL before confirming. + +### 03-vote-mail.sh + +Use this script to generate the `[VOTE]` email draft. It writes: + +- `vote-email.txt` +- `vote-email.eml` + +Review the draft and send it manually from the RM's `@apache.org` address to +`dev@doris.apache.org`. + +### 04-release-complete.sh + +Use this script only after the vote has passed and the `[RESULT]` email has +been sent manually. + +It checks the passed RC source artifacts in the dev SVN, verifies the source +tarball against the RC checksum and detached signature that voters approved, +regenerates and verifies the final checksum sidecar with the RC-free source +tarball name, then uses `svnmucc` to create the final release SVN directory, +move the source tarball and detached signature there, upload the final checksum, +and remove the dev RC folder in one SVN revision. + +It then writes: + +- `announce-email.txt` +- `announce-email.eml` + +Before sending the announce email, check that the release is visible on +`downloads.apache.org`, update and verify the Doris download page, and wait for +the download and CDN propagation window. Only after those checks pass, send the +`[ANNOUNCE]` email manually. + +Use `./04-release-complete.sh --mail-only` to regenerate the announce email +without touching SVN. + +## Tests + +Run the release-tool shell tests from this directory: + +```bash +./tests/run.sh +``` + +## Manual work not automated + +The RM must still do these tasks manually: + +- Prepare the release branch and tag. +- Validate the release before starting the RC upload flow. +- Send the `[VOTE]` email. +- Upload convenience binaries, if the release includes them. +- Tally the vote and send the `[RESULT]` email. +- Verify `downloads.apache.org` and the Doris download page before announcing. +- Wait for the download and CDN propagation window. +- Send the `[ANNOUNCE]` email. +- Publish Maven staging repositories, update the website and GitHub release + notes, and clean up old release versions when applicable. diff --git a/tools/release-tools/release.env b/tools/release-tools/release.env index 024296f9b8fcf5..f6f4c3d3e79e39 100644 --- a/tools/release-tools/release.env +++ b/tools/release-tools/release.env @@ -16,7 +16,7 @@ # under the License. # Shared config for the Apache Doris release helper scripts. -# Edit values here; 01/02/03 all `source` this file. +# Edit values here; 01/02/03/04 all `source` this file. # # This release: 4.0.6-rc02 (tag already created & pushed to apache/doris) @@ -30,10 +30,11 @@ TAG="${VERSION}-${RC}" # 4.0.6-rc02 GIT_REMOTE="upstream-apache" # remote pointing at github.com/apache/doris # --- Artifact naming --- -# Matches the live convention in release/doris/4.0/4.0.5/ : -# the rc IS kept in the file name; only the release *folder* drops it. +# Vote artifacts keep the RC in the file name. Step 04 publishes the final +# release artifacts with the RC removed from the file name. PKG_BASE="apache-doris-${TAG}-src" # -> apache-doris-4.0.6-rc02-src.tar.gz ARCHIVE_PREFIX="${PKG_BASE}/" # top-level dir inside the tarball +RELEASE_PKG_BASE="apache-doris-${VERSION}-src" # --- Work area (OUTSIDE the git repo: artifacts + svn checkouts live here) --- WORK_DIR="${ROOT}/${TAG}" @@ -67,6 +68,8 @@ SIGNING_KEY="" # use `gpg --list-keys` to get DEV_SVN_BASE="https://dist.apache.org/repos/dist/dev/doris" DEV_SVN_DIR="${DEV_SVN_BASE}/${TAG}" # <-- vote folder. VERIFY this URL before committing. RELEASE_SVN_BASE="https://dist.apache.org/repos/dist/release/doris" +RELEASE_SERIES="${VERSION%.*}" # e.g. 4.0.7 -> 4.0 +RELEASE_SVN_DIR="${RELEASE_SVN_BASE}/${RELEASE_SERIES}/${VERSION}" # final release folder, no RC suffix KEYS_URL="https://downloads.apache.org/doris/KEYS" # --- ASF credentials: export in your shell before running (NOT stored here) --- @@ -77,3 +80,7 @@ KEYS_URL="https://downloads.apache.org/doris/KEYS" DEV_LIST="dev@doris.apache.org" RELEASE_NOTES_URL="" # leave empty -> step 03 will prompt you for the issue link VERIFY_GUIDE_URL="https://doris.apache.org/community/release-and-verify/release-verify" + +# --- Announce email --- +DOWNLOAD_PAGE_URL="https://doris.apache.org/download/" +ANNOUNCE_RELEASE_NOTES_URL="" # leave empty -> step 04 will reuse RELEASE_NOTES_URL or prompt diff --git a/tools/release-tools/tests/run.sh b/tools/release-tools/tests/run.sh new file mode 100755 index 00000000000000..fec52ed29385a1 --- /dev/null +++ b/tools/release-tools/tests/run.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +for test_script in "${ROOT}"/tests/test-*.sh; do + echo "== ${test_script##*/} ==" + "$test_script" +done diff --git a/tools/release-tools/tests/test-check-env-required-tools.sh b/tools/release-tools/tests/test-check-env-required-tools.sh new file mode 100755 index 00000000000000..cde2a1ee455bcf --- /dev/null +++ b/tools/release-tools/tests/test-check-env-required-tools.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +required_tool_loop="$(awk '/^for t in /,/; do/' "${ROOT}/01-check-env.sh")" + +for tool in git gpg svn svnmucc sha512sum curl gzip; do + if ! grep -qw "$tool" <<<"$required_tool_loop"; then + echo "01-check-env.sh does not require tool: $tool" >&2 + exit 1 + fi +done diff --git a/tools/release-tools/tests/test-readme-release-complete-guidance.sh b/tools/release-tools/tests/test-readme-release-complete-guidance.sh new file mode 100755 index 00000000000000..18597a15ddee62 --- /dev/null +++ b/tools/release-tools/tests/test-readme-release-complete-guidance.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +readme="${ROOT}/README.md" + +grep -q 'downloads.apache.org' "$readme" +grep -q 'download page' "$readme" +grep -q 'propagation' "$readme" +grep -q 'Only after those checks pass, send the `\[ANNOUNCE\]` email' "$readme" diff --git a/tools/release-tools/tests/test-release-complete-checksum.sh b/tools/release-tools/tests/test-release-complete-checksum.sh new file mode 100755 index 00000000000000..0bde8c9ed31e69 --- /dev/null +++ b/tools/release-tools/tests/test-release-complete-checksum.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +cp "${ROOT}/04-release-complete.sh" "$tmp/" +cat > "$tmp/release.env" <<'EOF' +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +VERSION="9.9.9" +RC="rc01" +TAG="${VERSION}-${RC}" +PKG_BASE="apache-doris-${TAG}-src" +RELEASE_PKG_BASE="apache-doris-${VERSION}-src" +WORK_DIR="${ROOT}/${TAG}" +DEV_SVN_BASE="https://dist.example.test/dev/doris" +DEV_SVN_DIR="${DEV_SVN_BASE}/${TAG}" +RELEASE_SVN_BASE="https://dist.example.test/release/doris" +RELEASE_SERIES="${VERSION%.*}" +DOWNLOAD_PAGE_URL="https://doris.example.test/download/" +ANNOUNCE_RELEASE_NOTES_URL="https://doris.example.test/release-notes" +RELEASE_NOTES_URL="" +DEV_LIST="dev@example.test" +SIGNER_NAME="Release Manager" +EOF + +cat > "$tmp/svn" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +cmd="$1" +shift +case "$cmd" in + info) + url="${@: -1}" + case "$url" in + https://dist.example.test/release/doris/9.9|https://dist.example.test/release/doris/9.9/9.9.9) + exit 1 + ;; + esac + ;; + cat) + url="${@: -1}" + printf '%s\n' "$url" >> "$FAKE_SVN_CAT_LOG" + case "$url" in + *.tar.gz) + printf 'release artifact bytes\n' + ;; + *.sha512) + if [[ "${FAKE_BAD_SHA512:-0}" -eq 1 ]]; then + digest="00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + else + digest="$(printf 'release artifact bytes\n' | sha512sum | awk '{print $1}')" + fi + printf '%s apache-doris-9.9.9-rc01-src.tar.gz\n' "$digest" + ;; + *.asc) + printf 'fake detached signature\n' + ;; + *) + echo "unexpected svn cat url: $url" >&2 + exit 1 + ;; + esac + ;; + *) + echo "unexpected svn command: $cmd" >&2 + exit 1 + ;; +esac +EOF +chmod +x "$tmp/svn" + +cat > "$tmp/mktemp" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +[[ "$1" == "-d" ]] +dir="${FAKE_MKTEMP_ROOT}/checksum-dir-${RANDOM}-${RANDOM}" +mkdir -p "$dir" +printf '%s\n' "$dir" >> "$FAKE_MKTEMP_LOG" +printf '%s\n' "$dir" +EOF +chmod +x "$tmp/mktemp" + +cat > "$tmp/gpg" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" > "$FAKE_GPG_LOG" +[[ "$1" == "--verify" ]] +[[ "$2" == "apache-doris-9.9.9-rc01-src.tar.gz.asc" ]] +[[ "$3" == "apache-doris-9.9.9-rc01-src.tar.gz" ]] +[[ -f "$2" ]] +[[ -f "$3" ]] +EOF +chmod +x "$tmp/gpg" + +cat > "$tmp/svnmucc" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" > "$FAKE_SVNMUCC_LOG" +series_dir_created=0 +while (($#)); do + case "$1" in + mkdir) + if [[ "$2" == "https://dist.example.test/release/doris/9.9" ]]; then + series_dir_created=1 + elif [[ "$2" == "https://dist.example.test/release/doris/9.9/9.9.9" && "$series_dir_created" -eq 0 ]]; then + echo "release series directory must be created before release leaf directory" >&2 + exit 1 + fi + shift 2 + ;; + mv) + shift 3 + ;; + put) + checksum_file="$2" + cp "$checksum_file" "$FAKE_FINAL_SHA512" + shift 3 + ;; + rm) + shift 2 + ;; + -m|-u|-p) + shift 2 + ;; + --non-interactive|--no-auth-cache) + shift + ;; + *) + shift + ;; + esac +done +EOF +chmod +x "$tmp/svnmucc" + +export PATH="$tmp:$PATH" +export FAKE_SVN_CAT_LOG="$tmp/svn-cat.log" +export FAKE_GPG_LOG="$tmp/gpg.log" +export FAKE_SVNMUCC_LOG="$tmp/svnmucc.log" +export FAKE_FINAL_SHA512="$tmp/final.sha512" +export FAKE_MKTEMP_ROOT="$tmp/mktemp-root" +export FAKE_MKTEMP_LOG="$tmp/mktemp.log" +mkdir -p "$FAKE_MKTEMP_ROOT" + +printf 'y\n' | bash "$tmp/04-release-complete.sh" >/dev/null + +if grep -q 'mv https://dist.example.test/dev/doris/9.9.9-rc01/apache-doris-9.9.9-rc01-src.tar.gz.sha512' "$FAKE_SVNMUCC_LOG"; then + echo "release completion must not move the RC checksum sidecar unchanged" >&2 + exit 1 +fi + +grep -q 'apache-doris-9.9.9-rc01-src.tar.gz$' "$FAKE_SVN_CAT_LOG" +grep -q 'apache-doris-9.9.9-rc01-src.tar.gz.sha512$' "$FAKE_SVN_CAT_LOG" +grep -q 'apache-doris-9.9.9-rc01-src.tar.gz.asc$' "$FAKE_SVN_CAT_LOG" +grep -q -- '--verify apache-doris-9.9.9-rc01-src.tar.gz.asc apache-doris-9.9.9-rc01-src.tar.gz' "$FAKE_GPG_LOG" +grep -q 'mkdir https://dist.example.test/release/doris/9.9' "$FAKE_SVNMUCC_LOG" +grep -q 'https://dist.example.test/release/doris/9.9/9.9.9/apache-doris-9.9.9-src.tar.gz' "$FAKE_SVNMUCC_LOG" +grep -q 'put ' "$FAKE_SVNMUCC_LOG" +grep -q 'apache-doris-9.9.9-src.tar.gz$' "$FAKE_FINAL_SHA512" +if grep -q 'apache-doris-9.9.9-rc01-src.tar.gz' "$FAKE_FINAL_SHA512"; then + echo "final checksum sidecar still references the RC tarball name" >&2 + exit 1 +fi + +success_checksum_dir="$(tail -n 1 "$FAKE_MKTEMP_LOG")" +if [[ -e "$success_checksum_dir" ]]; then + echo "checksum temp directory was not removed after successful publish: $success_checksum_dir" >&2 + exit 1 +fi + +if FAKE_BAD_SHA512=1 bash "$tmp/04-release-complete.sh" >/dev/null 2>&1; then + echo "release completion unexpectedly succeeded with a bad RC checksum" >&2 + exit 1 +fi + +failed_checksum_dir="$(tail -n 1 "$FAKE_MKTEMP_LOG")" +if [[ -e "$failed_checksum_dir" ]]; then + echo "checksum temp directory was not removed after checksum verification failure: $failed_checksum_dir" >&2 + exit 1 +fi