From 3bbb7fe0269cf1fd0986fbe62b1ab04da928f363 Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Thu, 6 Aug 2026 10:18:56 -0700 Subject: [PATCH 1/2] Add Python pipelines for ARH artifact generation. --- .../jobs/apireview-hub-job-python.yml | 77 ++++++++++++++++ .../create-apireview-hub-artifacts-python.yml | 92 +++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 eng/pipelines/templates/jobs/apireview-hub-job-python.yml create mode 100644 eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml diff --git a/eng/pipelines/templates/jobs/apireview-hub-job-python.yml b/eng/pipelines/templates/jobs/apireview-hub-job-python.yml new file mode 100644 index 000000000000..64cd5de12566 --- /dev/null +++ b/eng/pipelines/templates/jobs/apireview-hub-job-python.yml @@ -0,0 +1,77 @@ +# Python job wrapper for API Review Hub requests. This file owns Python-specific +# setup and delegates shared job boilerplate to the base API Review Hub job. +# The caller supplies the create/update orchestration steps. +parameters: + - name: poolName + type: string + default: 'azsdk-pool' + - name: imageOverride + type: string + default: 'ubuntu-24.04' + - name: repositoryOwner + type: string + default: 'Azure' + - name: toolRef + type: string + default: 'main' + - name: pythonVersion + type: string + default: '3.12' + - name: steps + type: stepList + default: [] + +jobs: +# TODO: Once the base templates are merged and synced, this should be changed to refernce the local version +- template: /eng/common/pipelines/templates/jobs/apireview-hub-job-base.yml@self + parameters: + jobName: CreatePythonApiReviewArtifacts + displayName: 'Create Python API review artifacts' + poolName: ${{ parameters.poolName }} + imageOverride: ${{ parameters.imageOverride }} + sourceRepositoryFullName: ${{ format('{0}/azure-sdk-for-python', parameters.repositoryOwner) }} + sourceCheckoutDir: $(Pipeline.Workspace)/apireview/source + setupSteps: + - task: UsePythonVersion@0 + displayName: 'Use Python ${{ parameters.pythonVersion }}' + inputs: + versionSpec: ${{ parameters.pythonVersion }} + + # Install Python API review tooling once for the job. The base and target + # generation steps reuse this prepared environment. + - bash: | + set -Eeuo pipefail + + source_repo="$(ApiReviewSourceDir)" + tooling_dir="$(ApiReviewToolingDir)" + requirements_file="$source_repo/eng/apiview_reqs.txt" + azpysdk_package="$source_repo/eng/tools/azure-sdk-tools[build]" + + echo "Checking out Python API review tooling requirements from ${{ parameters.toolRef }}" + git -C "$source_repo" fetch --depth=1 origin "${{ parameters.toolRef }}" + git -C "$source_repo" checkout --force FETCH_HEAD + git -C "$source_repo" clean -ffd + + if [ ! -f "$requirements_file" ]; then + echo "Expected Python API review requirements file was not found at $requirements_file" >&2 + exit 1 + fi + + mkdir -p "$tooling_dir/eng/common/scripts" "$tooling_dir/eng/scripts" + cp "$source_repo/eng/common/scripts/Export-APIViewMarkdown.ps1" "$tooling_dir/eng/common/scripts/Export-APIViewMarkdown.ps1" + cp "$source_repo/eng/scripts/Extract-APIViewMetadata-Python.ps1" "$tooling_dir/eng/scripts/Extract-APIViewMetadata-Python.ps1" + + python -m pip install --upgrade pip + python -m pip install virtualenv + + echo "Installing Python API review requirements from $requirements_file" + python -m pip install -r "$requirements_file" --index-url="https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/" + + echo "Installing azpysdk tooling from ${{ parameters.toolRef }}" + python -m pip install "$azpysdk_package" + + echo "Installing setuptools runtime dependency for Python SDK common_tasks.py" + python -m pip install "setuptools<81" + PYTHONPATH="$source_repo/scripts/devops_tasks${PYTHONPATH:+:$PYTHONPATH}" python -c "import common_tasks; import pkg_resources; import azpysdk.main" + displayName: 'Install Python API review tooling' + steps: ${{ parameters.steps }} \ No newline at end of file diff --git a/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml b/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml new file mode 100644 index 000000000000..e5b8f11bb24a --- /dev/null +++ b/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml @@ -0,0 +1,92 @@ +# Generate one Python API review artifact bundle for one ref. This template +# supplies Python-specific generation and delegates shared artifact validation +# and metadata writing to the base API Review Hub artifact template. +parameters: + - name: requestMode + type: string + - name: operationId + type: string + - name: packageName + type: string + - name: ref + type: string + - name: kind + type: string + - name: outputDir + type: string + - name: workingDir + type: string +steps: + # TODO: Once the base templates are merged and synced, this should be changed to refernce the local version + - template: /eng/common/pipelines/templates/steps/create-apireview-hub-artifacts-base.yml@self + parameters: + requestMode: ${{ parameters.requestMode }} + operationId: ${{ parameters.operationId }} + packageName: ${{ parameters.packageName }} + ref: ${{ parameters.ref }} + kind: ${{ parameters.kind }} + outputDir: ${{ parameters.outputDir }} + workingDir: ${{ parameters.workingDir }} + sourceDir: $(ApiReviewSourceDir) + language: python + repositoryFullName: $(ApiReviewRepositoryFullName) + generationSteps: + - bash: | + set -Eeuo pipefail + + source_repo="$(ApiReviewSourceDir)" + tooling_dir="$(ApiReviewToolingDir)" + output_dir="${{ parameters.outputDir }}" + state_dir="${{ parameters.workingDir }}/state" + + mkdir -p "$source_repo" "$output_dir" "$state_dir" + + find_package_dir() { + local repo="$1" + local package_name="$2" + local matches + + matches=$(find "$repo/sdk" -mindepth 2 -maxdepth 2 -type d -name "$package_name" | sort) + if [ -z "$matches" ]; then + echo "Package '$package_name' was not found under sdk/*/." >&2 + return 1 + fi + + if [ "$(printf '%s\n' "$matches" | wc -l)" -ne 1 ]; then + echo "Package '$package_name' matched multiple directories:" >&2 + printf '%s\n' "$matches" >&2 + return 1 + fi + + printf '%s' "$matches" + } + + package_dir=$(find_package_dir "$source_repo" "${{ parameters.packageName }}") + package_relative_path=$(realpath --relative-to="$source_repo" "$package_dir") + + echo "Generating ${{ parameters.kind }} API review artifact from ${{ parameters.ref }}" + common_tasks_dir="$source_repo/scripts/devops_tasks" + if [ ! -f "$common_tasks_dir/common_tasks.py" ]; then + echo "Expected Python SDK common_tasks.py was not found at $common_tasks_dir/common_tasks.py" >&2 + exit 1 + fi + install -D "$tooling_dir/eng/common/scripts/Export-APIViewMarkdown.ps1" "$source_repo/eng/common/scripts/Export-APIViewMarkdown.ps1" + install -D "$tooling_dir/eng/scripts/Extract-APIViewMetadata-Python.ps1" "$source_repo/eng/scripts/Extract-APIViewMetadata-Python.ps1" + (cd "$source_repo" && PYTHONPATH="$common_tasks_dir${PYTHONPATH:+:$PYTHONPATH}" python -m azpysdk.main apistub --dest-dir "$package_dir" "${{ parameters.packageName }}") + + if [ ! -f "$package_dir/api.md" ]; then + echo "Expected api.md was not generated at $package_dir/api.md" >&2 + exit 1 + fi + if [ ! -f "$package_dir/api.metadata.yml" ]; then + echo "Expected api.metadata.yml was not generated at $package_dir/api.metadata.yml" >&2 + exit 1 + fi + + cp "$package_dir/api.md" "$output_dir/api.md" + cp "$package_dir/api.metadata.yml" "$output_dir/api.metadata.yml" + + version=$(python -c 'import pathlib, re, sys; package_dir = pathlib.Path(sys.argv[1]); pattern = re.compile(r"^\s*VERSION\s*[:=]\s*[\"'"'"']([^\"'"'"']+)[\"'"'"']", re.MULTILINE); candidates = sorted(package_dir.rglob("*_version.py")) + sorted(package_dir.rglob("version.py")); match = next((pattern.search(candidate.read_text(encoding="utf-8")) for candidate in candidates if "_generated" not in candidate.parts and pattern.search(candidate.read_text(encoding="utf-8"))), None); print(match.group(1) if match else "")' "$package_dir") + printf '%s' "$package_relative_path" > "$state_dir/package-relative-path.txt" + printf '%s' "$version" > "$state_dir/version.txt" + displayName: 'Generate ${{ parameters.kind }} Python API review bundle' \ No newline at end of file From 9972b56a934d3c8adb52bd4acaf1451bce32a207 Mon Sep 17 00:00:00 2001 From: Travis Prescott Date: Fri, 7 Aug 2026 10:24:47 -0700 Subject: [PATCH 2/2] Remove `@self` annotations. --- eng/pipelines/templates/jobs/apireview-hub-job-python.yml | 3 +-- .../templates/steps/create-apireview-hub-artifacts-python.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/templates/jobs/apireview-hub-job-python.yml b/eng/pipelines/templates/jobs/apireview-hub-job-python.yml index 64cd5de12566..2414fbb5b28f 100644 --- a/eng/pipelines/templates/jobs/apireview-hub-job-python.yml +++ b/eng/pipelines/templates/jobs/apireview-hub-job-python.yml @@ -22,8 +22,7 @@ parameters: default: [] jobs: -# TODO: Once the base templates are merged and synced, this should be changed to refernce the local version -- template: /eng/common/pipelines/templates/jobs/apireview-hub-job-base.yml@self +- template: /eng/common/pipelines/templates/jobs/apireview-hub-job-base.yml parameters: jobName: CreatePythonApiReviewArtifacts displayName: 'Create Python API review artifacts' diff --git a/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml b/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml index e5b8f11bb24a..353fc543cf9c 100644 --- a/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml +++ b/eng/pipelines/templates/steps/create-apireview-hub-artifacts-python.yml @@ -17,8 +17,7 @@ parameters: - name: workingDir type: string steps: - # TODO: Once the base templates are merged and synced, this should be changed to refernce the local version - - template: /eng/common/pipelines/templates/steps/create-apireview-hub-artifacts-base.yml@self + - template: /eng/common/pipelines/templates/steps/create-apireview-hub-artifacts-base.yml parameters: requestMode: ${{ parameters.requestMode }} operationId: ${{ parameters.operationId }}