diff --git a/.github/workflows/build-test-publish-wheel.yml b/.github/workflows/build-test-publish-wheel.yml deleted file mode 100644 index 02986742..00000000 --- a/.github/workflows/build-test-publish-wheel.yml +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Build, test, and publish a PyPi wheel (to testpypi) - -on: - push: - branches: - - "main" - - "[rv][0-9].[0-9].[0-9]" - - "[rv][0-9].[0-9].[0-9]rc[0-9]" - pull_request: - -defaults: - run: - shell: bash -x -e -u -o pipefail {0} - -jobs: - build-test-publish-wheel: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.88.1 - with: - dry-run: true - python-package: nemo_run - python-version: "3.10" - packaging: setuptools - secrets: - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ENDPOINT }} - SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }} diff --git a/.github/workflows/config/changelog-config.json b/.github/workflows/config/changelog-config.json index 0b78b096..0a2838c1 100644 --- a/.github/workflows/config/changelog-config.json +++ b/.github/workflows/config/changelog-config.json @@ -1,118 +1,194 @@ { "categories": [ - { - "title": "## Executors\n\n", - "labels": ["executor", "local", "slurm", "dgxcloud", "lepton", "skypilot", "docker"], - "exclude_labels": ["ignore"] - }, - { - "title": "\n## Ray Integration\n\n", - "labels": ["ray", "kuberay", "ray-slurm"], - "exclude_labels": ["ignore"] - }, - { - "title": "\n## CLI & Configuration\n\n", - "labels": ["cli", "config", "parsing"], - "exclude_labels": ["ignore"] - }, - { - "title": "\n## Experiment & Job Management\n\n", - "labels": ["experiment", "job", "task"], - "exclude_labels": ["ignore"] - }, - { - "title": "\n## Packaging & Deployment\n\n", - "labels": ["packaging", "deployment"], - "exclude_labels": ["ignore"] - }, - { - "title": "\n## Documentation\n\n", - "labels": ["docs", "documentation"], - "exclude_labels": ["ignore"] - }, - { - "title": "\n## CI/CD\n\n", - "labels": ["ci", "github-actions", "workflow"], - "exclude_labels": ["ignore"] - }, - { - "title": "\n## Bug Fixes\n\n", - "labels": ["bug", "bugfix", "fix"], - "exclude_labels": ["ignore"] - } + { + "title": "## Executors\n\n", + "labels": [ + "executor", + "local", + "slurm", + "dgxcloud", + "lepton", + "skypilot", + "docker" + ], + "exclude_labels": [ + "ignore" + ] + }, + { + "title": "\n## Ray Integration\n\n", + "labels": [ + "ray", + "kuberay", + "ray-slurm" + ], + "exclude_labels": [ + "ignore" + ] + }, + { + "title": "\n## CLI & Configuration\n\n", + "labels": [ + "cli", + "config", + "parsing" + ], + "exclude_labels": [ + "ignore" + ] + }, + { + "title": "\n## Experiment & Job Management\n\n", + "labels": [ + "experiment", + "job", + "task" + ], + "exclude_labels": [ + "ignore" + ] + }, + { + "title": "\n## Packaging & Deployment\n\n", + "labels": [ + "packaging", + "deployment" + ], + "exclude_labels": [ + "ignore" + ] + }, + { + "title": "\n## Documentation\n\n", + "labels": [ + "docs", + "documentation" + ], + "exclude_labels": [ + "ignore" + ] + }, + { + "title": "\n## CI/CD\n\n", + "labels": [ + "ci", + "github-actions", + "workflow" + ], + "exclude_labels": [ + "ignore" + ] + }, + { + "title": "\n## Bug Fixes\n\n", + "labels": [ + "bug", + "bugfix", + "fix" + ], + "exclude_labels": [ + "ignore" + ] + } ], "ignore_labels": [ - "ignore", - "skip-changelog" + "ignore", + "skip-changelog" ], "sort": "ASC", "template": "\n${{CHANGELOG}}\n## Others\n\n${{UNCATEGORIZED}}\n", "pr_template": "- ${{TITLE}} [#${{NUMBER}}](${{URL}})", "empty_template": "- No changes in this release", "label_extractor": [ - { - "pattern": "(.*executor.*)|(.*local.*)|(.*slurm.*)|(.*dgxcloud.*)|(.*lepton.*)|(.*skypilot.*)|(.*docker.*)", - "target": "executor", - "flags": "gimu", - "on_property": ["title", "body"] - }, - { - "pattern": "(.*ray.*)|(.*kuberay.*)", - "target": "ray", - "flags": "gimu", - "on_property": ["title", "body"] - }, - { - "pattern": "(.*cli.*)|(.*command.*)|(.*parse.*)|(.*argument.*)", - "target": "cli", - "flags": "gimu", - "on_property": ["title", "body"] - }, - { - "pattern": "(.*experiment.*)|(.*job.*)|(.*task.*)", - "target": "experiment", - "flags": "gimu", - "on_property": ["title", "body"] - }, - { - "pattern": "(.*packaging.*)|(.*package.*)|(.*deploy.*)|(.*archive.*)|(.*mount.*)", - "target": "packaging", - "flags": "gimu", - "on_property": ["title", "body"] - }, - { - "pattern": "(.*doc.*)|(.*readme.*)|(.*guide.*)|(.*tutorial.*)", - "target": "docs", - "flags": "gimu", - "on_property": ["title", "body"] - }, - { - "pattern": "(.*\\bci\\b.*)|(.*github.*)|(.*workflow.*)|(.*action.*)", - "target": "ci", - "flags": "gimu", - "on_property": ["title", "body"] - }, - { - "pattern": "(.*\\[bug.*)|(.*\\bfix\\b.*)|(.*bugfix.*)|(.*patch.*)", - "target": "bug", - "flags": "gimu", - "on_property": ["title", "body"] - } + { + "pattern": "(.*executor.*)|(.*local.*)|(.*slurm.*)|(.*dgxcloud.*)|(.*lepton.*)|(.*skypilot.*)|(.*docker.*)", + "target": "executor", + "flags": "gimu", + "on_property": [ + "title", + "body" + ] + }, + { + "pattern": "(.*ray.*)|(.*kuberay.*)", + "target": "ray", + "flags": "gimu", + "on_property": [ + "title", + "body" + ] + }, + { + "pattern": "(.*cli.*)|(.*command.*)|(.*parse.*)|(.*argument.*)", + "target": "cli", + "flags": "gimu", + "on_property": [ + "title", + "body" + ] + }, + { + "pattern": "(.*experiment.*)|(.*job.*)|(.*task.*)", + "target": "experiment", + "flags": "gimu", + "on_property": [ + "title", + "body" + ] + }, + { + "pattern": "(.*packaging.*)|(.*package.*)|(.*deploy.*)|(.*archive.*)|(.*mount.*)", + "target": "packaging", + "flags": "gimu", + "on_property": [ + "title", + "body" + ] + }, + { + "pattern": "(.*doc.*)|(.*readme.*)|(.*guide.*)|(.*tutorial.*)", + "target": "docs", + "flags": "gimu", + "on_property": [ + "title", + "body" + ] + }, + { + "pattern": "(.*\\bci\\b.*)|(.*github.*)|(.*workflow.*)|(.*action.*)", + "target": "ci", + "flags": "gimu", + "on_property": [ + "title", + "body" + ] + }, + { + "pattern": "(.*\\[bug.*)|(.*\\bfix\\b.*)|(.*bugfix.*)|(.*patch.*)", + "target": "bug", + "flags": "gimu", + "on_property": [ + "title", + "body" + ] + } ], "duplicate_filter": { - "pattern": ".+", - "on_property": "title", - "method": "match" + "pattern": ".+", + "on_property": "title", + "method": "match" }, "transformers": [ + { + "pattern": "^cp:\\s*`(.+?)`\\s*into\\s*\\S+", + "target": "$1" + } ], "max_tags_to_fetch": 100, "max_pull_requests": 500, "max_back_track_time_days": 365, - "exclude_merge_branches": [ - ], + "exclude_merge_branches": [], "tag_resolver": { - "method": "semver" + "method": "semver" } -} - +} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a7638cd6..a8ff162e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION. +# Copyright (c) 2020-2026, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,9 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -name: "Release NeMo Run" +name: "Build, validate, and release NeMo Run" on: + push: + branches: + - "main" + - "[rv][0-9].[0-9].[0-9]" + - "[rv][0-9].[0-9].[0-9]rc[0-9]" + - "deploy-release/*" + pull_request: workflow_dispatch: inputs: release-ref: @@ -21,7 +28,7 @@ on: required: true type: string dry-run: - description: Do not publish a wheel and GitHub release. + description: Compute the release but do not publish wheel, GH release, or docs. required: true default: true type: boolean @@ -39,35 +46,56 @@ on: type: string required: true description: Branch to target for version bump + +defaults: + run: + shell: bash -x -e -u -o pipefail {0} + +permissions: + id-token: write + contents: write + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: ${{ github.event_name != 'workflow_dispatch' }} + jobs: release: - uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_release_library.yml@v0.81.1 + if: '!cancelled()' + uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_release_library.yml@v1.0.0 with: - release-ref: ${{ inputs.release-ref }} + release-ref: ${{ inputs.release-ref || github.sha }} python-package: nemo_run python-version: "3.10" library-name: NeMo Run + validate-only: ${{ github.event_name != 'workflow_dispatch' }} + dry-run: ${{ inputs.dry-run || false }} + version-bump-branch: ${{ inputs.version-bump-branch || github.ref_name }} app-id: ${{ vars.BOT_ID }} - dry-run: ${{ inputs.dry-run }} - version-bump-branch: ${{ inputs.version-bump-branch }} - gh-release-use-changelog-builder: ${{ inputs.generate-changelog }} - publish-docs: ${{ inputs.publish-docs }} + packaging: setuptools + gh-release-use-changelog-builder: ${{ inputs.generate-changelog || false }} + publish-docs: ${{ inputs.publish-docs || true }} docs-target-path: nemo/run - packaging: hatch - secrets: - TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }} - TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} - SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }} - SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ENDPOINT }} - PAT: ${{ secrets.PAT }} - SSH_KEY: ${{ secrets.SSH_KEY }} - SSH_PWD: ${{ secrets.SSH_PWD }} - BOT_KEY: ${{ secrets.BOT_KEY }} - AWS_ASSUME_ROLE_ARN: ${{ secrets.AWS_ASSUME_ROLE_ARN }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AKAMAI_HOST: ${{ secrets.AKAMAI_HOST }} - AKAMAI_CLIENT_TOKEN: ${{ secrets.AKAMAI_CLIENT_TOKEN }} - AKAMAI_CLIENT_SECRET: ${{ secrets.AKAMAI_CLIENT_SECRET }} - AKAMAI_ACCESS_TOKEN: ${{ secrets.AKAMAI_ACCESS_TOKEN }} - S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} + restrict-to-admins: true + secrets: inherit # pragma: allowlist secret + + release-summary: + needs: [release] + if: ${{ !cancelled() }} + runs-on: ubuntu-latest + steps: + - name: Result + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --repo ${{ github.repository }} --json jobs --jq '[.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required")] | length') + + if [ "${FAILED_JOBS:-0}" -eq 0 ]; then + echo "✅ All previous jobs completed successfully" + exit 0 + else + echo "❌ Found $FAILED_JOBS failed job(s)" + gh run view $GITHUB_RUN_ID --repo ${{ github.repository }} --json jobs --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "action_required") | .name' + exit 1 + fi